diff --git a/.github/workflows/pr-approve-status.yml b/.github/workflows/pr-approve-status.yml index 2a6a64355d5b66e..cda6abc8f1c15b3 100644 --- a/.github/workflows/pr-approve-status.yml +++ b/.github/workflows/pr-approve-status.yml @@ -43,9 +43,7 @@ jobs: approves=() reviewers_unique=() for ((i=${#reviewers[@]}-1;i>=0;i--)); do - # shellcheck disable=SC2076 - # shellcheck disable=SC2199 - if [[ ! "${reviewers_unique[@]}" =~ "${reviewers[$i]}" ]]; then + if ! echo "${reviewers_unique[@]}" | grep -q -w "${reviewers[$i]}" && [ "${statuses[$i]}" != "COMMENTED" ]; then reviewers_unique+=( "${reviewers[$i]}" ) if [ "${statuses[$i]}" == "APPROVED" ]; then approves+=( "${reviewers[$i]}" ) diff --git a/.gitignore b/.gitignore index e6981ccb7a7b565..573eaf58baa5ea3 100644 --- a/.gitignore +++ b/.gitignore @@ -13,8 +13,8 @@ core.* .DS_Store .classpath nohup.out -custom_env.sh -custom_env_mac.sh +/custom_env.sh +/custom_env_mac.sh derby.log dependency-reduced-pom.xml yarn.lock @@ -33,6 +33,7 @@ package-lock.json .cache .settings/ **/.idea/ +!.idea/vcs.xml **/.vscode/ **/.fleet/ @@ -42,6 +43,7 @@ docs/.temp # output, thirdparty, extension output/ +output.bak/ rpc_data/ metastore_db/ thirdparty/src* diff --git a/.idea/vcs.xml b/.idea/vcs.xml new file mode 100644 index 000000000000000..77fcf54857d40e7 --- /dev/null +++ b/.idea/vcs.xml @@ -0,0 +1,32 @@ + + + + + + + + + + \ No newline at end of file diff --git a/be/src/common/config.cpp b/be/src/common/config.cpp index 1726a676b5ec860..fe5e4b5b8d1320c 100644 --- a/be/src/common/config.cpp +++ b/be/src/common/config.cpp @@ -506,25 +506,6 @@ DEFINE_Int32(min_buffer_size, "1024"); // 1024, The minimum read buffer size (in // With 1024B through 8MB buffers, this is up to ~2GB of buffers. DEFINE_Int32(max_free_io_buffers, "128"); -// Whether to disable the memory cache pool, -// including MemPool, ChunkAllocator, DiskIO free buffer. -DEFINE_Bool(disable_mem_pools, "false"); - -// The reserved bytes limit of Chunk Allocator, usually set as a percentage of mem_limit. -// defaults to bytes if no unit is given, the number of bytes must be a multiple of 2. -// must larger than 0. and if larger than physical memory size, it will be set to physical memory size. -// increase this variable can improve performance, -// but will acquire more free memory which can not be used by other modules. -DEFINE_mString(chunk_reserved_bytes_limit, "0"); -// 1024, The minimum chunk allocator size (in bytes) -DEFINE_Int32(min_chunk_reserved_bytes, "1024"); -// Disable Chunk Allocator in Vectorized Allocator, this will reduce memory cache. -// For high concurrent queries, using Chunk Allocator with vectorized Allocator can reduce the impact -// of gperftools tcmalloc central lock. -// Jemalloc or google tcmalloc have core cache, Chunk Allocator may no longer be needed after replacing -// gperftools tcmalloc. -DEFINE_mBool(disable_chunk_allocator_in_vec, "true"); - // The probing algorithm of partitioned hash table. // Enable quadratic probing hash table DEFINE_Bool(enable_quadratic_probing, "false"); @@ -1043,6 +1024,9 @@ DEFINE_Bool(enable_set_in_bitmap_value, "false"); DEFINE_Int64(max_hdfs_file_handle_cache_num, "20000"); DEFINE_Int64(max_external_file_meta_cache_num, "20000"); +// max_write_buffer_number for rocksdb +DEFINE_Int32(rocksdb_max_write_buffer_number, "5"); + #ifdef BE_TEST // test s3 DEFINE_String(test_s3_resource, "resource"); diff --git a/be/src/common/config.h b/be/src/common/config.h index 81b5308132a8447..639d4a350edfbf3 100644 --- a/be/src/common/config.h +++ b/be/src/common/config.h @@ -543,25 +543,6 @@ DECLARE_Int32(min_buffer_size); // 1024, The minimum read buffer size (in bytes) // With 1024B through 8MB buffers, this is up to ~2GB of buffers. DECLARE_Int32(max_free_io_buffers); -// Whether to disable the memory cache pool, -// including MemPool, ChunkAllocator, DiskIO free buffer. -DECLARE_Bool(disable_mem_pools); - -// The reserved bytes limit of Chunk Allocator, usually set as a percentage of mem_limit. -// defaults to bytes if no unit is given, the number of bytes must be a multiple of 2. -// must larger than 0. and if larger than physical memory size, it will be set to physical memory size. -// increase this variable can improve performance, -// but will acquire more free memory which can not be used by other modules. -DECLARE_mString(chunk_reserved_bytes_limit); -// 1024, The minimum chunk allocator size (in bytes) -DECLARE_Int32(min_chunk_reserved_bytes); -// Disable Chunk Allocator in Vectorized Allocator, this will reduce memory cache. -// For high concurrent queries, using Chunk Allocator with vectorized Allocator can reduce the impact -// of gperftools tcmalloc central lock. -// Jemalloc or google tcmalloc have core cache, Chunk Allocator may no longer be needed after replacing -// gperftools tcmalloc. -DECLARE_mBool(disable_chunk_allocator_in_vec); - // The probing algorithm of partitioned hash table. // Enable quadratic probing hash table DECLARE_Bool(enable_quadratic_probing); @@ -1059,6 +1040,9 @@ DECLARE_Int64(max_hdfs_file_handle_cache_num); // max number of meta info of external files, such as parquet footer DECLARE_Int64(max_external_file_meta_cache_num); +// max_write_buffer_number for rocksdb +DECLARE_Int32(rocksdb_max_write_buffer_number); + #ifdef BE_TEST // test s3 DECLARE_String(test_s3_resource); diff --git a/be/src/common/daemon.cpp b/be/src/common/daemon.cpp index 7c64596bac87d83..cfb002a4c45c17d 100644 --- a/be/src/common/daemon.cpp +++ b/be/src/common/daemon.cpp @@ -245,7 +245,7 @@ void Daemon::memory_gc_thread() { // No longer full gc and minor gc during sleep. memory_full_gc_sleep_time_ms = config::memory_gc_sleep_time_ms; memory_minor_gc_sleep_time_ms = config::memory_gc_sleep_time_ms; - doris::MemTrackerLimiter::print_log_process_usage("process full gc", false); + doris::MemTrackerLimiter::print_log_process_usage("Start Full GC", false); if (doris::MemInfo::process_full_gc()) { // If there is not enough memory to be gc, the process memory usage will not be printed in the next continuous gc. doris::MemTrackerLimiter::enable_print_log_process_usage(); @@ -255,7 +255,7 @@ void Daemon::memory_gc_thread() { proc_mem_no_allocator_cache >= doris::MemInfo::soft_mem_limit())) { // No minor gc during sleep, but full gc is possible. memory_minor_gc_sleep_time_ms = config::memory_gc_sleep_time_ms; - doris::MemTrackerLimiter::print_log_process_usage("process minor gc", false); + doris::MemTrackerLimiter::print_log_process_usage("Start Minor GC", false); if (doris::MemInfo::process_minor_gc()) { doris::MemTrackerLimiter::enable_print_log_process_usage(); } diff --git a/be/src/http/action/pprof_actions.cpp b/be/src/http/action/pprof_actions.cpp index 1d7128127379368..1cbe2163b9374f7 100644 --- a/be/src/http/action/pprof_actions.cpp +++ b/be/src/http/action/pprof_actions.cpp @@ -42,7 +42,7 @@ namespace doris { // pprof default sample time in seconds. -static const std::string SECOND_KEY = "seconds"; +[[maybe_unused]] static const std::string SECOND_KEY = "seconds"; static const int kPprofDefaultSampleSecs = 30; // Protect, only one thread can work diff --git a/be/src/io/fs/benchmark/base_benchmark.h b/be/src/io/fs/benchmark/base_benchmark.h index c28ad02de508e4a..41dae7cea23e1ef 100644 --- a/be/src/io/fs/benchmark/base_benchmark.h +++ b/be/src/io/fs/benchmark/base_benchmark.h @@ -27,6 +27,9 @@ #include #include "common/status.h" +#include "io/fs/file_reader.h" +#include "io/fs/file_writer.h" +#include "util/slice.h" namespace doris::io { @@ -44,24 +47,22 @@ void bm_log(const std::string& fmt, Args&&... args) { class BaseBenchmark { public: BaseBenchmark(const std::string& name, int threads, int iterations, size_t file_size, - int repetitions, const std::map& conf_map) + const std::map& conf_map) : _name(name), _threads(threads), _iterations(iterations), _file_size(file_size), - _repetitions(repetitions), _conf_map(conf_map) {} virtual ~BaseBenchmark() = default; virtual Status init() { return Status::OK(); } virtual Status run(benchmark::State& state) { return Status::OK(); } + void set_repetition(int rep) { _repetitions = rep; } + void register_bm() { auto bm = benchmark::RegisterBenchmark(_name.c_str(), [&](benchmark::State& state) { - Status st; - if (state.thread_index() == 0) { - st = this->init(); - } + Status st = this->init(); if (st != Status::OK()) { bm_log("Benchmark {} init error: {}", _name, st.to_string()); return; @@ -92,12 +93,114 @@ class BaseBenchmark { }); } + virtual std::string get_file_path(benchmark::State& state) { + std::string base_dir = _conf_map["base_dir"]; + std::string file_path; + if (base_dir.ends_with("/")) { + file_path = fmt::format("{}test_{}", base_dir, state.thread_index()); + } else { + file_path = fmt::format("{}/test_{}", base_dir, state.thread_index()); + } + bm_log("file_path: {}", file_path); + return file_path; + } + + Status read(benchmark::State& state, FileReaderSPtr reader) { + bm_log("begin to read {}", _name); + size_t buffer_size = + _conf_map.contains("buffer_size") ? std::stol(_conf_map["buffer_size"]) : 1000000L; + std::vector buffer; + buffer.resize(buffer_size); + doris::Slice data = {buffer.data(), buffer.size()}; + size_t offset = 0; + size_t bytes_read = 0; + + size_t read_size = reader->size(); + if (_file_size > 0) { + read_size = std::min(read_size, _file_size); + } + long remaining_size = read_size; + + Status status; + auto start = std::chrono::high_resolution_clock::now(); + while (remaining_size > 0) { + bytes_read = 0; + size_t size = std::min(buffer_size, (size_t)remaining_size); + data.size = size; + status = reader->read_at(offset, data, &bytes_read); + if (status != Status::OK() || bytes_read < 0) { + bm_log("reader read_at error: {}", status.to_string()); + break; + } + if (bytes_read == 0) { // EOF + break; + } + offset += bytes_read; + remaining_size -= bytes_read; + } + auto end = std::chrono::high_resolution_clock::now(); + auto elapsed_seconds = + std::chrono::duration_cast>(end - start); + state.SetIterationTime(elapsed_seconds.count()); + state.counters["ReadRate(B/S)"] = + benchmark::Counter(read_size, benchmark::Counter::kIsRate); + state.counters["ReadTotal(B)"] = read_size; + state.counters["ReadTime(S)"] = elapsed_seconds.count(); + + if (status.ok() && reader != nullptr) { + status = reader->close(); + } + bm_log("finish to read {}, size {}, seconds: {}, status: {}", _name, read_size, + elapsed_seconds.count(), status); + return status; + } + + Status write(benchmark::State& state, FileWriter* writer) { + bm_log("begin to write {}, size: {}", _name, _file_size); + size_t write_size = _file_size; + size_t buffer_size = + _conf_map.contains("buffer_size") ? std::stol(_conf_map["buffer_size"]) : 1000000L; + long remaining_size = write_size; + std::vector buffer; + buffer.resize(buffer_size); + doris::Slice data = {buffer.data(), buffer.size()}; + + Status status; + auto start = std::chrono::high_resolution_clock::now(); + while (remaining_size > 0) { + size_t size = std::min(buffer_size, (size_t)remaining_size); + data.size = size; + status = writer->append(data); + if (status != Status::OK()) { + bm_log("writer append error: {}", status.to_string()); + break; + } + remaining_size -= size; + } + if (status.ok() && writer != nullptr) { + status = writer->close(); + } + + auto end = std::chrono::high_resolution_clock::now(); + auto elapsed_seconds = + std::chrono::duration_cast>(end - start); + state.SetIterationTime(elapsed_seconds.count()); + state.counters["WriteRate(B/S)"] = + benchmark::Counter(write_size, benchmark::Counter::kIsRate); + state.counters["WriteTotal(B)"] = write_size; + state.counters["WriteTime(S)"] = elapsed_seconds.count(); + + bm_log("finish to write {}, size: {}, seconds: {}, status: {}", _name, write_size, + elapsed_seconds.count(), status); + return status; + } + protected: std::string _name; int _threads; int _iterations; size_t _file_size; - int _repetitions = 1; + int _repetitions = 3; std::map _conf_map; }; diff --git a/be/src/io/fs/benchmark/benchmark_factory.hpp b/be/src/io/fs/benchmark/benchmark_factory.hpp index 3e8c9314ca6dd14..0b8af3b96b5a892 100644 --- a/be/src/io/fs/benchmark/benchmark_factory.hpp +++ b/be/src/io/fs/benchmark/benchmark_factory.hpp @@ -38,8 +38,18 @@ Status BenchmarkFactory::getBm(const std::string fs_type, const std::string op_t const std::map& conf_map, BaseBenchmark** bm) { if (fs_type == "s3") { - if (op_type == "read") { - *bm = new S3ReadBenchmark(threads, iterations, file_size, conf_map); + if (op_type == "create_write") { + *bm = new S3CreateWriteBenchmark(threads, iterations, file_size, conf_map); + } else if (op_type == "open_read") { + *bm = new S3OpenReadBenchmark(threads, iterations, file_size, conf_map); + } else if (op_type == "single_read") { + *bm = new S3SingleReadBenchmark(threads, iterations, file_size, conf_map); + } else if (op_type == "rename") { + *bm = new S3RenameBenchmark(threads, iterations, file_size, conf_map); + } else if (op_type == "exists") { + *bm = new S3ExistsBenchmark(threads, iterations, file_size, conf_map); + } else if (op_type == "list") { + *bm = new S3ListBenchmark(threads, iterations, file_size, conf_map); } else { return Status::Error( "unknown params: fs_type: {}, op_type: {}, iterations: {}", fs_type, op_type, diff --git a/be/src/io/fs/benchmark/fs_benchmark_tool.cpp b/be/src/io/fs/benchmark/fs_benchmark_tool.cpp index a5be5db80a4433a..50085ae1e706d8c 100644 --- a/be/src/io/fs/benchmark/fs_benchmark_tool.cpp +++ b/be/src/io/fs/benchmark/fs_benchmark_tool.cpp @@ -20,6 +20,8 @@ #include #include "io/fs/benchmark/benchmark_factory.hpp" +#include "io/fs/s3_file_write_bufferpool.h" +#include "util/threadpool.h" DEFINE_string(fs_type, "hdfs", "Supported File System: s3, hdfs"); DEFINE_string(operation, "create_write", @@ -107,6 +109,15 @@ int main(int argc, char** argv) { return 1; } + // init s3 write buffer pool + std::unique_ptr buffered_reader_prefetch_thread_pool; + doris::ThreadPoolBuilder("BufferedReaderPrefetchThreadPool") + .set_min_threads(16) + .set_max_threads(64) + .build(&buffered_reader_prefetch_thread_pool); + doris::io::S3FileBufferPool* s3_buffer_pool = doris::io::S3FileBufferPool::GetInstance(); + s3_buffer_pool->init(524288000, 5242880, buffered_reader_prefetch_thread_pool.get()); + try { doris::io::MultiBenchmark multi_bm(FLAGS_fs_type, FLAGS_operation, std::stoi(FLAGS_threads), std::stoi(FLAGS_iterations), std::stol(FLAGS_file_size), diff --git a/be/src/io/fs/benchmark/hdfs_benchmark.hpp b/be/src/io/fs/benchmark/hdfs_benchmark.hpp index 1307ddc95a687fc..b508e14a24aa302 100644 --- a/be/src/io/fs/benchmark/hdfs_benchmark.hpp +++ b/be/src/io/fs/benchmark/hdfs_benchmark.hpp @@ -33,75 +33,30 @@ class HdfsOpenReadBenchmark : public BaseBenchmark { public: HdfsOpenReadBenchmark(int threads, int iterations, size_t file_size, const std::map& conf_map) - : BaseBenchmark("HdfsReadBenchmark", threads, iterations, file_size, 3, conf_map) {} + : BaseBenchmark("HdfsReadBenchmark", threads, iterations, file_size, conf_map) {} virtual ~HdfsOpenReadBenchmark() = default; - Status init() override { return Status::OK(); } - - virtual std::string get_file_path(benchmark::State& state) { - std::string base_dir = _conf_map["base_dir"]; - auto file_path = fmt::format("{}/test_{}", base_dir, state.thread_index()); - bm_log("file_path: {}", file_path); - return file_path; + virtual void set_default_file_size() { + if (_file_size <= 0) { + _file_size = 10 * 1024 * 1024; // default 10MB + } } Status run(benchmark::State& state) override { + auto file_path = get_file_path(state); + + auto start = std::chrono::high_resolution_clock::now(); std::shared_ptr fs; io::FileReaderSPtr reader; - bm_log("begin to init {}", _name); - size_t buffer_size = - _conf_map.contains("buffer_size") ? std::stol(_conf_map["buffer_size"]) : 1000000L; io::FileReaderOptions reader_opts = FileFactory::get_reader_options(nullptr); THdfsParams hdfs_params = parse_properties(_conf_map); - - auto file_path = get_file_path(state); RETURN_IF_ERROR( FileFactory::create_hdfs_reader(hdfs_params, file_path, &fs, &reader, reader_opts)); - bm_log("finish to init {}", _name); - - bm_log("begin to run {}", _name); - Status status; - std::vector buffer; - buffer.resize(buffer_size); - doris::Slice data = {buffer.data(), buffer.size()}; - size_t offset = 0; - size_t bytes_read = 0; - - size_t read_size = reader->size(); - if (_file_size > 0) { - read_size = std::min(read_size, _file_size); - } - long remaining_size = read_size; - - auto start = std::chrono::high_resolution_clock::now(); - while (remaining_size > 0) { - bytes_read = 0; - size_t size = std::min(buffer_size, (size_t)remaining_size); - data.size = size; - status = reader->read_at(offset, data, &bytes_read); - if (status != Status::OK() || bytes_read < 0) { - bm_log("reader read_at error: {}", status.to_string()); - break; - } - if (bytes_read == 0) { // EOF - break; - } - offset += bytes_read; - remaining_size -= bytes_read; - } - bm_log("finish to run {}", _name); auto end = std::chrono::high_resolution_clock::now(); - auto elapsed_seconds = std::chrono::duration_cast>(end - start); - - state.SetIterationTime(elapsed_seconds.count()); - state.counters["ReadRate"] = benchmark::Counter(read_size, benchmark::Counter::kIsRate); - - if (reader != nullptr) { - reader->close(); - } - return status; + state.counters["OpenReaderTime(S)"] = elapsed_seconds.count(); + return read(state, reader); } }; @@ -113,6 +68,10 @@ class HdfsSingleReadBenchmark : public HdfsOpenReadBenchmark { : HdfsOpenReadBenchmark(threads, iterations, file_size, conf_map) {} virtual ~HdfsSingleReadBenchmark() = default; + virtual void set_default_file_size() override { + // do nothing, default is 0, which means it will read the whole file + } + virtual std::string get_file_path(benchmark::State& state) override { std::string file_path = _conf_map["file_path"]; bm_log("file_path: {}", file_path); @@ -124,56 +83,20 @@ class HdfsCreateWriteBenchmark : public BaseBenchmark { public: HdfsCreateWriteBenchmark(int threads, int iterations, size_t file_size, const std::map& conf_map) - : BaseBenchmark("HdfsCreateWriteBenchmark", threads, iterations, file_size, 3, - conf_map) {} + : BaseBenchmark("HdfsCreateWriteBenchmark", threads, iterations, file_size, conf_map) {} virtual ~HdfsCreateWriteBenchmark() = default; - Status init() override { return Status::OK(); } - Status run(benchmark::State& state) override { - bm_log("begin to run {}", _name); - std::string base_dir = _conf_map["base_dir"]; - io::FileReaderOptions reader_opts = FileFactory::get_reader_options(nullptr); - THdfsParams hdfs_params = parse_properties(_conf_map); - auto file_path = fmt::format("{}/test_{}", base_dir, state.thread_index()); - bm_log("file_path: {}", file_path); - - auto start = std::chrono::high_resolution_clock::now(); + auto file_path = get_file_path(state); + if (_file_size <= 0) { + _file_size = 10 * 1024 * 1024; // default 10MB + } std::shared_ptr fs; io::FileWriterPtr writer; + THdfsParams hdfs_params = parse_properties(_conf_map); RETURN_IF_ERROR(io::HdfsFileSystem::create(hdfs_params, "", &fs)); RETURN_IF_ERROR(fs->create_file(file_path, &writer)); - Status status; - size_t write_size = _file_size; - size_t buffer_size = - _conf_map.contains("buffer_size") ? std::stol(_conf_map["buffer_size"]) : 1000000L; - long remaining_size = write_size; - std::vector buffer; - buffer.resize(buffer_size); - doris::Slice data = {buffer.data(), buffer.size()}; - while (remaining_size > 0) { - size_t size = std::min(buffer_size, (size_t)remaining_size); - data.size = size; - status = writer->append(data); - if (status != Status::OK()) { - bm_log("writer append error: {}", status.to_string()); - break; - } - remaining_size -= size; - } - auto end = std::chrono::high_resolution_clock::now(); - auto elapsed_seconds = - std::chrono::duration_cast>(end - start); - - state.SetIterationTime(elapsed_seconds.count()); - bm_log("finish to run {}", _name); - - state.counters["WriteRate"] = benchmark::Counter(write_size, benchmark::Counter::kIsRate); - - if (writer != nullptr) { - writer->close(); - } - return status; + return write(state, writer.get()); } }; @@ -181,75 +104,56 @@ class HdfsRenameBenchmark : public BaseBenchmark { public: HdfsRenameBenchmark(int threads, int iterations, size_t file_size, const std::map& conf_map) - : BaseBenchmark("HdfsRenameBenchmark", threads, 1, file_size, 1, conf_map) {} + : BaseBenchmark("HdfsRenameBenchmark", threads, iterations, file_size, conf_map) { + // rename can only do once + set_repetition(1); + } virtual ~HdfsRenameBenchmark() = default; - Status init() override { return Status::OK(); } - Status run(benchmark::State& state) override { - bm_log("begin to run {}", _name); - std::string base_dir = _conf_map["base_dir"]; - io::FileReaderOptions reader_opts = FileFactory::get_reader_options(nullptr); + auto file_path = get_file_path(state); + auto new_file_path = file_path + "_new"; THdfsParams hdfs_params = parse_properties(_conf_map); - auto file_path = fmt::format("{}/test_{}", base_dir, state.thread_index()); - auto new_file_path = fmt::format("{}/test_{}_new", base_dir, state.thread_index()); - bm_log("file_path: {}", file_path); - - auto start = std::chrono::high_resolution_clock::now(); std::shared_ptr fs; - io::FileWriterPtr writer; RETURN_IF_ERROR(io::HdfsFileSystem::create(hdfs_params, "", &fs)); + + auto start = std::chrono::high_resolution_clock::now(); RETURN_IF_ERROR(fs->rename(file_path, new_file_path)); auto end = std::chrono::high_resolution_clock::now(); auto elapsed_seconds = std::chrono::duration_cast>(end - start); - state.SetIterationTime(elapsed_seconds.count()); - bm_log("finish to run {}", _name); - state.counters["RenameCost"] = benchmark::Counter(1, benchmark::Counter::kIsRate | benchmark::Counter::kInvert); - if (writer != nullptr) { - writer->close(); - } return Status::OK(); } - -private: }; class HdfsExistsBenchmark : public BaseBenchmark { public: HdfsExistsBenchmark(int threads, int iterations, size_t file_size, const std::map& conf_map) - : BaseBenchmark("HdfsExistsBenchmark", threads, iterations, file_size, 3, conf_map) {} + : BaseBenchmark("HdfsExistsBenchmark", threads, iterations, file_size, conf_map) {} virtual ~HdfsExistsBenchmark() = default; - Status init() override { return Status::OK(); } - Status run(benchmark::State& state) override { - bm_log("begin to run {}", _name); - std::string base_dir = _conf_map["base_dir"]; - io::FileReaderOptions reader_opts = FileFactory::get_reader_options(nullptr); - THdfsParams hdfs_params = parse_properties(_conf_map); - auto file_path = fmt::format("{}/test_{}", base_dir, state.thread_index()); - bm_log("file_path: {}", file_path); + auto file_path = get_file_path(state); - auto start = std::chrono::high_resolution_clock::now(); std::shared_ptr fs; + THdfsParams hdfs_params = parse_properties(_conf_map); RETURN_IF_ERROR(io::HdfsFileSystem::create(hdfs_params, "", &fs)); + + auto start = std::chrono::high_resolution_clock::now(); bool res = false; RETURN_IF_ERROR(fs->exists(file_path, &res)); auto end = std::chrono::high_resolution_clock::now(); auto elapsed_seconds = std::chrono::duration_cast>(end - start); - state.SetIterationTime(elapsed_seconds.count()); - bm_log("finish to run {}", _name); - state.counters["ExistsCost"] = benchmark::Counter(1, benchmark::Counter::kIsRate | benchmark::Counter::kInvert); + return Status::OK(); } }; diff --git a/be/src/io/fs/benchmark/s3_benchmark.hpp b/be/src/io/fs/benchmark/s3_benchmark.hpp index 7e958cefdbc3837..c2ee8ddd99d1cb9 100644 --- a/be/src/io/fs/benchmark/s3_benchmark.hpp +++ b/be/src/io/fs/benchmark/s3_benchmark.hpp @@ -19,41 +19,193 @@ #include "io/file_factory.h" #include "io/fs/benchmark/base_benchmark.h" +#include "io/fs/file_writer.h" #include "io/fs/s3_file_reader.h" #include "io/fs/s3_file_system.h" +#include "runtime/exec_env.h" +#include "util/s3_uri.h" #include "util/slice.h" namespace doris::io { -class S3ReadBenchmark : public BaseBenchmark { +class S3Benchmark : public BaseBenchmark { public: - S3ReadBenchmark(int threads, int iterations, size_t file_size, - const std::map& conf_map) - : BaseBenchmark("S3ReadBenchmark", threads, iterations, file_size, 3, conf_map), - _result(buffer, 128) {} - virtual ~S3ReadBenchmark() = default; + S3Benchmark(const std::string& name, int threads, int iterations, size_t file_size, + const std::map& conf_map) + : BaseBenchmark(name, threads, iterations, file_size, conf_map) {} + virtual ~S3Benchmark() = default; - Status init() override { - bm_log("begin to init {}", _name); - std::string file_path = _conf_map["file"]; - io::FileReaderOptions reader_opts = FileFactory::get_reader_options(nullptr); + Status get_fs(const std::string& path) { + S3URI s3_uri(path); + RETURN_IF_ERROR(s3_uri.parse()); RETURN_IF_ERROR( - FileFactory::create_s3_reader(_conf_map, file_path, &_fs, &_reader, reader_opts)); - bm_log("finish to init {}", _name); + S3ClientFactory::convert_properties_to_s3_conf(_conf_map, s3_uri, &_s3_conf)); + return io::S3FileSystem::create(std::move(_s3_conf), "", &_fs); + } + +protected: + doris::S3Conf _s3_conf; + std::shared_ptr _fs; +}; + +class S3OpenReadBenchmark : public S3Benchmark { +public: + S3OpenReadBenchmark(int threads, int iterations, size_t file_size, + const std::map& conf_map) + : S3Benchmark("S3ReadBenchmark", threads, iterations, file_size, conf_map) {} + virtual ~S3OpenReadBenchmark() = default; + + virtual void set_default_file_size() { + if (_file_size <= 0) { + _file_size = 10 * 1024 * 1024; // default 10MB + } + } + + Status run(benchmark::State& state) override { + auto file_path = get_file_path(state); + RETURN_IF_ERROR(get_fs(file_path)); + + io::FileReaderSPtr reader; + io::FileReaderOptions reader_opts = FileFactory::get_reader_options(nullptr); + RETURN_IF_ERROR(FileFactory::create_s3_reader( + _conf_map, file_path, reinterpret_cast*>(&_fs), + &reader, reader_opts)); + + return read(state, reader); + } +}; + +// Read a single specified file +class S3SingleReadBenchmark : public S3OpenReadBenchmark { +public: + S3SingleReadBenchmark(int threads, int iterations, size_t file_size, + const std::map& conf_map) + : S3OpenReadBenchmark(threads, iterations, file_size, conf_map) {} + virtual ~S3SingleReadBenchmark() = default; + + virtual void set_default_file_size() override {} + + virtual std::string get_file_path(benchmark::State& state) override { + std::string file_path = _conf_map["file_path"]; + bm_log("file_path: {}", file_path); + return file_path; + } +}; + +class S3CreateWriteBenchmark : public S3Benchmark { +public: + S3CreateWriteBenchmark(int threads, int iterations, size_t file_size, + const std::map& conf_map) + : S3Benchmark("S3CreateWriteBenchmark", threads, iterations, file_size, conf_map) {} + virtual ~S3CreateWriteBenchmark() = default; + + Status run(benchmark::State& state) override { + auto file_path = get_file_path(state); + if (_file_size <= 0) { + _file_size = 10 * 1024 * 1024; // default 10MB + } + RETURN_IF_ERROR(get_fs(file_path)); + + io::FileWriterPtr writer; + RETURN_IF_ERROR(_fs->create_file(file_path, &writer)); + return write(state, writer.get()); + } +}; + +class S3ListBenchmark : public S3Benchmark { +public: + S3ListBenchmark(int threads, int iterations, size_t file_size, + const std::map& conf_map) + : S3Benchmark("S3ListBenchmark", threads, iterations, file_size, conf_map) {} + virtual ~S3ListBenchmark() = default; + + virtual std::string get_file_path(benchmark::State& state) override { + return _conf_map["base_dir"]; + } + + Status run(benchmark::State& state) override { + auto file_path = get_file_path(state); + RETURN_IF_ERROR(get_fs(file_path)); + + auto start = std::chrono::high_resolution_clock::now(); + std::vector files; + bool exists = true; + RETURN_IF_ERROR(_fs->list(file_path, true, &files, &exists)); + auto end = std::chrono::high_resolution_clock::now(); + auto elapsed_seconds = + std::chrono::duration_cast>(end - start); + state.SetIterationTime(elapsed_seconds.count()); + state.counters["ListCost"] = + benchmark::Counter(1, benchmark::Counter::kIsRate | benchmark::Counter::kInvert); + + std::stringstream ss; + int i = 0; + for (auto& file_info : files) { + if (i > 2) { + break; + } + ++i; + ss << "[" << file_info.file_name << ", " << file_info.file_size << ", " + << file_info.is_file << "] "; + } + bm_log("list files: {}", ss.str()); + return Status::OK(); } +}; + +class S3RenameBenchmark : public S3Benchmark { +public: + S3RenameBenchmark(int threads, int iterations, size_t file_size, + const std::map& conf_map) + : S3Benchmark("S3RenameBenchmark", threads, iterations, file_size, conf_map) { + // rename can only do once + set_repetition(1); + } + + virtual ~S3RenameBenchmark() = default; Status run(benchmark::State& state) override { - return _reader->read_at(0, _result, &_bytes_read); + auto file_path = get_file_path(state); + auto new_file_path = file_path + "_new"; + RETURN_IF_ERROR(get_fs(file_path)); + + auto start = std::chrono::high_resolution_clock::now(); + RETURN_IF_ERROR(_fs->rename(file_path, new_file_path)); + auto end = std::chrono::high_resolution_clock::now(); + auto elapsed_seconds = + std::chrono::duration_cast>(end - start); + state.SetIterationTime(elapsed_seconds.count()); + state.counters["RenameCost"] = + benchmark::Counter(1, benchmark::Counter::kIsRate | benchmark::Counter::kInvert); + + return Status::OK(); } +}; -private: - doris::S3Conf _s3_conf; - std::shared_ptr _fs; - io::FileReaderSPtr _reader; - char buffer[128]; - doris::Slice _result; - size_t _bytes_read = 0; +class S3ExistsBenchmark : public S3Benchmark { +public: + S3ExistsBenchmark(int threads, int iterations, size_t file_size, + const std::map& conf_map) + : S3Benchmark("S3ExistsBenchmark", threads, iterations, file_size, conf_map) {} + virtual ~S3ExistsBenchmark() = default; + + Status run(benchmark::State& state) override { + auto file_path = get_file_path(state); + RETURN_IF_ERROR(get_fs(file_path)); + + auto start = std::chrono::high_resolution_clock::now(); + bool res = false; + RETURN_IF_ERROR(_fs->exists(file_path, &res)); + auto end = std::chrono::high_resolution_clock::now(); + auto elapsed_seconds = + std::chrono::duration_cast>(end - start); + state.SetIterationTime(elapsed_seconds.count()); + state.counters["ExistsCost"] = + benchmark::Counter(1, benchmark::Counter::kIsRate | benchmark::Counter::kInvert); + + return Status::OK(); + } }; } // namespace doris::io diff --git a/be/src/io/fs/buffered_reader.cpp b/be/src/io/fs/buffered_reader.cpp index 6dcbad96b7c1d16..e6093612baf0d74 100644 --- a/be/src/io/fs/buffered_reader.cpp +++ b/be/src/io/fs/buffered_reader.cpp @@ -22,6 +22,7 @@ #include #include +#include // IWYU pragma: no_include #include "common/compiler_util.h" // IWYU pragma: keep @@ -363,13 +364,23 @@ Status MergeRangeFileReader::_fill_box(int range_index, size_t start_offset, siz return Status::OK(); } +// the condition variable would wait at most 10 seconds +// otherwise it would quit the procedure and treat it +// as one time out error status and would make the load +// task failed +constexpr static int WAIT_TIME_OUT_MS = 10000; + // there exists occasions where the buffer is already closed but // some prior tasks are still queued in thread pool, so we have to check whether // the buffer is closed each time the condition variable is notified. void PrefetchBuffer::reset_offset(size_t offset) { { std::unique_lock lck {_lock}; - _prefetched.wait(lck, [this]() { return _buffer_status != BufferStatus::PENDING; }); + if (!_prefetched.wait_for(lck, std::chrono::milliseconds(WAIT_TIME_OUT_MS), + [this]() { return _buffer_status != BufferStatus::PENDING; })) { + _prefetch_status = Status::TimedOut("time out when reset prefetch buffer"); + return; + } if (UNLIKELY(_buffer_status == BufferStatus::CLOSED)) { _prefetched.notify_all(); return; @@ -393,9 +404,13 @@ void PrefetchBuffer::reset_offset(size_t offset) { void PrefetchBuffer::prefetch_buffer() { { std::unique_lock lck {_lock}; - _prefetched.wait(lck, [this]() { - return _buffer_status == BufferStatus::RESET || _buffer_status == BufferStatus::CLOSED; - }); + if (!_prefetched.wait_for(lck, std::chrono::milliseconds(WAIT_TIME_OUT_MS), [this]() { + return _buffer_status == BufferStatus::RESET || + _buffer_status == BufferStatus::CLOSED; + })) { + _prefetch_status = Status::TimedOut("time out when invoking prefetch buffer"); + return; + } // in case buffer is already closed if (UNLIKELY(_buffer_status == BufferStatus::CLOSED)) { _prefetched.notify_all(); @@ -432,7 +447,11 @@ void PrefetchBuffer::prefetch_buffer() { _statis.prefetch_request_io += 1; _statis.prefetch_request_bytes += _len; std::unique_lock lck {_lock}; - _prefetched.wait(lck, [this]() { return _buffer_status == BufferStatus::PENDING; }); + if (!_prefetched.wait_for(lck, std::chrono::milliseconds(WAIT_TIME_OUT_MS), + [this]() { return _buffer_status == BufferStatus::PENDING; })) { + _prefetch_status = Status::TimedOut("time out when invoking prefetch buffer"); + return; + } if (!s.ok() && _offset < _reader->size()) { _prefetch_status = std::move(s); } @@ -509,10 +528,13 @@ Status PrefetchBuffer::read_buffer(size_t off, const char* out, size_t buf_len, { std::unique_lock lck {_lock}; // buffer must be prefetched or it's closed - _prefetched.wait(lck, [this]() { - return _buffer_status == BufferStatus::PREFETCHED || - _buffer_status == BufferStatus::CLOSED; - }); + if (!_prefetched.wait_for(lck, std::chrono::milliseconds(WAIT_TIME_OUT_MS), [this]() { + return _buffer_status == BufferStatus::PREFETCHED || + _buffer_status == BufferStatus::CLOSED; + })) { + _prefetch_status = Status::TimedOut("time out when read prefetch buffer"); + return _prefetch_status; + } if (UNLIKELY(BufferStatus::CLOSED == _buffer_status)) { return Status::OK(); } @@ -545,7 +567,11 @@ Status PrefetchBuffer::read_buffer(size_t off, const char* out, size_t buf_len, void PrefetchBuffer::close() { std::unique_lock lck {_lock}; // in case _reader still tries to write to the buf after we close the buffer - _prefetched.wait(lck, [this]() { return _buffer_status != BufferStatus::PENDING; }); + if (!_prefetched.wait_for(lck, std::chrono::milliseconds(WAIT_TIME_OUT_MS), + [this]() { return _buffer_status != BufferStatus::PENDING; })) { + _prefetch_status = Status::TimedOut("time out when close prefetch buffer"); + return; + } _buffer_status = BufferStatus::CLOSED; _prefetched.notify_all(); if (_sync_profile != nullptr) { diff --git a/be/src/io/fs/s3_file_write_bufferpool.cpp b/be/src/io/fs/s3_file_write_bufferpool.cpp index c6ec1a8b60c8a19..48887f9c6ea6a75 100644 --- a/be/src/io/fs/s3_file_write_bufferpool.cpp +++ b/be/src/io/fs/s3_file_write_bufferpool.cpp @@ -24,6 +24,7 @@ #include "io/fs/s3_common.h" #include "runtime/exec_env.h" #include "util/defer_op.h" +#include "util/threadpool.h" namespace doris { namespace io { @@ -59,26 +60,27 @@ void S3FileBuffer::submit() { _stream_ptr = std::make_shared(_buf.data, _size); } - ExecEnv::GetInstance()->buffered_reader_prefetch_thread_pool()->submit_func( - [buf = this->shared_from_this()]() { buf->_on_upload(); }); + _thread_pool->submit_func([buf = this->shared_from_this()]() { buf->_on_upload(); }); } -S3FileBufferPool::S3FileBufferPool() { +void S3FileBufferPool::init(int32_t s3_write_buffer_whole_size, int32_t s3_write_buffer_size, + doris::ThreadPool* thread_pool) { // the nums could be one configuration - size_t buf_num = config::s3_write_buffer_whole_size / config::s3_write_buffer_size; - DCHECK((config::s3_write_buffer_size >= 5 * 1024 * 1024) && - (config::s3_write_buffer_whole_size > config::s3_write_buffer_size)); + size_t buf_num = s3_write_buffer_whole_size / s3_write_buffer_size; + DCHECK((s3_write_buffer_size >= 5 * 1024 * 1024) && + (s3_write_buffer_whole_size > s3_write_buffer_size)); LOG_INFO("S3 file buffer pool with {} buffers", buf_num); - _whole_mem_buffer = std::make_unique(config::s3_write_buffer_whole_size); + _whole_mem_buffer = std::make_unique(s3_write_buffer_whole_size); for (size_t i = 0; i < buf_num; i++) { - Slice s {_whole_mem_buffer.get() + i * config::s3_write_buffer_size, - static_cast(config::s3_write_buffer_size)}; + Slice s {_whole_mem_buffer.get() + i * s3_write_buffer_size, + static_cast(s3_write_buffer_size)}; _free_raw_buffers.emplace_back(s); } + _thread_pool = thread_pool; } std::shared_ptr S3FileBufferPool::allocate(bool reserve) { - std::shared_ptr buf = std::make_shared(); + std::shared_ptr buf = std::make_shared(_thread_pool); // if need reserve then we must ensure return buf with memory preserved if (reserve) { { diff --git a/be/src/io/fs/s3_file_write_bufferpool.h b/be/src/io/fs/s3_file_write_bufferpool.h index b69964b48e3ad45..55fa53df4287788 100644 --- a/be/src/io/fs/s3_file_write_bufferpool.h +++ b/be/src/io/fs/s3_file_write_bufferpool.h @@ -31,13 +31,14 @@ #include "util/slice.h" namespace doris { +class ThreadPool; namespace io { // TODO(AlexYue): 1. support write into cache 2. unify write buffer and read buffer struct S3FileBuffer : public std::enable_shared_from_this { using Callback = std::function; - S3FileBuffer() = default; + S3FileBuffer(ThreadPool* pool) { _thread_pool = pool; } ~S3FileBuffer() = default; void rob_buffer(std::shared_ptr& other) { @@ -104,19 +105,26 @@ struct S3FileBuffer : public std::enable_shared_from_this { // caller of this buf could use this callback to do syncronization Callback _on_finish_upload = nullptr; Status _status; - size_t _offset; - size_t _size; + size_t _offset {0}; + size_t _size {0}; std::shared_ptr _stream_ptr; // only served as one reserved buffer Slice _buf; size_t _append_offset {0}; + // not owned + ThreadPool* _thread_pool = nullptr; }; class S3FileBufferPool { public: - S3FileBufferPool(); + S3FileBufferPool() = default; ~S3FileBufferPool() = default; + // should be called one and only once + // at startup + void init(int32_t s3_write_buffer_whole_size, int32_t s3_write_buffer_size, + doris::ThreadPool* thread_pool); + static S3FileBufferPool* GetInstance() { static S3FileBufferPool _pool; return &_pool; @@ -135,6 +143,8 @@ class S3FileBufferPool { std::condition_variable _cv; std::unique_ptr _whole_mem_buffer; std::list _free_raw_buffers; + // not owned + ThreadPool* _thread_pool = nullptr; }; } // namespace io } // namespace doris diff --git a/be/src/io/fs/s3_file_writer.h b/be/src/io/fs/s3_file_writer.h index 0716ac635637aaf..d8956da88acad3f 100644 --- a/be/src/io/fs/s3_file_writer.h +++ b/be/src/io/fs/s3_file_writer.h @@ -57,8 +57,6 @@ class S3FileWriter final : public FileWriter { return Status::NotSupported("not support"); } - size_t bytes_appended() const { return _bytes_appended; } - int64_t upload_cost_ms() const { return *_upload_cost_ms; } private: @@ -115,7 +113,6 @@ class S3FileWriter final : public FileWriter { std::shared_ptr _client; std::string _upload_id; - size_t _bytes_appended {0}; // Current Part Num for CompletedPart int _cur_part_num = 1; diff --git a/be/src/olap/block_column_predicate.cpp b/be/src/olap/block_column_predicate.cpp index ff99038012e7fd2..8cfb89363cdd245 100644 --- a/be/src/olap/block_column_predicate.cpp +++ b/be/src/olap/block_column_predicate.cpp @@ -52,6 +52,11 @@ bool SingleColumnBlockPredicate::evaluate_and(const segment_v2::BloomFilter* bf) return _predicate->evaluate_and(bf); } +bool SingleColumnBlockPredicate::evaluate_and(const StringRef* dict_words, + const size_t dict_num) const { + return _predicate->evaluate_and(dict_words, dict_num); +} + void SingleColumnBlockPredicate::evaluate_or(vectorized::MutableColumns& block, uint16_t* sel, uint16_t selected_size, bool* flags) const { auto column_id = _predicate->column_id(); @@ -158,6 +163,16 @@ bool AndBlockColumnPredicate::evaluate_and(const segment_v2::BloomFilter* bf) co return true; } +bool AndBlockColumnPredicate::evaluate_and(const StringRef* dict_words, + const size_t dict_num) const { + for (auto* predicate : _block_column_predicate_vec) { + if (!predicate->evaluate_and(dict_words, dict_num)) { + return false; + } + } + return true; +} + void AndBlockColumnPredicate::evaluate_or(vectorized::MutableColumns& block, uint16_t* sel, uint16_t selected_size, bool* flags) const { if (num_of_column_predicate() == 1) { diff --git a/be/src/olap/block_column_predicate.h b/be/src/olap/block_column_predicate.h index 467962e809cfe47..0069a62d2957b6b 100644 --- a/be/src/olap/block_column_predicate.h +++ b/be/src/olap/block_column_predicate.h @@ -81,6 +81,12 @@ class BlockColumnPredicate { LOG(FATAL) << "should not reach here"; return true; } + + virtual bool evaluate_and(const StringRef* dict_words, const size_t dict_num) const { + LOG(FATAL) << "should not reach here"; + return true; + } + virtual bool can_do_bloom_filter() const { return false; } //evaluate predicate on inverted @@ -109,6 +115,7 @@ class SingleColumnBlockPredicate : public BlockColumnPredicate { bool* flags) const override; bool evaluate_and(const std::pair& statistic) const override; bool evaluate_and(const segment_v2::BloomFilter* bf) const override; + bool evaluate_and(const StringRef* dict_words, const size_t dict_num) const override; void evaluate_or(vectorized::MutableColumns& block, uint16_t* sel, uint16_t selected_size, bool* flags) const override; @@ -179,6 +186,8 @@ class AndBlockColumnPredicate : public MutilColumnBlockPredicate { bool evaluate_and(const segment_v2::BloomFilter* bf) const override; + bool evaluate_and(const StringRef* dict_words, const size_t dict_num) const override; + bool can_do_bloom_filter() const override { for (auto& pred : _block_column_predicate_vec) { if (!pred->can_do_bloom_filter()) { diff --git a/be/src/olap/column_predicate.h b/be/src/olap/column_predicate.h index 31dbea7e56ad048..88f40c92c112c4f 100644 --- a/be/src/olap/column_predicate.h +++ b/be/src/olap/column_predicate.h @@ -179,6 +179,10 @@ class ColumnPredicate { virtual bool evaluate_and(const BloomFilter* bf) const { return true; } + virtual bool evaluate_and(const StringRef* dict_words, const size_t dict_count) const { + return true; + } + virtual bool can_do_bloom_filter() const { return false; } // used to evaluate pre read column in lazy materialization diff --git a/be/src/olap/comparison_predicate.h b/be/src/olap/comparison_predicate.h index 00de4632e7979c3..6524fdfc7df0029 100644 --- a/be/src/olap/comparison_predicate.h +++ b/be/src/olap/comparison_predicate.h @@ -259,6 +259,19 @@ class ComparisonPredicateBase : public ColumnPredicate { } } + bool evaluate_and(const StringRef* dict_words, const size_t count) const override { + if constexpr (std::is_same_v) { + for (size_t i = 0; i != count; ++i) { + if (_operator(dict_words[i], _value) ^ _opposite) { + return true; + } + } + return false; + } + + return true; + } + bool can_do_bloom_filter() const override { return PT == PredicateType::EQ; } void evaluate_or(const vectorized::IColumn& column, const uint16_t* sel, uint16_t size, diff --git a/be/src/olap/delta_writer.cpp b/be/src/olap/delta_writer.cpp index 16fc02dbc0f8c3a..3c84e67103d78e5 100644 --- a/be/src/olap/delta_writer.cpp +++ b/be/src/olap/delta_writer.cpp @@ -199,7 +199,7 @@ Status DeltaWriter::init() { context.segments_overlap = OVERLAPPING; context.tablet_schema = _tablet_schema; context.newest_write_timestamp = UnixSeconds(); - context.tablet_id = _tablet->table_id(); + context.tablet_id = _tablet->tablet_id(); context.tablet = _tablet; context.write_type = DataWriteType::TYPE_DIRECT; context.mow_context = std::make_shared(_cur_max_version, _req.txn_id, _rowset_ids, @@ -453,9 +453,24 @@ Status DeltaWriter::close_wait(const PSlaveTabletNodes& slave_tablet_nodes, RETURN_IF_ERROR(_tablet->calc_delete_bitmap_between_segments(_cur_rowset, segments, _delete_bitmap)); } + + // commit_phase_update_delete_bitmap() may generate new segments, we need to create a new + // transient rowset writer to write the new segments, then merge it back the original + // rowset. + std::unique_ptr rowset_writer; + _tablet->create_transient_rowset_writer(_cur_rowset, &rowset_writer); RETURN_IF_ERROR(_tablet->commit_phase_update_delete_bitmap( _cur_rowset, _rowset_ids, _delete_bitmap, segments, _req.txn_id, - _rowset_writer.get())); + rowset_writer.get())); + if (_cur_rowset->tablet_schema()->is_partial_update()) { + // build rowset writer and merge transient rowset + RETURN_IF_ERROR(rowset_writer->flush()); + RowsetSharedPtr transient_rowset = rowset_writer->build(); + _cur_rowset->merge_rowset_meta(transient_rowset->rowset_meta()); + + // erase segment cache cause we will add a segment to rowset + SegmentLoader::instance()->erase_segment(_cur_rowset->rowset_id()); + } } Status res = _storage_engine->txn_manager()->commit_txn(_req.partition_id, _tablet, _req.txn_id, _req.load_id, _cur_rowset, false); diff --git a/be/src/olap/in_list_predicate.h b/be/src/olap/in_list_predicate.h index 8af69efc66b5708..5f0f99f7ebd91c5 100644 --- a/be/src/olap/in_list_predicate.h +++ b/be/src/olap/in_list_predicate.h @@ -346,6 +346,17 @@ class InListPredicateBase : public ColumnPredicate { } } + bool evaluate_and(const StringRef* dict_words, const size_t count) const override { + for (size_t i = 0; i != count; ++i) { + const auto found = _values->find(dict_words[i].data, dict_words[i].size) ^ _opposite; + if (found == (PT == PredicateType::IN_LIST)) { + return true; + } + } + + return false; + } + bool evaluate_del(const std::pair& statistic) const override { if (statistic.first->is_null() || statistic.second->is_null()) { return false; diff --git a/be/src/olap/olap_common.h b/be/src/olap/olap_common.h index 1ebf306b41f20e3..86f59af14c50b26 100644 --- a/be/src/olap/olap_common.h +++ b/be/src/olap/olap_common.h @@ -324,6 +324,7 @@ struct OlapReaderStatistics { int64_t rows_key_range_filtered = 0; int64_t rows_stats_filtered = 0; int64_t rows_bf_filtered = 0; + int64_t rows_dict_filtered = 0; // Including the number of rows filtered out according to the Delete information in the Tablet, // and the number of rows filtered for marked deleted rows under the unique key model. // This metric is mainly used to record the number of rows filtered by the delete condition in Segment V1, diff --git a/be/src/olap/olap_meta.cpp b/be/src/olap/olap_meta.cpp index 889090d36c53d69..4df89c04dc31f2a 100644 --- a/be/src/olap/olap_meta.cpp +++ b/be/src/olap/olap_meta.cpp @@ -19,6 +19,7 @@ #include #include +#include #include #include #include @@ -62,12 +63,24 @@ OlapMeta::OlapMeta(const std::string& root_path) : _root_path(root_path) {} OlapMeta::~OlapMeta() = default; +class RocksdbLogger : public rocksdb::Logger { +public: + void Logv(const char* format, va_list ap) override { + char buf[1024]; + vsnprintf(buf, sizeof(buf), format, ap); + LOG(INFO) << "[Rocksdb] " << buf; + } +}; + Status OlapMeta::init() { // init db DBOptions options; options.IncreaseParallelism(); options.create_if_missing = true; options.create_missing_column_families = true; + options.info_log = std::make_shared(); + options.info_log_level = rocksdb::WARN_LEVEL; + std::string db_path = _root_path + META_POSTFIX; std::vector column_families; // default column family is required @@ -76,6 +89,7 @@ Status OlapMeta::init() { // meta column family add prefix extractor to improve performance and ensure correctness ColumnFamilyOptions meta_column_family; + meta_column_family.max_write_buffer_number = config::rocksdb_max_write_buffer_number; meta_column_family.prefix_extractor.reset(NewFixedPrefixTransform(PREFIX_LENGTH)); column_families.emplace_back(META_COLUMN_FAMILY, meta_column_family); diff --git a/be/src/olap/olap_server.cpp b/be/src/olap/olap_server.cpp index 57bf40a147b1dba..09875b9e5244fa2 100644 --- a/be/src/olap/olap_server.cpp +++ b/be/src/olap/olap_server.cpp @@ -1021,9 +1021,8 @@ Status StorageEngine::process_index_change_task(const TAlterInvertedIndexReq& re return Status::InternalError("tablet not exist, tablet_id={}.", tablet_id); } - IndexBuilderSharedPtr index_builder = - std::make_shared(tablet, request.columns, request.indexes_desc, - request.alter_inverted_indexes, request.is_drop_op); + IndexBuilderSharedPtr index_builder = std::make_shared( + tablet, request.columns, request.alter_inverted_indexes, request.is_drop_op); RETURN_IF_ERROR(_handle_index_change(index_builder)); return Status::OK(); } diff --git a/be/src/olap/primary_key_index.cpp b/be/src/olap/primary_key_index.cpp index 7b2f1593ca3d551..6c276dd262be8db 100644 --- a/be/src/olap/primary_key_index.cpp +++ b/be/src/olap/primary_key_index.cpp @@ -24,6 +24,7 @@ // IWYU pragma: no_include #include "common/compiler_util.h" // IWYU pragma: keep #include "common/config.h" +#include "io/fs/file_writer.h" #include "olap/olap_common.h" #include "olap/rowset/segment_v2/bloom_filter_index_reader.h" #include "olap/rowset/segment_v2/bloom_filter_index_writer.h" @@ -71,6 +72,7 @@ Status PrimaryKeyIndexBuilder::add_item(const Slice& key) { Status PrimaryKeyIndexBuilder::finalize(segment_v2::PrimaryKeyIndexMetaPB* meta) { // finish primary key index RETURN_IF_ERROR(_primary_key_index_builder->finish(meta->mutable_primary_key_index())); + _disk_size += _primary_key_index_builder->disk_size(); // set min_max key, the sequence column should be removed meta->set_min_key(min_key().to_string()); @@ -78,7 +80,11 @@ Status PrimaryKeyIndexBuilder::finalize(segment_v2::PrimaryKeyIndexMetaPB* meta) // finish bloom filter index RETURN_IF_ERROR(_bloom_filter_index_builder->flush()); - return _bloom_filter_index_builder->finish(_file_writer, meta->mutable_bloom_filter_index()); + uint64_t start_size = _file_writer->bytes_appended(); + RETURN_IF_ERROR( + _bloom_filter_index_builder->finish(_file_writer, meta->mutable_bloom_filter_index())); + _disk_size += _file_writer->bytes_appended() - start_size; + return Status::OK(); } Status PrimaryKeyIndexReader::parse_index(io::FileReaderSPtr file_reader, diff --git a/be/src/olap/primary_key_index.h b/be/src/olap/primary_key_index.h index 65cc64f0cd687aa..233644b4e07173f 100644 --- a/be/src/olap/primary_key_index.h +++ b/be/src/olap/primary_key_index.h @@ -51,7 +51,11 @@ class PrimaryKeyIndexMetaPB; class PrimaryKeyIndexBuilder { public: PrimaryKeyIndexBuilder(io::FileWriter* file_writer, size_t seq_col_length) - : _file_writer(file_writer), _num_rows(0), _size(0), _seq_col_length(seq_col_length) {} + : _file_writer(file_writer), + _num_rows(0), + _size(0), + _disk_size(0), + _seq_col_length(seq_col_length) {} Status init(); @@ -61,6 +65,8 @@ class PrimaryKeyIndexBuilder { uint64_t size() const { return _size; } + uint64_t disk_size() const { return _disk_size; } + Slice min_key() { return Slice(_min_key.data(), _min_key.size() - _seq_col_length); } Slice max_key() { return Slice(_max_key.data(), _max_key.size() - _seq_col_length); } @@ -70,6 +76,7 @@ class PrimaryKeyIndexBuilder { io::FileWriter* _file_writer = nullptr; uint32_t _num_rows; uint64_t _size; + uint64_t _disk_size; size_t _seq_col_length; faststring _min_key; diff --git a/be/src/olap/rowset/beta_rowset_reader.cpp b/be/src/olap/rowset/beta_rowset_reader.cpp index 5564c08740678eb..3ec62de7a879b2e 100644 --- a/be/src/olap/rowset/beta_rowset_reader.cpp +++ b/be/src/olap/rowset/beta_rowset_reader.cpp @@ -224,7 +224,7 @@ Status BetaRowsetReader::get_segment_iterators(RowsetReaderContext* read_context auto s = seg_ptr->new_iterator(_input_schema, _read_options, &iter); if (!s.ok()) { LOG(WARNING) << "failed to create iterator[" << seg_ptr->id() << "]: " << s.to_string(); - return Status::Error(); + return Status::Error(s.to_string()); } if (iter->empty()) { continue; @@ -268,7 +268,7 @@ Status BetaRowsetReader::init(RowsetReaderContext* read_context, if (!s.ok()) { LOG(WARNING) << "failed to init iterator: " << s.to_string(); _iterator.reset(); - return Status::Error(); + return Status::Error(s.to_string()); } return Status::OK(); } diff --git a/be/src/olap/rowset/segment_v2/column_reader.cpp b/be/src/olap/rowset/segment_v2/column_reader.cpp index 093eb365b097110..5c3e67e0a233b44 100644 --- a/be/src/olap/rowset/segment_v2/column_reader.cpp +++ b/be/src/olap/rowset/segment_v2/column_reader.cpp @@ -1170,23 +1170,8 @@ Status FileColumnIterator::_read_data_page(const OrdinalPageIndexIterator& iter) auto dict_page_decoder = reinterpret_cast(_page.data_decoder.get()); if (dict_page_decoder->is_dict_encoding()) { if (_dict_decoder == nullptr) { - // read dictionary page - Slice dict_data; - PageFooterPB dict_footer; - _opts.type = INDEX_PAGE; - RETURN_IF_ERROR(_reader->read_page(_opts, _reader->get_dict_page_pointer(), - &_dict_page_handle, &dict_data, &dict_footer, - _compress_codec)); - // ignore dict_footer.dict_page_footer().encoding() due to only - // PLAIN_ENCODING is supported for dict page right now - _dict_decoder = std::make_unique< - BinaryPlainPageDecoder>(dict_data); - RETURN_IF_ERROR(_dict_decoder->init()); - - auto* pd_decoder = (BinaryPlainPageDecoder*) - _dict_decoder.get(); - _dict_word_info.reset(new StringRef[pd_decoder->_num_elems]); - pd_decoder->get_dict_word_info(_dict_word_info.get()); + RETURN_IF_ERROR(_read_dict_data()); + CHECK_NOTNULL(_dict_decoder); } dict_page_decoder->set_dict_decoder(_dict_decoder.get(), _dict_word_info.get()); @@ -1195,6 +1180,27 @@ Status FileColumnIterator::_read_data_page(const OrdinalPageIndexIterator& iter) return Status::OK(); } +Status FileColumnIterator::_read_dict_data() { + CHECK_EQ(_reader->encoding_info()->encoding(), DICT_ENCODING); + // read dictionary page + Slice dict_data; + PageFooterPB dict_footer; + _opts.type = INDEX_PAGE; + RETURN_IF_ERROR(_reader->read_page(_opts, _reader->get_dict_page_pointer(), &_dict_page_handle, + &dict_data, &dict_footer, _compress_codec)); + // ignore dict_footer.dict_page_footer().encoding() due to only + // PLAIN_ENCODING is supported for dict page right now + _dict_decoder = + std::make_unique>(dict_data); + RETURN_IF_ERROR(_dict_decoder->init()); + + auto* pd_decoder = + (BinaryPlainPageDecoder*)_dict_decoder.get(); + _dict_word_info.reset(new StringRef[pd_decoder->_num_elems]); + pd_decoder->get_dict_word_info(_dict_word_info.get()); + return Status::OK(); +} + Status FileColumnIterator::get_row_ranges_by_zone_map( const AndBlockColumnPredicate* col_predicates, const std::vector* delete_predicates, RowRanges* row_ranges) { @@ -1213,6 +1219,23 @@ Status FileColumnIterator::get_row_ranges_by_bloom_filter( return Status::OK(); } +Status FileColumnIterator::get_row_ranges_by_dict(const AndBlockColumnPredicate* col_predicates, + RowRanges* row_ranges) { + if (!_is_all_dict_encoding) { + return Status::OK(); + } + + if (!_dict_decoder) { + RETURN_IF_ERROR(_read_dict_data()); + CHECK_NOTNULL(_dict_decoder); + } + + if (!col_predicates->evaluate_and(_dict_word_info.get(), _dict_decoder->count())) { + row_ranges->clear(); + } + return Status::OK(); +} + Status DefaultValueColumnIterator::init(const ColumnIteratorOptions& opts) { _opts = opts; // be consistent with segment v1 diff --git a/be/src/olap/rowset/segment_v2/column_reader.h b/be/src/olap/rowset/segment_v2/column_reader.h index bee0bcfb915d0d0..6cb9794b3b64b68 100644 --- a/be/src/olap/rowset/segment_v2/column_reader.h +++ b/be/src/olap/rowset/segment_v2/column_reader.h @@ -302,6 +302,11 @@ class ColumnIterator { return Status::OK(); } + virtual Status get_row_ranges_by_dict(const AndBlockColumnPredicate* col_predicates, + RowRanges* row_ranges) { + return Status::OK(); + } + virtual bool is_all_dict_encoding() const { return false; } protected: @@ -342,6 +347,9 @@ class FileColumnIterator final : public ColumnIterator { Status get_row_ranges_by_bloom_filter(const AndBlockColumnPredicate* col_predicates, RowRanges* row_ranges) override; + Status get_row_ranges_by_dict(const AndBlockColumnPredicate* col_predicates, + RowRanges* row_ranges) override; + ParsedPage* get_current_page() { return &_page; } bool is_nullable() { return _reader->is_nullable(); } @@ -352,8 +360,8 @@ class FileColumnIterator final : public ColumnIterator { void _seek_to_pos_in_page(ParsedPage* page, ordinal_t offset_in_page) const; Status _load_next_page(bool* eos); Status _read_data_page(const OrdinalPageIndexIterator& iter); + Status _read_dict_data(); -private: ColumnReader* _reader; // iterator owned compress codec, should NOT be shared by threads, initialized in init() diff --git a/be/src/olap/rowset/segment_v2/indexed_column_writer.cpp b/be/src/olap/rowset/segment_v2/indexed_column_writer.cpp index 28a44b7b757606e..acbbfd09346561f 100644 --- a/be/src/olap/rowset/segment_v2/indexed_column_writer.cpp +++ b/be/src/olap/rowset/segment_v2/indexed_column_writer.cpp @@ -23,6 +23,7 @@ #include #include "common/logging.h" +#include "io/fs/file_writer.h" #include "olap/key_coder.h" #include "olap/olap_common.h" #include "olap/rowset/segment_v2/encoding_info.h" @@ -45,6 +46,7 @@ IndexedColumnWriter::IndexedColumnWriter(const IndexedColumnWriterOptions& optio _file_writer(file_writer), _num_values(0), _num_data_pages(0), + _disk_size(0), _value_key_coder(nullptr), _compress_codec(nullptr) { _first_value.resize(_type_info->size()); @@ -116,10 +118,12 @@ Status IndexedColumnWriter::_finish_current_data_page(size_t& num_val) { footer.mutable_data_page_footer()->set_num_values(num_values_in_page); footer.mutable_data_page_footer()->set_nullmap_size(0); + uint64_t start_size = _file_writer->bytes_appended(); RETURN_IF_ERROR(PageIO::compress_and_write_page( _compress_codec, _options.compression_min_space_saving, _file_writer, {page_body.slice()}, footer, &_last_data_page)); _num_data_pages++; + _disk_size += (_file_writer->bytes_appended() - start_size); if (_options.write_ordinal_index) { std::string key; @@ -171,9 +175,11 @@ Status IndexedColumnWriter::_flush_index(IndexPageBuilder* index_builder, BTreeM index_builder->finish(&page_body, &page_footer); PagePointer pp; + uint64_t start_size = _file_writer->bytes_appended(); RETURN_IF_ERROR(PageIO::compress_and_write_page( _compress_codec, _options.compression_min_space_saving, _file_writer, {page_body.slice()}, page_footer, &pp)); + _disk_size += (_file_writer->bytes_appended() - start_size); meta->set_is_root_data_page(false); pp.to_proto(meta->mutable_root_page()); diff --git a/be/src/olap/rowset/segment_v2/indexed_column_writer.h b/be/src/olap/rowset/segment_v2/indexed_column_writer.h index a95a9fce7f76419..ba61708dd909362 100644 --- a/be/src/olap/rowset/segment_v2/indexed_column_writer.h +++ b/be/src/olap/rowset/segment_v2/indexed_column_writer.h @@ -83,6 +83,8 @@ class IndexedColumnWriter { Status finish(IndexedColumnMetaPB* meta); + uint64_t disk_size() const { return _disk_size; } + private: Status _finish_current_data_page(size_t& num_val); @@ -96,6 +98,7 @@ class IndexedColumnWriter { ordinal_t _num_values; uint32_t _num_data_pages; + uint64_t _disk_size; // remember the first value in current page faststring _first_value; PagePointer _last_data_page; diff --git a/be/src/olap/rowset/segment_v2/inverted_index_compound_directory.cpp b/be/src/olap/rowset/segment_v2/inverted_index_compound_directory.cpp index be19be580a520b8..4574d64abf92ecb 100644 --- a/be/src/olap/rowset/segment_v2/inverted_index_compound_directory.cpp +++ b/be/src/olap/rowset/segment_v2/inverted_index_compound_directory.cpp @@ -467,7 +467,6 @@ void DorisCompoundDirectory::init(const io::FileSystemSPtr& _fs, const char* _pa if (lock_factory == nullptr) { lock_factory = _CLNEW lucene::store::NoLockFactory(); - fs->create_directory(directory); } setLockFactory(lock_factory); @@ -476,17 +475,21 @@ void DorisCompoundDirectory::init(const io::FileSystemSPtr& _fs, const char* _pa lockFactory->setLockPrefix(nullptr); } + // It's fail checking directory existence in S3. + if (fs->type() == io::FileSystemType::S3) { + return; + } bool exists = false; Status status = fs->exists(directory, &exists); if (!status.ok()) { auto err = "File system error: " + status.to_string(); LOG(WARNING) << err; - _CLTHROWA_DEL(CL_ERR_IO, err.c_str()); + _CLTHROWA(CL_ERR_IO, err.c_str()); } if (!exists) { auto e = "Doris compound directory init error: " + directory + " is not a directory"; LOG(WARNING) << e; - _CLTHROWA_DEL(CL_ERR_IO, e.c_str()); + _CLTHROWA(CL_ERR_IO, e.c_str()); } } @@ -579,7 +582,7 @@ DorisCompoundDirectory* DorisCompoundDirectory::getDirectory( bool exists = false; _fs->exists(file, &exists); if (!exists) { - mkdir(file, 0777); + _fs->create_directory(file); } dir = _CLNEW DorisCompoundDirectory(); diff --git a/be/src/olap/rowset/segment_v2/inverted_index_writer.cpp b/be/src/olap/rowset/segment_v2/inverted_index_writer.cpp index ab5d3548df44b49..fcf125b2fa8f2de 100644 --- a/be/src/olap/rowset/segment_v2/inverted_index_writer.cpp +++ b/be/src/olap/rowset/segment_v2/inverted_index_writer.cpp @@ -409,7 +409,9 @@ class InvertedIndexColumnWriterImpl : public InvertedIndexColumnWriter { FINALLY_FINALIZE_OUTPUT(meta_out) FINALLY_FINALIZE_OUTPUT(data_out) FINALLY_FINALIZE_OUTPUT(index_out) - FINALLY_FINALIZE_OUTPUT(dir) + if constexpr (field_is_numeric_type(field_type)) { + FINALLY_FINALIZE_OUTPUT(dir) + } LOG(WARNING) << "Inverted index writer finish error occurred: " << e.what(); return Status::Error( "Inverted index writer finish error occurred"); diff --git a/be/src/olap/rowset/segment_v2/segment_iterator.cpp b/be/src/olap/rowset/segment_v2/segment_iterator.cpp index c15371368068fba..eaa3102d0b04f6b 100644 --- a/be/src/olap/rowset/segment_v2/segment_iterator.cpp +++ b/be/src/olap/rowset/segment_v2/segment_iterator.cpp @@ -374,6 +374,7 @@ Status SegmentIterator::_prepare_seek(const StorageReadOptions::KeyRange& key_ra &_column_iterators[unique_id])); ColumnIteratorOptions iter_opts; iter_opts.stats = _opts.stats; + iter_opts.use_page_cache = _opts.use_page_cache; iter_opts.file_reader = _file_reader.get(); iter_opts.io_ctx = _opts.io_ctx; RETURN_IF_ERROR(_column_iterators[unique_id]->init(iter_opts)); @@ -490,6 +491,26 @@ Status SegmentIterator::_get_row_ranges_from_conditions(RowRanges* condition_row RowRanges::ranges_intersection(*condition_row_ranges, zone_map_row_ranges, condition_row_ranges); _opts.stats->rows_stats_filtered += (pre_size - condition_row_ranges->count()); + + /// Low cardinality optimization is currently not very stable, so to prevent data corruption, + /// we are temporarily disabling its use in data compaction. + if (_opts.io_ctx.reader_type == ReaderType::READER_QUERY) { + RowRanges dict_row_ranges = RowRanges::create_single(num_rows()); + for (auto cid : cids) { + RowRanges tmp_row_ranges = RowRanges::create_single(num_rows()); + DCHECK(_opts.col_id_to_predicates.count(cid) > 0); + uint32_t unique_cid = _schema->unique_id(cid); + RETURN_IF_ERROR(_column_iterators[unique_cid]->get_row_ranges_by_dict( + _opts.col_id_to_predicates.at(cid).get(), &tmp_row_ranges)); + RowRanges::ranges_intersection(dict_row_ranges, tmp_row_ranges, &dict_row_ranges); + } + + pre_size = condition_row_ranges->count(); + RowRanges::ranges_intersection(*condition_row_ranges, dict_row_ranges, + condition_row_ranges); + _opts.stats->rows_dict_filtered += (pre_size - condition_row_ranges->count()); + } + return Status::OK(); } diff --git a/be/src/olap/rowset/segment_v2/segment_writer.cpp b/be/src/olap/rowset/segment_v2/segment_writer.cpp index 561da9c3c6c1819..d339b324a76aabc 100644 --- a/be/src/olap/rowset/segment_v2/segment_writer.cpp +++ b/be/src/olap/rowset/segment_v2/segment_writer.cpp @@ -330,7 +330,10 @@ void SegmentWriter::_serialize_block_to_row_column(vectorized::Block& block) { Status SegmentWriter::append_block_with_partial_content(const vectorized::Block* block, size_t row_pos, size_t num_rows) { CHECK(block->columns() > _tablet_schema->num_key_columns() && - block->columns() < _tablet_schema->num_columns()); + block->columns() < _tablet_schema->num_columns()) + << "block columns: " << block->columns() + << ", num key columns: " << _tablet_schema->num_key_columns() + << ", total schema columns: " << _tablet_schema->num_columns(); CHECK(_tablet_schema->keys_type() == UNIQUE_KEYS && _opts.enable_unique_key_merge_on_write); // find missing column cids @@ -365,12 +368,12 @@ Status SegmentWriter::append_block_with_partial_content(const vectorized::Block* bool has_default = false; std::vector use_default_flag; use_default_flag.reserve(num_rows); - std::unordered_map segment_caches; std::vector specified_rowsets; { std::shared_lock rlock(_tablet->get_header_lock()); specified_rowsets = _tablet->get_rowset_by_ids(&_mow_context->rowset_ids); } + std::vector> segment_caches(specified_rowsets.size()); // locate rows in base data { for (size_t pos = row_pos; pos < num_rows; pos++) { @@ -764,10 +767,13 @@ Status SegmentWriter::finalize_columns_index(uint64_t* index_size) { if (_has_key) { if (_tablet_schema->keys_type() == UNIQUE_KEYS && _opts.enable_unique_key_merge_on_write) { RETURN_IF_ERROR(_write_primary_key_index()); + // IndexedColumnWriter write data pages mixed with segment data, we should use + // the stat from primary key index builder. + *index_size += _primary_key_index_builder->disk_size(); } else { RETURN_IF_ERROR(_write_short_key_index()); + *index_size = _file_writer->bytes_appended() - index_start; } - *index_size = _file_writer->bytes_appended() - index_start; } _inverted_index_file_size = try_get_inverted_index_file_size(); // reset all column writers and data_conveter diff --git a/be/src/olap/schema_change.cpp b/be/src/olap/schema_change.cpp index 98f7e8e53527ce7..05d9c869c3ab671 100644 --- a/be/src/olap/schema_change.cpp +++ b/be/src/olap/schema_change.cpp @@ -288,19 +288,7 @@ Status BlockChanger::change_block(vectorized::Block* ref_block, for (int idx = 0; idx < column_size; idx++) { int ref_idx = _schema_mapping[idx].ref_column; - if (ref_idx < 0 && _type != ROLLUP) { - // new column, write default value - auto value = _schema_mapping[idx].default_value; - auto column = new_block->get_by_position(idx).column->assume_mutable(); - if (value->is_null()) { - DCHECK(column->is_nullable()); - column->insert_many_defaults(row_size); - } else { - auto type_info = get_type_info(_schema_mapping[idx].new_column); - DefaultValueColumnIterator::insert_default_data(type_info.get(), value->size(), - value->ptr(), column, row_size); - } - } else if (_schema_mapping[idx].expr != nullptr) { + if (_schema_mapping[idx].expr != nullptr) { vectorized::VExprContextSPtr ctx; RETURN_IF_ERROR(vectorized::VExpr::create_expr_tree(*_schema_mapping[idx].expr, ctx)); RETURN_IF_ERROR(ctx->prepare(state, row_desc)); @@ -322,6 +310,24 @@ Status BlockChanger::change_block(vectorized::Block* ref_block, ref_block->get_by_position(result_column_id).column)); } swap_idx_map[result_column_id] = idx; + } else if (ref_idx < 0) { + if (_type != ROLLUP) { + // new column, write default value + auto value = _schema_mapping[idx].default_value; + auto column = new_block->get_by_position(idx).column->assume_mutable(); + if (value->is_null()) { + DCHECK(column->is_nullable()); + column->insert_many_defaults(row_size); + } else { + auto type_info = get_type_info(_schema_mapping[idx].new_column); + DefaultValueColumnIterator::insert_default_data(type_info.get(), value->size(), + value->ptr(), column, row_size); + } + } else { + return Status::Error( + "rollup job meet invalid ref_column, new_column={}", + _schema_mapping[idx].new_column->name()); + } } else { // same type, just swap column swap_idx_map[ref_idx] = idx; diff --git a/be/src/olap/snapshot_manager.cpp b/be/src/olap/snapshot_manager.cpp index d19af18cca227d7..c9bc76ae6825875 100644 --- a/be/src/olap/snapshot_manager.cpp +++ b/be/src/olap/snapshot_manager.cpp @@ -603,7 +603,7 @@ Status SnapshotManager::_create_snapshot_files(const TabletSharedPtr& ref_tablet break; } - } while (0); + } while (false); if (!res.ok()) { LOG(WARNING) << "fail to make snapshot, try to delete the snapshot path. path=" diff --git a/be/src/olap/storage_engine.cpp b/be/src/olap/storage_engine.cpp index d10c11b1c27c4ae..60b0b023c6609f7 100644 --- a/be/src/olap/storage_engine.cpp +++ b/be/src/olap/storage_engine.cpp @@ -776,7 +776,11 @@ void StorageEngine::gc_binlogs(const std::unordered_map& gc_ta LOG(INFO) << fmt::format("start to gc binlogs for tablet_id: {}, version: {}", tablet_id, version); - TabletSharedPtr tablet = StorageEngine::instance()->tablet_manager()->get_tablet(tablet_id); + TabletSharedPtr tablet = _tablet_manager->get_tablet(tablet_id); + if (tablet == nullptr) { + LOG(WARNING) << fmt::format("tablet_id: {} not found", tablet_id); + continue; + } tablet->gc_binlogs(version); } } diff --git a/be/src/olap/tablet.cpp b/be/src/olap/tablet.cpp index 2babfed8a053615..2d4964323513a1a 100644 --- a/be/src/olap/tablet.cpp +++ b/be/src/olap/tablet.cpp @@ -1902,6 +1902,28 @@ Status Tablet::create_rowset_writer(RowsetWriterContext& context, return RowsetFactory::create_rowset_writer(context, false, rowset_writer); } +// create a rowset writer with rowset_id and seg_id +// after writer, merge this transient rowset with original rowset +Status Tablet::create_transient_rowset_writer(RowsetSharedPtr rowset_ptr, + std::unique_ptr* rowset_writer) { + RowsetWriterContext context; + context.rowset_state = PREPARED; + context.segments_overlap = OVERLAPPING; + context.tablet_schema = std::make_shared(); + context.tablet_schema->copy_from(*(rowset_ptr->tablet_schema())); + context.tablet_schema->set_partial_update_info(false, std::set()); + context.newest_write_timestamp = UnixSeconds(); + context.tablet_id = table_id(); + // ATTN: context.tablet is a shared_ptr, can't simply set it's value to `this`. We should + // get the shared_ptr from tablet_manager. + context.tablet = StorageEngine::instance()->tablet_manager()->get_tablet(tablet_id()); + context.write_type = DataWriteType::TYPE_DIRECT; + RETURN_IF_ERROR( + create_transient_rowset_writer(context, rowset_ptr->rowset_id(), rowset_writer)); + (*rowset_writer)->set_segment_start_id(rowset_ptr->num_segments()); + return Status::OK(); +} + Status Tablet::create_transient_rowset_writer(RowsetWriterContext& context, const RowsetId& rowset_id, std::unique_ptr* rowset_writer) { @@ -2699,12 +2721,11 @@ Status Tablet::lookup_row_data(const Slice& encoded_key, const RowLocation& row_ return Status::OK(); } -Status Tablet::lookup_row_key( - const Slice& encoded_key, bool with_seq_col, - const std::vector& specified_rowsets, RowLocation* row_location, - uint32_t version, - std::unordered_map& segment_caches, - RowsetSharedPtr* rowset) { +Status Tablet::lookup_row_key(const Slice& encoded_key, bool with_seq_col, + const std::vector& specified_rowsets, + RowLocation* row_location, uint32_t version, + std::vector>& segment_caches, + RowsetSharedPtr* rowset) { SCOPED_BVAR_LATENCY(g_tablet_lookup_rowkey_latency); size_t seq_col_length = 0; if (_schema->has_sequence_col() && with_seq_col) { @@ -2713,7 +2734,8 @@ Status Tablet::lookup_row_key( Slice key_without_seq = Slice(encoded_key.get_data(), encoded_key.get_size() - seq_col_length); RowLocation loc; - for (auto& rs : specified_rowsets) { + for (size_t i = 0; i < specified_rowsets.size(); i++) { + auto& rs = specified_rowsets[i]; auto& segments_key_bounds = rs->rowset_meta()->get_segments_key_bounds(); int num_segments = rs->num_segments(); DCHECK_EQ(segments_key_bounds.size(), num_segments); @@ -2729,14 +2751,12 @@ Status Tablet::lookup_row_key( continue; } - auto iter = segment_caches.find(rs->rowset_id()); - if (iter == segment_caches.end()) { - SegmentCacheHandle segment_cache_handle; + if (UNLIKELY(segment_caches[i] == nullptr)) { + segment_caches[i] = std::make_unique(); RETURN_IF_ERROR(SegmentLoader::instance()->load_segments( - std::static_pointer_cast(rs), &segment_cache_handle, true)); - iter = segment_caches.emplace(rs->rowset_id(), std::move(segment_cache_handle)).first; + std::static_pointer_cast(rs), segment_caches[i].get(), true)); } - auto& segments = iter->second.get_segments(); + auto& segments = segment_caches[i]->get_segments(); DCHECK_EQ(segments.size(), num_segments); for (auto id : picked_segments) { @@ -2845,7 +2865,7 @@ Status Tablet::calc_segment_delete_bitmap(RowsetSharedPtr rowset, // The data for each segment may be lookup multiple times. Creating a SegmentCacheHandle // will update the lru cache, and there will be obvious lock competition in multithreading // scenarios, so using a segment_caches to cache SegmentCacheHandle. - std::unordered_map segment_caches; + std::vector> segment_caches(specified_rowsets.size()); while (remaining > 0) { std::unique_ptr iter; RETURN_IF_ERROR(pk_idx->new_iterator(&iter)); diff --git a/be/src/olap/tablet.h b/be/src/olap/tablet.h index 7cccb03a313b8ca..724ab812430170f 100644 --- a/be/src/olap/tablet.h +++ b/be/src/olap/tablet.h @@ -328,6 +328,9 @@ class Tablet : public BaseTablet { Status create_rowset_writer(RowsetWriterContext& context, std::unique_ptr* rowset_writer); + + Status create_transient_rowset_writer(RowsetSharedPtr rowset_ptr, + std::unique_ptr* rowset_writer); Status create_transient_rowset_writer(RowsetWriterContext& context, const RowsetId& rowset_id, std::unique_ptr* rowset_writer); @@ -399,12 +402,11 @@ class Tablet : public BaseTablet { // Lookup the row location of `encoded_key`, the function sets `row_location` on success. // NOTE: the method only works in unique key model with primary key index, you will got a // not supported error in other data model. - Status lookup_row_key( - const Slice& encoded_key, bool with_seq_col, - const std::vector& specified_rowsets, RowLocation* row_location, - uint32_t version, - std::unordered_map& segment_caches, - RowsetSharedPtr* rowset = nullptr); + Status lookup_row_key(const Slice& encoded_key, bool with_seq_col, + const std::vector& specified_rowsets, + RowLocation* row_location, uint32_t version, + std::vector>& segment_caches, + RowsetSharedPtr* rowset = nullptr); // Lookup a row with TupleDescriptor and fill Block Status lookup_row_data(const Slice& encoded_key, const RowLocation& row_location, diff --git a/be/src/olap/tablet_schema.cpp b/be/src/olap/tablet_schema.cpp index f5f62e68578761f..414036ab58864c2 100644 --- a/be/src/olap/tablet_schema.cpp +++ b/be/src/olap/tablet_schema.cpp @@ -637,6 +637,17 @@ void TabletSchema::append_index(TabletIndex index) { _indexes.push_back(std::move(index)); } +void TabletSchema::remove_index(int64_t index_id) { + std::vector indexes; + for (auto index : _indexes) { + if (index.index_id() == index_id) { + continue; + } + indexes.emplace_back(std::move(index)); + } + _indexes = std::move(indexes); +} + void TabletSchema::clear_columns() { _field_name_to_index.clear(); _field_id_to_index.clear(); diff --git a/be/src/olap/tablet_schema.h b/be/src/olap/tablet_schema.h index ceadf76a0af6416..5a521f36107ef3f 100644 --- a/be/src/olap/tablet_schema.h +++ b/be/src/olap/tablet_schema.h @@ -207,6 +207,7 @@ class TabletSchema { void to_schema_pb(TabletSchemaPB* tablet_meta_pb) const; void append_column(TabletColumn column, bool is_dropped_column = false); void append_index(TabletIndex index); + void remove_index(int64_t index_id); // Must make sure the row column is always the last column void add_row_column(); void copy_from(const TabletSchema& tablet_schema); diff --git a/be/src/olap/task/engine_publish_version_task.cpp b/be/src/olap/task/engine_publish_version_task.cpp index 02af6bf674d17b4..249a7a424c8decf 100644 --- a/be/src/olap/task/engine_publish_version_task.cpp +++ b/be/src/olap/task/engine_publish_version_task.cpp @@ -72,8 +72,7 @@ EnginePublishVersionTask::EnginePublishVersionTask( const TPublishVersionRequest& publish_version_req, std::vector* error_tablet_ids, std::vector* succ_tablet_ids, std::vector>* discontinuous_version_tablets) - : _total_task_num(0), - _publish_version_req(publish_version_req), + : _publish_version_req(publish_version_req), _error_tablet_ids(error_tablet_ids), _succ_tablet_ids(succ_tablet_ids), _discontinuous_version_tablets(discontinuous_version_tablets) {} @@ -88,25 +87,14 @@ void EnginePublishVersionTask::add_succ_tablet_id(int64_t tablet_id) { _succ_tablet_ids->push_back(tablet_id); } -void EnginePublishVersionTask::wait() { - std::unique_lock lock(_tablet_finish_mutex); - _tablet_finish_cond.wait(lock); -} - -void EnginePublishVersionTask::notify() { - std::unique_lock lock(_tablet_finish_mutex); - _tablet_finish_cond.notify_one(); -} - -int64_t EnginePublishVersionTask::finish_task() { - return _total_task_num.fetch_sub(1); -} - Status EnginePublishVersionTask::finish() { Status res = Status::OK(); int64_t transaction_id = _publish_version_req.transaction_id; OlapStopWatch watch; VLOG_NOTICE << "begin to process publish version. transaction_id=" << transaction_id; + std::unique_ptr token = + StorageEngine::instance()->tablet_publish_txn_thread_pool()->new_token( + ThreadPool::ExecutionMode::CONCURRENT); // each partition for (auto& par_ver_info : _publish_version_req.partition_version_infos) { @@ -187,19 +175,13 @@ Status EnginePublishVersionTask::finish() { continue; } } - _total_task_num.fetch_add(1); auto tablet_publish_txn_ptr = std::make_shared( this, tablet, rowset, partition_id, transaction_id, version, tablet_info); - auto submit_st = - StorageEngine::instance()->tablet_publish_txn_thread_pool()->submit_func( - [=]() { tablet_publish_txn_ptr->handle(); }); + auto submit_st = token->submit_func([=]() { tablet_publish_txn_ptr->handle(); }); CHECK(submit_st.ok()) << submit_st; } } - // wait for all publish txn finished - while (_total_task_num.load() != 0) { - wait(); - } + token->wait(); // check if the related tablet remained all have the version for (auto& par_ver_info : _publish_version_req.partition_version_infos) { @@ -260,12 +242,7 @@ void TabletPublishTxnTask::handle() { _engine_publish_version_task->add_error_tablet_id(_tablet_info.tablet_id); return; } - Defer defer {[&] { - _rowset->finish_publish(); - if (_engine_publish_version_task->finish_task() == 1) { - _engine_publish_version_task->notify(); - } - }}; + Defer defer {[&] { _rowset->finish_publish(); }}; auto publish_status = StorageEngine::instance()->txn_manager()->publish_txn( _partition_id, _tablet, _transaction_id, _version, &_stats); if (publish_status != Status::OK()) { diff --git a/be/src/olap/task/engine_publish_version_task.h b/be/src/olap/task/engine_publish_version_task.h index c8a68dedea3a9d9..8acf8099ca244f8 100644 --- a/be/src/olap/task/engine_publish_version_task.h +++ b/be/src/olap/task/engine_publish_version_task.h @@ -93,21 +93,14 @@ class EnginePublishVersionTask : public EngineTask { void add_error_tablet_id(int64_t tablet_id); void add_succ_tablet_id(int64_t tablet_id); - void notify(); - void wait(); - int64_t finish_task(); private: - std::atomic _total_task_num; const TPublishVersionRequest& _publish_version_req; std::mutex _tablet_ids_mutex; vector* _error_tablet_ids; vector* _succ_tablet_ids; std::vector>* _discontinuous_version_tablets; - - std::mutex _tablet_finish_mutex; - std::condition_variable _tablet_finish_cond; }; class AsyncTabletPublishTask { diff --git a/be/src/olap/task/index_builder.cpp b/be/src/olap/task/index_builder.cpp index 05ba099f90e6e59..4717dff9b3638ad 100644 --- a/be/src/olap/task/index_builder.cpp +++ b/be/src/olap/task/index_builder.cpp @@ -30,12 +30,10 @@ namespace doris { IndexBuilder::IndexBuilder(const TabletSharedPtr& tablet, const std::vector& columns, - const std::vector exist_indexes, const std::vector& alter_inverted_indexes, bool is_drop_op) : _tablet(tablet), _columns(columns), - _exist_indexes(exist_indexes), _alter_inverted_indexes(alter_inverted_indexes), _is_drop_op(is_drop_op) { _olap_data_convertor = std::make_unique(); @@ -63,8 +61,16 @@ Status IndexBuilder::update_inverted_index_info() { auto input_rs_tablet_schema = input_rowset->tablet_schema(); output_rs_tablet_schema->copy_from(*input_rs_tablet_schema); if (_is_drop_op) { - output_rs_tablet_schema->update_indexes_from_thrift(_exist_indexes); + // base on input rowset's tablet_schema to build + // output rowset's tablet_schema which only remove + // the indexes specified in this drop index request + for (auto t_inverted_index : _alter_inverted_indexes) { + output_rs_tablet_schema->remove_index(t_inverted_index.index_id); + } } else { + // base on input rowset's tablet_schema to build + // output rowset's tablet_schema which only add + // the indexes specified in this build index request for (auto t_inverted_index : _alter_inverted_indexes) { TabletIndex index; index.init_from_thrift(t_inverted_index, *input_rs_tablet_schema); @@ -183,7 +189,7 @@ Status IndexBuilder::handle_single_rowset(RowsetMetaSharedPtr output_rowset_meta if (!res.ok()) { LOG(WARNING) << "failed to create iterator[" << seg_ptr->id() << "]: " << res.to_string(); - return Status::Error(); + return Status::Error(res.to_string()); } std::shared_ptr block = std::make_shared( @@ -427,6 +433,13 @@ Status IndexBuilder::do_build_inverted_index() { } Status IndexBuilder::modify_rowsets(const Merger::Statistics* stats) { + for (auto rowset_ptr : _output_rowsets) { + auto rowset_id = rowset_ptr->rowset_id(); + if (StorageEngine::instance()->check_rowset_id_in_unused_rowsets(rowset_id)) { + DCHECK(false) << "output rowset: " << rowset_id.to_string() << " in unused rowsets"; + } + } + if (_tablet->keys_type() == KeysType::UNIQUE_KEYS && _tablet->enable_unique_key_merge_on_write()) { std::lock_guard rwlock(_tablet->get_rowset_update_lock()); diff --git a/be/src/olap/task/index_builder.h b/be/src/olap/task/index_builder.h index 562cb1148d8a48e..9e406c22c12a223 100644 --- a/be/src/olap/task/index_builder.h +++ b/be/src/olap/task/index_builder.h @@ -36,7 +36,6 @@ using RowsetWriterUniquePtr = std::unique_ptr; class IndexBuilder { public: IndexBuilder(const TabletSharedPtr& tablet, const std::vector& columns, - const std::vector exist_indexes, const std::vector& alter_inverted_indexes, bool is_drop_op = false); ~IndexBuilder(); @@ -65,7 +64,6 @@ class IndexBuilder { private: TabletSharedPtr _tablet; std::vector _columns; - std::vector _exist_indexes; std::vector _alter_inverted_indexes; bool _is_drop_op; std::unordered_map> _rowset_alter_index_column_ids; diff --git a/be/src/olap/txn_manager.cpp b/be/src/olap/txn_manager.cpp index ad799868aa39090..ebcb2873893610b 100644 --- a/be/src/olap/txn_manager.cpp +++ b/be/src/olap/txn_manager.cpp @@ -372,7 +372,7 @@ Status TxnManager::publish_txn(OlapMeta* meta, TPartitionId partition_id, // update delete_bitmap if (tablet_txn_info.unique_key_merge_on_write) { std::unique_ptr rowset_writer; - _create_transient_rowset_writer(tablet, rowset, &rowset_writer); + tablet->create_transient_rowset_writer(rowset, &rowset_writer); int64_t t2 = MonotonicMicros(); RETURN_IF_ERROR(tablet->update_delete_bitmap(rowset, tablet_txn_info.rowset_ids, @@ -450,27 +450,6 @@ Status TxnManager::publish_txn(OlapMeta* meta, TPartitionId partition_id, return status; } -// create a rowset writer with rowset_id and seg_id -// after writer, merge this transient rowset with original rowset -Status TxnManager::_create_transient_rowset_writer(std::shared_ptr tablet, - RowsetSharedPtr rowset_ptr, - std::unique_ptr* rowset_writer) { - RowsetWriterContext context; - context.rowset_state = PREPARED; - context.segments_overlap = OVERLAPPING; - context.tablet_schema = std::make_shared(); - context.tablet_schema->copy_from(*(rowset_ptr->tablet_schema())); - context.tablet_schema->set_partial_update_info(false, std::set()); - context.newest_write_timestamp = UnixSeconds(); - context.tablet_id = tablet->table_id(); - context.tablet = tablet; - context.write_type = DataWriteType::TYPE_DIRECT; - RETURN_IF_ERROR(tablet->create_transient_rowset_writer(context, rowset_ptr->rowset_id(), - rowset_writer)); - (*rowset_writer)->set_segment_start_id(rowset_ptr->num_segments()); - return Status::OK(); -} - // txn could be rollbacked if it does not have related rowset // if the txn has related rowset then could not rollback it, because it // may be committed in another thread and our current thread meets errors when writing to data file diff --git a/be/src/olap/txn_manager.h b/be/src/olap/txn_manager.h index 36be3b03f54729b..fcebe2d9d1cfcf9 100644 --- a/be/src/olap/txn_manager.h +++ b/be/src/olap/txn_manager.h @@ -214,10 +214,6 @@ class TxnManager { void _insert_txn_partition_map_unlocked(int64_t transaction_id, int64_t partition_id); void _clear_txn_partition_map_unlocked(int64_t transaction_id, int64_t partition_id); - Status _create_transient_rowset_writer(std::shared_ptr tablet, - RowsetSharedPtr rowset_ptr, - std::unique_ptr* rowset_writer); - private: const int32_t _txn_map_shard_size; diff --git a/be/src/pipeline/exec/exchange_sink_buffer.cpp b/be/src/pipeline/exec/exchange_sink_buffer.cpp index e8b3f76fda5e1fb..0326929e5ca0116 100644 --- a/be/src/pipeline/exec/exchange_sink_buffer.cpp +++ b/be/src/pipeline/exec/exchange_sink_buffer.cpp @@ -62,6 +62,8 @@ void ExchangeSinkBuffer::close() { pair.second->release_finst_id(); pair.second->release_query_id(); } + _instance_to_broadcast_package_queue.clear(); + _instance_to_package_queue.clear(); _instance_to_request.clear(); } @@ -146,7 +148,7 @@ Status ExchangeSinkBuffer::add_block(BroadcastTransmitInfo&& request) { send_now = true; _instance_to_sending_by_pipeline[ins_id.lo] = false; } - _instance_to_broadcast_package_queue[ins_id.lo].emplace(std::move(request)); + _instance_to_broadcast_package_queue[ins_id.lo].emplace(request); } if (send_now) { RETURN_IF_ERROR(_send_rpc(ins_id.lo)); @@ -158,6 +160,8 @@ Status ExchangeSinkBuffer::add_block(BroadcastTransmitInfo&& request) { Status ExchangeSinkBuffer::_send_rpc(InstanceLoId id) { std::unique_lock lock(*_instance_to_package_queue_mutex[id]); + DCHECK(_instance_to_sending_by_pipeline[id] == false); + std::queue>& q = _instance_to_package_queue[id]; std::queue>& broadcast_q = _instance_to_broadcast_package_queue[id]; @@ -257,7 +261,6 @@ Status ExchangeSinkBuffer::_send_rpc(InstanceLoId id) { broadcast_q.pop(); } else { _instance_to_sending_by_pipeline[id] = true; - return Status::OK(); } return Status::OK(); diff --git a/be/src/runtime/descriptors.cpp b/be/src/runtime/descriptors.cpp index a75d3d0e71011b5..1855f5d58d79ce1 100644 --- a/be/src/runtime/descriptors.cpp +++ b/be/src/runtime/descriptors.cpp @@ -36,7 +36,6 @@ #include "vec/data_types/data_type_factory.hpp" namespace doris { -using boost::algorithm::join; const int RowDescriptor::INVALID_IDX = -1; std::string NullIndicatorOffset::debug_string() const { @@ -112,7 +111,7 @@ vectorized::DataTypePtr SlotDescriptor::get_data_type_ptr() const { std::string SlotDescriptor::debug_string() const { std::stringstream out; out << "Slot(id=" << _id << " type=" << _type << " col=" << _col_pos - << ", colname=" << _col_name << " null=" << _null_indicator_offset.debug_string() << ")"; + << ", colname=" << _col_name << ", nullable=" << is_nullable() << ")"; return out.str(); } @@ -140,7 +139,7 @@ std::string OlapTableDescriptor::debug_string() const { SchemaTableDescriptor::SchemaTableDescriptor(const TTableDescriptor& tdesc) : TableDescriptor(tdesc), _schema_table_type(tdesc.schemaTable.tableType) {} -SchemaTableDescriptor::~SchemaTableDescriptor() {} +SchemaTableDescriptor::~SchemaTableDescriptor() = default; std::string SchemaTableDescriptor::debug_string() const { std::stringstream out; @@ -151,7 +150,7 @@ std::string SchemaTableDescriptor::debug_string() const { BrokerTableDescriptor::BrokerTableDescriptor(const TTableDescriptor& tdesc) : TableDescriptor(tdesc) {} -BrokerTableDescriptor::~BrokerTableDescriptor() {} +BrokerTableDescriptor::~BrokerTableDescriptor() = default; std::string BrokerTableDescriptor::debug_string() const { std::stringstream out; @@ -161,7 +160,7 @@ std::string BrokerTableDescriptor::debug_string() const { HiveTableDescriptor::HiveTableDescriptor(const TTableDescriptor& tdesc) : TableDescriptor(tdesc) {} -HiveTableDescriptor::~HiveTableDescriptor() {} +HiveTableDescriptor::~HiveTableDescriptor() = default; std::string HiveTableDescriptor::debug_string() const { std::stringstream out; @@ -172,7 +171,7 @@ std::string HiveTableDescriptor::debug_string() const { IcebergTableDescriptor::IcebergTableDescriptor(const TTableDescriptor& tdesc) : TableDescriptor(tdesc) {} -IcebergTableDescriptor::~IcebergTableDescriptor() {} +IcebergTableDescriptor::~IcebergTableDescriptor() = default; std::string IcebergTableDescriptor::debug_string() const { std::stringstream out; @@ -189,7 +188,7 @@ MaxComputeTableDescriptor::MaxComputeTableDescriptor(const TTableDescriptor& tde _secret_key(tdesc.mcTable.secret_key), _public_access(tdesc.mcTable.public_access) {} -MaxComputeTableDescriptor::~MaxComputeTableDescriptor() {} +MaxComputeTableDescriptor::~MaxComputeTableDescriptor() = default; std::string MaxComputeTableDescriptor::debug_string() const { std::stringstream out; @@ -199,7 +198,7 @@ std::string MaxComputeTableDescriptor::debug_string() const { EsTableDescriptor::EsTableDescriptor(const TTableDescriptor& tdesc) : TableDescriptor(tdesc) {} -EsTableDescriptor::~EsTableDescriptor() {} +EsTableDescriptor::~EsTableDescriptor() = default; std::string EsTableDescriptor::debug_string() const { std::stringstream out; @@ -272,7 +271,6 @@ TupleDescriptor::TupleDescriptor(const TTupleDescriptor& tdesc, bool own_slots) _table_desc(nullptr), _num_null_bytes(tdesc.numNullBytes), _num_materialized_slots(0), - _slots(), _has_varlen_slots(false), _own_slots(own_slots) { if (false == tdesc.__isset.numNullSlots) { @@ -288,7 +286,6 @@ TupleDescriptor::TupleDescriptor(const PTupleDescriptor& pdesc, bool own_slots) _table_desc(nullptr), _num_null_bytes(pdesc.num_null_bytes()), _num_materialized_slots(0), - _slots(), _has_varlen_slots(false), _own_slots(own_slots) { if (!pdesc.has_num_null_slots()) { diff --git a/be/src/runtime/descriptors.h b/be/src/runtime/descriptors.h index 04e6571ef5c31b8..aff3b03a0f7011b 100644 --- a/be/src/runtime/descriptors.h +++ b/be/src/runtime/descriptors.h @@ -90,7 +90,6 @@ class SlotDescriptor { int col_pos() const { return _col_pos; } // Returns the field index in the generated llvm struct for this slot's tuple int field_idx() const { return _field_idx; } - const NullIndicatorOffset& null_indicator_offset() const { return _null_indicator_offset; } bool is_materialized() const { return _is_materialized; } bool is_nullable() const { return _null_indicator_offset.bit_mask != 0; } diff --git a/be/src/runtime/exec_env_init.cpp b/be/src/runtime/exec_env_init.cpp index 8875cf5463431be..a699dc402f08675 100644 --- a/be/src/runtime/exec_env_init.cpp +++ b/be/src/runtime/exec_env_init.cpp @@ -55,7 +55,6 @@ #include "runtime/heartbeat_flags.h" #include "runtime/load_channel_mgr.h" #include "runtime/load_path_mgr.h" -#include "runtime/memory/chunk_allocator.h" #include "runtime/memory/mem_tracker.h" #include "runtime/memory/mem_tracker_limiter.h" #include "runtime/memory/thread_mem_tracker_mgr.h" @@ -314,23 +313,6 @@ Status ExecEnv::_init_mem_env() { // 4. init other managers RETURN_IF_ERROR(_block_spill_mgr->init()); - - // 5. init chunk allocator - if (!BitUtil::IsPowerOf2(config::min_chunk_reserved_bytes)) { - ss << "Config min_chunk_reserved_bytes must be a power-of-two: " - << config::min_chunk_reserved_bytes; - return Status::InternalError(ss.str()); - } - - int64_t chunk_reserved_bytes_limit = - ParseUtil::parse_mem_spec(config::chunk_reserved_bytes_limit, MemInfo::mem_limit(), - MemInfo::physical_mem(), &is_percent); - chunk_reserved_bytes_limit = - BitUtil::RoundDown(chunk_reserved_bytes_limit, config::min_chunk_reserved_bytes); - ChunkAllocator::init_instance(chunk_reserved_bytes_limit); - LOG(INFO) << "Chunk allocator memory limit: " - << PrettyPrinter::print(chunk_reserved_bytes_limit, TUnit::BYTES) - << ", origin config value: " << config::chunk_reserved_bytes_limit; return Status::OK(); } diff --git a/be/src/runtime/memory/chunk_allocator.cpp b/be/src/runtime/memory/chunk_allocator.cpp deleted file mode 100644 index 53c51660750c53b..000000000000000 --- a/be/src/runtime/memory/chunk_allocator.cpp +++ /dev/null @@ -1,281 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#include "runtime/memory/chunk_allocator.h" - -#include -#include -#include - -#include - -#include "common/config.h" -#include "common/status.h" -#include "runtime/memory/chunk.h" -#include "runtime/memory/system_allocator.h" -#include "runtime/thread_context.h" -#include "util/bit_util.h" -#include "util/cpu_info.h" -#include "util/doris_metrics.h" -#include "util/metrics.h" -#include "util/runtime_profile.h" -#include "util/spinlock.h" - -namespace doris { - -// <= MIN_CHUNK_SIZE, A large number of small chunks will waste extra storage and increase lock time. -static constexpr size_t MIN_CHUNK_SIZE = 4096; // 4K -// >= MAX_CHUNK_SIZE, Large chunks may not be used for a long time, wasting memory. -static constexpr size_t MAX_CHUNK_SIZE = 64 * (1ULL << 20); // 64M - -ChunkAllocator* ChunkAllocator::_s_instance = nullptr; - -DEFINE_COUNTER_METRIC_PROTOTYPE_2ARG(chunk_pool_local_core_alloc_count, MetricUnit::NOUNIT); -DEFINE_COUNTER_METRIC_PROTOTYPE_2ARG(chunk_pool_other_core_alloc_count, MetricUnit::NOUNIT); -DEFINE_COUNTER_METRIC_PROTOTYPE_2ARG(chunk_pool_system_alloc_count, MetricUnit::NOUNIT); -DEFINE_COUNTER_METRIC_PROTOTYPE_2ARG(chunk_pool_system_free_count, MetricUnit::NOUNIT); -DEFINE_COUNTER_METRIC_PROTOTYPE_2ARG(chunk_pool_system_alloc_cost_ns, MetricUnit::NANOSECONDS); -DEFINE_COUNTER_METRIC_PROTOTYPE_2ARG(chunk_pool_system_free_cost_ns, MetricUnit::NANOSECONDS); -DEFINE_GAUGE_METRIC_PROTOTYPE_2ARG(chunk_pool_reserved_bytes, MetricUnit::NOUNIT); - -static IntCounter* chunk_pool_local_core_alloc_count; -static IntCounter* chunk_pool_other_core_alloc_count; -static IntCounter* chunk_pool_system_alloc_count; -static IntCounter* chunk_pool_system_free_count; -static IntCounter* chunk_pool_system_alloc_cost_ns; -static IntCounter* chunk_pool_system_free_cost_ns; -static IntGauge* chunk_pool_reserved_bytes; - -#ifdef BE_TEST -static std::mutex s_mutex; -ChunkAllocator* ChunkAllocator::instance() { - std::lock_guard l(s_mutex); - if (_s_instance == nullptr) { - CpuInfo::init(); - ChunkAllocator::init_instance(4096); - } - return _s_instance; -} -#endif - -// Keep free chunk's ptr in size separated free list. -// This class is thread-safe. -class ChunkArena { - int TRY_LOCK_TIMES = 3; - -public: - ChunkArena() : _chunk_lists(64) {} - - ~ChunkArena() { - for (int i = 0; i < 64; ++i) { - if (_chunk_lists[i].empty()) continue; - for (auto ptr : _chunk_lists[i]) { - SystemAllocator::free(ptr); - } - } - } - - // Try to pop a free chunk from corresponding free list. - // Return true if success - bool pop_free_chunk(size_t size, uint8_t** ptr) { - int idx = BitUtil::Log2Ceiling64(size); - auto& free_list = _chunk_lists[idx]; - - if (free_list.empty()) return false; - - for (int i = 0; i < TRY_LOCK_TIMES; ++i) { - if (_lock.try_lock()) { - if (free_list.empty()) { - _lock.unlock(); - return false; - } else { - *ptr = free_list.back(); - free_list.pop_back(); - ASAN_UNPOISON_MEMORY_REGION(*ptr, size); - _lock.unlock(); - return true; - } - } - } - return false; - } - - void push_free_chunk(uint8_t* ptr, size_t size) { - int idx = BitUtil::Log2Ceiling64(size); - // Poison this chunk to make asan can detect invalid access - ASAN_POISON_MEMORY_REGION(ptr, size); - std::lock_guard l(_lock); - _chunk_lists[idx].push_back(ptr); - } - - void clear() { - std::lock_guard l(_lock); - for (int i = 0; i < 64; ++i) { - if (_chunk_lists[i].empty()) { - continue; - } - for (auto ptr : _chunk_lists[i]) { - ::free(ptr); - } - std::vector().swap(_chunk_lists[i]); - } - } - -private: - SpinLock _lock; - std::vector> _chunk_lists; -}; - -void ChunkAllocator::init_instance(size_t reserve_limit) { - if (_s_instance != nullptr) return; - _s_instance = new ChunkAllocator(reserve_limit); -} - -ChunkAllocator::ChunkAllocator(size_t reserve_limit) - : _reserve_bytes_limit(reserve_limit), - _steal_arena_limit(reserve_limit * 0.1), - _reserved_bytes(0), - _arenas(CpuInfo::get_max_num_cores()) { - _mem_tracker = - std::make_unique(MemTrackerLimiter::Type::GLOBAL, "ChunkAllocator"); - for (int i = 0; i < _arenas.size(); ++i) { - _arenas[i].reset(new ChunkArena()); - } - - _chunk_allocator_metric_entity = - DorisMetrics::instance()->metric_registry()->register_entity("chunk_allocator"); - INT_COUNTER_METRIC_REGISTER(_chunk_allocator_metric_entity, chunk_pool_local_core_alloc_count); - INT_COUNTER_METRIC_REGISTER(_chunk_allocator_metric_entity, chunk_pool_other_core_alloc_count); - INT_COUNTER_METRIC_REGISTER(_chunk_allocator_metric_entity, chunk_pool_system_alloc_count); - INT_COUNTER_METRIC_REGISTER(_chunk_allocator_metric_entity, chunk_pool_system_free_count); - INT_COUNTER_METRIC_REGISTER(_chunk_allocator_metric_entity, chunk_pool_system_alloc_cost_ns); - INT_COUNTER_METRIC_REGISTER(_chunk_allocator_metric_entity, chunk_pool_system_free_cost_ns); - INT_GAUGE_METRIC_REGISTER(_chunk_allocator_metric_entity, chunk_pool_reserved_bytes); -} - -Status ChunkAllocator::allocate_align(size_t size, Chunk* chunk) { - CHECK(size > 0); - size = BitUtil::RoundUpToPowerOfTwo(size); - // fast path: allocate from current core arena - int core_id = CpuInfo::get_current_core(); - chunk->size = size; - chunk->core_id = core_id; - - if (_reserve_bytes_limit < 1) { - // allocate from system allocator - chunk->data = SystemAllocator::allocate(size); - return Status::OK(); - } - - if (_arenas[core_id]->pop_free_chunk(size, &chunk->data)) { - DCHECK_GE(_reserved_bytes, 0); - _reserved_bytes.fetch_sub(size); - chunk_pool_local_core_alloc_count->increment(1); - // transfer the memory ownership of allocate from ChunkAllocator::tracker to the tls tracker. - THREAD_MEM_TRACKER_TRANSFER_FROM(size, _mem_tracker.get()); - return Status::OK(); - } - // Second path: try to allocate from other core's arena - // When the reserved bytes is greater than the limit, the chunk is stolen from other arena. - // Otherwise, it is allocated from the system first, which can reserve enough memory as soon as possible. - // After that, allocate from current core arena as much as possible. - if (_reserved_bytes > _steal_arena_limit) { - ++core_id; - for (int i = 1; i < _arenas.size(); ++i, ++core_id) { - if (_arenas[core_id % _arenas.size()]->pop_free_chunk(size, &chunk->data)) { - DCHECK_GE(_reserved_bytes, 0); - _reserved_bytes.fetch_sub(size); - chunk_pool_other_core_alloc_count->increment(1); - // reset chunk's core_id to other - chunk->core_id = core_id % _arenas.size(); - // transfer the memory ownership of allocate from ChunkAllocator::tracker to the tls tracker. - THREAD_MEM_TRACKER_TRANSFER_FROM(size, _mem_tracker.get()); - return Status::OK(); - } - } - } - - int64_t cost_ns = 0; - { - SCOPED_RAW_TIMER(&cost_ns); - // allocate from system allocator - chunk->data = SystemAllocator::allocate(size); - } - chunk_pool_system_alloc_count->increment(1); - chunk_pool_system_alloc_cost_ns->increment(cost_ns); - if (chunk->data == nullptr) { - return Status::MemoryAllocFailed("ChunkAllocator failed to allocate chunk {} bytes", size); - } - return Status::OK(); -} - -void ChunkAllocator::free(const Chunk& chunk) { - DCHECK(chunk.core_id != -1); - CHECK((chunk.size & (chunk.size - 1)) == 0); - if (config::disable_mem_pools || _reserve_bytes_limit < 1) { - SystemAllocator::free(chunk.data); - return; - } - - int64_t old_reserved_bytes = _reserved_bytes; - int64_t new_reserved_bytes = 0; - do { - new_reserved_bytes = old_reserved_bytes + chunk.size; - if (chunk.size <= MIN_CHUNK_SIZE || chunk.size >= MAX_CHUNK_SIZE || - new_reserved_bytes > _reserve_bytes_limit) { - int64_t cost_ns = 0; - { - SCOPED_RAW_TIMER(&cost_ns); - SystemAllocator::free(chunk.data); - } - chunk_pool_system_free_count->increment(1); - chunk_pool_system_free_cost_ns->increment(cost_ns); - - return; - } - } while (!_reserved_bytes.compare_exchange_weak(old_reserved_bytes, new_reserved_bytes)); - - // The memory size of allocate/free is a multiple of 2, so `_reserved_bytes% 100 == 32` - // will definitely happen, and the latest `_reserved_bytes` value will be set every time. - // The real-time and accurate `_reserved_bytes` value is not required. Usually, - // the value of `_reserved_bytes` is equal to ChunkAllocator MemTracker. - // The `_reserved_bytes` metric is only concerned when verifying the accuracy of MemTracker. - // Therefore, reduce the number of sets and reduce the performance impact. - if (_reserved_bytes % 100 == 32) { - chunk_pool_reserved_bytes->set_value(_reserved_bytes); - } - // The chunk's memory ownership is transferred from tls tracker to ChunkAllocator. - THREAD_MEM_TRACKER_TRANSFER_TO(chunk.size, _mem_tracker.get()); - _arenas[chunk.core_id]->push_free_chunk(chunk.data, chunk.size); -} - -void ChunkAllocator::free(uint8_t* data, size_t size) { - Chunk chunk; - chunk.data = data; - chunk.size = size; - chunk.core_id = CpuInfo::get_current_core(); - free(chunk); -} - -void ChunkAllocator::clear() { - for (int i = 0; i < _arenas.size(); ++i) { - _arenas[i]->clear(); - } - THREAD_MEM_TRACKER_TRANSFER_FROM(_mem_tracker->consumption(), _mem_tracker.get()); -} - -} // namespace doris diff --git a/be/src/runtime/memory/chunk_allocator.h b/be/src/runtime/memory/chunk_allocator.h deleted file mode 100644 index 467317c19176f82..000000000000000 --- a/be/src/runtime/memory/chunk_allocator.h +++ /dev/null @@ -1,99 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#pragma once - -#include -#include -#include -#include -#include - -#include "runtime/memory/mem_tracker_limiter.h" - -namespace doris { - -struct Chunk; -class ChunkArena; -class MetricEntity; -class Status; - -// Used to allocate memory with power-of-two length. -// This Allocator allocate memory from system and cache free chunks for -// later use. -// -// ChunkAllocator has one ChunkArena for each CPU core, it will try to allocate -// memory from current core arena firstly. In this way, there will be no lock contention -// between concurrently-running threads. If this fails, ChunkAllocator will try to allocate -// memory from other core's arena. -// -// Memory Reservation -// ChunkAllocator has a limit about how much free chunk bytes it can reserve, above which -// chunk will released to system memory. For the worst case, when the limits is 0, it will -// act as allocating directly from system. -// -// ChunkArena will keep a separate free list for each chunk size. In common case, chunk will -// be allocated from current core arena. In this case, there is no lock contention. -// -// Must call CpuInfo::init() and DorisMetrics::instance()->initialize() to achieve good performance -// before first object is created. And call init_instance() before use instance is called. -class ChunkAllocator { -public: - static void init_instance(size_t reserve_limit); - -#ifdef BE_TEST - static ChunkAllocator* instance(); -#else - static ChunkAllocator* instance() { return _s_instance; } -#endif - - // Up size to 2^n length, allocate a chunk. - Status allocate_align(size_t size, Chunk* chunk); - - // Free chunk allocated from this allocator - void free(const Chunk& chunk); - - // Transfer the memory ownership to the chunk allocator. - // If the chunk allocator is full, then free to the system. - // Note: make sure that the length of 'data' is equal to size, - // otherwise the capacity of chunk allocator will be wrong. - void free(uint8_t* data, size_t size); - - void clear(); - - int64_t mem_consumption() { return _reserved_bytes; } - -private: - ChunkAllocator(size_t reserve_limit); - -private: - static ChunkAllocator* _s_instance; - - size_t _reserve_bytes_limit; - // When the reserved chunk memory size is greater than the limit, - // it is allowed to steal the chunks of other arenas. - size_t _steal_arena_limit; - std::atomic _reserved_bytes; - // each core has a ChunkArena - std::vector> _arenas; - - std::shared_ptr _chunk_allocator_metric_entity; - - std::unique_ptr _mem_tracker; -}; - -} // namespace doris diff --git a/be/src/runtime/memory/mem_tracker_limiter.cpp b/be/src/runtime/memory/mem_tracker_limiter.cpp index d03bd1ac005f12d..683971ecac4765d 100644 --- a/be/src/runtime/memory/mem_tracker_limiter.cpp +++ b/be/src/runtime/memory/mem_tracker_limiter.cpp @@ -137,7 +137,7 @@ void MemTrackerLimiter::make_process_snapshots(std::vector process_mem_sum += it.second->current_value(); } - snapshot.type = "tc/jemalloc_cache"; + snapshot.type = "tc/jemalloc_free_memory"; snapshot.label = ""; snapshot.limit = -1; snapshot.cur_consumption = MemInfo::allocator_cache_mem(); diff --git a/be/src/runtime/memory/system_allocator.cpp b/be/src/runtime/memory/system_allocator.cpp deleted file mode 100644 index 6acec3b104359a8..000000000000000 --- a/be/src/runtime/memory/system_allocator.cpp +++ /dev/null @@ -1,76 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#include "runtime/memory/system_allocator.h" - -#include -#include -#include - -#include -#include - -#if !defined(__APPLE__) || !defined(_POSIX_C_SOURCE) -#include -#else -#include -#endif - -#include "common/logging.h" -#include "runtime/memory/mem_tracker_limiter.h" -#include "runtime/thread_context.h" -#include "util/sse_util.hpp" - -namespace { - -int get_page_size() { -#if !defined(__APPLE__) || !defined(_POSIX_C_SOURCE) - return getpagesize(); -#else - return vm_page_size; -#endif -} - -} // namespace - -namespace doris { - -uint8_t* SystemAllocator::allocate(size_t length) { - return allocate_via_malloc(length); -} - -void SystemAllocator::free(uint8_t* ptr) { - ::free(ptr); -} - -uint8_t* SystemAllocator::allocate_via_malloc(size_t length) { - void* ptr = nullptr; - // try to use a whole page instead of parts of one page - int res = posix_memalign(&ptr, get_page_size(), length); - if (res != 0) { - char buf[64]; - auto err = fmt::format("fail to allocate mem via posix_memalign, res={}, errmsg={}.", res, - strerror_r(res, buf, 64)); - LOG(ERROR) << err; - if (enable_thread_catch_bad_alloc) throw std::bad_alloc {}; - MemTrackerLimiter::print_log_process_usage(err); - return nullptr; - } - return (uint8_t*)ptr; -} - -} // namespace doris diff --git a/be/src/runtime/memory/thread_mem_tracker_mgr.h b/be/src/runtime/memory/thread_mem_tracker_mgr.h index dd7b3b8b32aa197..deafcdc241733d3 100644 --- a/be/src/runtime/memory/thread_mem_tracker_mgr.h +++ b/be/src/runtime/memory/thread_mem_tracker_mgr.h @@ -173,7 +173,7 @@ inline void ThreadMemTrackerMgr::consume(int64_t size) { } // Large memory alloc should use allocator.h // Direct malloc or new large memory, unable to catch std::bad_alloc, BE may OOM. - if (size > 4294967296) { // 4G + if (size > 1024l * 1024 * 1024) { // 1G _stop_consume = true; LOG(WARNING) << fmt::format("MemHook alloc large memory: {}, stacktrace:\n{}", size, get_stack_trace()); diff --git a/be/src/runtime/runtime_state.cpp b/be/src/runtime/runtime_state.cpp index a422a81bea98348..66bc4a5743a6e44 100644 --- a/be/src/runtime/runtime_state.cpp +++ b/be/src/runtime/runtime_state.cpp @@ -300,17 +300,11 @@ Status RuntimeState::check_query_state(const std::string& msg) { return query_status(); } -const std::string ERROR_FILE_NAME = "error_log"; const int64_t MAX_ERROR_NUM = 50; Status RuntimeState::create_error_log_file() { _exec_env->load_path_mgr()->get_load_error_file_name( _db_name, _import_label, _fragment_instance_id, &_error_log_file_path); - // std::stringstream ss; - // ss << load_dir() << "/" << ERROR_FILE_NAME - // << "_" << std::hex << fragment_instance_id().hi - // << "_" << fragment_instance_id().lo; - // _error_log_file_path = ss.str(); std::string error_log_absolute_path = _exec_env->load_path_mgr()->get_load_error_absolute_path(_error_log_file_path); _error_log_file = new std::ofstream(error_log_absolute_path, std::ifstream::out); diff --git a/be/src/runtime/user_function_cache.cpp b/be/src/runtime/user_function_cache.cpp index 25e7405a0fee7df..f7ec0890a6427e3 100644 --- a/be/src/runtime/user_function_cache.cpp +++ b/be/src/runtime/user_function_cache.cpp @@ -140,8 +140,9 @@ Status UserFunctionCache::_load_entry_from_lib(const std::string& dir, const std } std::vector split_parts = strings::Split(file, "."); - if (split_parts.size() != 3) { - return Status::InternalError("user function's name should be function_id.checksum.so"); + if (split_parts.size() != 3 && split_parts.size() != 4) { + return Status::InternalError( + "user function's name should be function_id.checksum[.file_name].file_type"); } int64_t function_id = std::stol(split_parts[0]); std::string checksum = split_parts[1]; @@ -176,7 +177,7 @@ Status UserFunctionCache::_load_cached_lib() { auto st = _load_entry_from_lib(sub_dir, file.file_name); if (!st.ok()) { LOG(WARNING) << "load a library failed, dir=" << sub_dir - << ", file=" << file.file_name; + << ", file=" << file.file_name << ": " << st.to_string(); } return true; }; diff --git a/be/src/service/backend_service.cpp b/be/src/service/backend_service.cpp index 1b4a1ec9446bee4..76722ba858d9d67 100644 --- a/be/src/service/backend_service.cpp +++ b/be/src/service/backend_service.cpp @@ -384,6 +384,7 @@ void BackendService::check_storage_format(TCheckStorageFormatResult& result) { void BackendService::ingest_binlog(TIngestBinlogResult& result, const TIngestBinlogRequest& request) { + constexpr uint64_t kMaxTimeoutMs = 1000; TStatus tstatus; Defer defer {[&result, &tstatus]() { result.__set_status(tstatus); }}; @@ -485,7 +486,7 @@ void BackendService::ingest_binlog(TIngestBinlogResult& result, std::string binlog_info; auto get_binlog_info_cb = [&get_binlog_info_url, &binlog_info](HttpClient* client) { RETURN_IF_ERROR(client->init(get_binlog_info_url)); - client->set_timeout_ms(10); // 10ms + client->set_timeout_ms(kMaxTimeoutMs); return client->execute(&binlog_info); }; status = HttpClient::execute_with_retry(max_retry, 1, get_binlog_info_cb); @@ -509,7 +510,7 @@ void BackendService::ingest_binlog(TIngestBinlogResult& result, std::string rowset_meta_str; auto get_rowset_meta_cb = [&get_rowset_meta_url, &rowset_meta_str](HttpClient* client) { RETURN_IF_ERROR(client->init(get_rowset_meta_url)); - client->set_timeout_ms(10); // 10ms + client->set_timeout_ms(kMaxTimeoutMs); return client->execute(&rowset_meta_str); }; status = HttpClient::execute_with_retry(max_retry, 1, get_rowset_meta_cb); @@ -528,7 +529,7 @@ void BackendService::ingest_binlog(TIngestBinlogResult& result, } // rewrite rowset meta rowset_meta_pb.set_tablet_id(local_tablet_id); - rowset_meta_pb.set_partition_id(local_tablet->tablet_meta()->partition_id()); + rowset_meta_pb.set_partition_id(partition_id); rowset_meta_pb.set_tablet_schema_hash(local_tablet->tablet_meta()->schema_hash()); rowset_meta_pb.set_txn_id(txn_id); rowset_meta_pb.set_rowset_state(RowsetStatePB::COMMITTED); @@ -556,7 +557,7 @@ void BackendService::ingest_binlog(TIngestBinlogResult& result, auto get_segment_file_size_cb = [&get_segment_file_size_url, &segment_file_size](HttpClient* client) { RETURN_IF_ERROR(client->init(get_segment_file_size_url)); - client->set_timeout_ms(10); // 10ms + client->set_timeout_ms(kMaxTimeoutMs); RETURN_IF_ERROR(client->head()); return client->get_content_length(&segment_file_size); }; @@ -600,7 +601,7 @@ void BackendService::ingest_binlog(TIngestBinlogResult& result, auto get_segment_file_cb = [&get_segment_file_url, &local_segment_path, segment_file_size, estimate_timeout](HttpClient* client) { RETURN_IF_ERROR(client->init(get_segment_file_url)); - client->set_timeout_ms(estimate_timeout * 1000); // 10ms + client->set_timeout_ms(estimate_timeout * 1000); RETURN_IF_ERROR(client->download(local_segment_path)); std::error_code ec; diff --git a/be/src/service/doris_main.cpp b/be/src/service/doris_main.cpp index 699f5989676dc86..246032ef9ac9533 100644 --- a/be/src/service/doris_main.cpp +++ b/be/src/service/doris_main.cpp @@ -54,6 +54,7 @@ #include "common/signal_handler.h" #include "common/status.h" #include "io/cache/block/block_file_cache_factory.h" +#include "io/fs/s3_file_write_bufferpool.h" #include "olap/options.h" #include "olap/storage_engine.h" #include "runtime/exec_env.h" @@ -432,6 +433,12 @@ int main(int argc, char** argv) { doris::ExecEnv::init(exec_env, paths); doris::TabletSchemaCache::create_global_schema_cache(); + // init s3 write buffer pool + doris::io::S3FileBufferPool* s3_buffer_pool = doris::io::S3FileBufferPool::GetInstance(); + s3_buffer_pool->init(doris::config::s3_write_buffer_whole_size, + doris::config::s3_write_buffer_size, + exec_env->buffered_reader_prefetch_thread_pool()); + // init and open storage engine doris::EngineOptions options; options.store_paths = paths; diff --git a/be/src/service/internal_service.cpp b/be/src/service/internal_service.cpp index c0c6d4e4ea1ac64..e1ab48d2d884a9f 100644 --- a/be/src/service/internal_service.cpp +++ b/be/src/service/internal_service.cpp @@ -114,6 +114,7 @@ #include "vec/exec/format/json/new_json_reader.h" #include "vec/exec/format/orc/vorc_reader.h" #include "vec/exec/format/parquet/vparquet_reader.h" +#include "vec/exec/scan/avro_jni_reader.h" #include "vec/jsonb/serialize.h" #include "vec/runtime/vdata_stream_mgr.h" @@ -588,6 +589,14 @@ void PInternalServiceImpl::fetch_table_schema(google::protobuf::RpcController* c file_slots, &io_ctx); break; } + case TFileFormatType::FORMAT_AVRO: { + // file_slots is no use + std::vector file_slots; + reader = vectorized::AvroJNIReader::create_unique(profile.get(), params, range, + file_slots); + ((vectorized::AvroJNIReader*)(reader.get()))->init_fetch_table_schema_reader(); + break; + } default: st = Status::InternalError("Not supported file format in fetch table schema: {}", params.format_type); diff --git a/be/src/service/point_query_executor.cpp b/be/src/service/point_query_executor.cpp index 255d4c9a2c8650d..c6052c6a909af2d 100644 --- a/be/src/service/point_query_executor.cpp +++ b/be/src/service/point_query_executor.cpp @@ -245,12 +245,12 @@ Status PointQueryExecutor::_lookup_row_key() { SCOPED_TIMER(&_profile_metrics.lookup_key_ns); // 2. lookup row location Status st; - std::unordered_map segment_caches; std::vector specified_rowsets; { std::shared_lock rlock(_tablet->get_header_lock()); specified_rowsets = _tablet->get_rowset_by_ids(nullptr); } + std::vector> segment_caches(specified_rowsets.size()); for (size_t i = 0; i < _row_read_ctxs.size(); ++i) { RowLocation location; if (!config::disable_storage_row_cache) { diff --git a/be/src/util/faststring.cc b/be/src/util/faststring.cc index 8eb3b6c021edc6f..cf373efec4b568d 100644 --- a/be/src/util/faststring.cc +++ b/be/src/util/faststring.cc @@ -43,7 +43,7 @@ void faststring::GrowArray(size_t newcapacity) { } capacity_ = newcapacity; if (data_ != initial_data_) { - Allocator::free_no_munmap(data_); + Allocator::free(data_); } else { ASAN_POISON_MEMORY_REGION(initial_data_, arraysize(initial_data_)); } @@ -57,13 +57,13 @@ void faststring::ShrinkToFitInternal() { if (len_ <= kInitialCapacity) { ASAN_UNPOISON_MEMORY_REGION(initial_data_, len_); memcpy(initial_data_, &data_[0], len_); - Allocator::free_no_munmap(data_); + Allocator::free(data_); data_ = initial_data_; capacity_ = kInitialCapacity; } else { std::unique_ptr newdata(reinterpret_cast(Allocator::alloc(len_))); memcpy(&newdata[0], &data_[0], len_); - Allocator::free_no_munmap(data_); + Allocator::free(data_); data_ = newdata.release(); capacity_ = len_; } diff --git a/be/src/util/faststring.h b/be/src/util/faststring.h index c353ec8d55fa70b..09f325d1114b2fb 100644 --- a/be/src/util/faststring.h +++ b/be/src/util/faststring.h @@ -54,7 +54,7 @@ class faststring : private Allocator { ~faststring() { ASAN_UNPOISON_MEMORY_REGION(initial_data_, arraysize(initial_data_)); if (data_ != initial_data_) { - Allocator::free_no_munmap(data_); + Allocator::free(data_); } } diff --git a/be/src/util/jsonb_document.h b/be/src/util/jsonb_document.h index b18d3d47e006b3b..d54e7291dd4ab68 100644 --- a/be/src/util/jsonb_document.h +++ b/be/src/util/jsonb_document.h @@ -72,6 +72,7 @@ #include #include +#include #include #include #include @@ -144,6 +145,8 @@ constexpr char WILDCARD = '*'; constexpr char MINUS = '-'; constexpr char LAST[] = "last"; constexpr char ESCAPE = '\\'; +constexpr unsigned int MEMBER_CODE = 0; +constexpr unsigned int ARRAY_CODE = 1; /* * JsonbDocument is the main object that accesses and queries JSONB packed @@ -276,35 +279,43 @@ class Stream { skip_whitespace(); } - void clear_legPtr() { legPtr = nullptr; } + void clear_leg_ptr() { leg_ptr = nullptr; } - void set_legPtr(char* ptr) { - clear_legPtr(); - legPtr = ptr; + void set_leg_ptr(char* ptr) { + clear_leg_ptr(); + leg_ptr = ptr; } - char* get_legPtr() { return legPtr; } + char* get_leg_ptr() { return leg_ptr; } - void clear_legLen() { legLen = 0; } + void clear_leg_len() { leg_len = 0; } - void add_legLen() { legLen++; } + void add_leg_len() { leg_len++; } - unsigned int get_legLen() { return legLen; } + unsigned int get_leg_len() const { return leg_len; } void remove_escapes() { int new_len = 0; - for (int i = 0; i < legLen; i++) { - if (legPtr[i] != '\\') { - legPtr[new_len++] = legPtr[i]; + for (int i = 0; i < leg_len; i++) { + if (leg_ptr[i] != '\\') { + leg_ptr[new_len++] = leg_ptr[i]; } } - legPtr[new_len] = '\0'; - legLen = new_len; + leg_ptr[new_len] = '\0'; + leg_len = new_len; } - void set_hasEscapes(bool has) { hasEscapes = has; } + void set_has_escapes(bool has) { has_escapes = has; } - bool get_hasEscapes() { return hasEscapes; } + bool get_has_escapes() const { return has_escapes; } + + void set_is_invalid_json_path(bool has) { is_invalid_json_path = has; } + + bool get_is_invalid_json_path() const { return is_invalid_json_path; } + + void set_type(unsigned int code) { type = code; } + + bool get_type() const { return type; } private: /// The current position in the stream. @@ -314,19 +325,25 @@ class Stream { const char* const m_end; ///path leg ptr - char* legPtr; + char* leg_ptr; ///path leg len - unsigned int legLen; + unsigned int leg_len; + + ///Whether to contain escape characters + bool has_escapes = false; - /// - bool hasEscapes = false; + ///Is the json path valid + bool is_invalid_json_path = false; + + ///type: 0 is member 1 is array + unsigned int type; }; class JsonbPath { public: // parse json path - static bool parsePath(Stream* stream, JsonbValue* value); + static bool parsePath(Stream* stream); static bool parse_array(Stream* stream); static bool parse_member(Stream* stream); @@ -513,12 +530,14 @@ class JsonbValue { const char* getValuePtr() const; // find the JSONB value by a key path string (null terminated) - JsonbValue* findPath(const char* key_path, hDictFind handler = nullptr) { - return findPath(key_path, (unsigned int)strlen(key_path), handler); + JsonbValue* findPath(const char* key_path, bool& is_invalid_json_path, + hDictFind handler = nullptr) { + return findPath(key_path, (unsigned int)strlen(key_path), is_invalid_json_path, handler); } // find the JSONB value by a key path string (with length) - JsonbValue* findPath(const char* key_path, unsigned int len, hDictFind handler); + JsonbValue* findPath(const char* key_path, unsigned int len, bool& is_invalid_json_path, + hDictFind handler); friend class JsonbDocument; protected: @@ -1189,82 +1208,144 @@ inline const char* JsonbValue::getValuePtr() const { } inline JsonbValue* JsonbValue::findPath(const char* key_path, unsigned int kp_len, - hDictFind handler = nullptr) { + bool& is_invalid_json_path, hDictFind handler = nullptr) { if (!key_path) return nullptr; - if (kp_len == 0) return this; + if (kp_len == 0) { + is_invalid_json_path = true; + return nullptr; + } Stream stream(key_path, kp_len); stream.skip_whitespace(); - if (stream.exhausted() || stream.read() != SCOPE) return nullptr; + if (stream.exhausted() || stream.read() != SCOPE) { + is_invalid_json_path = true; + return nullptr; + } JsonbValue* pval = this; while (pval && !stream.exhausted()) { stream.skip_whitespace(); - stream.clear_legPtr(); - stream.clear_legLen(); + stream.clear_leg_ptr(); + stream.clear_leg_len(); - if (!JsonbPath::parsePath(&stream, pval)) { + if (!JsonbPath::parsePath(&stream)) { + is_invalid_json_path = stream.get_is_invalid_json_path(); return nullptr; } - if (stream.get_legLen() == 0) { + if (stream.get_leg_len() == 0) { return nullptr; } - if (LIKELY(pval->type_ == JsonbType::T_Object)) { - if (stream.get_legLen() == 1 && *stream.get_legPtr() == WILDCARD) { - return pval; - } else if (stream.get_hasEscapes()) { - stream.remove_escapes(); - } + if (stream.get_type() == MEMBER_CODE) { + if (LIKELY(pval->type_ == JsonbType::T_Object)) { + if (stream.get_leg_len() == 1 && *stream.get_leg_ptr() == WILDCARD) { + return pval; + } else if (stream.get_has_escapes()) { + stream.remove_escapes(); + } - pval = ((ObjectVal*)pval)->find(stream.get_legPtr(), stream.get_legLen(), handler); + pval = ((ObjectVal*)pval) + ->find(stream.get_leg_ptr(), stream.get_leg_len(), handler); - if (!pval) return nullptr; - } else if (LIKELY(pval->type_ == JsonbType::T_Array)) { + if (!pval) return nullptr; + } else { + return nullptr; + } + } else if (stream.get_type() == ARRAY_CODE) { int index = 0; - std::string idx_string(stream.get_legPtr(), stream.get_legLen()); + std::string_view idx_string(stream.get_leg_ptr(), stream.get_leg_len()); - if (stream.get_legLen() == 1 && *stream.get_legPtr() == WILDCARD) { - return pval; - } else if (std::string(stream.get_legPtr(), 4) == LAST) { + if (stream.get_leg_len() == 1 && *stream.get_leg_ptr() == WILDCARD) { + if (LIKELY(pval->type_ == JsonbType::T_Array)) { + stream.skip(1); + stream.skip_whitespace(); + continue; + } else { + return nullptr; + } + } else if (std::equal(LAST, LAST + 4, stream.get_leg_ptr(), + [](char c1, char c2) { + return std::tolower(c1) == std::tolower(c2); + }) && + stream.get_leg_len() >= 4) { auto pos = idx_string.find(MINUS); if (pos != std::string::npos) { idx_string = idx_string.substr(pos + 1); - size_t num = ((ArrayVal*)pval)->numElem(); - if (std::stoi(idx_string) > num) { - return nullptr; //invalid json path + + auto result = std::from_chars(idx_string.data(), + idx_string.data() + idx_string.size(), index); + if (result.ec != std::errc()) { + is_invalid_json_path = true; + return nullptr; } - index = num - 1 - std::stoi(idx_string); - } else if (stream.get_legLen() == 4) { - index = ((ArrayVal*)pval)->numElem() - 1; + + if (pval->type_ == JsonbType::T_Object) { + if (index == 0) { + continue; + } else { + return nullptr; + } + } else if (LIKELY(pval->type_ == JsonbType::T_Array)) { + size_t num = ((ArrayVal*)pval)->numElem(); + if (index > num) return nullptr; + index = num - 1 - index; + } else { + return nullptr; + } + } else if (stream.get_leg_len() == 4) { + if (pval->type_ == JsonbType::T_Object) { + continue; + } else if (LIKELY(pval->type_ == JsonbType::T_Array)) { + index = ((ArrayVal*)pval)->numElem() - 1; + } else { + return nullptr; + } + } else { - return nullptr; //invalid json path + is_invalid_json_path = true; + return nullptr; } } else { - std::string::size_type pos; - index = std::stoi(idx_string, &pos, 10); - if (pos != idx_string.size()) { - return nullptr; //invalid json path - } else if (index >= ((ArrayVal*)pval)->numElem()) { - return nullptr; //invalid json path + auto result = std::from_chars(idx_string.data(), + idx_string.data() + idx_string.size(), index); + if (result.ec != std::errc()) { + is_invalid_json_path = true; + return nullptr; + } + + if (pval->type_ == JsonbType::T_Object) { + if (index == 0) { + continue; + } else { + return nullptr; + } + } else if (LIKELY(pval->type_ == JsonbType::T_Array)) { + if (std::abs(index) >= ((ArrayVal*)pval)->numElem()) return nullptr; + } else { + return nullptr; } } - pval = ((ArrayVal*)pval)->get(index); + if (index >= 0) { + pval = ((ArrayVal*)pval)->get(index); + } else { + pval = ((ArrayVal*)pval)->get(((ArrayVal*)pval)->numElem() + index); + } } } return pval; } -inline bool JsonbPath::parsePath(Stream* stream, JsonbValue* value) { - if (stream->peek() == BEGIN_ARRAY && value->type() == JsonbType::T_Array) { +inline bool JsonbPath::parsePath(Stream* stream) { + if (stream->peek() == BEGIN_ARRAY) { return parse_array(stream); - } else if (stream->peek() == BEGIN_MEMBER && value->type() == JsonbType::T_Object) { + } else if (stream->peek() == BEGIN_MEMBER) { return parse_member(stream); } else { + stream->set_is_invalid_json_path(true); return false; //invalid json path } } @@ -1272,30 +1353,37 @@ inline bool JsonbPath::parsePath(Stream* stream, JsonbValue* value) { inline bool JsonbPath::parse_array(Stream* stream) { assert(stream->peek() == BEGIN_ARRAY); stream->skip(1); - if (stream->exhausted()) return false; //invalid json path + if (stream->exhausted()) { + stream->set_is_invalid_json_path(true); + return false; + } if (stream->peek() == WILDCARD) { - stream->set_legPtr(const_cast(stream->position())); - stream->add_legLen(); + stream->set_leg_ptr(const_cast(stream->position())); + stream->add_leg_len(); stream->skip(1); if (stream->peek() == END_ARRAY) { + stream->set_type(ARRAY_CODE); return true; } else { - return false; //invalid json path + stream->set_is_invalid_json_path(true); + return false; } } - stream->set_legPtr(const_cast(stream->position())); + stream->set_leg_ptr(const_cast(stream->position())); for (; !stream->exhausted() && stream->peek() != END_ARRAY; stream->skip(1)) { - stream->add_legLen(); + stream->add_leg_len(); } if (!stream->exhausted() && stream->peek() == END_ARRAY) { stream->skip(1); + stream->set_type(ARRAY_CODE); return true; } else { - return false; //invalid json path + stream->set_is_invalid_json_path(true); + return false; } } @@ -1303,31 +1391,35 @@ inline bool JsonbPath::parse_member(Stream* stream) { // advance past the . assert(stream->peek() == BEGIN_MEMBER); stream->skip(1); - if (stream->exhausted()) return false; //invalid json path + if (stream->exhausted()) { + stream->set_is_invalid_json_path(true); + return false; + } if (stream->peek() == WILDCARD) { - stream->set_legPtr(const_cast(stream->position())); - stream->add_legLen(); + stream->set_leg_ptr(const_cast(stream->position())); + stream->add_leg_len(); stream->skip(1); + stream->set_type(MEMBER_CODE); return true; } - stream->set_legPtr(const_cast(stream->position())); + stream->set_leg_ptr(const_cast(stream->position())); const char* left_quotation_marks = nullptr; const char* right_quotation_marks = nullptr; for (; !stream->exhausted(); stream->skip(1)) { if (stream->peek() == ESCAPE) { - stream->add_legLen(); + stream->add_leg_len(); stream->skip(1); - stream->add_legLen(); - stream->set_hasEscapes(true); + stream->add_leg_len(); + stream->set_has_escapes(true); continue; } else if (stream->peek() == DOUBLE_QUOTE) { if (left_quotation_marks == nullptr) { left_quotation_marks = stream->position(); - stream->set_legPtr(const_cast(++left_quotation_marks)); + stream->set_leg_ptr(const_cast(++left_quotation_marks)); continue; } else { right_quotation_marks = stream->position(); @@ -1340,13 +1432,16 @@ inline bool JsonbPath::parse_member(Stream* stream) { } } - stream->add_legLen(); + stream->add_leg_len(); } if (left_quotation_marks != nullptr && right_quotation_marks == nullptr) { + stream->set_is_invalid_json_path(true); return false; //invalid json path } + stream->set_type(MEMBER_CODE); + return true; } diff --git a/be/src/util/jsonb_error.h b/be/src/util/jsonb_error.h index 2ad632fb8bd2c69..49c061a32f966de 100644 --- a/be/src/util/jsonb_error.h +++ b/be/src/util/jsonb_error.h @@ -56,6 +56,7 @@ enum class JsonbErrType { E_NESTING_LVL_OVERFLOW, E_INVALID_DOCU_COMPAT, E_EXCEPTION, + E_INVALID_JSON_PATH, // new error code should always be added above E_NUM_ERRORS @@ -107,6 +108,7 @@ class JsonbErrMsg { "Object or array has too many nesting levels", "Invalid document", "Exception throwed", + "Invalid Json Path", nullptr /* E_NUM_ERRORS */ }; diff --git a/be/src/util/mem_info.cpp b/be/src/util/mem_info.cpp index 3acdc48ff086c10..200d346dede34a6 100644 --- a/be/src/util/mem_info.cpp +++ b/be/src/util/mem_info.cpp @@ -41,7 +41,6 @@ #include "olap/page_cache.h" #include "olap/rowset/segment_v2/inverted_index_cache.h" #include "olap/segment_loader.h" -#include "runtime/memory/chunk_allocator.h" #include "runtime/memory/mem_tracker_limiter.h" #include "runtime/task_group/task_group.h" #include "runtime/task_group/task_group_manager.h" @@ -78,14 +77,18 @@ int64_t MemInfo::_s_process_full_gc_size = -1; void MemInfo::refresh_allocator_mem() { #if defined(ADDRESS_SANITIZER) || defined(LEAK_SANITIZER) || defined(THREAD_SANITIZER) #elif defined(USE_JEMALLOC) + // 'epoch' is a special mallctl -- it updates the statistics. Without it, all + // the following calls will return stale values. It increments and returns + // the current epoch number, which might be useful to log as a sanity check. uint64_t epoch = 0; size_t sz = sizeof(epoch); jemallctl("epoch", &epoch, &sz, &epoch, sz); // https://jemalloc.net/jemalloc.3.html - _s_allocator_cache_mem = - get_je_metrics(fmt::format("stats.arenas.{}.tcache_bytes", MALLCTL_ARENAS_ALL)) + - get_je_metrics("stats.metadata"); + // https://www.bookstack.cn/read/aliyun-rds-core/4a0cdf677f62feb3.md + _s_allocator_cache_mem = get_je_all_arena_metrics("tcache_bytes") + + get_je_metrics("stats.metadata") + + get_je_all_arena_metrics("pdirty") * get_page_size(); _s_allocator_cache_mem_str = PrettyPrinter::print(static_cast(_s_allocator_cache_mem), TUnit::BYTES); _s_virtual_memory_used = get_je_metrics("stats.mapped"); @@ -104,10 +107,6 @@ void MemInfo::refresh_allocator_mem() { void MemInfo::process_cache_gc(int64_t& freed_mem) { // TODO, free more cache, and should free a certain percentage of capacity, not all. int32_t min_free_size = 33554432; // 32M - if (ChunkAllocator::instance()->mem_consumption() > min_free_size) { - freed_mem += ChunkAllocator::instance()->mem_consumption(); - ChunkAllocator::instance()->clear(); - } if (StoragePageCache::instance()->get_page_cache_mem_consumption(segment_v2::DATA_PAGE) > min_free_size) { @@ -130,6 +129,7 @@ void MemInfo::process_cache_gc(int64_t& freed_mem) { segment_v2::PRIMARY_KEY_INDEX_PAGE); StoragePageCache::instance()->prune(segment_v2::PRIMARY_KEY_INDEX_PAGE); } + je_purge_all_arena_dirty_pages(); } // step1: free all cache @@ -144,7 +144,8 @@ bool MemInfo::process_minor_gc() { std::string mem_available_str = MemInfo::sys_mem_available_str(); Defer defer {[&]() { - LOG(INFO) << fmt::format("Process Minor GC Free Memory {} Bytes. cost(us): {}", freed_mem, + je_purge_all_arena_dirty_pages(); + LOG(INFO) << fmt::format("End Minor GC, Free Memory {} Bytes. cost(us): {}", freed_mem, watch.elapsed_time() / 1000); }}; @@ -186,7 +187,8 @@ bool MemInfo::process_full_gc() { std::string mem_available_str = MemInfo::sys_mem_available_str(); Defer defer {[&]() { - LOG(INFO) << fmt::format("Process Full GC Free Memory {} Bytes. cost(us): {}", freed_mem, + je_purge_all_arena_dirty_pages(); + LOG(INFO) << fmt::format("End Full GC Free, Memory {} Bytes. cost(us): {}", freed_mem, watch.elapsed_time() / 1000); }}; diff --git a/be/src/util/mem_info.h b/be/src/util/mem_info.h index 12c70d8cc4dc3a4..89a66b06589e355 100644 --- a/be/src/util/mem_info.h +++ b/be/src/util/mem_info.h @@ -26,6 +26,12 @@ #include #include +#if !defined(__APPLE__) || !defined(_POSIX_C_SOURCE) +#include +#else +#include +#endif + #include "common/logging.h" #ifdef USE_JEMALLOC #include "jemalloc/jemalloc.h" @@ -46,6 +52,14 @@ class MemInfo { static inline bool initialized() { return _s_initialized; } + static int get_page_size() { +#if !defined(__APPLE__) || !defined(_POSIX_C_SOURCE) + return getpagesize(); +#else + return vm_page_size; +#endif + } + // Get total physical memory in bytes (if has cgroups memory limits, return the limits). static inline int64_t physical_mem() { DCHECK(_s_initialized); @@ -83,6 +97,22 @@ class MemInfo { #endif return 0; } + + static inline int64_t get_je_all_arena_metrics(const std::string& name) { +#ifdef USE_JEMALLOC + return get_je_metrics(fmt::format("stats.arenas.{}.{}", MALLCTL_ARENAS_ALL, name)); +#endif + return 0; + } + + static inline void je_purge_all_arena_dirty_pages() { +#ifdef USE_JEMALLOC + // Purge all unused dirty pages for arena , or for all arenas if equals MALLCTL_ARENAS_ALL. + jemallctl(fmt::format("arena.{}.purge", MALLCTL_ARENAS_ALL).c_str(), nullptr, nullptr, + nullptr, 0); +#endif + } + static inline size_t allocator_virtual_mem() { return _s_virtual_memory_used; } static inline size_t allocator_cache_mem() { return _s_allocator_cache_mem; } static inline std::string allocator_cache_mem_str() { return _s_allocator_cache_mem_str; } @@ -94,6 +124,13 @@ class MemInfo { // obtained by the process malloc, not the physical memory actually used by the process in the OS. static void refresh_allocator_mem(); + /** jemalloc pdirty is number of pages within unused extents that are potentially + * dirty, and for which madvise() or similar has not been called. + * + * So they will be subtracted from RSS to make accounting more + * accurate, since those pages are not really RSS but a memory + * that can be used at anytime via jemalloc. + */ static inline void refresh_proc_mem_no_allocator_cache() { _s_proc_mem_no_allocator_cache = PerfCounters::get_vm_rss() - static_cast(_s_allocator_cache_mem); diff --git a/be/src/util/network_util.cpp b/be/src/util/network_util.cpp index 20695bbfba90388..6841e257a320439 100644 --- a/be/src/util/network_util.cpp +++ b/be/src/util/network_util.cpp @@ -96,7 +96,15 @@ Status hostname_to_ip_addrs(const std::string& name, std::vector* a return Status::InternalError("Could not convert IPv4 address for: {}", name); } - addresses->push_back(std::string(addr_buf)); + // add address if not exists + std::string address = std::string(addr_buf); + if (std::find(addresses->begin(), addresses->end(), address) != addresses->end()) { + LOG(WARNING) << "Repeated ip addresses has been found for host: " << name + << ", ip address:" << address + << ", please check your network configuration"; + } else { + addresses->push_back(address); + } it = it->ai_next; } diff --git a/be/src/util/ref_count_closure.h b/be/src/util/ref_count_closure.h index fe6efa761517b65..d2fbd2fd14e863c 100644 --- a/be/src/util/ref_count_closure.h +++ b/be/src/util/ref_count_closure.h @@ -31,7 +31,7 @@ template class RefCountClosure : public google::protobuf::Closure { public: RefCountClosure() : _refs(0) {} - ~RefCountClosure() {} + ~RefCountClosure() override = default; void ref() { _refs.fetch_add(1); } diff --git a/be/src/util/runtime_profile.cpp b/be/src/util/runtime_profile.cpp index b7cd3a63af9a0d0..f385256ab210a29 100644 --- a/be/src/util/runtime_profile.cpp +++ b/be/src/util/runtime_profile.cpp @@ -38,8 +38,6 @@ namespace doris { // Thread counters name static const std::string THREAD_TOTAL_TIME = "TotalWallClockTime"; -static const std::string THREAD_USER_TIME = "UserTime"; -static const std::string THREAD_SYS_TIME = "SysTime"; static const std::string THREAD_VOLUNTARY_CONTEXT_SWITCHES = "VoluntaryContextSwitches"; static const std::string THREAD_INVOLUNTARY_CONTEXT_SWITCHES = "InvoluntaryContextSwitches"; diff --git a/be/src/util/slice.h b/be/src/util/slice.h index babedbe893d001d..57865b50e3e65db 100644 --- a/be/src/util/slice.h +++ b/be/src/util/slice.h @@ -279,7 +279,7 @@ class OwnedSlice : private Allocator { return *this; } - ~OwnedSlice() { Allocator::free_no_munmap(_slice.data); } + ~OwnedSlice() { Allocator::free(_slice.data); } const Slice& slice() const { return _slice; } diff --git a/be/src/util/system_metrics.cpp b/be/src/util/system_metrics.cpp index fa8f5a181a7bec9..ee7db9494c2e970 100644 --- a/be/src/util/system_metrics.cpp +++ b/be/src/util/system_metrics.cpp @@ -117,6 +117,12 @@ DEFINE_MEMORY_GAUGE_METRIC(jemalloc_metadata_bytes, MetricUnit::BYTES); DEFINE_MEMORY_GAUGE_METRIC(jemalloc_resident_bytes, MetricUnit::BYTES); DEFINE_MEMORY_GAUGE_METRIC(jemalloc_mapped_bytes, MetricUnit::BYTES); DEFINE_MEMORY_GAUGE_METRIC(jemalloc_retained_bytes, MetricUnit::BYTES); +DEFINE_MEMORY_GAUGE_METRIC(jemalloc_tcache_bytes, MetricUnit::BYTES); +DEFINE_MEMORY_GAUGE_METRIC(jemalloc_pactive_num, MetricUnit::NOUNIT); +DEFINE_MEMORY_GAUGE_METRIC(jemalloc_pdirty_num, MetricUnit::NOUNIT); +DEFINE_MEMORY_GAUGE_METRIC(jemalloc_pmuzzy_num, MetricUnit::NOUNIT); +DEFINE_MEMORY_GAUGE_METRIC(jemalloc_dirty_purged_num, MetricUnit::NOUNIT); +DEFINE_MEMORY_GAUGE_METRIC(jemalloc_muzzy_purged_num, MetricUnit::NOUNIT); #endif struct MemoryMetrics { @@ -142,6 +148,12 @@ struct MemoryMetrics { INT_GAUGE_METRIC_REGISTER(entity, memory_jemalloc_resident_bytes); INT_GAUGE_METRIC_REGISTER(entity, memory_jemalloc_mapped_bytes); INT_GAUGE_METRIC_REGISTER(entity, memory_jemalloc_retained_bytes); + INT_GAUGE_METRIC_REGISTER(entity, memory_jemalloc_tcache_bytes); + INT_GAUGE_METRIC_REGISTER(entity, memory_jemalloc_pactive_num); + INT_GAUGE_METRIC_REGISTER(entity, memory_jemalloc_pdirty_num); + INT_GAUGE_METRIC_REGISTER(entity, memory_jemalloc_pmuzzy_num); + INT_GAUGE_METRIC_REGISTER(entity, memory_jemalloc_dirty_purged_num); + INT_GAUGE_METRIC_REGISTER(entity, memory_jemalloc_muzzy_purged_num); #endif } @@ -167,6 +179,12 @@ struct MemoryMetrics { IntGauge* memory_jemalloc_resident_bytes; IntGauge* memory_jemalloc_mapped_bytes; IntGauge* memory_jemalloc_retained_bytes; + IntGauge* memory_jemalloc_tcache_bytes; + IntGauge* memory_jemalloc_pactive_num; + IntGauge* memory_jemalloc_pdirty_num; + IntGauge* memory_jemalloc_pmuzzy_num; + IntGauge* memory_jemalloc_dirty_purged_num; + IntGauge* memory_jemalloc_muzzy_purged_num; #endif }; @@ -457,6 +475,18 @@ void SystemMetrics::update_allocator_metrics() { MemInfo::get_je_metrics("stats.mapped")); _memory_metrics->memory_jemalloc_retained_bytes->set_value( MemInfo::get_je_metrics("stats.retained")); + _memory_metrics->memory_jemalloc_tcache_bytes->set_value( + MemInfo::get_je_all_arena_metrics("tcache_bytes")); + _memory_metrics->memory_jemalloc_pactive_num->set_value( + MemInfo::get_je_all_arena_metrics("pactive")); + _memory_metrics->memory_jemalloc_pdirty_num->set_value( + MemInfo::get_je_all_arena_metrics("pdirty")); + _memory_metrics->memory_jemalloc_pmuzzy_num->set_value( + MemInfo::get_je_all_arena_metrics("pmuzzy")); + _memory_metrics->memory_jemalloc_dirty_purged_num->set_value( + MemInfo::get_je_all_arena_metrics("dirty_purged")); + _memory_metrics->memory_jemalloc_muzzy_purged_num->set_value( + MemInfo::get_je_all_arena_metrics("muzzy_purged")); #else _memory_metrics->memory_tcmalloc_allocated_bytes->set_value( MemInfo::get_tc_metrics("generic.total_physical_bytes")); diff --git a/be/src/vec/aggregate_functions/aggregate_function_avg.h b/be/src/vec/aggregate_functions/aggregate_function_avg.h index 118c6b945fb992f..ff1de6b2e2761fc 100644 --- a/be/src/vec/aggregate_functions/aggregate_function_avg.h +++ b/be/src/vec/aggregate_functions/aggregate_function_avg.h @@ -237,8 +237,9 @@ class AggregateFunctionAvg final IColumn& to) const override { auto& col = assert_cast(to); col.set_item_size(sizeof(Data)); - col.resize(1); - *reinterpret_cast(col.get_data().data()) = this->data(place); + size_t old_size = col.size(); + col.resize(old_size + 1); + *(reinterpret_cast(col.get_data().data()) + old_size) = this->data(place); } MutableColumnPtr create_serialize_column() const override { diff --git a/be/src/vec/aggregate_functions/aggregate_function_count.h b/be/src/vec/aggregate_functions/aggregate_function_count.h index 1cea35958ab9f52..35da6c3406fcc03 100644 --- a/be/src/vec/aggregate_functions/aggregate_function_count.h +++ b/be/src/vec/aggregate_functions/aggregate_function_count.h @@ -150,8 +150,9 @@ class AggregateFunctionCount final auto& col = assert_cast(to); DCHECK(col.item_size() == sizeof(Data)) << "size is not equal: " << col.item_size() << " " << sizeof(Data); - col.resize(1); - reinterpret_cast(col.get_data().data())->count = + size_t old_size = col.size(); + col.resize(old_size + 1); + (reinterpret_cast(col.get_data().data()) + old_size)->count = AggregateFunctionCount::data(place).count; } diff --git a/be/src/vec/aggregate_functions/aggregate_function_min_max.h b/be/src/vec/aggregate_functions/aggregate_function_min_max.h index 550e83a7e3cc204..5d3bee95c99e2f6 100644 --- a/be/src/vec/aggregate_functions/aggregate_function_min_max.h +++ b/be/src/vec/aggregate_functions/aggregate_function_min_max.h @@ -628,8 +628,9 @@ class AggregateFunctionsSingleValue final IColumn& to) const override { if constexpr (Data::IsFixedLength) { auto& col = assert_cast(to); - col.resize(1); - *reinterpret_cast(col.get_data().data()) = this->data(place); + size_t old_size = col.size(); + col.resize(old_size + 1); + *(reinterpret_cast(col.get_data().data()) + old_size) = this->data(place); } else { Base::serialize_without_key_to_column(place, to); } diff --git a/be/src/vec/aggregate_functions/aggregate_function_sum.h b/be/src/vec/aggregate_functions/aggregate_function_sum.h index 3773efa5bb8fd9f..cab803bc7a3b821 100644 --- a/be/src/vec/aggregate_functions/aggregate_function_sum.h +++ b/be/src/vec/aggregate_functions/aggregate_function_sum.h @@ -182,8 +182,9 @@ class AggregateFunctionSum final auto& col = assert_cast(to); DCHECK(col.item_size() == sizeof(Data)) << "size is not equal: " << col.item_size() << " " << sizeof(Data); - col.resize(1); - reinterpret_cast(col.get_data().data())->sum = this->data(place).sum; + size_t old_size = col.size(); + col.resize(old_size + 1); + (reinterpret_cast(col.get_data().data()) + old_size)->sum = this->data(place).sum; } MutableColumnPtr create_serialize_column() const override { diff --git a/be/src/vec/aggregate_functions/aggregate_function_window.cpp b/be/src/vec/aggregate_functions/aggregate_function_window.cpp index bacdd45131584c1..fdc8a0bf861d634 100644 --- a/be/src/vec/aggregate_functions/aggregate_function_window.cpp +++ b/be/src/vec/aggregate_functions/aggregate_function_window.cpp @@ -43,7 +43,7 @@ AggregateFunctionPtr create_function_lead_lag_first_last(const String& name, if (which.idx == TypeIndex::TYPE) \ return std::make_shared>>>(argument_types); - TYPE_TO_BASIC_COLUMN_TYPE(DISPATCH) + TYPE_TO_COLUMN_TYPE(DISPATCH) #undef DISPATCH LOG(WARNING) << "with unknowed type, failed in create_aggregate_function_" << name @@ -99,4 +99,4 @@ void register_aggregate_function_window_lead_lag_first_last( factory.register_function_both("last_value", create_aggregate_function_window_last); } -} // namespace doris::vectorized \ No newline at end of file +} // namespace doris::vectorized diff --git a/be/src/vec/columns/column.h b/be/src/vec/columns/column.h index 00f1824b9976266..b291ba24430be84 100644 --- a/be/src/vec/columns/column.h +++ b/be/src/vec/columns/column.h @@ -365,7 +365,7 @@ class IColumn : public COW { /// On subsequent calls of this method for sequence of column values of arbitrary types, /// passed bytes to hash must identify sequence of values unambiguously. virtual void update_hash_with_value(size_t n, SipHash& hash) const { - LOG(FATAL) << "update_hash_with_value siphash not supported"; + LOG(FATAL) << get_name() << " update_hash_with_value siphash not supported"; } /// Update state of hash function with value of n elements to avoid the virtual function call @@ -374,7 +374,7 @@ class IColumn : public COW { /// do xxHash here, faster than other hash method virtual void update_hashes_with_value(std::vector& hashes, const uint8_t* __restrict null_data = nullptr) const { - LOG(FATAL) << "update_hashes_with_value siphash not supported"; + LOG(FATAL) << get_name() << " update_hashes_with_value siphash not supported"; } /// Update state of hash function with value of n elements to avoid the virtual function call @@ -383,7 +383,11 @@ class IColumn : public COW { /// do xxHash here, faster than other sip hash virtual void update_hashes_with_value(uint64_t* __restrict hashes, const uint8_t* __restrict null_data = nullptr) const { - LOG(FATAL) << "update_hashes_with_value xxhash not supported"; + LOG(FATAL) << get_name() << " update_hashes_with_value xxhash not supported"; + } + + virtual void update_xxHash_with_value(size_t n, uint64_t& hash) const { + LOG(FATAL) << get_name() << " update_hash_with_value xxhash not supported"; } /// Update state of crc32 hash function with value of n elements to avoid the virtual function call @@ -391,7 +395,11 @@ class IColumn : public COW { /// means all element need to do hash function, else only *null_data != 0 need to do hash func virtual void update_crcs_with_value(std::vector& hash, PrimitiveType type, const uint8_t* __restrict null_data = nullptr) const { - LOG(FATAL) << "update_crcs_with_value not supported"; + LOG(FATAL) << get_name() << "update_crcs_with_value not supported"; + } + + virtual void update_crc_with_value(size_t n, uint64_t& hash) const { + LOG(FATAL) << get_name() << " update_crc_with_value not supported"; } /** Removes elements that don't match the filter. diff --git a/be/src/vec/columns/column_array.cpp b/be/src/vec/columns/column_array.cpp index 4215bd36bdf6811..c5e35e5bb9b4ece 100644 --- a/be/src/vec/columns/column_array.cpp +++ b/be/src/vec/columns/column_array.cpp @@ -271,6 +271,69 @@ void ColumnArray::update_hash_with_value(size_t n, SipHash& hash) const { for (size_t i = 0; i < array_size; ++i) get_data().update_hash_with_value(offset + i, hash); } +void ColumnArray::update_hashes_with_value(std::vector& hashes, + const uint8_t* __restrict null_data) const { + SIP_HASHES_FUNCTION_COLUMN_IMPL(); +} + +// for every array row calculate xxHash +void ColumnArray::update_xxHash_with_value(size_t n, uint64_t& hash) const { + size_t elem_size = size_at(n); + size_t offset = offset_at(n); + hash = HashUtil::xxHash64WithSeed(reinterpret_cast(&elem_size), sizeof(elem_size), + hash); + for (auto i = 0; i < elem_size; ++i) { + get_data().update_xxHash_with_value(offset + i, hash); + } +} + +// for every array row calculate crcHash +void ColumnArray::update_crc_with_value(size_t n, uint64_t& crc) const { + size_t elem_size = size_at(n); + size_t offset = offset_at(n); + + crc = HashUtil::zlib_crc_hash(reinterpret_cast(&elem_size), sizeof(elem_size), + crc); + for (auto i = 0; i < elem_size; ++i) { + get_data().update_crc_with_value(offset + i, crc); + } +} + +void ColumnArray::update_hashes_with_value(uint64_t* __restrict hashes, + const uint8_t* __restrict null_data) const { + auto s = size(); + if (null_data) { + for (size_t i = 0; i < s; ++i) { + if (null_data[i] == 0) { + update_xxHash_with_value(i, hashes[i]); + } + } + } else { + for (size_t i = 0; i < s; ++i) { + update_xxHash_with_value(i, hashes[i]); + } + } +} + +void ColumnArray::update_crcs_with_value(std::vector& hash, PrimitiveType type, + const uint8_t* __restrict null_data) const { + auto s = hash.size(); + DCHECK(s == size()); + + if (null_data) { + for (size_t i = 0; i < s; ++i) { + // every row + if (null_data[i] == 0) { + update_crc_with_value(i, hash[i]); + } + } + } else { + for (size_t i = 0; i < s; ++i) { + update_crc_with_value(i, hash[i]); + } + } +} + void ColumnArray::insert(const Field& x) { const Array& array = doris::vectorized::get(x); size_t size = array.size(); diff --git a/be/src/vec/columns/column_array.h b/be/src/vec/columns/column_array.h index 4f08c269fb7e6e2..2e1c96a2c5e7f23 100644 --- a/be/src/vec/columns/column_array.h +++ b/be/src/vec/columns/column_array.h @@ -139,6 +139,18 @@ class ColumnArray final : public COWHelper { StringRef serialize_value_into_arena(size_t n, Arena& arena, char const*& begin) const override; const char* deserialize_and_insert_from_arena(const char* pos) override; void update_hash_with_value(size_t n, SipHash& hash) const override; + void update_xxHash_with_value(size_t n, uint64_t& hash) const override; + void update_crc_with_value(size_t n, uint64_t& crc) const override; + + void update_hashes_with_value(std::vector& hashes, + const uint8_t* __restrict null_data) const override; + + void update_hashes_with_value(uint64_t* __restrict hashes, + const uint8_t* __restrict null_data = nullptr) const override; + + void update_crcs_with_value(std::vector& hash, PrimitiveType type, + const uint8_t* __restrict null_data = nullptr) const override; + void insert_range_from(const IColumn& src, size_t start, size_t length) override; void insert(const Field& x) override; void insert_from(const IColumn& src_, size_t n) override; @@ -240,6 +252,8 @@ class ColumnArray final : public COWHelper { ColumnPtr index(const IColumn& indexes, size_t limit) const override; private: + // [[2,1,5,9,1], [1,2,4]] --> data column [2,1,5,9,1,1,2,4], offset[-1] = 0, offset[0] = 5, offset[1] = 8 + // [[[2,1,5],[9,1]], [[1,2]]] --> data column [3 column array], offset[-1] = 0, offset[0] = 2, offset[1] = 3 WrappedPtr data; WrappedPtr offsets; diff --git a/be/src/vec/columns/column_const.h b/be/src/vec/columns/column_const.h index bb17f7eb0417a3e..feeb0608a26d048 100644 --- a/be/src/vec/columns/column_const.h +++ b/be/src/vec/columns/column_const.h @@ -152,6 +152,19 @@ class ColumnConst final : public COWHelper { data->serialize_vec(keys, num_rows, max_row_byte_size); } + void update_xxHash_with_value(size_t n, uint64_t& hash) const override { + auto real_data = data->get_data_at(0); + if (real_data.data == nullptr) { + hash = HashUtil::xxHash64NullWithSeed(hash); + } else { + hash = HashUtil::xxHash64WithSeed(real_data.data, real_data.size, hash); + } + } + + void update_crc_with_value(size_t n, uint64_t& crc) const override { + get_data_column_ptr()->update_crc_with_value(n, crc); + } + void serialize_vec_with_null_map(std::vector& keys, size_t num_rows, const uint8_t* null_map, size_t max_row_byte_size) const override { @@ -165,6 +178,7 @@ class ColumnConst final : public COWHelper { void update_hashes_with_value(std::vector& hashes, const uint8_t* __restrict null_data) const override; + // (TODO.Amory) here may not use column_const update hash, and PrimitiveType is not used. void update_crcs_with_value(std::vector& hashes, PrimitiveType type, const uint8_t* __restrict null_data) const override; diff --git a/be/src/vec/columns/column_decimal.cpp b/be/src/vec/columns/column_decimal.cpp index 069f195c4a86e9b..e0b8fef05624470 100644 --- a/be/src/vec/columns/column_decimal.cpp +++ b/be/src/vec/columns/column_decimal.cpp @@ -137,6 +137,19 @@ void ColumnDecimal::update_hashes_with_value(std::vector& hashes, SIP_HASHES_FUNCTION_COLUMN_IMPL(); } +template +void ColumnDecimal::update_crc_with_value(size_t n, uint64_t& crc) const { + if constexpr (!IsDecimalV2) { + crc = HashUtil::zlib_crc_hash(&data[n], sizeof(T), crc); + } else { + const DecimalV2Value& dec_val = (const DecimalV2Value&)data[n]; + int64_t int_val = dec_val.int_value(); + int32_t frac_val = dec_val.frac_value(); + crc = HashUtil::zlib_crc_hash(&int_val, sizeof(int_val), crc); + crc = HashUtil::zlib_crc_hash(&frac_val, sizeof(frac_val), crc); + }; +} + template void ColumnDecimal::update_crcs_with_value(std::vector& hashes, PrimitiveType type, const uint8_t* __restrict null_data) const { @@ -146,27 +159,23 @@ void ColumnDecimal::update_crcs_with_value(std::vector& hashes, Pri if constexpr (!IsDecimalV2) { DO_CRC_HASHES_FUNCTION_COLUMN_IMPL() } else { - DCHECK(type == TYPE_DECIMALV2); - auto decimalv2_do_crc = [&](size_t i) { - const DecimalV2Value& dec_val = (const DecimalV2Value&)data[i]; - int64_t int_val = dec_val.int_value(); - int32_t frac_val = dec_val.frac_value(); - hashes[i] = HashUtil::zlib_crc_hash(&int_val, sizeof(int_val), hashes[i]); - hashes[i] = HashUtil::zlib_crc_hash(&frac_val, sizeof(frac_val), hashes[i]); - }; - if (null_data == nullptr) { for (size_t i = 0; i < s; i++) { - decimalv2_do_crc(i); + update_crc_with_value(i, hashes[i]); } } else { for (size_t i = 0; i < s; i++) { - if (null_data[i] == 0) decimalv2_do_crc(i); + if (null_data[i] == 0) update_crc_with_value(i, hashes[i]); } } } } +template +void ColumnDecimal::update_xxHash_with_value(size_t n, uint64_t& hash) const { + hash = HashUtil::xxHash64WithSeed(reinterpret_cast(&data[n]), sizeof(T), hash); +} + template void ColumnDecimal::update_hashes_with_value(uint64_t* __restrict hashes, const uint8_t* __restrict null_data) const { diff --git a/be/src/vec/columns/column_decimal.h b/be/src/vec/columns/column_decimal.h index f935c0c82686da2..973f0bea68873f2 100644 --- a/be/src/vec/columns/column_decimal.h +++ b/be/src/vec/columns/column_decimal.h @@ -184,6 +184,9 @@ class ColumnDecimal final : public COWHelper& hashes, PrimitiveType type, const uint8_t* __restrict null_data) const override; + void update_xxHash_with_value(size_t n, uint64_t& hash) const override; + void update_crc_with_value(size_t n, uint64_t& crc) const override; + int compare_at(size_t n, size_t m, const IColumn& rhs_, int nan_direction_hint) const override; void get_permutation(bool reverse, size_t limit, int nan_direction_hint, IColumn::Permutation& res) const override; diff --git a/be/src/vec/columns/column_map.cpp b/be/src/vec/columns/column_map.cpp index 4f06ee4dda55bc8..1924e2ba465b092 100644 --- a/be/src/vec/columns/column_map.cpp +++ b/be/src/vec/columns/column_map.cpp @@ -238,15 +238,79 @@ const char* ColumnMap::deserialize_and_insert_from_arena(const char* pos) { } void ColumnMap::update_hash_with_value(size_t n, SipHash& hash) const { - size_t array_size = size_at(n); + size_t kv_size = size_at(n); size_t offset = offset_at(n); - for (size_t i = 0; i < array_size; ++i) { + hash.update(reinterpret_cast(&kv_size), sizeof(kv_size)); + for (size_t i = 0; i < kv_size; ++i) { get_keys().update_hash_with_value(offset + i, hash); get_values().update_hash_with_value(offset + i, hash); } } +void ColumnMap::update_hashes_with_value(std::vector& hashes, + const uint8_t* __restrict null_data) const { + SIP_HASHES_FUNCTION_COLUMN_IMPL(); +} + +void ColumnMap::update_xxHash_with_value(size_t n, uint64_t& hash) const { + size_t kv_size = size_at(n); + size_t offset = offset_at(n); + + hash = HashUtil::xxHash64WithSeed(reinterpret_cast(&kv_size), sizeof(kv_size), + hash); + for (auto i = 0; i < kv_size; ++i) { + get_keys().update_xxHash_with_value(offset + i, hash); + get_values().update_xxHash_with_value(offset + i, hash); + } +} + +void ColumnMap::update_crc_with_value(size_t n, uint64_t& crc) const { + size_t kv_size = size_at(n); + size_t offset = offset_at(n); + + crc = HashUtil::zlib_crc_hash(reinterpret_cast(&kv_size), sizeof(kv_size), crc); + for (size_t i = 0; i < kv_size; ++i) { + get_keys().update_crc_with_value(offset + i, crc); + get_values().update_crc_with_value(offset + i, crc); + } +} + +void ColumnMap::update_hashes_with_value(uint64_t* hashes, const uint8_t* null_data) const { + size_t s = size(); + if (null_data) { + for (size_t i = 0; i < s; ++i) { + // every row + if (null_data[i] == 0) { + update_xxHash_with_value(i, hashes[i]); + } + } + } else { + for (size_t i = 0; i < s; ++i) { + update_xxHash_with_value(i, hashes[i]); + } + } +} + +void ColumnMap::update_crcs_with_value(std::vector& hash, PrimitiveType type, + const uint8_t* __restrict null_data) const { + auto s = hash.size(); + DCHECK(s == size()); + + if (null_data) { + for (size_t i = 0; i < s; ++i) { + // every row + if (null_data[i] == 0) { + update_crc_with_value(i, hash[i]); + } + } + } else { + for (size_t i = 0; i < s; ++i) { + update_crc_with_value(i, hash[i]); + } + } +} + void ColumnMap::insert_range_from(const IColumn& src, size_t start, size_t length) { if (length == 0) { return; diff --git a/be/src/vec/columns/column_map.h b/be/src/vec/columns/column_map.h index 6cb83adf2598e68..0d7bb2d0a7da24e 100644 --- a/be/src/vec/columns/column_map.h +++ b/be/src/vec/columns/column_map.h @@ -39,6 +39,7 @@ #include "vec/columns/column_vector.h" #include "vec/common/assert_cast.h" #include "vec/common/cow.h" +#include "vec/common/sip_hash.h" #include "vec/common/string_ref.h" #include "vec/core/field.h" #include "vec/core/types.h" @@ -166,6 +167,18 @@ class ColumnMap final : public COWHelper { size_t allocated_bytes() const override; void protect() override; + void update_xxHash_with_value(size_t n, uint64_t& hash) const override; + void update_crc_with_value(size_t n, uint64_t& crc) const override; + + void update_hashes_with_value(std::vector& hashes, + const uint8_t* __restrict null_data) const override; + + void update_hashes_with_value(uint64_t* __restrict hashes, + const uint8_t* __restrict null_data = nullptr) const override; + + void update_crcs_with_value(std::vector& hash, PrimitiveType type, + const uint8_t* __restrict null_data = nullptr) const override; + /******************** keys and values ***************/ const ColumnPtr& get_keys_ptr() const { return keys_column; } ColumnPtr& get_keys_ptr() { return keys_column; } diff --git a/be/src/vec/columns/column_nullable.cpp b/be/src/vec/columns/column_nullable.cpp index 42cda1d18d66ff1..ce5b68f3fb3a55f 100644 --- a/be/src/vec/columns/column_nullable.cpp +++ b/be/src/vec/columns/column_nullable.cpp @@ -65,6 +65,24 @@ MutableColumnPtr ColumnNullable::get_shrinked_column() { get_null_map_column_ptr()); } +void ColumnNullable::update_xxHash_with_value(size_t n, uint64_t& hash) const { + auto* __restrict real_null_data = assert_cast(*null_map).get_data().data(); + if (real_null_data[n] != 0) { + hash = HashUtil::xxHash64NullWithSeed(hash); + } else { + nested_column->update_xxHash_with_value(n, hash); + } +} + +void ColumnNullable::update_crc_with_value(size_t n, uint64_t& crc) const { + auto* __restrict real_null_data = assert_cast(*null_map).get_data().data(); + if (real_null_data[n] != 0) { + crc = HashUtil::zlib_crc_hash_null(crc); + } else { + nested_column->update_xxHash_with_value(n, crc); + } +} + void ColumnNullable::update_hash_with_value(size_t n, SipHash& hash) const { if (is_null_at(n)) hash.update(0); diff --git a/be/src/vec/columns/column_nullable.h b/be/src/vec/columns/column_nullable.h index d5ca7f844b3ea1b..be9ba7239953adb 100644 --- a/be/src/vec/columns/column_nullable.h +++ b/be/src/vec/columns/column_nullable.h @@ -98,11 +98,7 @@ class ColumnNullable final : public COWHelper { const char* get_family_name() const override { return "Nullable"; } std::string get_name() const override { return "Nullable(" + nested_column->get_name() + ")"; } MutableColumnPtr clone_resized(size_t size) const override; - size_t size() const override { - return nested_column->size( - - ); - } + size_t size() const override { return nested_column->size(); } bool is_null_at(size_t n) const override { return assert_cast(*null_map).get_data()[n] != 0; } @@ -219,6 +215,9 @@ class ColumnNullable final : public COWHelper { ColumnPtr replicate(const Offsets& replicate_offsets) const override; void replicate(const uint32_t* counts, size_t target_size, IColumn& column, size_t begin = 0, int count_sz = -1) const override; + void update_xxHash_with_value(size_t n, uint64_t& hash) const override; + void update_crc_with_value(size_t n, uint64_t& crc) const override; + void update_hash_with_value(size_t n, SipHash& hash) const override; void update_hashes_with_value(std::vector& hashes, const uint8_t* __restrict null_data) const override; diff --git a/be/src/vec/columns/column_string.h b/be/src/vec/columns/column_string.h index efd90fd8444ce91..703826cd24ea22a 100644 --- a/be/src/vec/columns/column_string.h +++ b/be/src/vec/columns/column_string.h @@ -398,6 +398,18 @@ class ColumnString final : public COWHelper { void deserialize_vec_with_null_map(std::vector& keys, const size_t num_rows, const uint8_t* null_map) override; + void update_xxHash_with_value(size_t n, uint64_t& hash) const override { + size_t string_size = size_at(n); + size_t offset = offset_at(n); + hash = HashUtil::xxHash64WithSeed(reinterpret_cast(&chars[offset]), + string_size, hash); + } + + void update_crc_with_value(size_t n, uint64_t& crc) const override { + auto data_ref = get_data_at(n); + crc = HashUtil::zlib_crc_hash(data_ref.data, data_ref.size, crc); + } + void update_hash_with_value(size_t n, SipHash& hash) const override { size_t string_size = size_at(n); size_t offset = offset_at(n); diff --git a/be/src/vec/columns/column_struct.cpp b/be/src/vec/columns/column_struct.cpp index 2a5a505fb8fd60c..58f5a4abaf462fa 100644 --- a/be/src/vec/columns/column_struct.cpp +++ b/be/src/vec/columns/column_struct.cpp @@ -191,6 +191,37 @@ void ColumnStruct::update_hash_with_value(size_t n, SipHash& hash) const { } } +void ColumnStruct::update_hashes_with_value(std::vector& hashes, + const uint8_t* __restrict null_data) const { + SIP_HASHES_FUNCTION_COLUMN_IMPL(); +} + +void ColumnStruct::update_xxHash_with_value(size_t n, uint64_t& hash) const { + for (const auto& column : columns) { + column->update_xxHash_with_value(n, hash); + } +} + +void ColumnStruct::update_crc_with_value(size_t n, uint64_t& crc) const { + for (const auto& column : columns) { + column->update_crc_with_value(n, crc); + } +} + +void ColumnStruct::update_hashes_with_value(uint64_t* __restrict hashes, + const uint8_t* __restrict null_data) const { + for (const auto& column : columns) { + column->update_hashes_with_value(hashes, null_data); + } +} + +void ColumnStruct::update_crcs_with_value(std::vector& hash, PrimitiveType type, + const uint8_t* __restrict null_data) const { + for (const auto& column : columns) { + column->update_crcs_with_value(hash, type, null_data); + } +} + void ColumnStruct::insert_indices_from(const IColumn& src, const int* indices_begin, const int* indices_end) { const ColumnStruct& src_concrete = assert_cast(src); diff --git a/be/src/vec/columns/column_struct.h b/be/src/vec/columns/column_struct.h index dfa8bd6f5dfd1ee..3771d29e48ec23b 100644 --- a/be/src/vec/columns/column_struct.h +++ b/be/src/vec/columns/column_struct.h @@ -35,6 +35,7 @@ #include "vec/columns/column.h" #include "vec/columns/column_impl.h" #include "vec/common/cow.h" +#include "vec/common/sip_hash.h" #include "vec/common/string_ref.h" #include "vec/core/field.h" #include "vec/core/types.h" @@ -103,7 +104,19 @@ class ColumnStruct final : public COWHelper { void pop_back(size_t n) override; StringRef serialize_value_into_arena(size_t n, Arena& arena, char const*& begin) const override; const char* deserialize_and_insert_from_arena(const char* pos) override; + void update_hash_with_value(size_t n, SipHash& hash) const override; + void update_xxHash_with_value(size_t n, uint64_t& hash) const override; + void update_crc_with_value(size_t n, uint64_t& crc) const override; + + void update_hashes_with_value(std::vector& hashes, + const uint8_t* __restrict null_data) const override; + + void update_hashes_with_value(uint64_t* __restrict hashes, + const uint8_t* __restrict null_data = nullptr) const override; + + void update_crcs_with_value(std::vector& hash, PrimitiveType type, + const uint8_t* __restrict null_data = nullptr) const override; void insert_indices_from(const IColumn& src, const int* indices_begin, const int* indices_end) override; diff --git a/be/src/vec/columns/column_vector.h b/be/src/vec/columns/column_vector.h index c30796792e33e5d..67f2827c927b25b 100644 --- a/be/src/vec/columns/column_vector.h +++ b/be/src/vec/columns/column_vector.h @@ -274,6 +274,24 @@ class ColumnVector final : public COWHelper> const uint8_t* null_map, size_t max_row_byte_size) const override; + void update_xxHash_with_value(size_t n, uint64_t& hash) const override { + hash = HashUtil::xxHash64WithSeed(reinterpret_cast(&data[n]), sizeof(T), hash); + } + + void update_crc_with_value(size_t n, uint64_t& crc) const override { + if constexpr (!std::is_same_v) { + crc = HashUtil::zlib_crc_hash(&data[n], sizeof(T), crc); + } else { + if (this->is_date_type() || this->is_datetime_type()) { + char buf[64]; + const VecDateTimeValue& date_val = (const VecDateTimeValue&)data[n]; + auto len = date_val.to_buffer(buf); + crc = HashUtil::zlib_crc_hash(buf, len, crc); + } else { + crc = HashUtil::zlib_crc_hash(&data[n], sizeof(T), crc); + } + } + } void update_hash_with_value(size_t n, SipHash& hash) const override; void update_hashes_with_value(std::vector& hashes, diff --git a/be/src/vec/common/allocator.h b/be/src/vec/common/allocator.h index 39fd1446da8388e..ae29eb916c1e380 100644 --- a/be/src/vec/common/allocator.h +++ b/be/src/vec/common/allocator.h @@ -28,8 +28,6 @@ #include "common/config.h" #include "common/status.h" -#include "runtime/memory/chunk.h" -#include "runtime/memory/chunk_allocator.h" #include "util/sse_util.hpp" #ifdef NDEBUG @@ -63,17 +61,6 @@ #define MAP_ANONYMOUS MAP_ANON #endif -/** - * Memory allocation between 4KB and 64MB will be through ChunkAllocator, - * those less than 4KB will be through malloc (for example, tcmalloc), - * and those greater than 64MB will be through MMAP. - * In the actual test, chunkallocator allocates less than 4KB of memory slower than malloc, - * and chunkallocator allocates more than 64MB of memory slower than MMAP, - * but the 4KB threshold is an empirical value, which needs to be determined - * by more detailed test later. - */ -static constexpr size_t CHUNK_THRESHOLD = 4096; - static constexpr size_t MMAP_MIN_ALIGNMENT = 4096; static constexpr size_t MALLOC_MIN_ALIGNMENT = 8; @@ -110,7 +97,7 @@ class Allocator { memory_check(size); void* buf; - if (size >= doris::config::mmap_threshold && use_mmap) { + if (use_mmap && size >= doris::config::mmap_threshold) { if (alignment > MMAP_MIN_ALIGNMENT) throw doris::Exception( doris::ErrorCode::INVALID_ARGUMENT, @@ -125,13 +112,6 @@ class Allocator { } /// No need for zero-fill, because mmap guarantees it. - } else if (!doris::config::disable_chunk_allocator_in_vec && size >= CHUNK_THRESHOLD) { - doris::Chunk chunk; - if (!doris::ChunkAllocator::instance()->allocate_align(size, &chunk)) { - throw_bad_alloc(fmt::format("Allocator: Cannot allocate chunk {}.", size)); - } - buf = chunk.data; - if constexpr (clear_memory) memset(buf, 0, chunk.size); } else { if (alignment <= MALLOC_MIN_ALIGNMENT) { if constexpr (clear_memory) @@ -158,28 +138,19 @@ class Allocator { } /// Free memory range. - void free(void* buf, size_t size) { - if (size >= doris::config::mmap_threshold && use_mmap) { + void free(void* buf, size_t size = -1) { + if (use_mmap && size >= doris::config::mmap_threshold) { + DCHECK(size != -1); if (0 != munmap(buf, size)) { throw_bad_alloc(fmt::format("Allocator: Cannot munmap {}.", size)); } else { release_memory(size); } - } else if (!doris::config::disable_chunk_allocator_in_vec && size >= CHUNK_THRESHOLD && - ((size & (size - 1)) == 0)) { - // Only power-of-two length are added to ChunkAllocator - doris::ChunkAllocator::instance()->free((uint8_t*)buf, size); } else { ::free(buf); } } - // Free memory range by ::free. - void free_no_munmap(void* buf) { - CHECK(!use_mmap); - ::free(buf); - } - /** Enlarge memory range. * Data from old range is moved to the beginning of new range. * Address of memory range could change. @@ -188,8 +159,9 @@ class Allocator { if (old_size == new_size) { /// nothing to do. /// BTW, it's not possible to change alignment while doing realloc. - } else if (old_size < CHUNK_THRESHOLD && new_size < CHUNK_THRESHOLD && - alignment <= MALLOC_MIN_ALIGNMENT) { + } else if (!use_mmap || (old_size < doris::config::mmap_threshold && + new_size < doris::config::mmap_threshold && + alignment <= MALLOC_MIN_ALIGNMENT)) { memory_check(new_size); /// Resize malloc'd memory region with no special alignment requirement. void* new_buf = ::realloc(buf, new_size); @@ -203,7 +175,7 @@ class Allocator { if (new_size > old_size) memset(reinterpret_cast(buf) + old_size, 0, new_size - old_size); } else if (old_size >= doris::config::mmap_threshold && - new_size >= doris::config::mmap_threshold && use_mmap) { + new_size >= doris::config::mmap_threshold) { memory_check(new_size); /// Resize mmap'd memory region. consume_memory(new_size - old_size); @@ -226,7 +198,6 @@ class Allocator { } } else { memory_check(new_size); - // CHUNK_THRESHOLD <= old_size <= MMAP_THRESHOLD use system realloc is slow, use ChunkAllocator. // Big allocs that requires a copy. void* new_buf = alloc(new_size, alignment); memcpy(new_buf, buf, std::min(old_size, new_size)); diff --git a/be/src/vec/common/format_ip.h b/be/src/vec/common/format_ip.h index f87be27b8034e4b..76091e39fbf085a 100644 --- a/be/src/vec/common/format_ip.h +++ b/be/src/vec/common/format_ip.h @@ -19,8 +19,11 @@ // and modified by Doris #pragma once + #include +#include #include +#include #include #include diff --git a/be/src/vec/common/hash_table/hash_table.h b/be/src/vec/common/hash_table/hash_table.h index 7ee35af64c193a5..0b9d6bc3e17af1d 100644 --- a/be/src/vec/common/hash_table/hash_table.h +++ b/be/src/vec/common/hash_table/hash_table.h @@ -445,8 +445,8 @@ class HashTable : private boost::noncopyable, using Self = HashTable; using cell_type = Cell; - size_t m_size = 0; /// Amount of elements - Cell* buf; /// A piece of memory for all elements except the element with zero key. + size_t m_size = 0; /// Amount of elements + Cell* buf {nullptr}; /// A piece of memory for all elements except the element with zero key. Grower grower; int64_t _resize_timer_ns; diff --git a/be/src/vec/common/hash_table/phmap_fwd_decl.h b/be/src/vec/common/hash_table/phmap_fwd_decl.h index e9ec663637ca510..623734109685490 100644 --- a/be/src/vec/common/hash_table/phmap_fwd_decl.h +++ b/be/src/vec/common/hash_table/phmap_fwd_decl.h @@ -26,7 +26,7 @@ namespace doris::vectorized { /// `Allocator_` implements several interfaces of `std::allocator` /// which `phmap::flat_hash_map` will use. template -class Allocator_ : private Allocator { +class Allocator_ : private Allocator { public: using value_type = T; using pointer = T*; diff --git a/be/src/vec/common/schema_util.cpp b/be/src/vec/common/schema_util.cpp index 703ee1bc7554a12..09fabbae0e9fe12 100644 --- a/be/src/vec/common/schema_util.cpp +++ b/be/src/vec/common/schema_util.cpp @@ -39,6 +39,7 @@ #include "olap/olap_common.h" #include "runtime/client_cache.h" #include "runtime/exec_env.h" +#include "util/string_util.h" #include "util/thrift_rpc_helper.h" #include "vec/columns/column.h" #include "vec/columns/column_array.h" diff --git a/be/src/vec/data_types/data_type_agg_state.h b/be/src/vec/data_types/data_type_agg_state.h index fb71e66ce8606ed..d48f2a88117b591 100644 --- a/be/src/vec/data_types/data_type_agg_state.h +++ b/be/src/vec/data_types/data_type_agg_state.h @@ -82,7 +82,9 @@ class DataTypeAggState : public DataTypeString { std::string res = "binary("; StringRef str = column.get_data_at(row_num); for (auto c : str.to_string()) { - res += std::to_string(int(c)); + for (int i = 0; i < 8; i++) { + res += (c & (1 << (7 - i))) ? "1" : "0"; + } res += ' '; } res += ")"; diff --git a/be/src/vec/data_types/serde/data_type_fixedlengthobject_serde.h b/be/src/vec/data_types/serde/data_type_fixedlengthobject_serde.h index 092d2202b021248..b61ed39075e2df3 100644 --- a/be/src/vec/data_types/serde/data_type_fixedlengthobject_serde.h +++ b/be/src/vec/data_types/serde/data_type_fixedlengthobject_serde.h @@ -38,10 +38,10 @@ class DataTypeFixedLengthObjectSerDe : public DataTypeSerDe { public: Status write_column_to_pb(const IColumn& column, PValues& result, int start, int end) const override { - LOG(FATAL) << "Not support write FixedLengthObject column to pb"; + return Status::NotSupported("Not support write FixedLengthObject column to pb"); } Status read_column_from_pb(IColumn& column, const PValues& arg) const override { - LOG(FATAL) << "Not support read from pb to FixedLengthObject"; + return Status::NotSupported("Not support read from pb to FixedLengthObject"); }; void write_one_cell_to_jsonb(const IColumn& column, JsonbWriter& result, Arena* mem_pool, int32_t col_id, int row_num) const override { @@ -63,12 +63,12 @@ class DataTypeFixedLengthObjectSerDe : public DataTypeSerDe { Status write_column_to_mysql(const IColumn& column, MysqlRowBuffer& row_buffer, int row_idx, bool col_const) const override { - LOG(FATAL) << "Not support write object column to mysql"; + return Status::NotSupported("Not support write object column to mysql"); } Status write_column_to_mysql(const IColumn& column, MysqlRowBuffer& row_buffer, int row_idx, bool col_const) const override { - LOG(FATAL) << "Not support write object column to mysql"; + return Status::NotSupported("Not support write object column to mysql"); } }; } // namespace vectorized diff --git a/be/src/vec/exec/format/orc/vorc_reader.cpp b/be/src/vec/exec/format/orc/vorc_reader.cpp index 8b1ccd8ccc6dbf2..140ca30987fb810 100644 --- a/be/src/vec/exec/format/orc/vorc_reader.cpp +++ b/be/src/vec/exec/format/orc/vorc_reader.cpp @@ -744,7 +744,7 @@ Status OrcReader::set_fill_columns( // create orc row reader _row_reader_options.range(_range_start_offset, _range_size); - _row_reader_options.setTimezoneName(_ctz); + _row_reader_options.setTimezoneName(_ctz == "CST" ? "Asia/Shanghai" : _ctz); _row_reader_options.include(_read_cols); if (_lazy_read_ctx.can_lazy_read) { _row_reader_options.filter(_lazy_read_ctx.predicate_orc_columns); diff --git a/be/src/vec/exec/jni_connector.cpp b/be/src/vec/exec/jni_connector.cpp index d3fafe59744e6c3..edb195479a046c0 100644 --- a/be/src/vec/exec/jni_connector.cpp +++ b/be/src/vec/exec/jni_connector.cpp @@ -72,12 +72,16 @@ Status JniConnector::open(RuntimeState* state, RuntimeProfile* profile) { // cannot put the env into fields, because frames in an env object is limited // to avoid limited frames in a thread, we should get local env in a method instead of in whole object. JNIEnv* env = nullptr; + int batch_size = 0; + if (!_is_table_schema) { + batch_size = _state->batch_size(); + } RETURN_IF_ERROR(JniUtil::GetJNIEnv(&env)); if (env == nullptr) { return Status::InternalError("Failed to get/create JVM"); } SCOPED_TIMER(_open_scanner_time); - RETURN_IF_ERROR(_init_jni_scanner(env, state->batch_size())); + RETURN_IF_ERROR(_init_jni_scanner(env, batch_size)); // Call org.apache.doris.common.jni.JniScanner#open env->CallVoidMethod(_jni_scanner_obj, _jni_scanner_open); RETURN_ERROR_IF_EXC(env); @@ -129,6 +133,18 @@ Status JniConnector::get_nex_block(Block* block, size_t* read_rows, bool* eof) { return Status::OK(); } +Status JniConnector::get_table_schema(std::string& table_schema_str) { + JNIEnv* env = nullptr; + RETURN_IF_ERROR(JniUtil::GetJNIEnv(&env)); + // Call org.apache.doris.jni.JniScanner#getTableSchema + // return the TableSchema information + jstring jstr = (jstring)env->CallObjectMethod(_jni_scanner_obj, _jni_scanner_get_table_schema); + RETURN_ERROR_IF_EXC(env); + table_schema_str = env->GetStringUTFChars(jstr, nullptr); + RETURN_ERROR_IF_EXC(env); + return Status::OK(); +} + std::map JniConnector::get_statistics(JNIEnv* env) { jobject metrics = env->CallObjectMethod(_jni_scanner_obj, _jni_scanner_get_statistics); std::map result = JniUtil::convert_to_cpp_map(env, metrics); @@ -197,6 +213,9 @@ Status JniConnector::_init_jni_scanner(JNIEnv* env, int batch_size) { _jni_scanner_open = env->GetMethodID(_jni_scanner_cls, "open", "()V"); _jni_scanner_get_next_batch = env->GetMethodID(_jni_scanner_cls, "getNextBatchMeta", "()J"); + _jni_scanner_get_table_schema = + env->GetMethodID(_jni_scanner_cls, "getTableSchema", "()Ljava/lang/String;"); + RETURN_ERROR_IF_EXC(env); _jni_scanner_close = env->GetMethodID(_jni_scanner_cls, "close", "()V"); _jni_scanner_release_column = env->GetMethodID(_jni_scanner_cls, "releaseColumn", "(I)V"); _jni_scanner_release_table = env->GetMethodID(_jni_scanner_cls, "releaseTable", "()V"); diff --git a/be/src/vec/exec/jni_connector.h b/be/src/vec/exec/jni_connector.h index 36be5379a9336e2..0f08fbe0f81d3e0 100644 --- a/be/src/vec/exec/jni_connector.h +++ b/be/src/vec/exec/jni_connector.h @@ -33,12 +33,13 @@ #include "runtime/define_primitive_type.h" #include "runtime/primitive_type.h" #include "runtime/types.h" +#include "util/runtime_profile.h" +#include "util/string_util.h" #include "vec/aggregate_functions/aggregate_function.h" #include "vec/common/string_ref.h" #include "vec/data_types/data_type.h" namespace doris { -class RuntimeProfile; class RuntimeState; namespace vectorized { @@ -168,6 +169,17 @@ class JniConnector { _connector_name = split(_connector_class, "/").back(); } + /** + * Just use to get the table schema. + * @param connector_class Java scanner class + * @param scanner_params Provided configuration map + */ + JniConnector(std::string connector_class, std::map scanner_params) + : _connector_class(std::move(connector_class)), + _scanner_params(std::move(scanner_params)) { + _is_table_schema = true; + } + /// Should release jni resources if other functions are failed. ~JniConnector(); @@ -205,6 +217,17 @@ class JniConnector { */ std::map get_statistics(JNIEnv* env); + /** + * Call java side function JniScanner.getTableSchema. + * + * The schema information are stored as a string. + * Use # between column names and column types. + * + * like: col_name1,col_name2,col_name3#col_type1,col_type2.col_type3 + * + */ + Status get_table_schema(std::string& table_schema_str); + /** * Close scanner and release jni resources. */ @@ -222,6 +245,7 @@ class JniConnector { std::string _connector_class; std::map _scanner_params; std::vector _column_names; + bool _is_table_schema = false; RuntimeState* _state; RuntimeProfile* _profile; @@ -237,6 +261,7 @@ class JniConnector { jobject _jni_scanner_obj; jmethodID _jni_scanner_open; jmethodID _jni_scanner_get_next_batch; + jmethodID _jni_scanner_get_table_schema; jmethodID _jni_scanner_close; jmethodID _jni_scanner_release_column; jmethodID _jni_scanner_release_table; diff --git a/be/src/vec/exec/scan/avro_jni_reader.cpp b/be/src/vec/exec/scan/avro_jni_reader.cpp new file mode 100644 index 000000000000000..5d1ef40cbc87aca --- /dev/null +++ b/be/src/vec/exec/scan/avro_jni_reader.cpp @@ -0,0 +1,165 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "avro_jni_reader.h" + +#include +#include + +#include "runtime/descriptors.h" +#include "runtime/types.h" + +namespace doris::vectorized { + +AvroJNIReader::AvroJNIReader(RuntimeState* state, RuntimeProfile* profile, + const TFileScanRangeParams& params, + const std::vector& file_slot_descs) + : _file_slot_descs(file_slot_descs), _state(state), _profile(profile), _params(params) {} + +AvroJNIReader::AvroJNIReader(RuntimeProfile* profile, const TFileScanRangeParams& params, + const TFileRangeDesc& range, + const std::vector& file_slot_descs) + : _file_slot_descs(file_slot_descs), _profile(profile), _params(params), _range(range) {} + +AvroJNIReader::~AvroJNIReader() = default; + +Status AvroJNIReader::get_next_block(Block* block, size_t* read_rows, bool* eof) { + RETURN_IF_ERROR(_jni_connector->get_nex_block(block, read_rows, eof)); + if (*eof) { + RETURN_IF_ERROR(_jni_connector->close()); + } + return Status::OK(); +} + +Status AvroJNIReader::get_columns(std::unordered_map* name_to_type, + std::unordered_set* missing_cols) { + for (auto& desc : _file_slot_descs) { + name_to_type->emplace(desc->col_name(), desc->type()); + } + return Status::OK(); +} + +Status AvroJNIReader::init_fetch_table_reader( + std::unordered_map* colname_to_value_range) { + _colname_to_value_range = colname_to_value_range; + std::ostringstream required_fields; + std::ostringstream columns_types; + std::vector column_names; + int index = 0; + for (auto& desc : _file_slot_descs) { + std::string field = desc->col_name(); + column_names.emplace_back(field); + std::string type = JniConnector::get_hive_type(desc->type()); + if (index == 0) { + required_fields << field; + columns_types << type; + } else { + required_fields << "," << field; + columns_types << "#" << type; + } + index++; + } + + TFileType::type type = _params.file_type; + std::map required_param = { + {"required_fields", required_fields.str()}, + {"columns_types", columns_types.str()}, + {"file_type", std::to_string(type)}, + {"is_get_table_schema", "false"}, + {"hive.serde", "org.apache.hadoop.hive.serde2.avro.AvroSerDe"}}; + switch (type) { + case TFileType::FILE_HDFS: + required_param.insert(std::make_pair("uri", _params.hdfs_params.hdfs_conf.data()->value)); + break; + case TFileType::FILE_S3: + required_param.insert(_params.properties.begin(), _params.properties.end()); + break; + default: + Status::InternalError("unsupported file reader type: {}", std::to_string(type)); + } + required_param.insert(_params.properties.begin(), _params.properties.end()); + _jni_connector = std::make_unique("org/apache/doris/avro/AvroJNIScanner", + required_param, column_names); + RETURN_IF_ERROR(_jni_connector->init(_colname_to_value_range)); + return _jni_connector->open(_state, _profile); +} + +Status AvroJNIReader::init_fetch_table_schema_reader() { + std::map required_param = {{"uri", _range.path}, + {"file_type", std::to_string(_params.file_type)}, + {"is_get_table_schema", "true"}}; + + required_param.insert(_params.properties.begin(), _params.properties.end()); + _jni_connector = + std::make_unique("org/apache/doris/avro/AvroJNIScanner", required_param); + return _jni_connector->open(nullptr, _profile); +} + +Status AvroJNIReader::get_parsed_schema(std::vector* col_names, + std::vector* col_types) { + std::string table_schema_str; + RETURN_IF_ERROR(_jni_connector->get_table_schema(table_schema_str)); + + rapidjson::Document document; + document.Parse(table_schema_str.c_str()); + if (document.IsArray()) { + for (int i = 0; i < document.Size(); ++i) { + rapidjson::Value& column_schema = document[i]; + col_names->push_back(column_schema["name"].GetString()); + col_types->push_back(convert_to_doris_type(column_schema)); + } + } + return _jni_connector->close(); +} + +TypeDescriptor AvroJNIReader::convert_to_doris_type(const rapidjson::Value& column_schema) { + ::doris::TPrimitiveType::type schema_type = + static_cast< ::doris::TPrimitiveType::type>(column_schema["type"].GetInt()); + switch (schema_type) { + case TPrimitiveType::INT: + case TPrimitiveType::STRING: + case TPrimitiveType::BIGINT: + case TPrimitiveType::BOOLEAN: + case TPrimitiveType::DOUBLE: + case TPrimitiveType::FLOAT: + return TypeDescriptor(thrift_to_type(schema_type)); + case TPrimitiveType::ARRAY: { + TypeDescriptor list_type(PrimitiveType::TYPE_ARRAY); + list_type.add_sub_type(convert_complex_type(column_schema["childColumn"].GetObject())); + return list_type; + } + case TPrimitiveType::MAP: { + TypeDescriptor map_type(PrimitiveType::TYPE_MAP); + + // The default type of AVRO MAP structure key is STRING + map_type.add_sub_type(PrimitiveType::TYPE_STRING); + map_type.add_sub_type(convert_complex_type(column_schema["childColumn"].GetObject())); + return map_type; + } + default: + return TypeDescriptor(PrimitiveType::INVALID_TYPE); + } +} + +TypeDescriptor AvroJNIReader::convert_complex_type( + const rapidjson::Document::ConstObject child_schema) { + ::doris::TPrimitiveType::type child_schema_type = + static_cast< ::doris::TPrimitiveType::type>(child_schema["type"].GetInt()); + return TypeDescriptor(thrift_to_type(child_schema_type)); +} + +} // namespace doris::vectorized diff --git a/be/src/vec/exec/scan/avro_jni_reader.h b/be/src/vec/exec/scan/avro_jni_reader.h new file mode 100644 index 000000000000000..70ef859fba5a288 --- /dev/null +++ b/be/src/vec/exec/scan/avro_jni_reader.h @@ -0,0 +1,96 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include + +#include +#include +#include +#include +#include + +#include "common/status.h" +#include "exec/olap_common.h" +#include "vec/exec/format/generic_reader.h" +#include "vec/exec/jni_connector.h" + +namespace doris { +class RuntimeProfile; + +class RuntimeState; + +class SlotDescriptor; +namespace vectorized { +class Block; +} // namespace vectorized +struct TypeDescriptor; +} // namespace doris + +namespace doris::vectorized { + +/** + * Read avro-format file + */ +class AvroJNIReader : public GenericReader { + ENABLE_FACTORY_CREATOR(AvroJNIReader); + +public: + /** + * Call java side by jni to get table data. + */ + AvroJNIReader(RuntimeState* state, RuntimeProfile* profile, const TFileScanRangeParams& params, + const std::vector& file_slot_descs); + + /** + * Call java side by jni to get table schema. + */ + AvroJNIReader(RuntimeProfile* profile, const TFileScanRangeParams& params, + const TFileRangeDesc& range, const std::vector& file_slot_descs); + + ~AvroJNIReader() override; + + Status get_next_block(Block* block, size_t* read_rows, bool* eof) override; + + Status get_columns(std::unordered_map* name_to_type, + std::unordered_set* missing_cols) override; + + Status init_fetch_table_reader( + std::unordered_map* colname_to_value_range); + + Status init_fetch_table_schema_reader(); + + Status get_parsed_schema(std::vector* col_names, + std::vector* col_types) override; + + TypeDescriptor convert_to_doris_type(const rapidjson::Value& column_schema); + + TypeDescriptor convert_complex_type(const rapidjson::Document::ConstObject child_schema); + +private: + const std::vector& _file_slot_descs; + RuntimeState* _state; + RuntimeProfile* _profile; + const TFileScanRangeParams _params; + const TFileRangeDesc _range; + std::unordered_map* _colname_to_value_range; + std::unique_ptr _jni_connector; +}; + +} // namespace doris::vectorized diff --git a/be/src/vec/exec/scan/new_olap_scan_node.cpp b/be/src/vec/exec/scan/new_olap_scan_node.cpp index 740a57e793b1537..011bd3a3e4065c9 100644 --- a/be/src/vec/exec/scan/new_olap_scan_node.cpp +++ b/be/src/vec/exec/scan/new_olap_scan_node.cpp @@ -144,6 +144,7 @@ Status NewOlapScanNode::_init_profile() { _stats_filtered_counter = ADD_COUNTER(_segment_profile, "RowsStatsFiltered", TUnit::UNIT); _bf_filtered_counter = ADD_COUNTER(_segment_profile, "RowsBloomFilterFiltered", TUnit::UNIT); + _dict_filtered_counter = ADD_COUNTER(_segment_profile, "RowsDictFiltered", TUnit::UNIT); _del_filtered_counter = ADD_COUNTER(_scanner_profile, "RowsDelFiltered", TUnit::UNIT); _conditions_filtered_counter = ADD_COUNTER(_segment_profile, "RowsConditionsFiltered", TUnit::UNIT); diff --git a/be/src/vec/exec/scan/new_olap_scan_node.h b/be/src/vec/exec/scan/new_olap_scan_node.h index 21e2540350810c9..fac6997c9d6a2ea 100644 --- a/be/src/vec/exec/scan/new_olap_scan_node.h +++ b/be/src/vec/exec/scan/new_olap_scan_node.h @@ -135,6 +135,7 @@ class NewOlapScanNode : public VScanNode { RuntimeProfile::Counter* _stats_filtered_counter = nullptr; RuntimeProfile::Counter* _bf_filtered_counter = nullptr; + RuntimeProfile::Counter* _dict_filtered_counter = nullptr; RuntimeProfile::Counter* _del_filtered_counter = nullptr; RuntimeProfile::Counter* _conditions_filtered_counter = nullptr; RuntimeProfile::Counter* _key_range_filtered_counter = nullptr; diff --git a/be/src/vec/exec/scan/new_olap_scanner.cpp b/be/src/vec/exec/scan/new_olap_scanner.cpp index ca56bc95bd245a9..e6763bf94b7bcc7 100644 --- a/be/src/vec/exec/scan/new_olap_scanner.cpp +++ b/be/src/vec/exec/scan/new_olap_scanner.cpp @@ -571,6 +571,7 @@ void NewOlapScanner::_update_counters_before_close() { } COUNTER_UPDATE(olap_parent->_stats_filtered_counter, stats.rows_stats_filtered); + COUNTER_UPDATE(olap_parent->_dict_filtered_counter, stats.rows_dict_filtered); COUNTER_UPDATE(olap_parent->_bf_filtered_counter, stats.rows_bf_filtered); COUNTER_UPDATE(olap_parent->_del_filtered_counter, stats.rows_del_filtered); COUNTER_UPDATE(olap_parent->_del_filtered_counter, stats.rows_del_by_bitmap); diff --git a/be/src/vec/exec/scan/pip_scanner_context.h b/be/src/vec/exec/scan/pip_scanner_context.h index 7edf634c1733a84..731c3bb926962fc 100644 --- a/be/src/vec/exec/scan/pip_scanner_context.h +++ b/be/src/vec/exec/scan/pip_scanner_context.h @@ -17,7 +17,6 @@ #pragma once -#include "concurrentqueue.h" #include "runtime/descriptors.h" #include "scanner_context.h" @@ -167,63 +166,10 @@ class PipScannerContext : public vectorized::ScannerContext { } } - vectorized::BlockUPtr get_free_block(bool* has_free_block, - bool get_not_empty_block = false) override { - { - vectorized::BlockUPtr block; - if (_free_blocks_queues.try_dequeue(block)) { - if (!get_not_empty_block || block->mem_reuse()) { - _total_free_block_num--; - _free_blocks_memory_usage->add(-block->allocated_bytes()); - return block; - } - } - } - *has_free_block = false; - - COUNTER_UPDATE(_newly_create_free_blocks_num, 1); - return vectorized::Block::create_unique(_real_tuple_desc->slots(), _batch_size, - true /*ignore invalid slots*/); - } - - void return_free_block(std::unique_ptr block) override { - block->clear_column_data(); - _free_blocks_memory_usage->add(block->allocated_bytes()); - _free_blocks_queues.enqueue(std::move(block)); - _total_free_block_num++; - } - - void _init_free_block(int pre_alloc_block_count, int real_block_size) override { - // The free blocks is used for final output block of scanners. - // So use _output_tuple_desc; - int64_t free_blocks_memory_usage = 0; - auto block = vectorized::Block::create_unique(_output_tuple_desc->slots(), real_block_size, - true /*ignore invalid slots*/); - free_blocks_memory_usage += block->allocated_bytes(); - _free_blocks_queues.enqueue(std::move(block)); - _total_free_block_num = pre_alloc_block_count; - _free_blocks_memory_usage->add(free_blocks_memory_usage); - } - - int cal_thread_slot_num_by_free_block_num() override { - // For pipeline engine, we don't promise `thread_slot_num` is exact (e.g. scanners to - // schedule may need more free blocks than available free blocks). - // This is because we don't want a heavy lock for free block queues. - int local_val = _total_free_block_num; - int thread_slot_num = local_val / _block_per_scanner; - thread_slot_num += (local_val % _block_per_scanner != 0); - thread_slot_num = std::min(thread_slot_num, _max_thread_num - _num_running_scanners); - if (thread_slot_num <= 0) { - thread_slot_num = 1; - } - return thread_slot_num; - } - private: int _next_queue_to_feed = 0; std::vector> _blocks_queues; std::atomic_int64_t _current_used_bytes = 0; - std::atomic_int32_t _total_free_block_num = 0; const std::vector& _col_distribute_ids; const bool _need_colocate_distribute; @@ -231,8 +177,6 @@ class PipScannerContext : public vectorized::ScannerContext { std::vector> _colocate_mutable_blocks; std::vector> _colocate_block_mutexs; - moodycamel::ConcurrentQueue _free_blocks_queues; - void _add_rows_colocate_blocks(vectorized::Block* block, int loc, const std::vector& rows) { int row_wait_add = rows.size(); diff --git a/be/src/vec/exec/scan/scanner_context.cpp b/be/src/vec/exec/scan/scanner_context.cpp index fed8215ad286b52..b8e5847a153e896 100644 --- a/be/src/vec/exec/scan/scanner_context.cpp +++ b/be/src/vec/exec/scan/scanner_context.cpp @@ -133,28 +133,19 @@ void ScannerContext::_init_free_block(int pre_alloc_block_count, int real_block_ auto block = vectorized::Block::create_unique(_output_tuple_desc->slots(), real_block_size, true /*ignore invalid slots*/); free_blocks_memory_usage += block->allocated_bytes(); - _free_blocks.emplace_back(std::move(block)); + _free_blocks.enqueue(std::move(block)); } _free_blocks_memory_usage->add(free_blocks_memory_usage); } vectorized::BlockUPtr ScannerContext::get_free_block(bool* has_free_block, bool get_block_not_empty) { - { - std::lock_guard l(_free_blocks_lock); - *has_free_block = _free_blocks_capacity > 0; - // Always reduce _free_blocks_capacity by one since we always return a block - if (_free_blocks_capacity > 0) { - --_free_blocks_capacity; - } - - if (!_free_blocks.empty()) { - if (!get_block_not_empty || _free_blocks.back()->mem_reuse()) { - auto block = std::move(_free_blocks.back()); - _free_blocks.pop_back(); - _free_blocks_memory_usage->add(-block->allocated_bytes()); - return block; - } + vectorized::BlockUPtr block; + if (_free_blocks.try_dequeue(block)) { + if (!get_block_not_empty || block->mem_reuse()) { + _free_blocks_capacity--; + _free_blocks_memory_usage->add(-block->allocated_bytes()); + return block; } } @@ -166,8 +157,7 @@ vectorized::BlockUPtr ScannerContext::get_free_block(bool* has_free_block, void ScannerContext::return_free_block(std::unique_ptr block) { block->clear_column_data(); _free_blocks_memory_usage->add(block->allocated_bytes()); - std::lock_guard l(_free_blocks_lock); - _free_blocks.emplace_back(std::move(block)); + _free_blocks.enqueue(std::move(block)); ++_free_blocks_capacity; } @@ -315,8 +305,6 @@ void ScannerContext::clear_and_join(VScanNode* node, RuntimeState* state) { _close_and_clear_scanners(node, state); _blocks_queue.clear(); - std::unique_lock lock(_free_blocks_lock); - _free_blocks.clear(); } bool ScannerContext::no_schedule() { @@ -331,8 +319,9 @@ std::string ScannerContext::debug_string() { " limit: {}, _num_running_scanners: {}, _num_scheduling_ctx: {}, _max_thread_num: {}," " _block_per_scanner: {}, _cur_bytes_in_queue: {}, MAX_BYTE_OF_QUEUE: {}", ctx_id, _scanners.size(), _blocks_queue.size(), _process_status.ok(), _should_stop, - _is_finished, _free_blocks.size(), limit, _num_running_scanners, _num_scheduling_ctx, - _max_thread_num, _block_per_scanner, _cur_bytes_in_queue, _max_bytes_in_queue); + _is_finished, _free_blocks.size_approx(), limit, _num_running_scanners, + _num_scheduling_ctx, _max_thread_num, _block_per_scanner, _cur_bytes_in_queue, + _max_bytes_in_queue); } void ScannerContext::reschedule_scanner_ctx() { diff --git a/be/src/vec/exec/scan/scanner_context.h b/be/src/vec/exec/scan/scanner_context.h index 5986384edde0f29..fa264c756a0924f 100644 --- a/be/src/vec/exec/scan/scanner_context.h +++ b/be/src/vec/exec/scan/scanner_context.h @@ -29,6 +29,7 @@ #include "common/factory_creator.h" #include "common/status.h" +#include "concurrentqueue.h" #include "util/lock.h" #include "util/runtime_profile.h" #include "vec/core/block.h" @@ -70,9 +71,8 @@ class ScannerContext { virtual ~ScannerContext() = default; virtual Status init(); - virtual vectorized::BlockUPtr get_free_block(bool* has_free_block, - bool get_not_empty_block = false); - virtual void return_free_block(std::unique_ptr block); + vectorized::BlockUPtr get_free_block(bool* has_free_block, bool get_not_empty_block = false); + void return_free_block(std::unique_ptr block); // Append blocks from scanners to the blocks queue. virtual void append_blocks_to_queue(std::vector& blocks); @@ -124,7 +124,7 @@ class ScannerContext { void clear_and_join(VScanNode* node, RuntimeState* state); - virtual bool no_schedule(); + bool no_schedule(); std::string debug_string(); @@ -143,9 +143,8 @@ class ScannerContext { return _cur_bytes_in_queue < _max_bytes_in_queue / 2; } - virtual int cal_thread_slot_num_by_free_block_num() { + int cal_thread_slot_num_by_free_block_num() { int thread_slot_num = 0; - std::lock_guard f(_free_blocks_lock); thread_slot_num = (_free_blocks_capacity + _block_per_scanner - 1) / _block_per_scanner; thread_slot_num = std::min(thread_slot_num, _max_thread_num - _num_running_scanners); if (thread_slot_num <= 0) { @@ -169,7 +168,7 @@ class ScannerContext { protected: virtual void _dispose_coloate_blocks_not_in_queue() {} - virtual void _init_free_block(int pre_alloc_block_count, int real_block_size); + void _init_free_block(int pre_alloc_block_count, int real_block_size); RuntimeState* _state; VScanNode* _parent; @@ -213,12 +212,11 @@ class ScannerContext { std::atomic_bool _is_finished = false; // Lazy-allocated blocks for all scanners to share, for memory reuse. - doris::Mutex _free_blocks_lock; - std::vector _free_blocks; + moodycamel::ConcurrentQueue _free_blocks; // The current number of free blocks available to the scanners. // Used to limit the memory usage of the scanner. // NOTE: this is NOT the size of `_free_blocks`. - int32_t _free_blocks_capacity = 0; + std::atomic_int32_t _free_blocks_capacity = 0; int _batch_size; // The limit from SQL's limit clause diff --git a/be/src/vec/exec/scan/vfile_scanner.cpp b/be/src/vec/exec/scan/vfile_scanner.cpp index 43c6147dad64343..359e05624ed53bd 100644 --- a/be/src/vec/exec/scan/vfile_scanner.cpp +++ b/be/src/vec/exec/scan/vfile_scanner.cpp @@ -61,6 +61,7 @@ #include "vec/exec/format/parquet/vparquet_reader.h" #include "vec/exec/format/table/iceberg_reader.h" #include "vec/exec/format/table/transactional_hive_reader.h" +#include "vec/exec/scan/avro_jni_reader.h" #include "vec/exec/scan/hudi_jni_reader.h" #include "vec/exec/scan/max_compute_jni_reader.h" #include "vec/exec/scan/new_file_scan_node.h" @@ -715,6 +716,12 @@ Status VFileScanner::_get_next_reader() { ((NewJsonReader*)(_cur_reader.get()))->init_reader(_col_default_value_ctx); break; } + case TFileFormatType::FORMAT_AVRO: { + _cur_reader = AvroJNIReader::create_unique(_state, _profile, _params, _file_slot_descs); + init_status = ((AvroJNIReader*)(_cur_reader.get())) + ->init_fetch_table_reader(_colname_to_value_range); + break; + } default: return Status::InternalError("Not supported file format: {}", _params.format_type); } diff --git a/be/src/vec/exec/scan/vmeta_scan_node.cpp b/be/src/vec/exec/scan/vmeta_scan_node.cpp index b94049697ddd307..3bdcbfbaaee3d96 100644 --- a/be/src/vec/exec/scan/vmeta_scan_node.cpp +++ b/be/src/vec/exec/scan/vmeta_scan_node.cpp @@ -37,6 +37,9 @@ VMetaScanNode::VMetaScanNode(ObjectPool* pool, const TPlanNode& tnode, const Des _tuple_id(tnode.meta_scan_node.tuple_id), _scan_params(tnode.meta_scan_node) { _output_tuple_id = _tuple_id; + if (_scan_params.__isset.current_user_ident) { + _user_identity = _scan_params.current_user_ident; + } } Status VMetaScanNode::init(const TPlanNode& tnode, RuntimeState* state) { @@ -62,9 +65,11 @@ Status VMetaScanNode::_init_scanners(std::list* scanners) { if (_eos == true) { return Status::OK(); } + for (auto& scan_range : _scan_ranges) { - std::shared_ptr scanner = VMetaScanner::create_shared( - _state, this, _tuple_id, scan_range, _limit_per_scanner, runtime_profile()); + std::shared_ptr scanner = + VMetaScanner::create_shared(_state, this, _tuple_id, scan_range, _limit_per_scanner, + runtime_profile(), _user_identity); RETURN_IF_ERROR(scanner->prepare(_state, _conjuncts)); scanners->push_back(scanner); } diff --git a/be/src/vec/exec/scan/vmeta_scan_node.h b/be/src/vec/exec/scan/vmeta_scan_node.h index b432d74760b805a..caad8b1b7f94f1c 100644 --- a/be/src/vec/exec/scan/vmeta_scan_node.h +++ b/be/src/vec/exec/scan/vmeta_scan_node.h @@ -19,6 +19,7 @@ #include #include +#include #include #include @@ -55,6 +56,7 @@ class VMetaScanNode : public VScanNode { Status _process_conjuncts() override; TupleId _tuple_id; + TUserIdentity _user_identity; TMetaScanNode _scan_params; std::vector _scan_ranges; }; diff --git a/be/src/vec/exec/scan/vmeta_scanner.cpp b/be/src/vec/exec/scan/vmeta_scanner.cpp index 030f710eba54eaf..eb1bc857a25363f 100644 --- a/be/src/vec/exec/scan/vmeta_scanner.cpp +++ b/be/src/vec/exec/scan/vmeta_scanner.cpp @@ -23,7 +23,6 @@ #include #include #include -#include #include #include @@ -58,10 +57,11 @@ namespace doris::vectorized { VMetaScanner::VMetaScanner(RuntimeState* state, VMetaScanNode* parent, int64_t tuple_id, const TScanRangeParams& scan_range, int64_t limit, - RuntimeProfile* profile) + RuntimeProfile* profile, TUserIdentity user_identity) : VScanner(state, static_cast(parent), limit, profile), _meta_eos(false), _tuple_id(tuple_id), + _user_identity(user_identity), _scan_range(scan_range.scan_range) {} Status VMetaScanner::open(RuntimeState* state) { @@ -317,6 +317,7 @@ Status VMetaScanner::_build_workload_groups_metadata_request( // create TMetadataTableRequestParams TMetadataTableRequestParams metadata_table_params; metadata_table_params.__set_metadata_type(TMetadataType::WORKLOAD_GROUPS); + metadata_table_params.__set_current_user_ident(_user_identity); request->__set_metada_table_params(metadata_table_params); return Status::OK(); diff --git a/be/src/vec/exec/scan/vmeta_scanner.h b/be/src/vec/exec/scan/vmeta_scanner.h index ae3505d19f34d03..22f3dfe6817710e 100644 --- a/be/src/vec/exec/scan/vmeta_scanner.h +++ b/be/src/vec/exec/scan/vmeta_scanner.h @@ -18,6 +18,7 @@ #pragma once #include +#include #include #include @@ -51,7 +52,8 @@ class VMetaScanner : public VScanner { public: VMetaScanner(RuntimeState* state, VMetaScanNode* parent, int64_t tuple_id, - const TScanRangeParams& scan_range, int64_t limit, RuntimeProfile* profile); + const TScanRangeParams& scan_range, int64_t limit, RuntimeProfile* profile, + TUserIdentity user_identity); Status open(RuntimeState* state) override; Status close(RuntimeState* state) override; @@ -74,6 +76,7 @@ class VMetaScanner : public VScanner { bool _meta_eos; TupleId _tuple_id; + TUserIdentity _user_identity; const TupleDescriptor* _tuple_desc; std::vector _batch_data; const TScanRange& _scan_range; diff --git a/be/src/vec/exprs/vectorized_agg_fn.cpp b/be/src/vec/exprs/vectorized_agg_fn.cpp index a6895ff7281a42c..2dcf12207a8b141 100644 --- a/be/src/vec/exprs/vectorized_agg_fn.cpp +++ b/be/src/vec/exprs/vectorized_agg_fn.cpp @@ -168,7 +168,12 @@ Status AggFnEvaluator::prepare(RuntimeState* state, const RowDescriptor& desc, "Agg state function input type must be agg_state but get {}", argument_types[0]->get_family_name()); } - if (match_suffix(_fn.name.function_name, AGG_UNION_SUFFIX)) { + + std::string type_function_name = + assert_cast(argument_types[0].get()) + ->get_nested_function() + ->get_name(); + if (type_function_name + AGG_UNION_SUFFIX == _fn.name.function_name) { if (_data_type->is_nullable()) { return Status::InternalError( "Union function return type must be not nullable, real={}", @@ -180,11 +185,19 @@ Status AggFnEvaluator::prepare(RuntimeState* state, const RowDescriptor& desc, _data_type->get_name()); } _function = get_agg_state_function(argument_types, _data_type); - } else if (match_suffix(_fn.name.function_name, AGG_MERGE_SUFFIX)) { + } else if (type_function_name + AGG_MERGE_SUFFIX == _fn.name.function_name) { + auto type = assert_cast(argument_types[0].get()) + ->get_nested_function() + ->get_return_type(); + if (!type->equals(*_data_type)) { + return Status::InternalError("{}'s expect return type is {}, but input {}", + argument_types[0]->get_name(), type->get_name(), + _data_type->get_name()); + } _function = get_agg_state_function(argument_types, _data_type); } else { - return Status::InternalError( - "Aggregate Function {} is not endwith '_merge' or '_union'", _fn.signature); + return Status::InternalError("{} not match function {}", argument_types[0]->get_name(), + _fn.name.function_name); } } else { _function = AggregateFunctionSimpleFactory::instance().get( diff --git a/be/src/vec/functions/array/function_array_enumerate_uniq.cpp b/be/src/vec/functions/array/function_array_enumerate_uniq.cpp index d40185f90300d16..e3b8aa4bf6a963a 100644 --- a/be/src/vec/functions/array/function_array_enumerate_uniq.cpp +++ b/be/src/vec/functions/array/function_array_enumerate_uniq.cpp @@ -105,6 +105,16 @@ class FunctionArrayEnumerateUniq : public IFunction { return return_type; } +// When compiling `FunctionArrayEnumerateUniq::_execute_by_hash`, `AllocatorWithStackMemory::free(buf)` +// will be called when `delete HashMapContainer`. the gcc compiler will think that `size > N` and `buf` is not heap memory, +// and report an error `' void free(void*)' called on unallocated object 'hash_map'` +// This only fails on doris docker + gcc 11.1, no problem on doris docker + clang 16.0.1, +// no problem on ldb_toolchanin gcc 11.1 and clang 16.0.1. +#ifdef __GNUC__ +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wfree-nonheap-object" +#endif // __GNUC__ + Status execute_impl(FunctionContext* context, Block& block, const ColumnNumbers& arguments, size_t result, size_t input_rows_count) override { ColumnRawPtrs data_columns(arguments.size()); @@ -285,6 +295,10 @@ class FunctionArrayEnumerateUniq : public IFunction { } }; +#ifdef __GNUC__ +#pragma GCC diagnostic pop +#endif // __GNUC__ + void register_function_array_enumerate_uniq(SimpleFunctionFactory& factory) { factory.register_function(); } diff --git a/be/src/vec/functions/function_java_udf.cpp b/be/src/vec/functions/function_java_udf.cpp index fe96c92051d8383..9305bae94960d9d 100644 --- a/be/src/vec/functions/function_java_udf.cpp +++ b/be/src/vec/functions/function_java_udf.cpp @@ -54,16 +54,24 @@ Status JavaFunctionCall::open(FunctionContext* context, FunctionContext::Functio if (env == nullptr) { return Status::InternalError("Failed to get/create JVM"); } - RETURN_IF_ERROR(JniUtil::GetGlobalClassRef(env, EXECUTOR_CLASS, &executor_cl_)); - executor_ctor_id_ = env->GetMethodID(executor_cl_, "", EXECUTOR_CTOR_SIGNATURE); - RETURN_ERROR_IF_EXC(env); - executor_evaluate_id_ = env->GetMethodID(executor_cl_, "evaluate", EXECUTOR_EVALUATE_SIGNATURE); - RETURN_ERROR_IF_EXC(env); - executor_close_id_ = env->GetMethodID(executor_cl_, "close", EXECUTOR_CLOSE_SIGNATURE); - RETURN_ERROR_IF_EXC(env); - - std::shared_ptr jni_ctx = - std::make_shared(_argument_types.size(), this); + if (scope == FunctionContext::FunctionStateScope::FRAGMENT_LOCAL) { + std::shared_ptr jni_env = std::make_shared(); + RETURN_IF_ERROR(JniUtil::GetGlobalClassRef(env, EXECUTOR_CLASS, &jni_env->executor_cl)); + jni_env->executor_ctor_id = + env->GetMethodID(jni_env->executor_cl, "", EXECUTOR_CTOR_SIGNATURE); + RETURN_ERROR_IF_EXC(env); + jni_env->executor_evaluate_id = + env->GetMethodID(jni_env->executor_cl, "evaluate", EXECUTOR_EVALUATE_SIGNATURE); + RETURN_ERROR_IF_EXC(env); + jni_env->executor_close_id = + env->GetMethodID(jni_env->executor_cl, "close", EXECUTOR_CLOSE_SIGNATURE); + RETURN_ERROR_IF_EXC(env); + context->set_function_state(FunctionContext::FRAGMENT_LOCAL, jni_env); + } + JniEnv* jni_env = + reinterpret_cast(context->get_function_state(FunctionContext::FRAGMENT_LOCAL)); + std::shared_ptr jni_ctx = std::make_shared( + _argument_types.size(), jni_env->executor_cl, jni_env->executor_close_id); context->set_function_state(FunctionContext::THREAD_LOCAL, jni_ctx); // Add a scoped cleanup jni reference object. This cleans up local refs made below. @@ -99,7 +107,9 @@ Status JavaFunctionCall::open(FunctionContext* context, FunctionContext::Functio RETURN_IF_ERROR(jni_frame.push(env)); RETURN_IF_ERROR(SerializeThriftMsg(env, &ctor_params, &ctor_params_bytes)); - jni_ctx->executor = env->NewObject(executor_cl_, executor_ctor_id_, ctor_params_bytes); + + jni_ctx->executor = + env->NewObject(jni_env->executor_cl, jni_env->executor_ctor_id, ctor_params_bytes); jbyte* pBytes = env->GetByteArrayElements(ctor_params_bytes, nullptr); env->ReleaseByteArrayElements(ctor_params_bytes, pBytes, JNI_ABORT); @@ -118,6 +128,8 @@ Status JavaFunctionCall::execute(FunctionContext* context, Block& block, RETURN_IF_ERROR(JniUtil::GetJNIEnv(&env)); JniContext* jni_ctx = reinterpret_cast( context->get_function_state(FunctionContext::THREAD_LOCAL)); + JniEnv* jni_env = + reinterpret_cast(context->get_function_state(FunctionContext::FRAGMENT_LOCAL)); int arg_idx = 0; ColumnPtr data_cols[arguments.size()]; ColumnPtr null_cols[arguments.size()]; @@ -192,105 +204,105 @@ Status JavaFunctionCall::execute(FunctionContext* context, Block& block, *(jni_ctx->output_null_value) = reinterpret_cast(null_col->get_data().data()); #ifndef EVALUATE_JAVA_UDF -#define EVALUATE_JAVA_UDF \ - if (data_col->is_column_string()) { \ - const ColumnString* str_col = assert_cast(data_col.get()); \ - ColumnString::Chars& chars = const_cast(str_col->get_chars()); \ - ColumnString::Offsets& offsets = \ - const_cast(str_col->get_offsets()); \ - int increase_buffer_size = 0; \ - int64_t buffer_size = JniUtil::IncreaseReservedBufferSize(increase_buffer_size); \ - chars.resize(buffer_size); \ - offsets.resize(num_rows); \ - *(jni_ctx->output_value_buffer) = reinterpret_cast(chars.data()); \ - *(jni_ctx->output_offsets_ptr) = reinterpret_cast(offsets.data()); \ - jni_ctx->output_intermediate_state_ptr->row_idx = 0; \ - jni_ctx->output_intermediate_state_ptr->buffer_size = buffer_size; \ - env->CallNonvirtualVoidMethodA(jni_ctx->executor, executor_cl_, executor_evaluate_id_, \ - nullptr); \ - while (jni_ctx->output_intermediate_state_ptr->row_idx < num_rows) { \ - increase_buffer_size++; \ - buffer_size = JniUtil::IncreaseReservedBufferSize(increase_buffer_size); \ - chars.resize(buffer_size); \ - *(jni_ctx->output_value_buffer) = reinterpret_cast(chars.data()); \ - jni_ctx->output_intermediate_state_ptr->buffer_size = buffer_size; \ - env->CallNonvirtualVoidMethodA(jni_ctx->executor, executor_cl_, executor_evaluate_id_, \ - nullptr); \ - } \ - } else if (data_col->is_numeric() || data_col->is_column_decimal()) { \ - data_col->resize(num_rows); \ - *(jni_ctx->output_value_buffer) = \ - reinterpret_cast(data_col->get_raw_data().data); \ - env->CallNonvirtualVoidMethodA(jni_ctx->executor, executor_cl_, executor_evaluate_id_, \ - nullptr); \ - } else if (data_col->is_column_array()) { \ - ColumnArray* array_col = assert_cast(data_col.get()); \ - ColumnNullable& array_nested_nullable = \ - assert_cast(array_col->get_data()); \ - auto data_column_null_map = array_nested_nullable.get_null_map_column_ptr(); \ - auto data_column = array_nested_nullable.get_nested_column_ptr(); \ - auto& offset_column = array_col->get_offsets_column(); \ - int increase_buffer_size = 0; \ - int64_t buffer_size = JniUtil::IncreaseReservedBufferSize(increase_buffer_size); \ - offset_column.resize(num_rows); \ - *(jni_ctx->output_offsets_ptr) = \ - reinterpret_cast(offset_column.get_raw_data().data); \ - data_column_null_map->resize(buffer_size); \ - auto& null_map_data = \ - assert_cast*>(data_column_null_map.get())->get_data(); \ - *(jni_ctx->output_array_null_ptr) = reinterpret_cast(null_map_data.data()); \ - jni_ctx->output_intermediate_state_ptr->row_idx = 0; \ - jni_ctx->output_intermediate_state_ptr->buffer_size = buffer_size; \ - if (data_column->is_column_string()) { \ - ColumnString* str_col = assert_cast(data_column.get()); \ - ColumnString::Chars& chars = assert_cast(str_col->get_chars()); \ - ColumnString::Offsets& offsets = \ - assert_cast(str_col->get_offsets()); \ - chars.resize(buffer_size); \ - offsets.resize(buffer_size); \ - *(jni_ctx->output_value_buffer) = reinterpret_cast(chars.data()); \ - *(jni_ctx->output_array_string_offsets_ptr) = \ - reinterpret_cast(offsets.data()); \ - env->CallNonvirtualVoidMethodA(jni_ctx->executor, executor_cl_, executor_evaluate_id_, \ - nullptr); \ - while (jni_ctx->output_intermediate_state_ptr->row_idx < num_rows) { \ - increase_buffer_size++; \ - buffer_size = JniUtil::IncreaseReservedBufferSize(increase_buffer_size); \ - null_map_data.resize(buffer_size); \ - chars.resize(buffer_size); \ - offsets.resize(buffer_size); \ - *(jni_ctx->output_array_null_ptr) = \ - reinterpret_cast(null_map_data.data()); \ - *(jni_ctx->output_value_buffer) = reinterpret_cast(chars.data()); \ - *(jni_ctx->output_array_string_offsets_ptr) = \ - reinterpret_cast(offsets.data()); \ - jni_ctx->output_intermediate_state_ptr->buffer_size = buffer_size; \ - env->CallNonvirtualVoidMethodA(jni_ctx->executor, executor_cl_, \ - executor_evaluate_id_, nullptr); \ - } \ - } else { \ - data_column->resize(buffer_size); \ - *(jni_ctx->output_value_buffer) = \ - reinterpret_cast(data_column->get_raw_data().data); \ - env->CallNonvirtualVoidMethodA(jni_ctx->executor, executor_cl_, executor_evaluate_id_, \ - nullptr); \ - while (jni_ctx->output_intermediate_state_ptr->row_idx < num_rows) { \ - increase_buffer_size++; \ - buffer_size = JniUtil::IncreaseReservedBufferSize(increase_buffer_size); \ - null_map_data.resize(buffer_size); \ - data_column->resize(buffer_size); \ - *(jni_ctx->output_array_null_ptr) = \ - reinterpret_cast(null_map_data.data()); \ - *(jni_ctx->output_value_buffer) = \ - reinterpret_cast(data_column->get_raw_data().data); \ - jni_ctx->output_intermediate_state_ptr->buffer_size = buffer_size; \ - env->CallNonvirtualVoidMethodA(jni_ctx->executor, executor_cl_, \ - executor_evaluate_id_, nullptr); \ - } \ - } \ - } else { \ - return Status::InvalidArgument(strings::Substitute( \ - "Java UDF doesn't support return type $0 now !", return_type->get_name())); \ +#define EVALUATE_JAVA_UDF \ + if (data_col->is_column_string()) { \ + const ColumnString* str_col = assert_cast(data_col.get()); \ + ColumnString::Chars& chars = const_cast(str_col->get_chars()); \ + ColumnString::Offsets& offsets = \ + const_cast(str_col->get_offsets()); \ + int increase_buffer_size = 0; \ + int64_t buffer_size = JniUtil::IncreaseReservedBufferSize(increase_buffer_size); \ + chars.resize(buffer_size); \ + offsets.resize(num_rows); \ + *(jni_ctx->output_value_buffer) = reinterpret_cast(chars.data()); \ + *(jni_ctx->output_offsets_ptr) = reinterpret_cast(offsets.data()); \ + jni_ctx->output_intermediate_state_ptr->row_idx = 0; \ + jni_ctx->output_intermediate_state_ptr->buffer_size = buffer_size; \ + env->CallNonvirtualVoidMethodA(jni_ctx->executor, jni_env->executor_cl, \ + jni_env->executor_evaluate_id, nullptr); \ + while (jni_ctx->output_intermediate_state_ptr->row_idx < num_rows) { \ + increase_buffer_size++; \ + buffer_size = JniUtil::IncreaseReservedBufferSize(increase_buffer_size); \ + chars.resize(buffer_size); \ + *(jni_ctx->output_value_buffer) = reinterpret_cast(chars.data()); \ + jni_ctx->output_intermediate_state_ptr->buffer_size = buffer_size; \ + env->CallNonvirtualVoidMethodA(jni_ctx->executor, jni_env->executor_cl, \ + jni_env->executor_evaluate_id, nullptr); \ + } \ + } else if (data_col->is_numeric() || data_col->is_column_decimal()) { \ + data_col->resize(num_rows); \ + *(jni_ctx->output_value_buffer) = \ + reinterpret_cast(data_col->get_raw_data().data); \ + env->CallNonvirtualVoidMethodA(jni_ctx->executor, jni_env->executor_cl, \ + jni_env->executor_evaluate_id, nullptr); \ + } else if (data_col->is_column_array()) { \ + ColumnArray* array_col = assert_cast(data_col.get()); \ + ColumnNullable& array_nested_nullable = \ + assert_cast(array_col->get_data()); \ + auto data_column_null_map = array_nested_nullable.get_null_map_column_ptr(); \ + auto data_column = array_nested_nullable.get_nested_column_ptr(); \ + auto& offset_column = array_col->get_offsets_column(); \ + int increase_buffer_size = 0; \ + int64_t buffer_size = JniUtil::IncreaseReservedBufferSize(increase_buffer_size); \ + offset_column.resize(num_rows); \ + *(jni_ctx->output_offsets_ptr) = \ + reinterpret_cast(offset_column.get_raw_data().data); \ + data_column_null_map->resize(buffer_size); \ + auto& null_map_data = \ + assert_cast*>(data_column_null_map.get())->get_data(); \ + *(jni_ctx->output_array_null_ptr) = reinterpret_cast(null_map_data.data()); \ + jni_ctx->output_intermediate_state_ptr->row_idx = 0; \ + jni_ctx->output_intermediate_state_ptr->buffer_size = buffer_size; \ + if (data_column->is_column_string()) { \ + ColumnString* str_col = assert_cast(data_column.get()); \ + ColumnString::Chars& chars = assert_cast(str_col->get_chars()); \ + ColumnString::Offsets& offsets = \ + assert_cast(str_col->get_offsets()); \ + chars.resize(buffer_size); \ + offsets.resize(buffer_size); \ + *(jni_ctx->output_value_buffer) = reinterpret_cast(chars.data()); \ + *(jni_ctx->output_array_string_offsets_ptr) = \ + reinterpret_cast(offsets.data()); \ + env->CallNonvirtualVoidMethodA(jni_ctx->executor, jni_env->executor_cl, \ + jni_env->executor_evaluate_id, nullptr); \ + while (jni_ctx->output_intermediate_state_ptr->row_idx < num_rows) { \ + increase_buffer_size++; \ + buffer_size = JniUtil::IncreaseReservedBufferSize(increase_buffer_size); \ + null_map_data.resize(buffer_size); \ + chars.resize(buffer_size); \ + offsets.resize(buffer_size); \ + *(jni_ctx->output_array_null_ptr) = \ + reinterpret_cast(null_map_data.data()); \ + *(jni_ctx->output_value_buffer) = reinterpret_cast(chars.data()); \ + *(jni_ctx->output_array_string_offsets_ptr) = \ + reinterpret_cast(offsets.data()); \ + jni_ctx->output_intermediate_state_ptr->buffer_size = buffer_size; \ + env->CallNonvirtualVoidMethodA(jni_ctx->executor, jni_env->executor_cl, \ + jni_env->executor_evaluate_id, nullptr); \ + } \ + } else { \ + data_column->resize(buffer_size); \ + *(jni_ctx->output_value_buffer) = \ + reinterpret_cast(data_column->get_raw_data().data); \ + env->CallNonvirtualVoidMethodA(jni_ctx->executor, jni_env->executor_cl, \ + jni_env->executor_evaluate_id, nullptr); \ + while (jni_ctx->output_intermediate_state_ptr->row_idx < num_rows) { \ + increase_buffer_size++; \ + buffer_size = JniUtil::IncreaseReservedBufferSize(increase_buffer_size); \ + null_map_data.resize(buffer_size); \ + data_column->resize(buffer_size); \ + *(jni_ctx->output_array_null_ptr) = \ + reinterpret_cast(null_map_data.data()); \ + *(jni_ctx->output_value_buffer) = \ + reinterpret_cast(data_column->get_raw_data().data); \ + jni_ctx->output_intermediate_state_ptr->buffer_size = buffer_size; \ + env->CallNonvirtualVoidMethodA(jni_ctx->executor, jni_env->executor_cl, \ + jni_env->executor_evaluate_id, nullptr); \ + } \ + } \ + } else { \ + return Status::InvalidArgument(strings::Substitute( \ + "Java UDF doesn't support return type $0 now !", return_type->get_name())); \ } #endif EVALUATE_JAVA_UDF; diff --git a/be/src/vec/functions/function_java_udf.h b/be/src/vec/functions/function_java_udf.h index 1f47394c69573f8..605d7c01984ff53 100644 --- a/be/src/vec/functions/function_java_udf.h +++ b/be/src/vec/functions/function_java_udf.h @@ -23,6 +23,7 @@ #include #include +#include #include #include "common/logging.h" @@ -83,13 +84,6 @@ class JavaFunctionCall : public IFunctionBase { const DataTypes _argument_types; const DataTypePtr _return_type; - /// Global class reference to the UdfExecutor Java class and related method IDs. Set in - /// Init(). These have the lifetime of the process (i.e. 'executor_cl_' is never freed). - jclass executor_cl_; - jmethodID executor_ctor_id_; - jmethodID executor_evaluate_id_; - jmethodID executor_close_id_; - struct IntermediateState { size_t buffer_size; size_t row_idx; @@ -97,6 +91,15 @@ class JavaFunctionCall : public IFunctionBase { IntermediateState() : buffer_size(0), row_idx(0) {} }; + struct JniEnv { + /// Global class reference to the UdfExecutor Java class and related method IDs. Set in + /// Init(). These have the lifetime of the process (i.e. 'executor_cl_' is never freed). + jclass executor_cl; + jmethodID executor_ctor_id; + jmethodID executor_evaluate_id; + jmethodID executor_close_id; + }; + struct JniContext { // Do not save parent directly, because parent is in VExpr, but jni context is in FunctionContext // The deconstruct sequence is not determined, it will core. @@ -124,9 +127,9 @@ class JavaFunctionCall : public IFunctionBase { // intermediate_state includes two parts: reserved / used buffer size and rows std::unique_ptr output_intermediate_state_ptr; - JniContext(int64_t num_args, JavaFunctionCall* parent) - : executor_cl_(parent->executor_cl_), - executor_close_id_(parent->executor_close_id_), + JniContext(int64_t num_args, jclass executor_cl, jmethodID executor_close_id) + : executor_cl_(executor_cl), + executor_close_id_(executor_close_id), input_values_buffer_ptr(new int64_t[num_args]), input_nulls_buffer_ptr(new int64_t[num_args]), input_offsets_ptrs(new int64_t[num_args]), diff --git a/be/src/vec/functions/function_jsonb.cpp b/be/src/vec/functions/function_jsonb.cpp index fbe5f113133990e..42bdbe4b8564678 100644 --- a/be/src/vec/functions/function_jsonb.cpp +++ b/be/src/vec/functions/function_jsonb.cpp @@ -78,8 +78,10 @@ enum class JsonbParseErrorMode { FAIL = 0, RETURN_NULL, RETURN_VALUE, RETURN_INV template class FunctionJsonbParseBase : public IFunction { private: - JsonbParser default_value_parser; - bool has_const_default_value = false; + struct FunctionJsonbParseState { + JsonbParser default_value_parser; + bool has_const_default_value = false; + }; public: static constexpr auto name = "json_parse"; @@ -152,20 +154,31 @@ class FunctionJsonbParseBase : public IFunction { bool use_default_implementation_for_nulls() const override { return false; } Status open(FunctionContext* context, FunctionContext::FunctionStateScope scope) override { + if (scope == FunctionContext::FunctionStateScope::FRAGMENT_LOCAL) { + std::shared_ptr state = + std::make_shared(); + context->set_function_state(FunctionContext::FRAGMENT_LOCAL, state); + } if constexpr (parse_error_handle_mode == JsonbParseErrorMode::RETURN_VALUE) { if (context->is_col_constant(1)) { const auto default_value_col = context->get_constant_col(1)->column_ptr; const auto& default_value = default_value_col->get_data_at(0); JsonbErrType error = JsonbErrType::E_NONE; - if (!default_value_parser.parse(default_value.data, default_value.size)) { - error = default_value_parser.getErrorCode(); - return Status::InvalidArgument( - "invalid default json value: {} , error: {}", - std::string_view(default_value.data, default_value.size), - JsonbErrMsg::getErrMsg(error)); + if (scope == FunctionContext::FunctionStateScope::FRAGMENT_LOCAL) { + FunctionJsonbParseState* state = reinterpret_cast( + context->get_function_state(FunctionContext::FRAGMENT_LOCAL)); + + if (!state->default_value_parser.parse(default_value.data, + default_value.size)) { + error = state->default_value_parser.getErrorCode(); + return Status::InvalidArgument( + "invalid default json value: {} , error: {}", + std::string_view(default_value.data, default_value.size), + JsonbErrMsg::getErrMsg(error)); + } + state->has_const_default_value = true; } - has_const_default_value = true; } } return Status::OK(); @@ -257,10 +270,14 @@ class FunctionJsonbParseBase : public IFunction { continue; } case JsonbParseErrorMode::RETURN_VALUE: { - if (has_const_default_value) { + FunctionJsonbParseState* state = reinterpret_cast( + context->get_function_state(FunctionContext::FRAGMENT_LOCAL)); + if (state->has_const_default_value) { col_to->insert_data( - default_value_parser.getWriter().getOutput()->getBuffer(), - (size_t)default_value_parser.getWriter().getOutput()->getSize()); + state->default_value_parser.getWriter().getOutput()->getBuffer(), + (size_t)state->default_value_parser.getWriter() + .getOutput() + ->getSize()); } else { auto val = block.get_by_position(arguments[1]).column->get_data_at(i); if (parser.parse(val.data, val.size)) { @@ -358,6 +375,8 @@ class FunctionJsonbExtract : public IFunction { auto& rdata = jsonb_path_column->get_chars(); auto& roffsets = jsonb_path_column->get_offsets(); + bool is_invalid_json_path = false; + // execute Impl if constexpr (std::is_same_v || std::is_same_v) { @@ -365,26 +384,36 @@ class FunctionJsonbExtract : public IFunction { auto& res_offsets = res->get_offsets(); if (col_const[0]) { Impl::scalar_vector(context, jsonb_data_column->get_data_at(0), rdata, roffsets, - res_data, res_offsets, null_map->get_data()); + res_data, res_offsets, null_map->get_data(), + is_invalid_json_path); } else if (col_const[1]) { Impl::vector_scalar(context, ldata, loffsets, jsonb_path_column->get_data_at(0), - res_data, res_offsets, null_map->get_data()); + res_data, res_offsets, null_map->get_data(), + is_invalid_json_path); } else { Impl::vector_vector(context, ldata, loffsets, rdata, roffsets, res_data, - res_offsets, null_map->get_data()); + res_offsets, null_map->get_data(), is_invalid_json_path); } } else { if (col_const[0]) { Impl::scalar_vector(context, jsonb_data_column->get_data_at(0), rdata, roffsets, - res->get_data(), null_map->get_data()); + res->get_data(), null_map->get_data(), is_invalid_json_path); } else if (col_const[1]) { Impl::vector_scalar(context, ldata, loffsets, jsonb_path_column->get_data_at(0), - res->get_data(), null_map->get_data()); + res->get_data(), null_map->get_data(), is_invalid_json_path); } else { Impl::vector_vector(context, ldata, loffsets, rdata, roffsets, res->get_data(), - null_map->get_data()); + null_map->get_data(), is_invalid_json_path); } } + + if (is_invalid_json_path) { + return Status::InvalidArgument( + "Json path error: {} for value: {}", + JsonbErrMsg::getErrMsg(JsonbErrType::E_INVALID_JSON_PATH), + std::string_view(reinterpret_cast(rdata.data()), rdata.size())); + } + block.get_by_position(result).column = ColumnNullable::create(std::move(res), std::move(null_map)); return Status::OK(); @@ -403,7 +432,7 @@ struct JsonbExtractStringImpl { const std::unique_ptr& writer, std::unique_ptr& formater, const char* l_raw, int l_size, const char* r_raw, - int r_size) { + int r_size, bool& is_invalid_json_path) { String path(r_raw, r_size); if (null_map[i]) { @@ -419,8 +448,9 @@ struct JsonbExtractStringImpl { } // value is NOT necessary to be deleted since JsonbValue will not allocate memory - JsonbValue* value = doc->getValue()->findPath(r_raw, r_size, nullptr); - if (UNLIKELY(!value)) { + JsonbValue* value = doc->getValue()->findPath(r_raw, r_size, is_invalid_json_path, nullptr); + + if (UNLIKELY(!value) || is_invalid_json_path) { StringOP::push_null_string(i, res_data, res_offsets, null_map); return; } @@ -477,7 +507,8 @@ struct JsonbExtractStringImpl { const ColumnString::Offsets& loffsets, const ColumnString::Chars& rdata, const ColumnString::Offsets& roffsets, ColumnString::Chars& res_data, - ColumnString::Offsets& res_offsets, NullMap& null_map) { + ColumnString::Offsets& res_offsets, NullMap& null_map, + bool& is_invalid_json_path) { size_t input_rows_count = loffsets.size(); res_offsets.resize(input_rows_count); @@ -496,13 +527,13 @@ struct JsonbExtractStringImpl { const char* r_raw = reinterpret_cast(&rdata[roffsets[i - 1]]); inner_loop_impl(i, res_data, res_offsets, null_map, writer, formater, l_raw, l_size, - r_raw, r_size); + r_raw, r_size, is_invalid_json_path); } //for } //function static void vector_scalar(FunctionContext* context, const ColumnString::Chars& ldata, const ColumnString::Offsets& loffsets, const StringRef& rdata, ColumnString::Chars& res_data, ColumnString::Offsets& res_offsets, - NullMap& null_map) { + NullMap& null_map, bool& is_invalid_json_path) { size_t input_rows_count = loffsets.size(); res_offsets.resize(input_rows_count); @@ -518,13 +549,14 @@ struct JsonbExtractStringImpl { const char* l_raw = reinterpret_cast(&ldata[loffsets[i - 1]]); inner_loop_impl(i, res_data, res_offsets, null_map, writer, formater, l_raw, l_size, - rdata.data, rdata.size); + rdata.data, rdata.size, is_invalid_json_path); } //for } //function static void scalar_vector(FunctionContext* context, const StringRef& ldata, const ColumnString::Chars& rdata, const ColumnString::Offsets& roffsets, ColumnString::Chars& res_data, - ColumnString::Offsets& res_offsets, NullMap& null_map) { + ColumnString::Offsets& res_offsets, NullMap& null_map, + bool& is_invalid_json_path) { size_t input_rows_count = roffsets.size(); res_offsets.resize(input_rows_count); @@ -540,7 +572,7 @@ struct JsonbExtractStringImpl { const char* r_raw = reinterpret_cast(&rdata[roffsets[i - 1]]); inner_loop_impl(i, res_data, res_offsets, null_map, writer, formater, ldata.data, - ldata.size, r_raw, r_size); + ldata.size, r_raw, r_size, is_invalid_json_path); } //for } //function }; @@ -555,7 +587,8 @@ struct JsonbExtractImpl { private: static ALWAYS_INLINE void inner_loop_impl(size_t i, Container& res, NullMap& null_map, const char* l_raw_str, int l_str_size, - const char* r_raw_str, int r_str_size) { + const char* r_raw_str, int r_str_size, + bool& is_invalid_json_path) { if (null_map[i]) { res[i] = 0; return; @@ -570,9 +603,10 @@ struct JsonbExtractImpl { } // value is NOT necessary to be deleted since JsonbValue will not allocate memory - JsonbValue* value = doc->getValue()->findPath(r_raw_str, r_str_size, nullptr); + JsonbValue* value = + doc->getValue()->findPath(r_raw_str, r_str_size, is_invalid_json_path, nullptr); - if (UNLIKELY(!value)) { + if (UNLIKELY(!value) || is_invalid_json_path) { if constexpr (!only_check_exists) { null_map[i] = 1; } @@ -636,7 +670,7 @@ struct JsonbExtractImpl { const ColumnString::Offsets& loffsets, const ColumnString::Chars& rdata, const ColumnString::Offsets& roffsets, Container& res, - NullMap& null_map) { + NullMap& null_map, bool& is_invalid_json_path) { size_t size = loffsets.size(); res.resize(size); @@ -651,13 +685,14 @@ struct JsonbExtractImpl { const char* r_raw_str = reinterpret_cast(&rdata[roffsets[i - 1]]); int r_str_size = roffsets[i] - roffsets[i - 1]; - inner_loop_impl(i, res, null_map, l_raw_str, l_str_size, r_raw_str, r_str_size); + inner_loop_impl(i, res, null_map, l_raw_str, l_str_size, r_raw_str, r_str_size, + is_invalid_json_path); } //for } //function static void scalar_vector(FunctionContext* context, const StringRef& ldata, const ColumnString::Chars& rdata, const ColumnString::Offsets& roffsets, Container& res, - NullMap& null_map) { + NullMap& null_map, bool& is_invalid_json_path) { size_t size = roffsets.size(); res.resize(size); @@ -669,12 +704,13 @@ struct JsonbExtractImpl { const char* r_raw_str = reinterpret_cast(&rdata[roffsets[i - 1]]); int r_str_size = roffsets[i] - roffsets[i - 1]; - inner_loop_impl(i, res, null_map, ldata.data, ldata.size, r_raw_str, r_str_size); + inner_loop_impl(i, res, null_map, ldata.data, ldata.size, r_raw_str, r_str_size, + is_invalid_json_path); } //for } //function static void vector_scalar(FunctionContext* context, const ColumnString::Chars& ldata, const ColumnString::Offsets& loffsets, const StringRef& rdata, - Container& res, NullMap& null_map) { + Container& res, NullMap& null_map, bool& is_invalid_json_path) { size_t size = loffsets.size(); res.resize(size); @@ -686,7 +722,8 @@ struct JsonbExtractImpl { const char* l_raw_str = reinterpret_cast(&ldata[loffsets[i - 1]]); int l_str_size = loffsets[i] - loffsets[i - 1]; - inner_loop_impl(i, res, null_map, l_raw_str, l_str_size, rdata.data, rdata.size); + inner_loop_impl(i, res, null_map, l_raw_str, l_str_size, rdata.data, rdata.size, + is_invalid_json_path); } //for } //function }; diff --git a/be/src/vec/functions/function_rpc.cpp b/be/src/vec/functions/function_rpc.cpp index dbd92eeec2f6daf..44c5e1b369048b0 100644 --- a/be/src/vec/functions/function_rpc.cpp +++ b/be/src/vec/functions/function_rpc.cpp @@ -90,16 +90,20 @@ FunctionRPC::FunctionRPC(const TFunction& fn, const DataTypes& argument_types, : _argument_types(argument_types), _return_type(return_type), _tfn(fn) {} Status FunctionRPC::open(FunctionContext* context, FunctionContext::FunctionStateScope scope) { - _fn = std::make_unique(_tfn); - - if (!_fn->available()) { - return Status::InternalError("rpc env init error"); + if (scope == FunctionContext::FRAGMENT_LOCAL) { + std::shared_ptr fn = std::make_shared(_tfn); + if (!fn->available()) { + return Status::InternalError("rpc env init error"); + } + context->set_function_state(FunctionContext::FRAGMENT_LOCAL, fn); } return Status::OK(); } Status FunctionRPC::execute(FunctionContext* context, Block& block, const ColumnNumbers& arguments, size_t result, size_t input_rows_count, bool dry_run) { - return _fn->vec_call(context, block, arguments, result, input_rows_count); + RPCFnImpl* fn = reinterpret_cast( + context->get_function_state(FunctionContext::FRAGMENT_LOCAL)); + return fn->vec_call(context, block, arguments, result, input_rows_count); } } // namespace doris::vectorized diff --git a/be/src/vec/functions/function_rpc.h b/be/src/vec/functions/function_rpc.h index 5623183470e95bf..d10b9be546bbd71 100644 --- a/be/src/vec/functions/function_rpc.h +++ b/be/src/vec/functions/function_rpc.h @@ -107,7 +107,6 @@ class FunctionRPC : public IFunctionBase { DataTypes _argument_types; DataTypePtr _return_type; TFunction _tfn; - std::unique_ptr _fn; }; } // namespace doris::vectorized diff --git a/be/src/vec/runtime/vdatetime_value.cpp b/be/src/vec/runtime/vdatetime_value.cpp index f0ef48f1f39a2e0..a8db4d6f66fd834 100644 --- a/be/src/vec/runtime/vdatetime_value.cpp +++ b/be/src/vec/runtime/vdatetime_value.cpp @@ -1965,6 +1965,9 @@ bool DateV2Value::from_date_str(const char* date_str, int len, int scale) { } if (num_field < 3) return false; + if (is_invalid(date_val[0], date_val[1], date_val[2], 0, 0, 0, 0)) { + return false; + } format_datetime(date_val, carry_bits); return check_range_and_set_time(date_val[0], date_val[1], date_val[2], date_val[3], date_val[4], date_val[5], date_val[6]); diff --git a/be/src/vec/sink/vdata_stream_sender.cpp b/be/src/vec/sink/vdata_stream_sender.cpp index df0bb396a3a52d6..b49c6ec92e108f9 100644 --- a/be/src/vec/sink/vdata_stream_sender.cpp +++ b/be/src/vec/sink/vdata_stream_sender.cpp @@ -40,6 +40,7 @@ #include "runtime/types.h" #include "util/proto_util.h" #include "util/telemetry/telemetry.h" +#include "vec/columns/column_const.h" #include "vec/common/sip_hash.h" #include "vec/exprs/vexpr.h" #include "vec/runtime/vdata_stream_mgr.h" @@ -629,7 +630,9 @@ Status VDataStreamSender::send(RuntimeState* state, Block* block, bool eos) { std::vector siphashs(rows); // result[j] means column index, i means rows index for (int j = 0; j < result_size; ++j) { - block->get_by_position(result[j]).column->update_hashes_with_value(siphashs); + // complex type most not implement get_data_at() method which column_const will call + unpack_if_const(block->get_by_position(result[j]).column) + .first->update_hashes_with_value(siphashs); } for (int i = 0; i < rows; i++) { hashes[i] = siphashs[i].get64() % element_size; @@ -638,7 +641,9 @@ Status VDataStreamSender::send(RuntimeState* state, Block* block, bool eos) { SCOPED_TIMER(_split_block_hash_compute_timer); // result[j] means column index, i means rows index, here to calculate the xxhash value for (int j = 0; j < result_size; ++j) { - block->get_by_position(result[j]).column->update_hashes_with_value(hashes); + // complex type most not implement get_data_at() method which column_const will call + unpack_if_const(block->get_by_position(result[j]).column) + .first->update_hashes_with_value(hashes); } for (int i = 0; i < rows; i++) { @@ -654,8 +659,10 @@ Status VDataStreamSender::send(RuntimeState* state, Block* block, bool eos) { RETURN_IF_ERROR(channel_add_rows(state, _channels, element_size, hashes, rows, block)); } else { for (int j = 0; j < result_size; ++j) { - block->get_by_position(result[j]).column->update_crcs_with_value( - hash_vals, _partition_expr_ctxs[j]->root()->type().type); + // complex type most not implement get_data_at() method which column_const will call + unpack_if_const(block->get_by_position(result[j]).column) + .first->update_crcs_with_value( + hash_vals, _partition_expr_ctxs[j]->root()->type().type); } element_size = _channel_shared_ptrs.size(); for (int i = 0; i < rows; i++) { diff --git a/be/test/olap/primary_key_index_test.cpp b/be/test/olap/primary_key_index_test.cpp index 81cea0530dd12cb..ac277fececb76f0 100644 --- a/be/test/olap/primary_key_index_test.cpp +++ b/be/test/olap/primary_key_index_test.cpp @@ -71,6 +71,7 @@ TEST_F(PrimaryKeyIndexTest, builder) { EXPECT_EQ("9998", builder.max_key().to_string()); segment_v2::PrimaryKeyIndexMetaPB index_meta; EXPECT_TRUE(builder.finalize(&index_meta)); + EXPECT_EQ(builder.disk_size(), file_writer->bytes_appended()); EXPECT_TRUE(file_writer->close().ok()); EXPECT_EQ(num_rows, builder.num_rows()); diff --git a/be/test/runtime/memory/allocator_test.cpp b/be/test/runtime/memory/allocator_test.cpp new file mode 100644 index 000000000000000..2e48dcb5ed05309 --- /dev/null +++ b/be/test/runtime/memory/allocator_test.cpp @@ -0,0 +1,62 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "vec/common/allocator.h" + +#include +#include + +#include + +#include "gtest/gtest_pred_impl.h" +#include "vec/common/allocator_fwd.h" + +namespace doris { + +template +void test_allocator(T allocator) { + auto ptr = allocator.alloc(4096); + EXPECT_NE(nullptr, ptr); + ptr = allocator.realloc(ptr, 4096, 4096 * 1024); + EXPECT_NE(nullptr, ptr); + allocator.free(ptr, 4096 * 1024); + + ptr = allocator.alloc(100); + EXPECT_NE(nullptr, ptr); + ptr = allocator.realloc(ptr, 100, 100 * 1024); + EXPECT_NE(nullptr, ptr); + allocator.free(ptr, 100 * 1024); +} + +void test_normal() { + { + test_allocator(Allocator()); + test_allocator(Allocator()); + test_allocator(Allocator()); + test_allocator(Allocator()); + test_allocator(Allocator()); + test_allocator(Allocator()); + test_allocator(Allocator()); + test_allocator(Allocator()); + } +} + +TEST(AllocatorTest, TestNormal) { + test_normal(); +} + +} // namespace doris diff --git a/be/test/vec/columns/column_hash_func_test.cpp b/be/test/vec/columns/column_hash_func_test.cpp new file mode 100644 index 000000000000000..0e409b46406399a --- /dev/null +++ b/be/test/vec/columns/column_hash_func_test.cpp @@ -0,0 +1,190 @@ + +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include +#include + +#include "gtest/gtest_pred_impl.h" +#include "vec/columns/column_const.h" +#include "vec/core/field.h" +#include "vec/data_types/data_type.h" +#include "vec/data_types/data_type_array.h" +#include "vec/data_types/data_type_date.h" +#include "vec/data_types/data_type_date_time.h" +#include "vec/data_types/data_type_decimal.h" +#include "vec/data_types/data_type_map.h" +#include "vec/data_types/data_type_nullable.h" +#include "vec/data_types/data_type_number.h" +#include "vec/data_types/data_type_string.h" +#include "vec/data_types/data_type_struct.h" + +namespace doris::vectorized { + +DataTypes create_scala_data_types() { + DataTypePtr dt = std::make_shared(std::make_shared()); + DataTypePtr d = std::make_shared(std::make_shared()); + DataTypePtr dc = std::make_shared(vectorized::create_decimal(10, 2, false)); + DataTypePtr dcv2 = std::make_shared( + std::make_shared>(27, 9)); + DataTypePtr n3 = std::make_shared(std::make_shared()); + DataTypePtr n1 = std::make_shared(std::make_shared()); + DataTypePtr s1 = std::make_shared(std::make_shared()); + + DataTypes dataTypes; + dataTypes.push_back(dt); + dataTypes.push_back(d); + dataTypes.push_back(dc); + dataTypes.push_back(dcv2); + dataTypes.push_back(n3); + dataTypes.push_back(n1); + dataTypes.push_back(s1); + + return dataTypes; +} + +TEST(HashFuncTest, ArrayTypeTest) { + DataTypes dataTypes = create_scala_data_types(); + + std::vector sip_hash_vals(1); + std::vector xx_hash_vals(1); + std::vector crc_hash_vals(1); + auto* __restrict sip_hashes = sip_hash_vals.data(); + auto* __restrict xx_hashes = xx_hash_vals.data(); + auto* __restrict crc_hashes = crc_hash_vals.data(); + + for (auto d : dataTypes) { + DataTypePtr a = std::make_shared(d); + ColumnPtr col_a = a->create_column_const_with_default_value(1); + // sipHash + std::vector siphashs(1); + col_a->update_hashes_with_value(siphashs); + EXPECT_NO_FATAL_FAILURE(col_a->update_hashes_with_value(siphashs)); + sip_hashes[0] = siphashs[0].get64(); + std::cout << sip_hashes[0] << std::endl; + // xxHash + EXPECT_NO_FATAL_FAILURE(col_a->update_hashes_with_value(xx_hashes)); + std::cout << xx_hashes[0] << std::endl; + // crcHash + EXPECT_NO_FATAL_FAILURE( + col_a->update_crcs_with_value(crc_hash_vals, PrimitiveType::TYPE_ARRAY)); + std::cout << crc_hashes[0] << std::endl; + } +} + +TEST(HashFuncTest, ArrayCornerCaseTest) { + DataTypes dataTypes = create_scala_data_types(); + + DataTypePtr d = std::make_shared(); + DataTypePtr a = std::make_shared(d); + MutableColumnPtr array_mutable_col = a->create_column(); + Array a1, a2; + a1.push_back(Int64(1)); + a1.push_back(Int64(2)); + a1.push_back(Int64(3)); + array_mutable_col->insert(a1); + array_mutable_col->insert(a1); + a2.push_back(Int64(11)); + a2.push_back(Int64(12)); + a2.push_back(Int64(13)); + array_mutable_col->insert(a2); + + EXPECT_EQ(array_mutable_col->size(), 3); + + std::vector sip_hash_vals(3); + std::vector xx_hash_vals(3); + std::vector crc_hash_vals(3); + auto* __restrict sip_hashes = sip_hash_vals.data(); + auto* __restrict xx_hashes = xx_hash_vals.data(); + auto* __restrict crc_hashes = crc_hash_vals.data(); + + // sipHash + std::vector siphashs(3); + array_mutable_col->update_hashes_with_value(siphashs); + EXPECT_NO_FATAL_FAILURE(array_mutable_col->update_hashes_with_value(siphashs)); + sip_hashes[0] = siphashs[0].get64(); + sip_hashes[1] = siphashs[1].get64(); + sip_hashes[2] = siphashs[2].get64(); + EXPECT_EQ(sip_hashes[0], sip_hash_vals[1]); + EXPECT_TRUE(sip_hash_vals[0] != sip_hash_vals[2]); + // xxHash + EXPECT_NO_FATAL_FAILURE(array_mutable_col->update_hashes_with_value(xx_hashes)); + EXPECT_EQ(xx_hashes[0], xx_hashes[1]); + EXPECT_TRUE(xx_hashes[0] != xx_hashes[2]); + // crcHash + EXPECT_NO_FATAL_FAILURE( + array_mutable_col->update_crcs_with_value(crc_hash_vals, PrimitiveType::TYPE_ARRAY)); + EXPECT_EQ(crc_hashes[0], crc_hashes[1]); + EXPECT_TRUE(xx_hashes[0] != xx_hashes[2]); +} + +TEST(HashFuncTest, MapTypeTest) { + DataTypes dataTypes = create_scala_data_types(); + + std::vector sip_hash_vals(1); + std::vector xx_hash_vals(1); + std::vector crc_hash_vals(1); + auto* __restrict sip_hashes = sip_hash_vals.data(); + auto* __restrict xx_hashes = xx_hash_vals.data(); + auto* __restrict crc_hashes = crc_hash_vals.data(); + // data_type_map + for (int i = 0; i < dataTypes.size() - 1; ++i) { + DataTypePtr a = std::make_shared(dataTypes[i], dataTypes[i + 1]); + ColumnPtr col_a = a->create_column_const_with_default_value(1); + // sipHash + std::vector siphashs(1); + EXPECT_NO_FATAL_FAILURE(unpack_if_const(col_a).first->update_hashes_with_value(siphashs)); + sip_hashes[0] = siphashs[0].get64(); + std::cout << sip_hashes[0] << std::endl; + // xxHash + EXPECT_NO_FATAL_FAILURE(unpack_if_const(col_a).first->update_hashes_with_value(xx_hashes)); + std::cout << xx_hashes[0] << std::endl; + // crcHash + EXPECT_NO_FATAL_FAILURE(unpack_if_const(col_a).first->update_crcs_with_value( + crc_hash_vals, PrimitiveType::TYPE_MAP)); + std::cout << crc_hashes[0] << std::endl; + } +} + +TEST(HashFuncTest, StructTypeTest) { + DataTypes dataTypes = create_scala_data_types(); + + std::vector sip_hash_vals(1); + std::vector xx_hash_vals(1); + std::vector crc_hash_vals(1); + auto* __restrict sip_hashes = sip_hash_vals.data(); + auto* __restrict xx_hashes = xx_hash_vals.data(); + auto* __restrict crc_hashes = crc_hash_vals.data(); + + // data_type_struct + DataTypePtr a = std::make_shared(dataTypes); + ColumnPtr col_a = a->create_column_const_with_default_value(1); + // sipHash + std::vector siphashs(1); + EXPECT_NO_FATAL_FAILURE(unpack_if_const(col_a).first->update_hashes_with_value(siphashs)); + sip_hashes[0] = siphashs[0].get64(); + std::cout << sip_hashes[0] << std::endl; + // xxHash + EXPECT_NO_FATAL_FAILURE(unpack_if_const(col_a).first->update_hashes_with_value(xx_hashes)); + std::cout << xx_hashes[0] << std::endl; + // crcHash + EXPECT_NO_FATAL_FAILURE(unpack_if_const(col_a).first->update_crcs_with_value( + crc_hash_vals, PrimitiveType::TYPE_STRUCT)); + std::cout << crc_hashes[0] << std::endl; +} + +} // namespace doris::vectorized diff --git a/be/test/vec/exprs/vexpr_test.cpp b/be/test/vec/exprs/vexpr_test.cpp index 51ae892af7ee2b2..6fedf5e8e331881 100644 --- a/be/test/vec/exprs/vexpr_test.cpp +++ b/be/test/vec/exprs/vexpr_test.cpp @@ -38,7 +38,6 @@ #include "runtime/descriptors.h" #include "runtime/jsonb_value.h" #include "runtime/large_int_value.h" -#include "runtime/memory/chunk_allocator.h" #include "runtime/runtime_state.h" #include "testutil/desc_tbl_builder.h" #include "vec/core/field.h" @@ -49,7 +48,6 @@ #include "vec/utils/util.hpp" TEST(TEST_VEXPR, ABSTEST) { - doris::ChunkAllocator::init_instance(4096); doris::ObjectPool object_pool; doris::DescriptorTblBuilder builder(&object_pool); builder.declare_tuple() << doris::TYPE_INT << doris::TYPE_DOUBLE; diff --git a/be/test/vec/function/function_jsonb_test.cpp b/be/test/vec/function/function_jsonb_test.cpp index fe2052e9f19a157..3efc33a9f938898 100644 --- a/be/test/vec/function/function_jsonb_test.cpp +++ b/be/test/vec/function/function_jsonb_test.cpp @@ -559,15 +559,16 @@ TEST(FunctionJsonbTEST, JsonbExtractTest) { {{STRING("null"), STRING("$[0]")}, Null()}, {{STRING("true"), STRING("$[0]")}, Null()}, {{STRING("false"), STRING("$[0]")}, Null()}, - {{STRING("100"), STRING("$[0]")}, Null()}, //int8 - {{STRING("10000"), STRING("$[0]")}, Null()}, // int16 - {{STRING("1000000000"), STRING("$[0]")}, Null()}, // int32 - {{STRING("1152921504606846976"), STRING("$[0]")}, Null()}, // int64 - {{STRING("6.18"), STRING("$[0]")}, Null()}, // double - {{STRING(R"("abcd")"), STRING("$[0]")}, Null()}, // string - {{STRING("{}"), STRING("$[0]")}, Null()}, // empty object - {{STRING(R"({"k1":"v31", "k2": 300})"), STRING("$[0]")}, Null()}, // object - {{STRING("[]"), STRING("$[0]")}, Null()}, // empty array + {{STRING("100"), STRING("$[0]")}, Null()}, //int8 + {{STRING("10000"), STRING("$[0]")}, Null()}, // int16 + {{STRING("1000000000"), STRING("$[0]")}, Null()}, // int32 + {{STRING("1152921504606846976"), STRING("$[0]")}, Null()}, // int64 + {{STRING("6.18"), STRING("$[0]")}, Null()}, // double + {{STRING(R"("abcd")"), STRING("$[0]")}, Null()}, // string + {{STRING("{}"), STRING("$[0]")}, STRING("{}")}, // empty object + {{STRING(R"({"k1":"v31", "k2": 300})"), STRING("$[0]")}, + STRING(R"({"k1":"v31","k2":300})")}, // object + {{STRING("[]"), STRING("$[0]")}, Null()}, // empty array {{STRING("null"), STRING("$[1]")}, Null()}, {{STRING("true"), STRING("$[1]")}, Null()}, {{STRING("false"), STRING("$[1]")}, Null()}, @@ -620,17 +621,18 @@ TEST(FunctionJsonbTEST, JsonbExtractTest) { {{STRING("null"), STRING("$[0].k1")}, Null()}, {{STRING("true"), STRING("$[0].k1")}, Null()}, {{STRING("false"), STRING("$[0].k1")}, Null()}, - {{STRING("100"), STRING("$[0].k1")}, Null()}, //int8 - {{STRING("10000"), STRING("$[0].k1")}, Null()}, // int16 - {{STRING("1000000000"), STRING("$[0].k1")}, Null()}, // int32 - {{STRING("1152921504606846976"), STRING("$[0].k1")}, Null()}, // int64 - {{STRING("6.18"), STRING("$[0].k1")}, Null()}, // double - {{STRING(R"("abcd")"), STRING("$[0].k1")}, Null()}, // string - {{STRING("{}"), STRING("$[0].k1")}, Null()}, // empty object - {{STRING(R"({"k1":"v31", "k2": 300})"), STRING("$[0].k1")}, Null()}, // object - {{STRING("[]"), STRING("$[0].k1")}, Null()}, // empty array - {{STRING("[123, 456]"), STRING("$[0].k1")}, Null()}, // int array - {{STRING(R"(["abc", "def"])"), STRING("$[0].k1")}, Null()}, // string array + {{STRING("100"), STRING("$[0].k1")}, Null()}, //int8 + {{STRING("10000"), STRING("$[0].k1")}, Null()}, // int16 + {{STRING("1000000000"), STRING("$[0].k1")}, Null()}, // int32 + {{STRING("1152921504606846976"), STRING("$[0].k1")}, Null()}, // int64 + {{STRING("6.18"), STRING("$[0].k1")}, Null()}, // double + {{STRING(R"("abcd")"), STRING("$[0].k1")}, Null()}, // string + {{STRING("{}"), STRING("$[0].k1")}, Null()}, // empty object + {{STRING(R"({"k1":"v31", "k2": 300})"), STRING("$[0].k1")}, + STRING(R"("v31")")}, // object + {{STRING("[]"), STRING("$[0].k1")}, Null()}, // empty array + {{STRING("[123, 456]"), STRING("$[0].k1")}, Null()}, // int array + {{STRING(R"(["abc", "def"])"), STRING("$[0].k1")}, Null()}, // string array {{STRING(R"([null, true, false, 100, 6.18, "abc"])"), STRING("$[0].k1")}, Null()}, // multi type array {{STRING(R"([{"k1":"v41", "k2": 400}, 1, "a", 3.14])"), STRING("$[0].k1")}, @@ -702,15 +704,16 @@ TEST(FunctionJsonbTEST, JsonbExtractStringTest) { {{STRING("null"), STRING("$[0]")}, Null()}, {{STRING("true"), STRING("$[0]")}, Null()}, {{STRING("false"), STRING("$[0]")}, Null()}, - {{STRING("100"), STRING("$[0]")}, Null()}, //int8 - {{STRING("10000"), STRING("$[0]")}, Null()}, // int16 - {{STRING("1000000000"), STRING("$[0]")}, Null()}, // int32 - {{STRING("1152921504606846976"), STRING("$[0]")}, Null()}, // int64 - {{STRING("6.18"), STRING("$[0]")}, Null()}, // double - {{STRING(R"("abcd")"), STRING("$[0]")}, Null()}, // string - {{STRING("{}"), STRING("$[0]")}, Null()}, // empty object - {{STRING(R"({"k1":"v31", "k2": 300})"), STRING("$[0]")}, Null()}, // object - {{STRING("[]"), STRING("$[0]")}, Null()}, // empty array + {{STRING("100"), STRING("$[0]")}, Null()}, //int8 + {{STRING("10000"), STRING("$[0]")}, Null()}, // int16 + {{STRING("1000000000"), STRING("$[0]")}, Null()}, // int32 + {{STRING("1152921504606846976"), STRING("$[0]")}, Null()}, // int64 + {{STRING("6.18"), STRING("$[0]")}, Null()}, // double + {{STRING(R"("abcd")"), STRING("$[0]")}, Null()}, // string + {{STRING("{}"), STRING("$[0]")}, STRING("{}")}, // empty object + {{STRING(R"({"k1":"v31", "k2": 300})"), STRING("$[0]")}, + STRING(R"({"k1":"v31","k2":300})")}, // object + {{STRING("[]"), STRING("$[0]")}, Null()}, // empty array {{STRING("null"), STRING("$[1]")}, Null()}, {{STRING("true"), STRING("$[1]")}, Null()}, {{STRING("false"), STRING("$[1]")}, Null()}, @@ -763,17 +766,17 @@ TEST(FunctionJsonbTEST, JsonbExtractStringTest) { {{STRING("null"), STRING("$[0].k1")}, Null()}, {{STRING("true"), STRING("$[0].k1")}, Null()}, {{STRING("false"), STRING("$[0].k1")}, Null()}, - {{STRING("100"), STRING("$[0].k1")}, Null()}, //int8 - {{STRING("10000"), STRING("$[0].k1")}, Null()}, // int16 - {{STRING("1000000000"), STRING("$[0].k1")}, Null()}, // int32 - {{STRING("1152921504606846976"), STRING("$[0].k1")}, Null()}, // int64 - {{STRING("6.18"), STRING("$[0].k1")}, Null()}, // double - {{STRING(R"("abcd")"), STRING("$[0].k1")}, Null()}, // string - {{STRING("{}"), STRING("$[0].k1")}, Null()}, // empty object - {{STRING(R"({"k1":"v31", "k2": 300})"), STRING("$[0].k1")}, Null()}, // object - {{STRING("[]"), STRING("$[0].k1")}, Null()}, // empty array - {{STRING("[123, 456]"), STRING("$[0].k1")}, Null()}, // int array - {{STRING(R"(["abc", "def"])"), STRING("$[0].k1")}, Null()}, // string array + {{STRING("100"), STRING("$[0].k1")}, Null()}, //int8 + {{STRING("10000"), STRING("$[0].k1")}, Null()}, // int16 + {{STRING("1000000000"), STRING("$[0].k1")}, Null()}, // int32 + {{STRING("1152921504606846976"), STRING("$[0].k1")}, Null()}, // int64 + {{STRING("6.18"), STRING("$[0].k1")}, Null()}, // double + {{STRING(R"("abcd")"), STRING("$[0].k1")}, Null()}, // string + {{STRING("{}"), STRING("$[0].k1")}, Null()}, // empty object + {{STRING(R"({"k1":"v31", "k2": 300})"), STRING("$[0].k1")}, STRING(R"(v31)")}, // object + {{STRING("[]"), STRING("$[0].k1")}, Null()}, // empty array + {{STRING("[123, 456]"), STRING("$[0].k1")}, Null()}, // int array + {{STRING(R"(["abc", "def"])"), STRING("$[0].k1")}, Null()}, // string array {{STRING(R"([null, true, false, 100, 6.18, "abc"])"), STRING("$[0].k1")}, Null()}, // multi type array {{STRING(R"([{"k1":"v41", "k2": 400}, 1, "a", 3.14])"), STRING("$[0].k1")}, diff --git a/build-for-release.sh b/build-for-release.sh index 10046bf98dd590d..0d9e6efb4d2f741 100755 --- a/build-for-release.sh +++ b/build-for-release.sh @@ -110,6 +110,17 @@ echo "Get params: ARCH="$(uname -m)" +if [[ "${ARCH}" == "aarch64" ]]; then + ARCH="arm64" +elif [[ "${ARCH}" == "x86_64" ]]; then + ARCH="x64" +else + echo "Unknown arch: ${ARCH}" + exit 1 +fi + +echo "ARCH: ${ARCH}" + ORI_OUTPUT="${ROOT}/output" FE="fe" @@ -117,7 +128,7 @@ BE="be" EXT="extensions" PACKAGE="apache-doris-${VERSION}-bin-${ARCH}" -if [[ "${_USE_AVX2}" == "0" && "${ARCH}" == "x86_64" ]]; then +if [[ "${_USE_AVX2}" == "0" ]]; then PACKAGE="${PACKAGE}-noavx2" fi diff --git a/build.sh b/build.sh index dfc16b79365b864..260bc1dd11ce095 100755 --- a/build.sh +++ b/build.sh @@ -436,6 +436,7 @@ if [[ "${BUILD_BE_JAVA_EXTENSIONS}" -eq 1 ]]; then modules+=("be-java-extensions/jdbc-scanner") modules+=("be-java-extensions/paimon-scanner") modules+=("be-java-extensions/max-compute-scanner") + modules+=("be-java-extensions/avro-scanner") fi FE_MODULES="$( IFS=',' @@ -647,6 +648,7 @@ EOF extensions_modules+=("hudi-scanner") extensions_modules+=("paimon-scanner") extensions_modules+=("max-compute-scanner") + extensions_modules+=("avro-scanner") BE_JAVA_EXTENSIONS_DIR="${DORIS_OUTPUT}/be/lib/java_extensions/" rm -rf "${BE_JAVA_EXTENSIONS_DIR}" diff --git a/conf/be.conf b/conf/be.conf index e79efd55da62942..220a1122add2d2e 100644 --- a/conf/be.conf +++ b/conf/be.conf @@ -54,13 +54,11 @@ enable_auth = false # priority_networks = 10.10.10.0/24;192.168.0.0/16 # data root path, separate by ';' -# you can specify the storage medium of each root path, HDD or SSD -# you can add capacity limit at the end of each root path, separate by ',' +# You can specify the storage type for each root path, HDD (cold data) or SSD (hot data) # eg: -# storage_root_path = /home/disk1/doris.HDD,50;/home/disk2/doris.SSD,1;/home/disk2/doris -# /home/disk1/doris.HDD, capacity limit is 50GB, HDD; -# /home/disk2/doris.SSD, capacity limit is 1GB, SSD; -# /home/disk2/doris, capacity limit is disk capacity, HDD(default) +# storage_root_path = /home/disk1/doris;/home/disk2/doris;/home/disk2/doris +# storage_root_path = /home/disk1/doris,medium:SSD;/home/disk2/doris,medium:SSD;/home/disk2/doris,medium:HDD +# /home/disk2/doris,medium:HDD(default) # # you also can specify the properties by setting ':', separate by ',' # property 'medium' has a higher priority than the extension of path diff --git a/docker/thirdparties/docker-compose/mysql/init/03-create-table.sql b/docker/thirdparties/docker-compose/mysql/init/03-create-table.sql index 9845db17bf29b1d..f1c45b7b1e8e914 100644 --- a/docker/thirdparties/docker-compose/mysql/init/03-create-table.sql +++ b/docker/thirdparties/docker-compose/mysql/init/03-create-table.sql @@ -291,12 +291,16 @@ CREATE TABLE `doris_test`.`auto_default_t` ( PRIMARY KEY (`id`) ) engine=innodb charset=utf8; -create table doris_test.dt ( - `timestamp0` timestamp(0), - `timestamp1` timestamp(1), - `timestamp2` timestamp(2), - `timestamp3` timestamp(3), - `timestamp4` timestamp(4), - `timestamp5` timestamp(5), - `timestamp6` timestamp(6) -) engine=innodb charset=utf8; \ No newline at end of file +CREATE TABLE doris_test.dt ( + `timestamp0` timestamp(0) DEFAULT CURRENT_TIMESTAMP(0), + `timestamp1` timestamp(1) DEFAULT CURRENT_TIMESTAMP(1), + `timestamp2` timestamp(2) DEFAULT CURRENT_TIMESTAMP(2), + `timestamp3` timestamp(3) DEFAULT CURRENT_TIMESTAMP(3), + `timestamp4` timestamp(4) DEFAULT CURRENT_TIMESTAMP(4), + `timestamp5` timestamp(5) DEFAULT CURRENT_TIMESTAMP(5), + `timestamp6` timestamp(6) DEFAULT CURRENT_TIMESTAMP(6) +) ENGINE=INNODB CHARSET=utf8; + +CREATE TABLE doris_test.dt_null ( + `dt` datetime NOT NULL +) ENGINE=INNODB CHARSET=utf8; \ No newline at end of file diff --git a/docker/thirdparties/docker-compose/mysql/init/04-insert.sql b/docker/thirdparties/docker-compose/mysql/init/04-insert.sql index c164bb3b8209a9d..fbbe2221bb4295f 100644 --- a/docker/thirdparties/docker-compose/mysql/init/04-insert.sql +++ b/docker/thirdparties/docker-compose/mysql/init/04-insert.sql @@ -1143,3 +1143,6 @@ INSERT INTO doris_test.dt (`timestamp0`, `timestamp1`, `timestamp2`, `timestamp3 VALUES ('2023-06-17 10:00:00', '2023-06-17 10:00:01.1', '2023-06-17 10:00:02.22', '2023-06-17 10:00:03.333', '2023-06-17 10:00:04.4444', '2023-06-17 10:00:05.55555', '2023-06-17 10:00:06.666666'); +SET SESSION sql_mode=(SELECT REPLACE(@@sql_mode,'STRICT_TRANS_TABLES','')); +INSERT INTO doris_test.dt_null +VALUES ('2023-06-17 10:00:00'),('0000-00-00 00:00:00'); diff --git a/docker/thirdparties/docker-compose/oracle/init/03-create-table.sql b/docker/thirdparties/docker-compose/oracle/init/03-create-table.sql index 03aa1d511461a2a..7d2b2921254409a 100644 --- a/docker/thirdparties/docker-compose/oracle/init/03-create-table.sql +++ b/docker/thirdparties/docker-compose/oracle/init/03-create-table.sql @@ -72,11 +72,12 @@ t3 interval day(3) to second(6) create table doris_test.test_timestamp( id int, t1 date, -t2 timestamp(6), -t3 timestamp(9), -t4 timestamp, -t5 interval year(3) to month, -t6 interval day(3) to second(6) +t2 timestamp(3), +t3 timestamp(6), +t4 timestamp(9), +t5 timestamp, +t6 interval year(3) to month, +t7 interval day(3) to second(6) ); create table doris_test.test_insert( diff --git a/docker/thirdparties/docker-compose/oracle/init/04-insert.sql b/docker/thirdparties/docker-compose/oracle/init/04-insert.sql index d4638d0348dad3f..a5eb3bd5d143d3e 100644 --- a/docker/thirdparties/docker-compose/oracle/init/04-insert.sql +++ b/docker/thirdparties/docker-compose/oracle/init/04-insert.sql @@ -47,12 +47,13 @@ insert into doris_test.test_date (id, t3) values (5, interval '12 10:23:01.12345 insert into doris_test.test_timestamp (id, t1) values (1, to_date('2013-1-21 5:23:01','yyyy-mm-dd hh24:mi:ss')); insert into doris_test.test_timestamp (id, t1) values (2, to_date('20131112203256', 'yyyymmddhh24miss')); -insert into doris_test.test_timestamp (id, t2) values (3, to_timestamp('20191112203357.999997623', 'yyyymmddhh24miss.ff')); -insert into doris_test.test_timestamp (id, t3) values (4, to_timestamp_tz('20191112203357.999996623', 'yyyymmddhh24miss.ff')); +insert into doris_test.test_timestamp (id, t2) values (3, to_timestamp('20191112203357.999', 'yyyymmddhh24miss.ff')); +insert into doris_test.test_timestamp (id, t3) values (4, to_timestamp('20191112203357.999997623', 'yyyymmddhh24miss.ff')); insert into doris_test.test_timestamp (id, t4) values (5, to_timestamp_tz('20191112203357.999996623', 'yyyymmddhh24miss.ff')); -insert into doris_test.test_timestamp (id, t5) values (6, interval '11' year); -insert into doris_test.test_timestamp (id, t5) values (7, interval '223-9' year(3) to month); -insert into doris_test.test_timestamp (id, t6) values (8, interval '12 10:23:01.1234568' day to second); +insert into doris_test.test_timestamp (id, t5) values (6, to_timestamp_tz('20191112203357.999996623', 'yyyymmddhh24miss.ff')); +insert into doris_test.test_timestamp (id, t6) values (7, interval '11' year); +insert into doris_test.test_timestamp (id, t6) values (8, interval '223-9' year(3) to month); +insert into doris_test.test_timestamp (id, t7) values (9, interval '12 10:23:01.1234568' day to second); insert into doris_test.test_number values (1, 123.45, 12345, 0.0012345); insert into doris_test.test_number values (2, 123.45, 12345, 0.0099999); diff --git a/docker/thirdparties/docker-compose/trino/gen_env.sh.tpl b/docker/thirdparties/docker-compose/trino/gen_env.sh.tpl new file mode 100644 index 000000000000000..dc1357540a03df0 --- /dev/null +++ b/docker/thirdparties/docker-compose/trino/gen_env.sh.tpl @@ -0,0 +1,39 @@ +#!/bin/bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +#################################################################### +# This script will generate hadoop-hive.env from hadoop-hive.env.tpl +#################################################################### + +set -eo pipefail + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")" &>/dev/null && pwd)" + +FS_PORT=8120 +HMS_PORT=9183 + +# Need to set hostname of container to same as host machine's. +# Otherwise, the doris process can not connect to namenode directly. +HOST_NAME="doris--" + +{ + echo "FS_PORT=${FS_PORT}" + echo "HMS_PORT=${HMS_PORT}" + echo "CORE_CONF_fs_defaultFS=hdfs://doris--namenode:${FS_PORT}" + echo "HOST_NAME=${HOST_NAME}" +} >>"${ROOT}"/trino_hive.env diff --git a/docker/thirdparties/docker-compose/trino/hive.properties.tpl b/docker/thirdparties/docker-compose/trino/hive.properties.tpl new file mode 100644 index 000000000000000..4f11682864718e6 --- /dev/null +++ b/docker/thirdparties/docker-compose/trino/hive.properties.tpl @@ -0,0 +1,19 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +connector.name=hive +hive.metastore.uri=thrift://metastore_ip:9083 diff --git a/docker/thirdparties/docker-compose/trino/scripts/create_trino_table.sql b/docker/thirdparties/docker-compose/trino/scripts/create_trino_table.sql new file mode 100644 index 000000000000000..ea9749f18a2bd7e --- /dev/null +++ b/docker/thirdparties/docker-compose/trino/scripts/create_trino_table.sql @@ -0,0 +1,222 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +create schema hive.doris_test; +create table hive.doris_test.orc_basic_data_type +( + T_BOOLEAN BOOLEAN, + T_TINYINT TINYINT, + T_SMALLINT SMALLINT, + T_INTEGER INTEGER, + T_BIGINT BIGINT, + T_REAL REAL, + T_DOUBLE DOUBLE, + T_DECIMAL DECIMAL(38,12), + T_CHAR CHAR, + T_VARCHAR VARCHAR, + T_DATE DATE, + T_TIMESTAMP TIMESTAMP +) WITH (format = 'ORC'); + +create table hive.doris_test.parquet_basic_data_type +( + T_BOOLEAN BOOLEAN, + T_TINYINT TINYINT, + T_SMALLINT SMALLINT, + T_INTEGER INTEGER, + T_BIGINT BIGINT, + T_REAL REAL, + T_DOUBLE DOUBLE, + T_DECIMAL DECIMAL(38,12), + T_CHAR CHAR, + T_VARCHAR VARCHAR, + T_DATE DATE, + T_TIMESTAMP TIMESTAMP +) WITH (format = 'PARQUET'); + + +insert into hive.doris_test.orc_basic_data_type +values (true, tinyint '1', smallint '1', integer '126', bigint '123456789', real '12.34', double '12.3456', decimal + '12.3456789', char 'A', varchar 'Beijing,Shanghai', date '2023-05-23', timestamp '2023-05-24 12:00:00.123'), + (false, tinyint '1', smallint '1', integer '126', bigint '1234567890123456', real '12.34', double '12.3456', decimal + '12.345678901', char 'A', varchar 'Beijing,Shanghai', date '2023-05-24', timestamp '2023-05-24 13:00:00.123'), + (false, tinyint '1', smallint '1', integer '126', bigint '123456789012345678', real '12', double '10', decimal + '12.3456789012', char 'A', varchar 'Beijing,Shanghai', date '2023-05-25', timestamp '2023-05-24 13:00:00.123'), + (null, null, null, null, null, null, null, null, null, null, null, null); + +insert into hive.doris_test.parquet_basic_data_type +select * +from hive.doris_test.orc_basic_data_type; + + +CREATE TABLE hive.doris_test.orc_array_data_type +( + t_int_array array(integer), + t_tinyint_array array(tinyint), + t_smallint_array array(smallint), + t_bigint_array array(bigint), + t_real_array array(real), + t_double_array array(double), + t_string_array array(varchar), + t_boolean_array array(boolean), + t_timestamp_array array(timestamp (3)), + t_date_array array(date), + t_decimal_array array(decimal (38, 12)) +) + WITH ( + format = 'ORC' + ); + +CREATE TABLE hive.doris_test.parquet_array_data_type +( + t_int_array array( integer), + t_tinyint_array array(tinyint), + t_smallint_array array( smallint), + t_bigint_array array(bigint), + t_real_array array( real), + t_double_array array( double), + t_string_array array( varchar), + t_boolean_array array(boolean), + t_timestamp_array array( timestamp (3)), + t_date_array array( date), + t_decimal_array array( decimal (38, 12)) +) + WITH ( + format = 'PARQUET' + ); + +insert into hive.doris_test.orc_array_data_type +values (ARRAY[1,2,3,4,5,6,7],ARRAY[1,2,3,4,5,6,7],ARRAY[1,2,3,4,5,6,7],ARRAY[1234567890123,12345678901234], + ARRAY[45.123,123.45,11.0],ARRAY[45.12344,123.4544,11.0],ARRAY['TEST','TEST#12123123'],ARRAY[TRUE,FALSE,TRUE,FALSE], + ARRAY[TIMESTAMP '2023-05-24 13:00:00.123',TIMESTAMP '2023-05-24 14:00:00.123'], + ARRAY[DATE '2023-05-24',DATE '2023-05-26'], + ARRAY[DECIMAL '10001.11122233344']); + +insert into hive.doris_test.parquet_array_data_type select * from hive.doris_test.orc_array_data_type; + + +create table hive.doris_test.orc_string_complex +( + t_string_array array(varchar), + t_string_map map(varchar,varchar), + t_string_struct row(f_string varchar,f_int varchar) +)WITH ( + FORMAT = 'ORC' + ); + +create table hive.doris_test.parquet_string_complex +( + t_string_array array(varchar), + t_string_map map(varchar,varchar), + t_string_struct row(f_string varchar,f_int varchar) +)WITH ( + FORMAT = 'PARQUET' + ); + +insert into hive.doris_test.orc_string_complex +values (array['1', '2', '3', '北京', 'beijing'], + map(array['1', '2', '3'], array['1', 'beijing', '北京']), + row('beijing', '1')); + +insert into hive.doris_test.parquet_string_complex +select * +from hive.doris_test.orc_string_complex; + +CREATE TABLE hive.doris_test.orc_supplier_partitioned +( + suppkey bigint, + name varchar(25), + address varchar(40), + phone varchar(15), + acctbal double, + comment varchar(101), + nationkey bigint +) + WITH ( + format = 'ORC', + partitioned_by = ARRAY['nationkey'] + ); + +CREATE TABLE hive.doris_test.parquet_supplier_partitioned +( + suppkey bigint, + name varchar(25), + address varchar(40), + phone varchar(15), + acctbal double, + comment varchar(101), + nationkey bigint +) + WITH ( + format = 'PARQUET', + partitioned_by = ARRAY['nationkey'] + ); + +insert into hive.doris_test.orc_supplier_partitioned +select suppkey, name, address, phone, acctbal, comment, nationkey +from tpch.sf100.supplier; + +insert into hive.doris_test.parquet_supplier_partitioned +select * +from hive.doris_test.orc_supplier_partitioned; + +-- partition and bucket +CREATE TABLE hive.doris_test.orc_supplier_partitioned_bucketed +( + suppkey bigint, + name varchar(25), + address varchar(40), + phone varchar(15), + acctbal double, + comment varchar(101), + nationkey bigint +) + WITH ( + format = 'ORC', + partitioned_by = ARRAY['nationkey'], + bucketed_by = ARRAY['suppkey'], + bucket_count = 10 + ); + +CREATE TABLE hive.doris_test.parquet_supplier_partitioned_bucketed +( + suppkey bigint, + name varchar(25), + address varchar(40), + phone varchar(15), + acctbal double, + comment varchar(101), + nationkey bigint +) + WITH ( + format = 'PARQUET', + partitioned_by = ARRAY['nationkey'], + bucketed_by = ARRAY['suppkey'], + bucket_count = 10 + ); + +insert into hive.doris_test.orc_supplier_partitioned_bucketed +select suppkey, name, address, phone, acctbal, comment, nationkey +from tpch.sf100.supplier; + +insert into hive.doris_test.parquet_supplier_partitioned_bucketed +select * +from hive.doris_test.orc_supplier_partitioned_bucketed; + + + + diff --git a/docker/thirdparties/docker-compose/trino/trino_hive.env.tpl b/docker/thirdparties/docker-compose/trino/trino_hive.env.tpl new file mode 100644 index 000000000000000..0f2ce0a443f38ca --- /dev/null +++ b/docker/thirdparties/docker-compose/trino/trino_hive.env.tpl @@ -0,0 +1,53 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +DOCKER_TRINO_EXTERNAL_PORT=8080 + +HIVE_SITE_CONF_javax_jdo_option_ConnectionURL=jdbc:postgresql://doris--hive-metastore-postgresql:5432/metastore +HIVE_SITE_CONF_javax_jdo_option_ConnectionDriverName=org.postgresql.Driver +HIVE_SITE_CONF_javax_jdo_option_ConnectionUserName=hive +HIVE_SITE_CONF_javax_jdo_option_ConnectionPassword=hive +HIVE_SITE_CONF_datanucleus_autoCreateSchema=false +HIVE_SITE_CONF_hive_metastore_uris=thrift://doris--hive-metastore:9083 +HDFS_CONF_dfs_namenode_datanode_registration_ip___hostname___check=false +HIVE_SITE_CONF_hive_server2_thrift_bind_host=0.0.0.0 +HIVE_SITE_CONF_hive_server2_thrift_port=10000 + +CORE_CONF_hadoop_http_staticuser_user=root +CORE_CONF_hadoop_proxyuser_hue_hosts=* +CORE_CONF_hadoop_proxyuser_hue_groups=* +CORE_CONF_hadoop_proxyuser_hive_hosts=* + +HDFS_CONF_dfs_webhdfs_enabled=true +HDFS_CONF_dfs_permissions_enabled=false + +YARN_CONF_yarn_log___aggregation___enable=true +YARN_CONF_yarn_resourcemanager_recovery_enabled=true +YARN_CONF_yarn_resourcemanager_store_class=org.apache.hadoop.yarn.server.resourcemanager.recovery.FileSystemRMStateStore +YARN_CONF_yarn_resourcemanager_fs_state___store_uri=/rmstate +YARN_CONF_yarn_nodemanager_remote___app___log___dir=/app-logs +YARN_CONF_yarn_log_server_url=http://historyserver:8188/applicationhistory/logs/ +YARN_CONF_yarn_timeline___service_enabled=true +YARN_CONF_yarn_timeline___service_generic___application___history_enabled=true +YARN_CONF_yarn_resourcemanager_system___metrics___publisher_enabled=true +YARN_CONF_yarn_resourcemanager_hostname=resourcemanager +YARN_CONF_yarn_timeline___service_hostname=historyserver +YARN_CONF_yarn_resourcemanager_address=resourcemanager:8032 +YARN_CONF_yarn_resourcemanager_scheduler_address=resourcemanager:8030 +YARN_CONF_yarn_resourcemanager_resource__tracker_address=resourcemanager:8031 + diff --git a/docker/thirdparties/docker-compose/trino/trino_hive.yaml.tpl b/docker/thirdparties/docker-compose/trino/trino_hive.yaml.tpl new file mode 100644 index 000000000000000..f034a0c7bda9603 --- /dev/null +++ b/docker/thirdparties/docker-compose/trino/trino_hive.yaml.tpl @@ -0,0 +1,141 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +version: "3.8" + +networks: + doris--network: + driver: bridge + +services: + + doris--trino: + image: trinodb/trino:418 + hostname: doris--trino + container_name: doris--trino + env_file: + - ./trino_hive.env + ports: + - "${DOCKER_TRINO_EXTERNAL_PORT}:8080" + volumes: + - ./scripts:/scripts + healthcheck: + test: [ "CMD", "curl", "-f", "http://localhost:8080/" ] + interval: 5s + timeout: 120s + retries: 120 + networks: + - doris--network + + doris--namenode: + image: bde2020/hadoop-namenode:2.0.0-hadoop2.7.4-java8 + environment: + - CLUSTER_NAME=test + env_file: + - ./trino_hive.env + hostname: doris--namenode + container_name: doris--namenode + expose: + - "50070" + - "8020" + - "9000" + - "${FS_PORT}" + ports: + - "${FS_PORT}:${FS_PORT}" + healthcheck: + test: [ "CMD", "curl", "http://localhost:50070/" ] + interval: 5s + timeout: 120s + retries: 120 + networks: + - doris--network + + doris--datanode: + image: bde2020/hadoop-datanode:2.0.0-hadoop2.7.4-java8 + env_file: + - ./trino_hive.env + environment: + SERVICE_PRECONDITION: "doris--namenode:50070" + hostname: doris--datanode + container_name: doris--datanode + expose: + - "50075" + healthcheck: + test: [ "CMD", "curl", "http://localhost:50075" ] + interval: 5s + timeout: 60s + retries: 120 + networks: + - doris--network + + doris--hive-server: + image: bde2020/hive:2.3.2-postgresql-metastore + env_file: + - ./trino_hive.env + environment: + HIVE_CORE_CONF_javax_jdo_option_ConnectionURL: "jdbc:postgresql://doris--hive-metastore-postgresql:5432/metastore" + SERVICE_PRECONDITION: "doris--hive-metastore:9083" + hostname: doris--hive-server + container_name: doris--hive-server + expose: + - "10000" + depends_on: + - doris--datanode + - doris--namenode + healthcheck: + test: beeline -u "jdbc:hive2://127.0.0.1:10000/default" -n health_check -e "show databases;" + interval: 10s + timeout: 120s + retries: 120 + networks: + - doris--network + + + doris--hive-metastore: + image: bde2020/hive:2.3.2-postgresql-metastore + env_file: + - ./trino_hive.env + command: /opt/hive/bin/hive --service metastore + environment: + SERVICE_PRECONDITION: "doris--namenode:50070 doris--datanode:50075 doris--hive-metastore-postgresql:5432" + hostname: doris--hive-metastore + container_name: doris--hive-metastore + expose: + - "9083" + ports: + - "${HMS_PORT}:9083" + volumes: + - ./scripts:/mnt/scripts + depends_on: + - doris--hive-metastore-postgresql + networks: + - doris--network + + doris--hive-metastore-postgresql: + image: bde2020/hive-metastore-postgresql:2.3.0 + restart: always + hostname: doris--hive-metastore-postgresql + container_name: doris--hive-metastore-postgresql + expose: + - "5432" + healthcheck: + test: ["CMD-SHELL", "pg_isready -U postgres"] + interval: 5s + timeout: 60s + retries: 120 + networks: + - doris--network diff --git a/docker/thirdparties/run-thirdparties-docker.sh b/docker/thirdparties/run-thirdparties-docker.sh index 283ed7b35b718ce..5d0821da18f8368 100755 --- a/docker/thirdparties/run-thirdparties-docker.sh +++ b/docker/thirdparties/run-thirdparties-docker.sh @@ -37,7 +37,7 @@ Usage: $0 --stop stop the specified components All valid components: - mysql,pg,oracle,sqlserver,clickhouse,es,hive,iceberg,hudi + mysql,pg,oracle,sqlserver,clickhouse,es,hive,iceberg,hudi,trino " exit 1 } @@ -60,7 +60,7 @@ STOP=0 if [[ "$#" == 1 ]]; then # default - COMPONENTS="mysql,pg,oracle,sqlserver,clickhouse,hive,iceberg,hudi" + COMPONENTS="mysql,pg,oracle,sqlserver,clickhouse,hive,iceberg,hudi,trino" else while true; do case "$1" in @@ -92,7 +92,7 @@ else done if [[ "${COMPONENTS}"x == ""x ]]; then if [[ "${STOP}" -eq 1 ]]; then - COMPONENTS="mysql,pg,oracle,sqlserver,clickhouse,hive,iceberg,hudi" + COMPONENTS="mysql,pg,oracle,sqlserver,clickhouse,hive,iceberg,hudi,trino" fi fi fi @@ -129,6 +129,8 @@ RUN_HIVE=0 RUN_ES=0 RUN_ICEBERG=0 RUN_HUDI=0 +RUN_TRINO=0 + for element in "${COMPONENTS_ARR[@]}"; do if [[ "${element}"x == "mysql"x ]]; then RUN_MYSQL=1 @@ -148,6 +150,8 @@ for element in "${COMPONENTS_ARR[@]}"; do RUN_ICEBERG=1 elif [[ "${element}"x == "hudi"x ]]; then RUN_HUDI=1 + elif [[ "${element}"x == "trino"x ]];then + RUN_TRINO=1 else echo "Invalid component: ${element}" usage @@ -290,3 +294,80 @@ if [[ "${RUN_HUDI}" -eq 1 ]]; then docker exec -it adhoc-2 /bin/bash /var/scripts/setup_demo_container_adhoc_2.sh fi fi + +if [[ "${RUN_TRINO}" -eq 1 ]]; then + # trino + trino_docker="${ROOT}"/docker-compose/trino + TRINO_CONTAINER_ID="${CONTAINER_UID}trino" + NAMENODE_CONTAINER_ID="${CONTAINER_UID}namenode" + HIVE_METASTORE_CONTAINER_ID=${CONTAINER_UID}hive-metastore + for file in trino_hive.yaml trino_hive.env gen_env.sh hive.properties + do + cp "${trino_docker}/$file.tpl" "${trino_docker}/$file" + if [[ $file != "hive.properties" ]]; then + sed -i "s/doris--/${CONTAINER_UID}/g" "${trino_docker}/$file" + fi + done + + bash "${trino_docker}"/gen_env.sh + sudo docker compose -f "${trino_docker}"/trino_hive.yaml --env-file "${trino_docker}"/trino_hive.env down + if [[ "${STOP}" -ne 1 ]]; then + sudo sed -i "/${NAMENODE_CONTAINER_ID}/d" /etc/hosts + sudo docker compose -f "${trino_docker}"/trino_hive.yaml --env-file "${trino_docker}"/trino_hive.env up --build --remove-orphans -d + sudo echo "127.0.0.1 ${NAMENODE_CONTAINER_ID}" >> /etc/hosts + sleep 20s + hive_metastore_ip=$(docker inspect --format='{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' ${HIVE_METASTORE_CONTAINER_ID}) + + if [ -z "$hive_metastore_ip" ]; then + echo "Failed to get Hive Metastore IP address" >&2 + exit 1 + else + echo "Hive Metastore IP address is: $hive_metastore_ip" + fi + + sed -i "s/metastore_ip/${hive_metastore_ip}/g" "${trino_docker}"/hive.properties + docker cp "${trino_docker}"/hive.properties "${CONTAINER_UID}trino":/etc/trino/catalog/ + + # trino load hive catalog need restart server + max_retries=3 + + function control_container() { + max_retries=3 + operation=$1 + expected_status=$2 + retries=0 + + while [ $retries -lt $max_retries ] + do + status=$(docker inspect --format '{{.State.Running}}' ${TRINO_CONTAINER_ID}) + if [ "${status}" == "${expected_status}" ]; then + echo "Container ${TRINO_CONTAINER_ID} has ${operation}ed successfully." + break + else + echo "Waiting for container ${TRINO_CONTAINER_ID} to ${operation}..." + sleep 5s + ((retries++)) + fi + sleep 3s + done + + if [ $retries -eq $max_retries ]; then + echo "${operation} operation failed to complete after $max_retries attempts." + exit 1 + fi + } + # Stop the container + docker stop ${TRINO_CONTAINER_ID} + sleep 5s + control_container "stop" "false" + + # Start the container + docker start ${TRINO_CONTAINER_ID} + control_container "start" "true" + + # waite trino init + sleep 20s + # execute create table sql + docker exec -it ${TRINO_CONTAINER_ID} /bin/bash -c 'trino -f /scripts/create_trino_table.sql' + fi +fi diff --git a/docs/en/docs/admin-manual/config/be-config.md b/docs/en/docs/admin-manual/config/be-config.md index 864d1cab84669b5..f97cdb83cc56d65 100644 --- a/docs/en/docs/admin-manual/config/be-config.md +++ b/docs/en/docs/admin-manual/config/be-config.md @@ -315,6 +315,12 @@ There are two ways to configure BE configuration items: * Description: When using the odbc external table, if a column type of the odbc source table is not HLL, CHAR or VARCHAR, and the length of the column value exceeds this value, the query will report an error 'column value length longer than buffer length'. You can increase this value * Default value: 100 +#### `jsonb_type_length_soft_limit_bytes` + +* Type: int32 +* Description: The soft limit of the maximum length of JSONB type. +* Default value: 1,048,576 + ### Query #### `fragment_pool_queue_size` diff --git a/docs/en/docs/admin-manual/maint-monitor/monitor-alert.md b/docs/en/docs/admin-manual/maint-monitor/monitor-alert.md index 4d6993b5f371798..f1680b21e6b8a96 100644 --- a/docs/en/docs/admin-manual/maint-monitor/monitor-alert.md +++ b/docs/en/docs/admin-manual/maint-monitor/monitor-alert.md @@ -230,7 +230,7 @@ Prometheus Start Grafana with the following command - `nohuo ./bin/grafana-server &` + `nohup ./bin/grafana-server &` This command runs Grafana in the background, and the access port is 8182 configured above. diff --git a/docs/en/docs/admin-manual/workload-group.md b/docs/en/docs/admin-manual/workload-group.md index 242bb9dd13dac42..05f5aa81d9b4333 100644 --- a/docs/en/docs/admin-manual/workload-group.md +++ b/docs/en/docs/admin-manual/workload-group.md @@ -34,7 +34,7 @@ The workload group can limit the use of compute and memory resources on a single * cpu_share: Required, used to set how much cpu time the workload group can acquire, which can achieve soft isolation of cpu resources. cpu_share is a relative value indicating the weight of cpu resources available to the running workload group. For example, if a user creates 3 workload groups rg-a, rg-b and rg-c with cpu_share of 10, 30 and 40 respectively, and at a certain moment rg-a and rg-b are running tasks while rg-c has no tasks, then rg-a can get 25% (10 / (10 + 30)) of the cpu resources while workload group rg-b can get 75% of the cpu resources. If the system has only one workload group running, it gets all the cpu resources regardless of the value of its cpu_share. -* memory_limit: Required, set the percentage of be memory that can be used by the workload group. The absolute value of the workload group memory limit is: physical_memory * mem_limit * memory_limit, where mem_limit is a be configuration item. The total memory_limit of all workload groups in the system must not exceed 100%. Workload groups are guaranteed to use the memory_limit for the tasks in the group in most cases. When the workload group memory usage exceeds this limit, tasks in the group with larger memory usage may be canceled to release the excess memory, refer to enable_memory_overcommit. +* memory_limit: Required, set the percentage of be memory that can be used by the workload group. The absolute value of the workload group memory limit is: `physical_memory * mem_limit * memory_limit`, where mem_limit is a be configuration item. The total memory_limit of all workload groups in the system must not exceed 100%. Workload groups are guaranteed to use the memory_limit for the tasks in the group in most cases. When the workload group memory usage exceeds this limit, tasks in the group with larger memory usage may be canceled to release the excess memory, refer to enable_memory_overcommit. * enable_memory_overcommit: Optional, enable soft memory isolation for the workload group, default is false. if set to false, the workload group is hard memory isolated and the tasks with the largest memory usage will be canceled immediately after the workload group memory usage exceeds the limit to release the excess memory. if set to true, the workload group is hard memory isolated and the tasks with the largest memory usage will be canceled immediately after the workload group memory usage exceeds the limit to release the excess memory. if set to true, the workload group is softly isolated, if the system has free memory resources, the workload group can continue to use system memory after exceeding the memory_limit limit, and when the total system memory is tight, it will cancel several tasks in the group with the largest memory occupation, releasing part of the excess memory to relieve the system memory pressure. It is recommended that when this configuration is enabled for a workload group, the total memory_limit of all workload groups should be less than 100%, and the remaining portion should be used for workload group memory overcommit. diff --git a/docs/en/docs/benchmark/ssb.md b/docs/en/docs/benchmark/ssb.md index 7d3720eeca3b1f9..acb9b06c11705ec 100644 --- a/docs/en/docs/benchmark/ssb.md +++ b/docs/en/docs/benchmark/ssb.md @@ -67,7 +67,7 @@ On the SQL test with standard SSB, the overall performance of Apache Doris 1.2.0 | customer | 3,000,000 | Customer Information | | part | 1,400,000 | Parts Information | | supplier | 200,000 | Supplier Information | -| date | 2,556 | Date | +| dates | 2,556 | Date | | lineorder_flat | 600,037,902 | Wide Table after Data Flattening | ## 4. Test Results @@ -169,7 +169,7 @@ With the `-s 100` parameter, the resulting dataset size is: | customer | 3,000,000 | 277M | 1 | | part | 1,400,000 | 116M | 1 | | supplier | 200,000 | 17M | 1 | -| date | 2,556 | 228K | 1 | +| dates | 2,556 | 228K | 1 | ### 7.3 Create Table @@ -283,7 +283,7 @@ We use the following command to complete all data import of SSB test set and SSB select count(*) from part; select count(*) from customer; select count(*) from supplier; -select count(*) from date; +select count(*) from dates; select count(*) from lineorder; select count(*) from lineorder_flat; ``` @@ -297,7 +297,7 @@ The amount of data should be consistent with the number of rows of generated dat | customer | 3,000,000 | 277 MB | 138.247 MB | | part | 1,400,000 | 116 MB | 12.759 MB | | supplier | 200,000 | 17 MB | 9.143 MB | -| date | 2,556 | 228 KB | 34.276 KB | +| dates | 2,556 | 228 KB | 34.276 KB | ### 7.6 Query Test diff --git a/docs/en/docs/data-table/index/inverted-index.md b/docs/en/docs/data-table/index/inverted-index.md index 06102632f8f9cf9..016c4aa57ffa554 100644 --- a/docs/en/docs/data-table/index/inverted-index.md +++ b/docs/en/docs/data-table/index/inverted-index.md @@ -98,6 +98,15 @@ CREATE TABLE table_name table_properties; ``` +:::tip + +Inverted indexes have different limitations in different data models: +- Aggregate model: Inverted indexes can only be created for the Key column. +- Unique model: The merge on write feature needs to be enabled. After enabling it, an inverted index can be created for any column. +- Duplicate model: An inverted index can be created for any column. + +::: + - add an inverted index to existed table **Before version 2.0-beta:** diff --git a/docs/en/docs/ecosystem/flink-doris-connector.md b/docs/en/docs/ecosystem/flink-doris-connector.md index 142cc7f66209b68..7a07793ec4d2f9e 100644 --- a/docs/en/docs/ecosystem/flink-doris-connector.md +++ b/docs/en/docs/ecosystem/flink-doris-connector.md @@ -407,6 +407,8 @@ insert into doris_sink select id,name from cdc_mysql_source; - **--sink-conf** All configurations of Doris Sink, you can view the complete configuration items [here](https://doris.apache.org/zh-CN/docs/dev/ecosystem/flink-doris-connector/#%E9%80%9A%E7%94%A8%E9%85%8D%E7%BD%AE%E9%A1%B9). - **--table-conf** The configuration item of the Doris table, that is, the content contained in properties. For example --table-conf replication_num=1 +>Note: flink-sql-connector-mysql-cdc-2.3.0.jar needs to be added in the $FLINK_HOME/lib directory + ### Example ``` /bin/flink run \ @@ -468,7 +470,7 @@ CREATE TABLE DORIS_SINK( 'username' = 'root', 'password' = '', 'sink.enable-delete' = 'false', -- false means not to get the event type from RowKind - 'sink.properties.columns' = 'name,age,__DORIS_DELETE_SIGN__' -- Display the import column of the specified streamload + 'sink.properties.columns' = 'id, name, __DORIS_DELETE_SIGN__' -- Display the import column of the specified streamload ); INSERT INTO KAFKA_SOURCE diff --git a/docs/en/docs/ecosystem/udf/native-user-defined-function.md b/docs/en/docs/ecosystem/udf/native-user-defined-function.md deleted file mode 100644 index abb3bf348927055..000000000000000 --- a/docs/en/docs/ecosystem/udf/native-user-defined-function.md +++ /dev/null @@ -1,271 +0,0 @@ ---- -{ - "title": "Native User Defined Function", - "language": "en" -} ---- - - - -# Native User Defined Function -UDF is mainly suitable for scenarios where the analytical capabilities that users need do not possess. Users can implement custom functions according to their own needs, and register with Doris through the UDF framework to expand Doris' capabilities and solve user analysis needs. - -There are two types of analysis requirements that UDF can meet: UDF and UDAF. UDF in this article refers to both. - -1. UDF: User-defined function, this function will operate on a single line and output a single line result. When users use UDFs for queries, each row of data will eventually appear in the result set. Typical UDFs are string operations such as concat(). -2. UDAF: User-defined aggregation function. This function operates on multiple lines and outputs a single line of results. When the user uses UDAF in the query, each group of data after grouping will finally calculate a value and expand the result set. A typical UDAF is the set operation sum(). Generally speaking, UDAF will be used together with group by. - -This document mainly describes how to write a custom UDF function and how to use it in Doris. - -If users use the UDF function and extend Doris' function analysis, and want to contribute their own UDF functions back to the Doris community for other users, please see the document [Contribute UDF](./contribute-udf.md). - -## Writing UDF functions - -Before using UDF, users need to write their own UDF functions under Doris' UDF framework. In the `contrib/udf/src/udf_samples/udf_sample.h|cpp` file is a simple UDF Demo. - -Writing a UDF function requires the following steps. - -### Writing functions - -Create the corresponding header file and CPP file, and implement the logic you need in the CPP file. Correspondence between the implementation function format and UDF in the CPP file. - -Users can put their own source code in a folder. Taking udf_sample as an example, the directory structure is as follows: - -``` -└── udf_samples - ├── uda_sample.cpp - ├── uda_sample.h - ├── udf_sample.cpp - └── udf_sample.h -``` - -#### Non-variable parameters - -For UDFs with non-variable parameters, the correspondence between the two is straightforward. -For example, the UDF of `INT MyADD(INT, INT)` will correspond to `IntVal AddUdf(FunctionContext* context, const IntVal& arg1, const IntVal& arg2)`. - -1. `AddUdf` can be any name, as long as it is specified when creating UDF. -2. The first parameter in the implementation function is always `FunctionContext*`. The implementer can obtain some query-related content through this structure, and apply for some memory to be used. The specific interface used can refer to the definition in `udf/udf.h`. -3. In the implementation function, the second parameter needs to correspond to the UDF parameter one by one, for example, `IntVal` corresponds to `INT` type. All types in this part must be referenced with `const`. -4. The return parameter must correspond to the type of UDF parameter. - -#### variable parameter - -For variable parameters, you can refer to the following example, corresponding to UDF`String md5sum(String, ...)` -The implementation function is `StringVal md5sumUdf(FunctionContext* ctx, int num_args, const StringVal* args)` - -1. `md5sumUdf` can also be changed arbitrarily, just specify it when creating. -2. The first parameter is the same as the non-variable parameter function, and the passed in is a `FunctionContext*`. -3. The variable parameter part consists of two parts. First, an integer is passed in, indicating that there are several parameters behind. An array of variable parameter parts is passed in later. - -#### Type correspondence - -|UDF Type|Argument Type| -|----|---------| -|TinyInt|TinyIntVal| -|SmallInt|SmallIntVal| -|Int|IntVal| -|BigInt|BigIntVal| -|LargeInt|LargeIntVal| -|Float|FloatVal| -|Double|DoubleVal| -|Date|DateTimeVal| -|Datetime|DateTimeVal| -|Char|StringVal| -|Varchar|StringVal| -|Decimal|DecimalVal| - - -## Compile UDF function - -Since the UDF implementation relies on Doris' UDF framework, the first step in compiling UDF functions is to compile Doris, that is, the UDF framework. - -After the compilation is completed, the static library file of the UDF framework will be generated. Then introduce the UDF framework dependency and compile the UDF. - -### Compile Doris - -Running `sh build.sh` in the root directory of Doris will generate a static library file of the UDF framework `headers|libs` in `output/udf/` - -``` -├── output -│ └── udf -│ ├── include -│ │ ├── uda_test_harness.h -│ │ └── udf.h -│ └── lib -│ └── libDorisUdf.a - -``` - -### Writing UDF compilation files - -1. Prepare thirdparty - - The thirdparty folder is mainly used to store thirdparty libraries that users' UDF functions depend on, including header files and static libraries. It must contain the two files `udf.h` and `libDorisUdf.a` in the dependent Doris UDF framework. - - Taking udf_sample as an example here, the source code is stored in the user's own `udf_samples` directory. Create a thirdparty folder in the same directory to store the static library. The directory structure is as follows: - - ``` - ├── thirdparty - │ │── include - │ │ └── udf.h - │ └── lib - │ └── libDorisUdf.a - └── udf_samples - - ``` - - `udf.h` is the UDF frame header file. The storage path is `doris/output/udf/include/udf.h`. Users need to copy the header file in the Doris compilation output to their include folder of `thirdparty`. - - `libDorisUdf.a` is a static library of UDF framework. After Doris is compiled, the file is stored in `doris/output/udf/lib/libDorisUdf.a`. The user needs to copy the file to the lib folder of his `thirdparty`. - - *Note: The static library of UDF framework will not be generated until Doris is compiled. - -2. Prepare to compile UDF's CMakeFiles.txt - - CMakeFiles.txt is used to declare how UDF functions are compiled. Stored in the source code folder, level with user code. Here, taking udf_samples as an example, the directory structure is as follows: - - ``` - ├── thirdparty - └── udf_samples - ├── CMakeLists.txt - ├── uda_sample.cpp - ├── uda_sample.h - ├── udf_sample.cpp - └── udf_sample.h - ``` - - + Need to show declaration reference `libDorisUdf.a` - + Declare `udf.h` header file location - - -Take udf_sample as an example - -``` -# Include udf -include_directories(../thirdparty/include) - -# Set all libraries -add_library(udf STATIC IMPORTED) -set_target_properties(udf PROPERTIES IMPORTED_LOCATION ../thirdparty/lib/libDorisUdf.a) - -# where to put generated libraries -set(LIBRARY_OUTPUT_PATH "src/udf_samples") - -# where to put generated binaries -set(EXECUTABLE_OUTPUT_PATH "src/udf_samples") - -add_library(udfsample SHARED udf_sample.cpp) - target_link_libraries(udfsample - udf - -static-libstdc++ - -static-libgcc -) - -add_library(udasample SHARED uda_sample.cpp) - target_link_libraries(udasample - udf - -static-libstdc++ - -static-libgcc -) -``` - -If the user's UDF function also depends on other thirdparty libraries, you need to declare include, lib, and add dependencies in `add_library`. - -The complete directory structure after all files are prepared is as follows: - -``` - ├── thirdparty - │ │── include - │ │ └── udf.h - │ └── lib - │ └── libDorisUdf.a - └── udf_samples - ├── CMakeLists.txt - ├── uda_sample.cpp - ├── uda_sample.h - ├── udf_sample.cpp - └── udf_sample.h -``` - -Prepare the above files and you can compile UDF directly - -### Execute compilation - -Create a build folder under the udf_samples folder to store the compilation output. - -Run the command `cmake ../` in the build folder to generate a Makefile, and execute make to generate the corresponding dynamic library. - -``` -├── thirdparty -├── udf_samples - └── build -``` - -### Compilation result - -After the compilation is completed, the UDF dynamic link library is successfully generated. Under `build/src/`, taking udf_samples as an example, the directory structure is as follows: - -``` -├── thirdparty -├── udf_samples - └── build - └── src - └── udf_samples - ├── libudasample.so -   └── libudfsample.so - -``` - -## Create UDF function - -After following the above steps, you can get the UDF dynamic library (that is, the `.so` file in the compilation result). You need to put this dynamic library in a location that can be accessed through the HTTP protocol. - -Then log in to the Doris system and create a UDF function in the mysql-client through the `CREATE FUNCTION` syntax. You need to have ADMIN authority to complete this operation. At this time, there will be a UDF created in the Doris system. - -``` -CREATE [AGGREGATE] FUNCTION -name ([argtype][,...]) -[RETURNS] rettype -PROPERTIES (["key"="value"][,...]) -``` -Description: - -1. "Symbol" in PROPERTIES means that the symbol corresponding to the entry function is executed. This parameter must be set. You can get the corresponding symbol through the `nm` command, for example, `_ZN9doris_udf6AddUdfEPNS_15FunctionContextERKNS_6IntValES4_` obtained by `nm libudfsample.so | grep AddUdf` is the corresponding symbol. -2. The object_file in PROPERTIES indicates where it can be downloaded to the corresponding dynamic library. This parameter must be set. -3. name: A function belongs to a certain DB, and the name is in the form of `dbName`.`funcName`. When `dbName` is not explicitly specified, the db where the current session is located is used as `dbName`. - -For specific use, please refer to `CREATE FUNCTION` for more detailed information. - -## Use UDF - -Users must have the `SELECT` permission of the corresponding database to use UDF/UDAF. - -The use of UDF is consistent with ordinary function methods. The only difference is that the scope of built-in functions is global, and the scope of UDF is internal to DB. When the link session is inside the data, directly using the UDF name will find the corresponding UDF inside the current DB. Otherwise, the user needs to display the specified UDF database name, such as `dbName`.`funcName`. - -In current version, vectorization needs to be turned off to use native udf -``` -set enable_vectorized_engine = false; -``` - - -## Delete UDF - -When you no longer need UDF functions, you can delete a UDF function by the following command, you can refer to `DROP FUNCTION`. diff --git a/docs/en/docs/install/standard-deployment.md b/docs/en/docs/install/standard-deployment.md index ebb945ee72b9631..d6e9bb04c9dfc6a 100644 --- a/docs/en/docs/install/standard-deployment.md +++ b/docs/en/docs/install/standard-deployment.md @@ -99,7 +99,7 @@ Both ext4 and xfs file systems are supported. > > 1. FE nodes are divided into Followers and Observers based on their roles. (Leader is an elected role in the Follower group, hereinafter referred to as Follower, too.) > 2. The number of FE nodes should be at least 1 (1 Follower). If you deploy 1 Follower and 1 Observer, you can achieve high read availability; if you deploy 3 Followers, you can achieve high read-write availability (HA). -> 3. The number of Followers **must be** odd, and there is no limit on the number of Observers. +> 3. Although multiple BEs can be deployed on one machine, **only one instance** is recommended to be deployed, and **only one FE** can be deployed at the same time. If 3 copies of data are required, at least 3 machines are required to deploy a BE instance (instead of 1 machine deploying 3 BE instances). **The clocks of the servers where multiple FEs are located must be consistent (up to 5 seconds of clock deviation is allowed)**. > 4. According to past experience, for business that requires high cluster availability (e.g. online service providers), we recommend that you deploy 3 Followers and 1-3 Observers; for offline business, we recommend that you deploy 1 Follower and 1-3 Observers. * **Usually we recommend 10 to 100 machines to give full play to Doris' performance (deploy FE on 3 of them (HA) and BE on the rest).** @@ -187,80 +187,77 @@ See the `lower_case_table_names` section in [Variables](../advanced/variables.md * For details about deployment of multiple FEs, see the [FE scaling](https://doris.apache.org/docs/dev/admin-manual/cluster-management/elastic-expansion/) section. -#### Deploy BE +#### BE Deployment -* Copy the BE deployment file into all nodes that are to deploy BE on +* Copy the BE deployment file to all nodes to deploy BE - Find the BE folder under the output generated by source code compilation, copy it into to the specified deployment paths of the BE nodes. + Copy the be folder under the output generated by source code compilation to the specified deployment path of the BE node. -* Modify all BE configurations - - Modify be/conf/be.conf, which mainly involves configuring `storage_root_path`: data storage directory. By default, under be/storage, the directory needs to be **created manually**. Use `;` to separate multiple paths (do not add `;` after the last directory). + > Note: The `output/be/lib/debug_info/` directory contains debugging information files, which are relatively large, but these files are not needed for actual operation and can not be deployed. +* Modify all BE configurations - You may specify the directory storage medium in the path: HDD or SSD. You may also add capacity limit to the end of every path and use `,` for separation. Unless you use a mix of SSD and HDD disks, you do not need to follow the configuration methods in Example 1 or Example 2 below, but only need to specify the storage directory; you do not need to modify the default storage medium configuration of FE, either. + Modify be/conf/be.conf. Mainly configure `storage_root_path`: data storage directory. By default, it is under be/storage. If you need to specify a directory, you need to **pre-create the directory**. Multiple paths are separated by a semicolon `;` in English (**do not add `;`** after the last directory). + The hot and cold data storage directories in the node can be distinguished by path, HDD (cold data directory) or SSD (hot data directory). If you don't need the hot and cold mechanism in the BE node, you only need to configure the path without specifying the medium type; and you don't need to modify the default storage medium configuration of FE - Example 1: + **Notice:** + 1. If you specify the storage type of the storage path, at least one path must have a storage type of HDD (cold data directory)! + 2. If the storage type of the storage path is not specified, all are HDD (cold data directory) by default. + 3. The HDD and SSD here have nothing to do with the physical storage medium, but only to distinguish the storage type of the storage path, that is, you can mark a certain directory on the disk of the HDD medium as SSD (hot data directory). + 4. Here HDD and SSD **MUST** be capitalized! - Note: For SSD disks, add `.SSD` to the end of the directory; for HDD disks, add `.HDD`. + Example 1 is as follows: - `storage_root_path=/home/disk1/doris.HDD;/home/disk2/doris.SSD;/home/disk2/doris` + `storage_root_path=/home/disk1/doris;/home/disk2/doris;/home/disk2/doris` - **Description** + Example 2 is as follows: - * 1./home/disk1/doris.HDD: The storage medium is HDD; - * 2./home/disk2/doris.SSD: The storage medium is SSD; - * 3./home/disk2/doris: The storage medium is HDD (default). + **Use the storage_root_path parameter to specify medium** - Example 2: + `storage_root_path=/home/disk1/doris,medium:HDD;/home/disk2/doris,medium:SSD` - Note: You do not need to add the `.SSD` or `.HDD` suffix, but to specify the medium in the `storage_root_path` parameter + **illustrate** - `storage_root_path=/home/disk1/doris,medium:HDD;/home/disk2/doris,medium:SSD` - - **Description** - + - /home/disk1/doris,medium:HDD: Indicates that the directory stores cold data; + - /home/disk2/doris,medium:SSD: Indicates that the directory stores hot data; - * 1./home/disk1/doris,medium:HDD : The storage medium is HDD; - * 2./home/disk2/doris,medium:SSD : The storage medium is SSD. +* BE webserver_port port configuration -* BE webserver_port configuration + If be is deployed in a hadoop cluster, pay attention to adjusting `webserver_port = 8040` in be.conf to avoid port conflicts - If the BE component is installed in hadoop cluster, you need to change configuration `webserver_port=8040` to avoid port used. +* Configure the JAVA_HOME environment variable -* Set JAVA_HOME environment variable + + Since Java UDF functions are supported from version 1.2, BE depends on the Java environment. So to pre-configure the `JAVA_HOME` environment variable, you can also add `export JAVA_HOME=your_java_home_path` to the first line of the `start_be.sh` startup script to add the environment variable. - - Java UDF is supported since version 1.2, so BEs are dependent on the Java environment. It is necessary to set the `JAVA_HOME` environment variable before starting. You can also do this by adding `export JAVA_HOME=your_java_home_path` to the first line of the `start_be.sh` startup script. +* Install Java UDF functions -* Install Java UDF + Install Java UDF functions + Because Java UDF functions are supported from version 1.2, you need to download the JAR package of Java UDF functions from the official website and put them in the lib directory of BE, otherwise it may fail to start. - - Because Java UDF is supported since version 1.2, you need to download the JAR package of Java UDF from the official website and put them under the lib directory of BE, otherwise it may fail to start. - -* Add all BE nodes to FE +* Add all BE nodes in FE - BE nodes need to be added in FE before they can join the cluster. You can use mysql-client ([Download MySQL 5.7](https://dev.mysql.com/downloads/mysql/5.7.html)) to connect to FE: + BE nodes need to be added in FE before they can join the cluster. You can use mysql-client ([Download MySQL 5.7](https://dev.mysql.com/downloads/mysql/5.7.html)) to connect to FE: - `./mysql-client -h fe_host -P query_port -uroot` + `./mysql-client -h fe_host -P query_port -uroot` - `fe_host` is the node IP where FE is located; `query_port` is in fe/conf/fe.conf; the root account is used by default and no password is required in login. + Among them, fe_host is the ip of the node where FE is located; query_port is in fe/conf/fe.conf; the root account is used by default, and there is no password to log in. - After login, execute the following command to add all the BE host and heartbeat service port: + Once logged in, execute the following command to add each BE: - `ALTER SYSTEM ADD BACKEND "be_host:heartbeat_service_port";` + `ALTER SYSTEM ADD BACKEND "be_host:heartbeat-service_port";` - `be_host` is the node IP where BE is located; `heartbeat_service_port` is in be/conf/be.conf. + Where be_host is the node ip where BE is located; heartbeat_service_port is in be/conf/be.conf. * Start BE - `bin/start_be.sh --daemon` + `bin/start_be.sh --daemon` - The BE process will start and go into the background for execution. Logs are stored in be/log/directory by default. If startup fails, you can view error messages by checking out be/log/be.log or be/log/be.out. + The BE process will start and enter the background execution. Logs are stored in the be/log/ directory by default. If the startup fails, you can view the error message by viewing be/log/be.log or be/log/be.out. * View BE status - Connect to FE using mysql-client and execute `SHOW PROC '/backends'; ` to view BE operation status. If everything goes well, the `isAlive`column should be `true`. + Use mysql-client to connect to FE, and execute `SHOW PROC '/backends';` to check the running status of BE. If all is well, the `isAlive` column should be `true`. #### (Optional) FS_Broker Deployment diff --git a/docs/en/docs/lakehouse/multi-catalog/faq.md b/docs/en/docs/lakehouse/faq.md similarity index 100% rename from docs/en/docs/lakehouse/multi-catalog/faq.md rename to docs/en/docs/lakehouse/faq.md diff --git a/docs/en/docs/lakehouse/fs_benchmark_tool.md b/docs/en/docs/lakehouse/fs_benchmark_tool.md new file mode 100644 index 000000000000000..9c3ad4c77ea4082 --- /dev/null +++ b/docs/en/docs/lakehouse/fs_benchmark_tool.md @@ -0,0 +1,233 @@ +--- +{ + "title": "File system benchmark tools", + "language": "en" +} +--- + + + + + +# Introduction + +`fs_benchmark_tool` can be used to test the basic service performance of remote storage systems including hdfs and object storage, such as read and write performance. This tool is mainly used to analyze or troubleshoot the performance problems of remote storage systems. + +# Compile and install + +`fs_benchmark_tool` is part of the `BE` code and does not compile by default. To compile, execute the following command: + +``` +cd doris +BUILD_FS_BENCHMARK=ON ./build.sh --be +``` +After compilation, the following contents will be generated in the `output/be/` directory: +``` +bin/run-fs-benchmark.sh +lib/fs_benchmark_tool +``` +> Note that `fs_benchmark_tool` it needs to be used in the BE running environment directory, because it depends on the BE-related jar package, environment variables, etc. + +# Use + +Command format: + +```shell +sh run-fs-benchmark.sh \ + --conf= configuration file \ + --fs_type= file system \ + --operation= operations on the file system \ + --file_size= file size \ + --threads= the number of threads \ + --iterations= the number of iterations +``` + +## Parameter parsing + + `--conf` Required parameter + + +Configuration file corresponding to the operation file. It is mainly used to add the relevant connection information of the remote storage system. See examples below. + +If you want to connect `hdfs`, please put the `hdfs-site.xml` `core-site.xml` file in the `be/conf` directory. + +In addition to the connection information, there are the following additional parameters: + +- `file_size`: Specifies the size of the file to read or write. + +- `buffer_size`: The block size of the file read by one read operation. + +- `base_dir`: Specifies the base path to read or write to the file. + +`--fs_type` Required parameter + +The type of file system on which the operation is required. Currently supported `hdfs`,`s3`. + +`--operation` Required parameter + +Specifies the type of operation + +- `create_write`: Each thread creates a file named `test_${current thread number}` in the `base_dir(set in conf file)` directory and writes to the file with a write size `file_size` of. + +- `open_read`: On `create_write` the basis of the created file, each thread reads the file with the name of `test_${current thread number}` and the read size of `file_size`. + +- `exists`: Each thread queries whether a file with `test_${current thread number}` filename exists. + +- `rename`: On `create_write` the basis of the created file, each thread changes the `test_${current thread number}` filename to `test_${current thread number}_new`. + +- `single_read`: Read `file_path(set in conf file)` file, read size is `file_size`. + +- `list`: Get `base_dir(set in conf file)` the list of files in the directory. + +`--file_size` + +The file size of the operation, in bytes. + +- `create_write`: Default is 10 MB. + +- `open_read`: Default is 10 MB. + +- `single_read`: The default is 0, that is, the full file is read. + +`--threads` + +The number of threads for the operation. The default number is 1. + +`--iterations` + +The number of iterations ( The number of times the function was executed ) per thread. The default number is 1. + +## Result analysis + +Except for `rename` the operation, the other operations are repeated three times, and the average value, the median value, the standard deviation, and the like are calculated. +``` +-------------------------------------------------------------------------------------------------------------------------------- +Benchmark Time CPU Iterations UserCounters... +-------------------------------------------------------------------------------------------------------------------------------- +HdfsReadBenchmark/iterations:1/repeats:3/manual_time/threads:1 13642 ms 2433 ms 1 OpenReaderTime(S)=4.80734 ReadRate(B/S)=101.104M/s ReadTime(S)=13.642 ReadTotal(B)=1.37926G +HdfsReadBenchmark/iterations:1/repeats:3/manual_time/threads:1 3918 ms 1711 ms 1 OpenReaderTime(S)=22.041u ReadRate(B/S)=352.011M/s ReadTime(S)=3.91824 ReadTotal(B)=1.37926G +HdfsReadBenchmark/iterations:1/repeats:3/manual_time/threads:1 3685 ms 1697 ms 1 OpenReaderTime(S)=35.837u ReadRate(B/S)=374.313M/s ReadTime(S)=3.68479 ReadTotal(B)=1.37926G +HdfsReadBenchmark/iterations:1/repeats:3/manual_time/threads:1_mean 7082 ms 1947 ms 3 OpenReaderTime(S)=1.60247 ReadRate(B/S)=275.809M/s ReadTime(S)=7.08166 ReadTotal(B)=1.37926G +HdfsReadBenchmark/iterations:1/repeats:3/manual_time/threads:1_median 3918 ms 1711 ms 3 OpenReaderTime(S)=35.837u ReadRate(B/S)=352.011M/s ReadTime(S)=3.91824 ReadTotal(B)=1.37926G +HdfsReadBenchmark/iterations:1/repeats:3/manual_time/threads:1_stddev 5683 ms 421 ms 3 OpenReaderTime(S)=2.7755 ReadRate(B/S)=151.709M/s ReadTime(S)=5.68258 ReadTotal(B)=0 +HdfsReadBenchmark/iterations:1/repeats:3/manual_time/threads:1_cv 80.24 % 21.64 % 3 OpenReaderTime(S)=173.20% ReadRate(B/S)=55.01% ReadTime(S)=80.24% ReadTotal(B)=0.00% +HdfsReadBenchmark/iterations:1/repeats:3/manual_time/threads:1_max 13642 ms 2433 ms 3 OpenReaderTime(S)=4.80734 ReadRate(B/S)=374.313M/s ReadTime(S)=13.642 ReadTotal(B)=1.37926G +HdfsReadBenchmark/iterations:1/repeats:3/manual_time/threads:1_min 3685 ms 1697 ms 3 OpenReaderTime(S)=22.041u ReadRate(B/S)=101.104M/s ReadTime(S)=3.68479 ReadTotal(B)=1.37926G +``` + +Focus on the first three lines, the result of three repeated executions of the code. The first time involves some operations such as connection initialization, so it will take a long time. The latter two times usually represent normal performance. + +Focus `UserCounters` on information in: +- `OpenReaderTime`: Time to open the file. +- `ReadRate`: read rate. The overall throughput is recorded here. If it is multithreaded, it can be divided by the number of threads, which represents the average rate per thread. +- `ReadTime`: Read time consuming. What is recorded here is the accumulated time of multiple threads. Divided by the number of threads, it represents the average time spent per thread. +- `ReadTotal`: Total amount read. What is recorded here is the accumulated value of multiple threads. Divided by the number of threads, this represents the average reads per thread. +- `WriteRate`: Same as `ReadRate`. Represents the write rate. +- `WriteTime`: Same as `ReadTime`. Represents time to write. +- `WriteTotal`: Same as `ReadTotal`. Represents the total amount written. +- `ListCost/RenameCost/ExistsCost`: A single operation of the corresponding operation takes time. + +# Examples + +## HDFS + +Command: +``` +sh run-fs-benchmark.sh \ + --conf=hdfs.conf \ + --fs_type=hdfs \ + --operation=create_write \ + --file_size=1024000 \ + --threads=3 \ + --iterations=5 +``` +Using `hdfs.conf` the configuration file,`create_write` operate on the `hdfs` file system , using three threads, write 1MB per operation, and iterate 5 times. + + `hdfs.conf` Profile: +``` +fs.defaultFS=hdfs://HDFS8000871 +hadoop.username=hadoop +dfs.nameservices=HDFS8000871 +dfs.ha.namenodes.HDFS8000871=nn1,nn2 +dfs.namenode.rpc-address.HDFS8000871.nn1=102.22.10.56:4007 +dfs.namenode.rpc-address.HDFS8000871.nn2=102.22.10.57:4007 +dfs.client.failover.proxy.provider.HDFS8000871=org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider +base_dir=hdfs://HDFS8000871/benchmarks/TestDFSIO/io_data/ +``` +Operation result: +``` +--------------------------------------------------------------------------------------------------------------------------------------- +Benchmark Time CPU Iterations UserCounters... +--------------------------------------------------------------------------------------------------------------------------------------- +HdfsCreateWriteBenchmark/iterations:5/repeats:3/manual_time/threads:3 61.7 ms 38.7 ms 15 WriteRate(B/S)=3.31902M/s WriteTime(S)=0.387954 WriteTotal(B)=3.072M +HdfsCreateWriteBenchmark/iterations:5/repeats:3/manual_time/threads:3 49.6 ms 3.09 ms 15 WriteRate(B/S)=4.12967M/s WriteTime(S)=0.427992 WriteTotal(B)=3.072M +HdfsCreateWriteBenchmark/iterations:5/repeats:3/manual_time/threads:3 45.2 ms 2.72 ms 15 WriteRate(B/S)=4.53148M/s WriteTime(S)=0.362854 WriteTotal(B)=3.072M +HdfsCreateWriteBenchmark/iterations:5/repeats:3/manual_time/threads:3_mean 52.2 ms 14.8 ms 3 WriteRate(B/S)=3.99339M/s WriteTime(S)=0.392933 WriteTotal(B)=3.072M +HdfsCreateWriteBenchmark/iterations:5/repeats:3/manual_time/threads:3_median 49.6 ms 3.09 ms 3 WriteRate(B/S)=4.12967M/s WriteTime(S)=0.387954 WriteTotal(B)=3.072M +HdfsCreateWriteBenchmark/iterations:5/repeats:3/manual_time/threads:3_stddev 8.55 ms 20.7 ms 3 WriteRate(B/S)=617.61k/s WriteTime(S)=0.0328536 WriteTotal(B)=0 +HdfsCreateWriteBenchmark/iterations:5/repeats:3/manual_time/threads:3_cv 16.39 % 139.34 % 3 WriteRate(B/S)=15.47% WriteTime(S)=8.36% WriteTotal(B)=0.00% +HdfsCreateWriteBenchmark/iterations:5/repeats:3/manual_time/threads:3_max 61.7 ms 38.7 ms 3 WriteRate(B/S)=4.53148M/s WriteTime(S)=0.427992 WriteTotal(B)=3.072M +HdfsCreateWriteBenchmark/iterations:5/repeats:3/manual_time/threads:3_min 45.2 ms 2.72 ms 3 WriteRate(B/S)=3.31902M/s WriteTime(S)=0.362854 WriteTotal(B)=3.072M +HDFS 上生成的文件： +[hadoop@172 ~]$ hadoop fs -ls -h /benchmarks/TestDFSIO/io_data/ +Found 3 items +-rw-r--r-- 3 hadoop supergroup 100 2023-06-27 11:55 /benchmarks/TestDFSIO/io_data/test_0 +-rw-r--r-- 3 hadoop supergroup 100 2023-06-27 11:55 /benchmarks/TestDFSIO/io_data/test_1 +-rw-r--r-- 3 hadoop supergroup 100 2023-06-27 11:55 /benchmarks/TestDFSIO/io_data/test_2 +``` + +## Object storage + +Command: +``` +sh bin/run-fs-benchmark.sh \ + --conf=s3.conf \ + --fs_type=s3 \ + --operation=single_read \ + --threads=1 \ + --iterations=1 +``` + +Using `s3.conf` the configuration file, operate on the `s3` file system `single_read`, using 1 thread, with 1 iteration. + + `s3.conf` Profile: +``` +AWS_ACCESS_KEY=ak +AWS_SECRET_KEY=sk +AWS_ENDPOINT=cos.ap-beijing.myqcloud.com +AWS_REGION=ap-beijing +file_path=s3://bucket-123/test_data/parquet/000016_0 +``` +Operation result: +``` +------------------------------------------------------------------------------------------------------------------------------ +Benchmark Time CPU Iterations UserCounters... +------------------------------------------------------------------------------------------------------------------------------ +S3ReadBenchmark/iterations:1/repeats:3/manual_time/threads:1 7534 ms 140 ms 1 ReadRate(B/S)=11.9109M/s ReadTime(S)=7.53353 ReadTotal(B)=89.7314M +S3ReadBenchmark/iterations:1/repeats:3/manual_time/threads:1 5988 ms 118 ms 1 ReadRate(B/S)=14.985M/s ReadTime(S)=5.98808 ReadTotal(B)=89.7314M +S3ReadBenchmark/iterations:1/repeats:3/manual_time/threads:1 6060 ms 124 ms 1 ReadRate(B/S)=14.8081M/s ReadTime(S)=6.05961 ReadTotal(B)=89.7314M +S3ReadBenchmark/iterations:1/repeats:3/manual_time/threads:1_mean 6527 ms 127 ms 3 ReadRate(B/S)=13.9014M/s ReadTime(S)=6.52707 ReadTotal(B)=89.7314M +S3ReadBenchmark/iterations:1/repeats:3/manual_time/threads:1_median 6060 ms 124 ms 3 ReadRate(B/S)=14.8081M/s ReadTime(S)=6.05961 ReadTotal(B)=89.7314M +S3ReadBenchmark/iterations:1/repeats:3/manual_time/threads:1_stddev 872 ms 11.4 ms 3 ReadRate(B/S)=1.72602M/s ReadTime(S)=0.87235 ReadTotal(B)=0 +S3ReadBenchmark/iterations:1/repeats:3/manual_time/threads:1_cv 13.37 % 8.94 % 3 ReadRate(B/S)=12.42% ReadTime(S)=13.37% ReadTotal(B)=0.00% +S3ReadBenchmark/iterations:1/repeats:3/manual_time/threads:1_max 7534 ms 140 ms 3 ReadRate(B/S)=14.985M/s ReadTime(S)=7.53353 ReadTotal(B)=89.7314M +S3ReadBenchmark/iterations:1/repeats:3/manual_time/threads:1_min 5988 ms 118 ms 3 ReadRate(B/S)=11.9109M/s ReadTime(S)=5.98808 ReadTotal(B)=89.7314M +``` + diff --git a/docs/en/docs/lakehouse/multi-catalog/jdbc.md b/docs/en/docs/lakehouse/multi-catalog/jdbc.md index 9e1d80492850d2a..7be1ae455788c90 100644 --- a/docs/en/docs/lakehouse/multi-catalog/jdbc.md +++ b/docs/en/docs/lakehouse/multi-catalog/jdbc.md @@ -675,4 +675,16 @@ When Doris connects to OceanBase, it will automatically recognize that OceanBase unable to find valid certification path to requested target". ClientConnectionId:a92f3817-e8e6-4311-bc21-7c66 ``` - In the create Catalog `jdbc_url` the JDBC connection string finally increase `encrypt=false`, such as `"jdbc_url" = "jdbc:sqlserver://127.0.0.1:1433;DataBaseName=doris_test;encrypt=false"` \ No newline at end of file + In the create Catalog `jdbc_url` the JDBC connection string finally increase `encrypt=false`, such as `"jdbc_url" = "jdbc:sqlserver://127.0.0.1:1433;DataBaseName=doris_test;encrypt=false"` + +11. Error encountered when reading MySQL datetime type + + ``` + ERROR 1105 (HY000): errCode = 2, detailMessage = (10.16.10.6)[INTERNAL_ERROR]UdfRuntimeException: get next block failed: + CAUSED BY: SQLException: Zero date value prohibited + CAUSED BY: DataReadException: Zero date value prohibited + ``` + + This happens because JDBC can't handle the datetime format 0000-00-00 00:00:00. + To address this, append zeroDateTimeBehavior=convertToNull to the jdbc_url when creating the Catalog, e.g., "jdbc_url" = "jdbc:mysql://127.0.0.1:3306/test?zeroDateTimeBehavior=convertToNull". + In this case, JDBC will convert 0000-00-00 00:00:00 to null, and then Doris will handle the DateTime column as a nullable type, allowing for successful reading. \ No newline at end of file diff --git a/docs/en/docs/lakehouse/multi-catalog/multi-catalog.md b/docs/en/docs/lakehouse/multi-catalog/multi-catalog.md index d9fc1ba480d957f..e2ff1c26ee1aef5 100644 --- a/docs/en/docs/lakehouse/multi-catalog/multi-catalog.md +++ b/docs/en/docs/lakehouse/multi-catalog/multi-catalog.md @@ -1,6 +1,6 @@ --- { - "title": "Multi Catalog", + "title": "Overview", "language": "en" } --- @@ -25,7 +25,7 @@ under the License. --> -# Multi Catalog +# Overview Multi-Catalog is designed to make it easier to connect to external data catalogs to enhance Doris's data lake analysis and federated data query capabilities. diff --git a/docs/en/docs/query-acceleration/join-optimization/doris-join-optimization.md b/docs/en/docs/query-acceleration/join-optimization/doris-join-optimization.md index 057975f7a78f110..2c3d8266d3fa2f9 100644 --- a/docs/en/docs/query-acceleration/join-optimization/doris-join-optimization.md +++ b/docs/en/docs/query-acceleration/join-optimization/doris-join-optimization.md @@ -48,7 +48,7 @@ As a distributed MPP database, data shuffle needs to be performed during the Joi When Hash Join is performed, the corresponding Hash value can be calculated through the Join column, and Hash bucketing can be performed. - Its network overhead is: T(R) + T(N), but it can only support Hash Join, because it also calculates buckets according to the conditions of Join. + Its network overhead is: T(S) + T(R), but it can only support Hash Join, because it also calculates buckets according to the conditions of Join. ![image-20220523151902368](/images/join/image-20220523151902368.png) diff --git a/docs/en/docs/query-acceleration/statistics.md b/docs/en/docs/query-acceleration/statistics.md index e769177753b15cb..6fb9de9c99642fa 100644 --- a/docs/en/docs/query-acceleration/statistics.md +++ b/docs/en/docs/query-acceleration/statistics.md @@ -609,11 +609,12 @@ mysql> SHOW TABLE STATS stats_test.example_tbl PARTITION (p_201701); The syntax is as follows: ```SQL -SHOW COLUMN STATS table_name [ (column_name [, ...]) ] [ PARTITION (partition_name) ]; +SHOW COLUMN [cached] STATS table_name [ (column_name [, ...]) ] [ PARTITION (partition_name) ]; ``` Explanation: +- cached: Cached means to show statistics in current FE memory cache. - Table_name: The target table for collecting statistics. It can be a `db_name.table_name` form. - Column_name: Specified destination column. `table_name` Must be a column that exists in. Multiple column names are separated by commas. - Partition_name: The specified target partition `table_name` must exist in. Only one partition can be specified. diff --git a/docs/en/docs/releasenotes/release-1.2.0.md b/docs/en/docs/releasenotes/release-1.2.0.md index b0a7ad4df9f4a07..2529ce7e58aa254 100644 --- a/docs/en/docs/releasenotes/release-1.2.0.md +++ b/docs/en/docs/releasenotes/release-1.2.0.md @@ -115,7 +115,7 @@ When creating a table, set `"light_schema_change"="true"` in properties. 2. When datev2 and datetimev2 are calculated with the original date and datetime (for example, equivalent connection), the original type will be cast into a new type for calculation 3. The example is in the documentation - Documentation: https://doris.apache.org/docs/dev/sql-manual/sql-reference/Data-Types/DATEV2 + Documentation: https://doris.apache.org/docs/1.2/sql-manual/sql-reference/Data-Types/DATEV2 ## More diff --git a/docs/en/docs/sql-manual/sql-functions/combinators/merge.md b/docs/en/docs/sql-manual/sql-functions/combinators/merge.md new file mode 100644 index 000000000000000..6f9ae643b5ca416 --- /dev/null +++ b/docs/en/docs/sql-manual/sql-functions/combinators/merge.md @@ -0,0 +1,50 @@ +--- +{ + "title": "MERGE", + "language": "en" +} +--- + + + +## MERGE + + + + + +### description +#### Syntax + +`AGGREGATE_FUNCTION_MERGE(agg_state)` +The aggregated intermediate results are aggregated and calculated to obtain the actual result. +The type of the result is consistent with `AGGREGATE_FUNCTION`. + +### example +``` +mysql [test]>select avg_merge(avg_state(1)) from d_table; ++-------------------------+ +| avg_merge(avg_state(1)) | ++-------------------------+ +| 1 | ++-------------------------+ +``` +### keywords +AGG_STATE, MERGE diff --git a/docs/en/docs/sql-manual/sql-functions/combinators/state.md b/docs/en/docs/sql-manual/sql-functions/combinators/state.md new file mode 100644 index 000000000000000..6202da018eec444 --- /dev/null +++ b/docs/en/docs/sql-manual/sql-functions/combinators/state.md @@ -0,0 +1,50 @@ +--- +{ + "title": "STATE", + "language": "en" +} +--- + + + +## STATE + + + + + +### description +#### Syntax + +`AGGREGATE_FUNCTION_STATE(arg...)` +Returns the intermediate result of the aggregation function, which can be used for subsequent aggregation or to obtain the actual calculation result through the merge combiner, or can be directly written into the agg_state type table and saved. +The type of the result is agg_state, and the function signature in agg_state is `AGGREGATE_FUNCTION(arg...)`. + +### example +``` +mysql [test]>select avg_merge(t) from (select avg_union(avg_state(1)) as t from d_table group by k1)p; ++----------------+ +| avg_merge(`t`) | ++----------------+ +| 1 | ++----------------+ +``` +### keywords +AGG_STATE,STATE diff --git a/docs/en/docs/sql-manual/sql-functions/combinators/union.md b/docs/en/docs/sql-manual/sql-functions/combinators/union.md new file mode 100644 index 000000000000000..e3c249ca1a31a89 --- /dev/null +++ b/docs/en/docs/sql-manual/sql-functions/combinators/union.md @@ -0,0 +1,50 @@ +--- +{ + "title": "UNION", + "language": "en" +} +--- + + + +## UNION + + + + + +### description +#### Syntax + +`AGGREGATE_FUNCTION_UNION(agg_state)` +Aggregate multiple aggregation intermediate results into one. +The type of the result is agg_state, and the function signature is consistent with the input parameter. + +### example +``` +mysql [test]>select avg_merge(t) from (select avg_union(avg_state(1)) as t from d_table group by k1)p; ++----------------+ +| avg_merge(`t`) | ++----------------+ +| 1 | ++----------------+ +``` +### keywords +AGG_STATE, UNION diff --git a/docs/en/docs/sql-manual/sql-functions/table-functions/hdfs.md b/docs/en/docs/sql-manual/sql-functions/table-functions/hdfs.md index 7e259bc639d16b6..d741fb70019b189 100644 --- a/docs/en/docs/sql-manual/sql-functions/table-functions/hdfs.md +++ b/docs/en/docs/sql-manual/sql-functions/table-functions/hdfs.md @@ -61,6 +61,12 @@ Related parameters for accessing hdfs: - `dfs.client.read.shortcircuit`: (optional) - `dfs.domain.socket.path`: (optional) +Related parameters for accessing HDFS in HA mode: +- `dfs.nameservices`: (optional) +- `dfs.ha.namenodes.your-nameservices`: (optional) +- `dfs.namenode.rpc-address.your-nameservices.your-namenode`: (optional) +- `dfs.client.failover.proxy.provider.your-nameservices`: (optional) + File format parameters: - `format`: (required) Currently support `csv/csv_with_names/csv_with_names_and_types/json/parquet/orc` @@ -103,6 +109,29 @@ MySQL [(none)]> select * from hdfs( +------+---------+------+ ``` +Read and access csv format files on hdfs storage in HA mode. +```sql +MySQL [(none)]> select * from hdfs( + "uri" = "hdfs://127.0.0.1:842/user/doris/csv_format_test/student.csv", + "fs.defaultFS" = "hdfs://127.0.0.1:8424", + "hadoop.username" = "doris", + "format" = "csv", + "dfs.nameservices" = "my_hdfs", + "dfs.ha.namenodes.my_hdfs" = "nn1,nn2", + "dfs.namenode.rpc-address.my_hdfs.nn1" = "nanmenode01:8020", + "dfs.namenode.rpc-address.my_hdfs.nn2" = "nanmenode02:8020", + "dfs.client.failover.proxy.provider.my_hdfs" = "org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider"); ++------+---------+------+ +| c1 | c2 | c3 | ++------+---------+------+ +| 1 | alice | 18 | +| 2 | bob | 20 | +| 3 | jack | 24 | +| 4 | jackson | 19 | +| 5 | liming | 18 | ++------+---------+------+ +``` + Can be used with `desc function` : ```sql diff --git a/docs/en/docs/sql-manual/sql-functions/table-functions/s3.md b/docs/en/docs/sql-manual/sql-functions/table-functions/s3.md index bb937a9624f01a8..574381402811448 100644 --- a/docs/en/docs/sql-manual/sql-functions/table-functions/s3.md +++ b/docs/en/docs/sql-manual/sql-functions/table-functions/s3.md @@ -168,7 +168,7 @@ select * from s3( ``` -**csv foramt** +**csv format** `csv` format: Read the file on S3 and process it as a csv file, read the first line in the file to parse out the table schema. The number of columns in the first line of the file `n` will be used as the number of columns in the table schema, and the column names of the table schema will be automatically named `c1, c2, ..., cn`, and the column type is set to `String` , for example: @@ -214,7 +214,7 @@ MySQL [(none)]> Desc function s3("uri" = "http://127.0.0.1:9312/test2/student1.c +-------+------+------+-------+---------+-------+ ``` -**csv_with_names foramt** +**csv_with_names format** `csv_with_names` format: The first line of the file is used as the number and name of the columns of the table schema, and the column type is set to `String`, for example: The file content of student_with_names.csv: @@ -258,9 +258,9 @@ MySQL [(none)]> Desc function s3("uri" = "http://127.0.0.1:9312/test2/student_wi +-------+------+------+-------+---------+-------+ ``` -**csv_with_names_and_types foramt** +**csv_with_names_and_types format** -`csv_with_names_and_types` foramt: Currently, it does not support parsing the column type from a csv file. When using this format, S3 tvf will parse the first line of the file as the number and name of the columns of the table schema, and set the column type to String. Meanwhile, the second line of the file is ignored. +`csv_with_names_and_types` format: Currently, it does not support parsing the column type from a csv file. When using this format, S3 tvf will parse the first line of the file as the number and name of the columns of the table schema, and set the column type to String. Meanwhile, the second line of the file is ignored. The file content of student_with_names_and_types.csv: @@ -304,7 +304,7 @@ MySQL [(none)]> Desc function s3("uri" = "http://127.0.0.1:9312/test2/student_wi +-------+------+------+-------+---------+-------+ ``` -**json foramt** +**json format** `json` format: The json format involves many optional parameters, and the meaning of each parameter can be referred to: [Json Load](../../../data-operate/import/import-way/load-json-format.md). When S3 tvf queries the json format file, it locates a json object according to the `json_root` and `jsonpaths` parameters, and uses the `key` in the object as the column name of the table schema, and sets the column type to String. For example: @@ -352,7 +352,7 @@ MySQL [(none)]> select * from s3( +------+------+ ``` -**parquet foramt** +**parquet format** `parquet` format: S3 tvf supports parsing the column names and column types of the table schema from the parquet file. Example: @@ -396,7 +396,7 @@ MySQL [(none)]> desc function s3( +---------------+--------------+------+-------+---------+-------+ ``` -**orc foramt** +**orc format** `orc` format: Same as `parquet` format, set `format` parameter to orc. diff --git a/docs/en/docs/sql-manual/sql-functions/table-functions/workload-group.md b/docs/en/docs/sql-manual/sql-functions/table-functions/workload-group.md index 0e222e7a0a589d1..2371c7c2b2aab00 100644 --- a/docs/en/docs/sql-manual/sql-functions/table-functions/workload-group.md +++ b/docs/en/docs/sql-manual/sql-functions/table-functions/workload-group.md @@ -36,7 +36,7 @@ workload_groups ### description -Table-Value-Function, generate a temporary table named `workload_groups`. This tvf is used to view informations about current workload groups. +Table-Value-Function, generate a temporary table named `workload_groups`. This tvf is used to view information about workload groups for which current user has permission. This function is used in `FROM` clauses. diff --git a/docs/en/docs/sql-manual/sql-reference/Data-Definition-Statements/Create/CREATE-WORKLOAD-GROUP.md b/docs/en/docs/sql-manual/sql-reference/Data-Definition-Statements/Create/CREATE-WORKLOAD-GROUP.md index eda88cad5d6448b..186d073a612a6af 100644 --- a/docs/en/docs/sql-manual/sql-reference/Data-Definition-Statements/Create/CREATE-WORKLOAD-GROUP.md +++ b/docs/en/docs/sql-manual/sql-reference/Data-Definition-Statements/Create/CREATE-WORKLOAD-GROUP.md @@ -51,7 +51,7 @@ Properties supported by property_list: * cpu_share: Required, used to set how much cpu time the workload group can acquire, which can achieve soft isolation of cpu resources. cpu_share is a relative value indicating the weight of cpu resources available to the running workload group. For example, if a user creates 3 workload groups rg-a, rg-b and rg-c with cpu_share of 10, 30 and 40 respectively, and at a certain moment rg-a and rg-b are running tasks while rg-c has no tasks, then rg-a can get 25% (10 / (10 + 30)) of the cpu resources while workload group rg-b can get 75% of the cpu resources. If the system has only one workload group running, it gets all the cpu resources regardless of the value of its cpu_share. -* memory_limit: Required, set the percentage of be memory that can be used by the workload group. The absolute value of the workload group memory limit is: physical_memory * mem_limit * memory_limit, where mem_limit is a be configuration item. The total memory_limit of all workload groups in the system must not exceed 100%. Workload groups are guaranteed to use the memory_limit for the tasks in the group in most cases. When the workload group memory usage exceeds this limit, tasks in the group with larger memory usage may be canceled to release the excess memory, refer to enable_memory_overcommit. +* memory_limit: Required, set the percentage of be memory that can be used by the workload group. The absolute value of the workload group memory limit is: `physical_memory * mem_limit * memory_limit`, where mem_limit is a be configuration item. The total memory_limit of all workload groups in the system must not exceed 100%. Workload groups are guaranteed to use the memory_limit for the tasks in the group in most cases. When the workload group memory usage exceeds this limit, tasks in the group with larger memory usage may be canceled to release the excess memory, refer to enable_memory_overcommit. * enable_memory_overcommit: Optional, enable soft memory isolation for the workload group, default is false. if set to false, the workload group is hard memory isolated and the tasks with the largest memory usage will be canceled immediately after the workload group memory usage exceeds the limit to release the excess memory. if set to true, the workload group is hard memory isolated and the tasks with the largest memory usage will be canceled immediately after the workload group memory usage exceeds the limit to release the excess memory. if set to true, the workload group is softly isolated, if the system has free memory resources, the workload group can continue to use system memory after exceeding the memory_limit limit, and when the total system memory is tight, it will cancel several tasks in the group with the largest memory occupation, releasing part of the excess memory to relieve the system memory pressure. It is recommended that when this configuration is enabled for a workload group, the total memory_limit of all workload groups should be less than 100%, and the remaining portion should be used for workload group memory overcommit. diff --git a/docs/en/docs/sql-manual/sql-reference/Data-Manipulation-Statements/Load/STREAM-LOAD.md b/docs/en/docs/sql-manual/sql-reference/Data-Manipulation-Statements/Load/STREAM-LOAD.md index 9170c9eae3a8716..5087ec02ee7d942 100644 --- a/docs/en/docs/sql-manual/sql-reference/Data-Manipulation-Statements/Load/STREAM-LOAD.md +++ b/docs/en/docs/sql-manual/sql-reference/Data-Manipulation-Statements/Load/STREAM-LOAD.md @@ -127,17 +127,18 @@ Parameter introduction: 16. merge_type: The merge type of data, which supports three types: APPEND, DELETE, and MERGE. Among them, APPEND is the default value, which means that this batch of data needs to be appended to the existing data, and DELETE means to delete all the data with the same key as this batch of data. Line, the MERGE semantics need to be used in conjunction with the delete condition, which means that the data that meets the delete condition is processed according to the DELETE semantics and the rest is processed according to the APPEND semantics, for example: `-H "merge_type: MERGE" -H "delete: flag=1"` 17. delete: Only meaningful under MERGE, indicating the deletion condition of the data - function_column.sequence_col: Only applicable to UNIQUE_KEYS. Under the same key column, ensure that the value column is REPLACEed according to the source_sequence column. The source_sequence can be a column in the data source or a column in the table structure. + +18. function_column.sequence_col: Only applicable to UNIQUE_KEYS. Under the same key column, ensure that the value column is REPLACEed according to the source_sequence column. The source_sequence can be a column in the data source or a column in the table structure. -18. fuzzy_parse: Boolean type, true means that json will be parsed with the schema of the first row. Enabling this option can improve the efficiency of json import, but requires that the order of the keys of all json objects is the same as the first row, the default is false, only use in json format +19. fuzzy_parse: Boolean type, true means that json will be parsed with the schema of the first row. Enabling this option can improve the efficiency of json import, but requires that the order of the keys of all json objects is the same as the first row, the default is false, only use in json format -19. num_as_string: Boolean type, true means that when parsing json data, the numeric type will be converted to a string, and then imported without losing precision. +20. num_as_string: Boolean type, true means that when parsing json data, the numeric type will be converted to a string, and then imported without losing precision. -20. read_json_by_line: Boolean type, true to support reading one json object per line, the default value is false. +21. read_json_by_line: Boolean type, true to support reading one json object per line, the default value is false. -21. send_batch_parallelism: Integer, used to set the parallelism of sending batch data. If the value of parallelism exceeds `max_send_batch_parallelism_per_job` in the BE configuration, the BE as a coordination point will use the value of `max_send_batch_parallelism_per_job`. +22. send_batch_parallelism: Integer, used to set the parallelism of sending batch data. If the value of parallelism exceeds `max_send_batch_parallelism_per_job` in the BE configuration, the BE as a coordination point will use the value of `max_send_batch_parallelism_per_job`. -22. hidden_columns: Specify hidden column when no `columns` in Headers，multi hidden column shoud be +23. hidden_columns: Specify hidden column when no `columns` in Headers，multi hidden column shoud be separated by commas. ``` @@ -145,45 +146,15 @@ separated by commas. The system will use the order specified by user. in case above, data should be ended with __DORIS_SEQUENCE_COL__. ``` -23. load_to_single_tablet: Boolean type, True means that one task can only load data to one tablet in the corresponding partition at a time. The default value is false. This parameter can only be set when loading data into the OLAP table with random partition. - - RETURN VALUES - After the import is complete, the related content of this import will be returned in Json format. Currently includes the following fields - Status: Import the last status. - Success: Indicates that the import is successful and the data is already visible; - Publish Timeout: Indicates that the import job has been successfully committed, but is not immediately visible for some reason. The user can consider the import to be successful and not have to retry the import - Label Already Exists: Indicates that the Label has been occupied by other jobs. It may be imported successfully or it may be being imported. - The user needs to determine the subsequent operation through the get label state command - Others: The import failed, the user can specify the Label to retry the job - Message: Detailed description of the import status. On failure, the specific failure reason is returned. - NumberTotalRows: The total number of rows read from the data stream - NumberLoadedRows: The number of data rows imported this time, only valid in Success - NumberFilteredRows: The number of rows filtered out by this import, that is, the number of rows with unqualified data quality - NumberUnselectedRows: This import, the number of rows filtered out by the where condition - LoadBytes: The size of the source file data imported this time - LoadTimeMs: The time taken for this import - BeginTxnTimeMs: The time it takes to request Fe to start a transaction, in milliseconds. - StreamLoadPutTimeMs: The time it takes to request Fe to obtain the execution plan for importing data, in milliseconds. - ReadDataTimeMs: Time spent reading data, in milliseconds. - WriteDataTimeMs: The time taken to perform the write data operation, in milliseconds. - CommitAndPublishTimeMs: The time it takes to submit a request to Fe and publish the transaction, in milliseconds. - ErrorURL: The specific content of the filtered data, only the first 1000 items are retained - -ERRORS: - Import error details can be viewed with the following statement: - ```` - SHOW LOAD WARNINGS ON 'url' - ```` - where url is the url given by ErrorURL. - -24. compress_type - - Specify compress type file. Only support compressed csv file now. Support gz, lzo, bz2, lz4, lzop, deflate. - -25. trim_double_quotes: Boolean type, The default value is false. True means that the outermost double quotes of each field in the csv file are trimmed. - -26. skip_lines: Integer type, the default value is 0. It will skip some lines in the head of csv file. It will be disabled when format is `csv_with_names` or `csv_with_names_and_types`. -27. comment: String type, the default value is "". +24. load_to_single_tablet: Boolean type, True means that one task can only load data to one tablet in the corresponding partition at a time. The default value is false. This parameter can only be set when loading data into the OLAP table with random partition. + +25. compress_type: Specify compress type file. Only support compressed csv file now. Support gz, lzo, bz2, lz4, lzop, deflate. + +26. trim_double_quotes: Boolean type, The default value is false. True means that the outermost double quotes of each field in the csv file are trimmed. + +27. skip_lines: Integer type, the default value is 0. It will skip some lines in the head of csv file. It will be disabled when format is `csv_with_names` or `csv_with_names_and_types`. + +28. comment: String type, the default value is "". ### Example @@ -390,52 +361,53 @@ ERRORS: } ```` - The field definitions are as follows: + The following main explanations are given for the Stream load import result parameters: + + + TxnId: The imported transaction ID. Users do not perceive. + + + Label: Import Label. User specified or automatically generated by the system. - - TxnId: Import transaction ID, which is automatically generated by the system and is globally unique. + + Status: Import completion status. - - Label: Import Label, if not specified, the system will generate a UUID. + "Success": Indicates successful import. - - Status: + "Publish Timeout": This state also indicates that the import has been completed, except that the data may be delayed and visible without retrying. - Import results. Has the following values: + "Label Already Exists": Label duplicate, need to be replaced Label. - - Success: Indicates that the import was successful and the data is already visible. - - Publish Timeout: This status also means that the import has completed, but the data may be visible with a delay. - - Label Already Exists: The Label is duplicated and needs to be replaced. - - Fail: Import failed. + "Fail": Import failed. - - ExistingJobStatus: + + ExistingJobStatus: The state of the load job corresponding to the existing Label. - The status of the import job corresponding to the existing Label. + This field is displayed only when the status is "Label Already Exists". The user can know the status of the load job corresponding to Label through this state. "RUNNING" means that the job is still executing, and "FINISHED" means that the job is successful. - This field is only displayed when the Status is "Label Already Exists". The user can know the status of the import job corresponding to the existing Label through this status. "RUNNING" means the job is still executing, "FINISHED" means the job was successful. + + Message: Import error messages. - - Message: Import error message. + + NumberTotalRows: Number of rows imported for total processing. - - NumberTotalRows: The total number of rows processed by the import. + + NumberLoadedRows: Number of rows successfully imported. - - NumberLoadedRows: The number of rows successfully imported. + + NumberFilteredRows: Number of rows that do not qualify for data quality. - - NumberFilteredRows: The number of rows with unqualified data quality. + + NumberUnselectedRows: Number of rows filtered by where condition. - - NumberUnselectedRows: The number of rows filtered by the where condition. + + LoadBytes: Number of bytes imported. - - LoadBytes: Number of bytes imported. + + LoadTimeMs: Import completion time. Unit milliseconds. - - LoadTimeMs: Import completion time. The unit is milliseconds. + + BeginTxnTimeMs: The time cost for RPC to Fe to begin a transaction, Unit milliseconds. - - BeginTxnTimeMs: The time it takes to request the FE to start a transaction, in milliseconds. + + StreamLoadPutTimeMs: The time cost for RPC to Fe to get a stream load plan, Unit milliseconds. - - StreamLoadPutTimeMs: The time taken to request the FE to obtain the execution plan for importing data, in milliseconds. + + ReadDataTimeMs: Read data time, Unit milliseconds. - - ReadDataTimeMs: Time spent reading data, in milliseconds. + + WriteDataTimeMs: Write data time, Unit milliseconds. - - WriteDataTimeMs: The time spent performing the write data operation, in milliseconds. + + CommitAndPublishTimeMs: The time cost for RPC to Fe to commit and publish a transaction, Unit milliseconds. - - CommitAndPublishTimeMs: The time it takes to submit a request to Fe and publish the transaction, in milliseconds. + + ErrorURL: If you have data quality problems, visit this URL to see specific error lines. - - ErrorURL: If there is a data quality problem, visit this URL to view the specific error line. + > Note: Since Stream load is a synchronous import mode, import information will not be recorded in Doris system. Users cannot see Stream load asynchronously by looking at import commands. You need to listen for the return value of the create import request to get the import result. 2. How to correctly submit the Stream Load job and process the returned results. diff --git a/docs/en/docs/sql-manual/sql-reference/Data-Manipulation-Statements/OUTFILE.md b/docs/en/docs/sql-manual/sql-reference/Data-Manipulation-Statements/OUTFILE.md index a92c6570cad140a..0b87f9bc9cecdc3 100644 --- a/docs/en/docs/sql-manual/sql-reference/Data-Manipulation-Statements/OUTFILE.md +++ b/docs/en/docs/sql-manual/sql-reference/Data-Manipulation-Statements/OUTFILE.md @@ -75,8 +75,8 @@ illustrate: The following properties are supported: File related properties - column_separator: column separator. Support mulit-bytes, such as: "\\x01", "abc" - line_delimiter: line delimiter. Support mulit-bytes, such as: "\\x01", "abc" + column_separator: column separator,is only for CSV format Support mulit-bytes, such as: "\\x01", "abc" + line_delimiter: line delimiter,is only for CSV format Support mulit-bytes, such as: "\\x01", "abc" max_file_size: the size limit of a single file, if the result exceeds this value, it will be cut into multiple files. delete_existing_files: default `false`. If it is specified as true, you will first delete all files specified in the directory specified by the file_path, and then export the data to the directory.For example: "file_path" = "/user/tmp", then delete all files and directory under "/user/"; "file_path" = "/user/tmp/", then delete all files and directory under "/user/tmp/" diff --git a/docs/en/docs/sql-manual/sql-reference/Data-Types/AGG_STATE.md b/docs/en/docs/sql-manual/sql-reference/Data-Types/AGG_STATE.md new file mode 100644 index 000000000000000..69f6f2bb9c88921 --- /dev/null +++ b/docs/en/docs/sql-manual/sql-reference/Data-Types/AGG_STATE.md @@ -0,0 +1,87 @@ +--- +{ + "title": "AGG_STATE", + "language": "en" +} +--- + + + +## AGG_STATE +### description + AGG_STATE cannot be used as a key column, and the signature of the aggregation function must be declared at the same time when creating the table. + User does not need to specify length and default value. The actual stored data size is related to the function implementation. + + AGG_STATE can only be used with [state](../../sql-functions/combinators/state.md) + /[merge](../../sql-functions/combinators/merge.md)/[union](../..//sql-functions/combinators/union.md) function combiner usage. + + It should be noted that the signature of the aggregation function is also part of the type, and agg_state with different signatures cannot be mixed. For example, if the signature of the table creation statement is `max_by(int,int)`, then `max_by(bigint,int)` or `group_concat(varchar)` cannot be inserted. + The nullable attribute here is also part of the signature. If you can confirm that you will not enter a null value, you can declare the parameter as not null, which can obtain a smaller storage size and reduce serialization/deserialization overhead. + +### example + +Create table example: +```sql + create table a_table( + k1 int null, + k2 agg_state max_by(int not null,int), + k3 agg_state group_concat(string) + ) + aggregate key (k1) + distributed BY hash(k1) buckets 3 + properties("replication_num" = "1"); +``` +Here k2 and k3 use max_by and group_concat as aggregation types respectively. + +Insert data example: +```sql + insert into a_table values(1,max_by_state(3,1),group_concat_state('a')); + insert into a_table values(1,max_by_state(2,2),group_concat_state('bb')); + insert into a_table values(2,max_by_state(1,3),group_concat_state('ccc')); +``` +For the agg_state column, the insert statement must use the [state](../../sql-functions/combinators/state.md) function to generate the corresponding agg_state data, where the functions and input parameter types must completely correspond to agg_state. + +Select data example: +```sql + mysql [test]>select k1,max_by_merge(k2),group_concat_merge(k3) from a_table group by k1 order by k1; + +------+--------------------+--------------------------+ + | k1 | max_by_merge(`k2`) | group_concat_merge(`k3`) | + +------+--------------------+--------------------------+ + | 1 | 2 | bb,a | + | 2 | 1 | ccc | + +------+--------------------+--------------------------+ +``` +If you need to get the actual result, you need to use the corresponding [merge](../../sql-functions/combinators/merge.md) function. + +```sql + mysql [test]>select max_by_merge(u2),group_concat_merge(u3) from ( + select k1,max_by_union(k2) as u2,group_concat_union(k3) u3 from a_table group by k1 order by k1 + ) t; + +--------------------+--------------------------+ + | max_by_merge(`u2`) | group_concat_merge(`u3`) | + +--------------------+--------------------------+ + | 1 | ccc,bb,a | + +--------------------+--------------------------+ +``` +If you want to aggregate only the agg_state without getting the actual result during the process, you can use the [union](../..//sql-functions/combinators/union.md) function. + +### keywords + + AGG_STATE diff --git a/docs/en/docs/sql-manual/sql-reference/Data-Types/JSON.md b/docs/en/docs/sql-manual/sql-reference/Data-Types/JSON.md index 1f722758b89d988..fdba861d3c96219 100644 --- a/docs/en/docs/sql-manual/sql-reference/Data-Types/JSON.md +++ b/docs/en/docs/sql-manual/sql-reference/Data-Types/JSON.md @@ -34,7 +34,7 @@ NOTICE: In version 1.2.x the data type name is JSONB. It's renamed to JSON to be ### description JSON (Binary) datatype. - Use binary JSON format for storage and json function to extract field. + Use binary JSON format for storage and json function to extract field.The maximum (default) support is 1048576 bytes (1MB), and the JSONB type is also limited by the be configuration `jsonb_type_length_soft_limit_bytes` ### note There are some advantanges for JSON over plain JSON STRING. diff --git a/docs/sidebars.json b/docs/sidebars.json index 87a7f7fe3e7b6f2..0b138025e8e6145 100644 --- a/docs/sidebars.json +++ b/docs/sidebars.json @@ -207,14 +207,15 @@ "lakehouse/multi-catalog/iceberg", "lakehouse/multi-catalog/hudi", "lakehouse/multi-catalog/paimon", - "lakehouse/multi-catalog/es", - "lakehouse/multi-catalog/jdbc", "lakehouse/multi-catalog/dlf", - "lakehouse/multi-catalog/faq" + "lakehouse/multi-catalog/es", + "lakehouse/multi-catalog/jdbc" ] }, "lakehouse/file", - "lakehouse/filecache" + "lakehouse/filecache", + "lakehouse/faq", + "lakehouse/fs_benchmark_tool" ] }, { @@ -239,7 +240,6 @@ "items": [ "ecosystem/udf/contribute-udf", "ecosystem/udf/remote-user-defined-function", - "ecosystem/udf/native-user-defined-function", "ecosystem/udf/java-user-defined-function" ] }, @@ -495,6 +495,15 @@ "sql-manual/sql-functions/struct-functions/struct_element" ] }, + { + "type": "category", + "label": "Combinators", + "items": [ + "sql-manual/sql-functions/combinators/state", + "sql-manual/sql-functions/combinators/merge", + "sql-manual/sql-functions/combinators/union" + ] + }, { "type": "category", "label": "Aggregate Functions", @@ -741,6 +750,24 @@ "type": "category", "label": "SQL Reference", "items": [ + { + "type": "category", + "label": "Cluster management", + "items": [ + "sql-manual/sql-reference/Cluster-Management-Statements/ALTER-SYSTEM-ADD-FOLLOWER", + "sql-manual/sql-reference/Cluster-Management-Statements/ALTER-SYSTEM-ADD-OBSERVER", + "sql-manual/sql-reference/Cluster-Management-Statements/ALTER-SYSTEM-ADD-BACKEND", + "sql-manual/sql-reference/Cluster-Management-Statements/ALTER-SYSTEM-ADD-BROKER", + "sql-manual/sql-reference/Cluster-Management-Statements/ALTER-SYSTEM-MODIFY-BACKEND", + "sql-manual/sql-reference/Cluster-Management-Statements/ALTER-SYSTEM-MODIFY-BROKER", + "sql-manual/sql-reference/Cluster-Management-Statements/ALTER-SYSTEM-DECOMMISSION-BACKEND", + "sql-manual/sql-reference/Cluster-Management-Statements/ALTER-SYSTEM-DROP-FOLLOWER", + "sql-manual/sql-reference/Cluster-Management-Statements/ALTER-SYSTEM-DROP-OBSERVER", + "sql-manual/sql-reference/Cluster-Management-Statements/ALTER-SYSTEM-DROP-BACKEND", + "sql-manual/sql-reference/Cluster-Management-Statements/ALTER-SYSTEM-DROP-BROKER", + "sql-manual/sql-reference/Cluster-Management-Statements/CANCEL-ALTER-SYSTEM" + ] + }, { "type": "category", "label": "Account Management", @@ -759,20 +786,55 @@ }, { "type": "category", - "label": "Cluster management", + "label": "Database Administration", "items": [ - "sql-manual/sql-reference/Cluster-Management-Statements/ALTER-SYSTEM-ADD-FOLLOWER", - "sql-manual/sql-reference/Cluster-Management-Statements/ALTER-SYSTEM-ADD-OBSERVER", - "sql-manual/sql-reference/Cluster-Management-Statements/ALTER-SYSTEM-ADD-BACKEND", - "sql-manual/sql-reference/Cluster-Management-Statements/ALTER-SYSTEM-ADD-BROKER", - "sql-manual/sql-reference/Cluster-Management-Statements/ALTER-SYSTEM-MODIFY-BACKEND", - "sql-manual/sql-reference/Cluster-Management-Statements/ALTER-SYSTEM-MODIFY-BROKER", - "sql-manual/sql-reference/Cluster-Management-Statements/ALTER-SYSTEM-DECOMMISSION-BACKEND", - "sql-manual/sql-reference/Cluster-Management-Statements/ALTER-SYSTEM-DROP-FOLLOWER", - "sql-manual/sql-reference/Cluster-Management-Statements/ALTER-SYSTEM-DROP-OBSERVER", - "sql-manual/sql-reference/Cluster-Management-Statements/ALTER-SYSTEM-DROP-BACKEND", - "sql-manual/sql-reference/Cluster-Management-Statements/ALTER-SYSTEM-DROP-BROKER", - "sql-manual/sql-reference/Cluster-Management-Statements/CANCEL-ALTER-SYSTEM" + "sql-manual/sql-reference/Database-Administration-Statements/ADMIN-SHOW-CONFIG", + "sql-manual/sql-reference/Database-Administration-Statements/ADMIN-SET-CONFIG", + "sql-manual/sql-reference/Database-Administration-Statements/SET-VARIABLE", + "sql-manual/sql-reference/Database-Administration-Statements/INSTALL-PLUGIN", + "sql-manual/sql-reference/Database-Administration-Statements/UNINSTALL-PLUGIN", + "sql-manual/sql-reference/Database-Administration-Statements/ADMIN-SET-REPLICA-STATUS", + "sql-manual/sql-reference/Database-Administration-Statements/ADMIN-SHOW-REPLICA-DISTRIBUTION", + "sql-manual/sql-reference/Database-Administration-Statements/ADMIN-SHOW-REPLICA-STATUS", + "sql-manual/sql-reference/Database-Administration-Statements/ADMIN-REPAIR-TABLE", + "sql-manual/sql-reference/Database-Administration-Statements/ADMIN-CANCEL-REPAIR", + "sql-manual/sql-reference/Database-Administration-Statements/ADMIN-CHECK-TABLET", + "sql-manual/sql-reference/Database-Administration-Statements/ADMIN-DIAGNOSE-TABLET", + "sql-manual/sql-reference/Database-Administration-Statements/ADMIN-COPY-TABLET", + "sql-manual/sql-reference/Database-Administration-Statements/ADMIN-SHOW-TABLET-STORAGE-FORMAT", + "sql-manual/sql-reference/Database-Administration-Statements/ADMIN-CLEAN-TRASH", + "sql-manual/sql-reference/Database-Administration-Statements/RECOVER", + "sql-manual/sql-reference/Database-Administration-Statements/KILL", + "sql-manual/sql-reference/Database-Administration-Statements/ADMIN-REBALANCE-DISK", + "sql-manual/sql-reference/Database-Administration-Statements/ADMIN-CANCEL-REBALANCE-DISK" + ] + }, + { + "type": "category", + "label": "Data Types", + "items": [ + "sql-manual/sql-reference/Data-Types/BOOLEAN", + "sql-manual/sql-reference/Data-Types/TINYINT", + "sql-manual/sql-reference/Data-Types/SMALLINT", + "sql-manual/sql-reference/Data-Types/INT", + "sql-manual/sql-reference/Data-Types/BIGINT", + "sql-manual/sql-reference/Data-Types/LARGEINT", + "sql-manual/sql-reference/Data-Types/FLOAT", + "sql-manual/sql-reference/Data-Types/DOUBLE", + "sql-manual/sql-reference/Data-Types/DECIMAL", + "sql-manual/sql-reference/Data-Types/DATE", + "sql-manual/sql-reference/Data-Types/DATETIME", + "sql-manual/sql-reference/Data-Types/CHAR", + "sql-manual/sql-reference/Data-Types/VARCHAR", + "sql-manual/sql-reference/Data-Types/STRING", + "sql-manual/sql-reference/Data-Types/HLL", + "sql-manual/sql-reference/Data-Types/BITMAP", + "sql-manual/sql-reference/Data-Types/QUANTILE_STATE", + "sql-manual/sql-reference/Data-Types/ARRAY", + "sql-manual/sql-reference/Data-Types/MAP", + "sql-manual/sql-reference/Data-Types/STRUCT", + "sql-manual/sql-reference/Data-Types/JSON", + "sql-manual/sql-reference/Data-Types/AGG_STATE" ] }, { @@ -805,8 +867,8 @@ "type": "category", "label": "Alter", "items": [ - "sql-manual/sql-reference/Data-Definition-Statements/Alter/ALTER-DATABASE", "sql-manual/sql-reference/Data-Definition-Statements/Alter/ALTER-CATALOG", + "sql-manual/sql-reference/Data-Definition-Statements/Alter/ALTER-DATABASE", "sql-manual/sql-reference/Data-Definition-Statements/Alter/ALTER-TABLE-COLUMN", "sql-manual/sql-reference/Data-Definition-Statements/Alter/ALTER-TABLE-PARTITION", "sql-manual/sql-reference/Data-Definition-Statements/Alter/ALTER-TABLE-BITMAP", @@ -817,10 +879,10 @@ "sql-manual/sql-reference/Data-Definition-Statements/Alter/ALTER-TABLE-COMMENT", "sql-manual/sql-reference/Data-Definition-Statements/Alter/CANCEL-ALTER-TABLE", "sql-manual/sql-reference/Data-Definition-Statements/Alter/ALTER-VIEW", + "sql-manual/sql-reference/Data-Definition-Statements/Alter/ALTER-STORAGE-POLICY", "sql-manual/sql-reference/Data-Definition-Statements/Alter/ALTER-RESOURCE", "sql-manual/sql-reference/Data-Definition-Statements/Alter/ALTER-WORKLOAD-GROUP", - "sql-manual/sql-reference/Data-Definition-Statements/Alter/ALTER-SQL-BLOCK-RULE", - "sql-manual/sql-reference/Data-Definition-Statements/Alter/ALTER-STORAGE-POLICY" + "sql-manual/sql-reference/Data-Definition-Statements/Alter/ALTER-SQL-BLOCK-RULE" ] }, { @@ -898,32 +960,6 @@ "sql-manual/sql-reference/Data-Manipulation-Statements/OUTFILE" ] }, - { - "type": "category", - "label": "Database Administration", - "items": [ - "sql-manual/sql-reference/Database-Administration-Statements/ADMIN-SHOW-CONFIG", - "sql-manual/sql-reference/Database-Administration-Statements/ADMIN-SET-CONFIG", - "sql-manual/sql-reference/Database-Administration-Statements/SET-VARIABLE", - "sql-manual/sql-reference/Database-Administration-Statements/INSTALL-PLUGIN", - "sql-manual/sql-reference/Database-Administration-Statements/UNINSTALL-PLUGIN", - "sql-manual/sql-reference/Database-Administration-Statements/ENABLE-FEATURE", - "sql-manual/sql-reference/Database-Administration-Statements/ADMIN-SET-REPLICA-STATUS", - "sql-manual/sql-reference/Database-Administration-Statements/ADMIN-SHOW-REPLICA-DISTRIBUTION", - "sql-manual/sql-reference/Database-Administration-Statements/ADMIN-SHOW-REPLICA-STATUS", - "sql-manual/sql-reference/Database-Administration-Statements/ADMIN-REPAIR-TABLE", - "sql-manual/sql-reference/Database-Administration-Statements/ADMIN-CANCEL-REPAIR", - "sql-manual/sql-reference/Database-Administration-Statements/ADMIN-CHECK-TABLET", - "sql-manual/sql-reference/Database-Administration-Statements/ADMIN-DIAGNOSE-TABLET", - "sql-manual/sql-reference/Database-Administration-Statements/ADMIN-COPY-TABLET", - "sql-manual/sql-reference/Database-Administration-Statements/ADMIN-SHOW-TABLET-STORAGE-FORMAT", - "sql-manual/sql-reference/Database-Administration-Statements/ADMIN-CLEAN-TRASH", - "sql-manual/sql-reference/Database-Administration-Statements/RECOVER", - "sql-manual/sql-reference/Database-Administration-Statements/KILL", - "sql-manual/sql-reference/Database-Administration-Statements/ADMIN-REBALANCE-DISK", - "sql-manual/sql-reference/Database-Administration-Statements/ADMIN-CANCEL-REBALANCE-DISK" - ] - }, { "type": "category", "label": "Show", @@ -1005,33 +1041,6 @@ "sql-manual/sql-reference/Show-Statements/SHOW-QUERY-STATS" ] }, - { - "type": "category", - "label": "Data Types", - "items": [ - "sql-manual/sql-reference/Data-Types/BOOLEAN", - "sql-manual/sql-reference/Data-Types/TINYINT", - "sql-manual/sql-reference/Data-Types/SMALLINT", - "sql-manual/sql-reference/Data-Types/INT", - "sql-manual/sql-reference/Data-Types/BIGINT", - "sql-manual/sql-reference/Data-Types/LARGEINT", - "sql-manual/sql-reference/Data-Types/FLOAT", - "sql-manual/sql-reference/Data-Types/DOUBLE", - "sql-manual/sql-reference/Data-Types/DECIMAL", - "sql-manual/sql-reference/Data-Types/DATE", - "sql-manual/sql-reference/Data-Types/DATETIME", - "sql-manual/sql-reference/Data-Types/CHAR", - "sql-manual/sql-reference/Data-Types/VARCHAR", - "sql-manual/sql-reference/Data-Types/STRING", - "sql-manual/sql-reference/Data-Types/HLL", - "sql-manual/sql-reference/Data-Types/BITMAP", - "sql-manual/sql-reference/Data-Types/QUANTILE_STATE", - "sql-manual/sql-reference/Data-Types/ARRAY", - "sql-manual/sql-reference/Data-Types/MAP", - "sql-manual/sql-reference/Data-Types/STRUCT", - "sql-manual/sql-reference/Data-Types/JSON" - ] - }, { "type": "category", "label": "Operators", diff --git a/docs/zh-CN/community/developer-guide/regression-testing.md b/docs/zh-CN/community/developer-guide/regression-testing.md index 3617b4d769235a0..a9243496c4ec6ca 100644 --- a/docs/zh-CN/community/developer-guide/regression-testing.md +++ b/docs/zh-CN/community/developer-guide/regression-testing.md @@ -605,7 +605,7 @@ Doris 支持一些外部署数据源的查询。所以回归框架也提供了 1. 启动 Container - Doris 目前支持 es, mysql, pg, hive, sqlserver, oracle, iceberg, hudi 等数据源的 Docker compose。相关文件存放在 `docker/thirdparties/docker-compose` 目录下。 + Doris 目前支持 es, mysql, pg, hive, sqlserver, oracle, iceberg, hudi, trino 等数据源的 Docker compose。相关文件存放在 `docker/thirdparties/docker-compose` 目录下。 默认情况下，可以直接通过以下命令启动所有外部数据源的 Docker container： （注意，hive和hudi container 需要下载预制的数据文件，请参阅下面 hive和hudi 相关的文档。） @@ -795,6 +795,20 @@ Doris 支持一些外部署数据源的查询。所以回归框架也提供了 ``` 更多使用方式可参阅 [Hudi 官方文档](https://hudi.apache.org/docs/docker_demo)。 + + 10. Trino + Trino 相关的 Docker compose 文件存放在 docker/thirdparties/docker-compose/trino 下。 + 模版文件： + * gen_env.sh.tpl ：用于生成 HDFS相关端口号，无需修改，若出现端口冲突，可以对端口号进行修改。 + * hive.properties.tpl ：用于配置trino catalog 信息，无需修改。 + * trino_hive.env.tpl ：Hive 的环境配置信息，无需修改。 + * trino_hive.yaml.tpl ：Docker compose 文件，无需修改。 + 启动 Trino docker 后，会配置一套 Trino + hive catalog 环境，此时 Trino 拥有两个catalog + 1. hive + 2. tpch（trino docker 自带） + + 更多使用方式可参阅 [Trino 官方文档](https://trino.io/docs/current/installation/containers.html) + 2. 运行回归测试 外表相关的回归测试默认是关闭的，可以修改 `regression-test/conf/regression-conf.groovy` 中的以下配置来开启： diff --git a/docs/zh-CN/docs/admin-manual/config/be-config.md b/docs/zh-CN/docs/admin-manual/config/be-config.md index a01b81a21e54792..253c23416102597 100644 --- a/docs/zh-CN/docs/admin-manual/config/be-config.md +++ b/docs/zh-CN/docs/admin-manual/config/be-config.md @@ -326,6 +326,12 @@ BE 重启后该配置将失效。如果想持久化修改结果，使用如下 * 描述：当使用odbc外表时，如果odbc源表的某一列类型不是HLL, CHAR或者VARCHAR，并且列值长度超过该值，则查询报错'column value length longer than buffer length'. 可增大该值 * 默认值：100 +#### `jsonb_type_length_soft_limit_bytes` + +* 类型: int32 +* 描述: JSONB 类型最大长度的软限，单位是字节 +* 默认值: 1048576 + ### 查询 #### `fragment_pool_queue_size` diff --git a/docs/zh-CN/docs/admin-manual/workload-group.md b/docs/zh-CN/docs/admin-manual/workload-group.md index 775b10c199f774d..aaeab10b81b1455 100644 --- a/docs/zh-CN/docs/admin-manual/workload-group.md +++ b/docs/zh-CN/docs/admin-manual/workload-group.md @@ -32,11 +32,11 @@ workload group 可限制组内任务在单个be节点上的计算资源和内存 ## workload group属性 -* cpu_share：必选，用于设置workload group获取cpu时间的多少，可以实现cpu资源软隔离。cpu_share 是相对值，表示正在运行的workload group可获取cpu资源的权重。例如，用户创建了3个workload group g-a、g-b和g-c，cpu_share 分别为 10、30、40，某一时刻g-a和g-b正在跑任务，而g-c没有任务，此时g-a可获得 25% (10 / (10 + 30))的cpu资源，而g-b可获得75%的cpu资源。如果系统只有一个workload group正在运行，则不管其cpu_share的值为多少，它都可获取全部的cpu资源。 +* cpu_share: 必选，用于设置workload group获取cpu时间的多少，可以实现cpu资源软隔离。cpu_share 是相对值，表示正在运行的workload group可获取cpu资源的权重。例如，用户创建了3个workload group g-a、g-b和g-c，cpu_share 分别为 10、30、40，某一时刻g-a和g-b正在跑任务，而g-c没有任务，此时g-a可获得 25% (10 / (10 + 30))的cpu资源，而g-b可获得75%的cpu资源。如果系统只有一个workload group正在运行，则不管其cpu_share的值为多少，它都可获取全部的cpu资源。 -* memory_limit: 必选，用于设置workload group可以使用be内存的百分比。workload group内存限制的绝对值为： 物理内存 * mem_limit * memory_limit，其中 mem_limit 为be配置项。系统所有workload group的 memory_limit总合不可超过100%。workload group在绝大多数情况下保证组内任务可使用memory_limit的内存，当workload group内存使用超出该限制后，组内内存占用较大的任务可能会被cancel以释放超出的内存，参考 enable_memory_overcommit。 +* memory_limit: 必选，用于设置workload group可以使用be内存的百分比。workload group内存限制的绝对值为：`物理内存 * mem_limit * memory_limit`，其中 mem_limit 为be配置项。系统所有workload group的 memory_limit总合不可超过100%。workload group在绝大多数情况下保证组内任务可使用memory_limit的内存，当workload group内存使用超出该限制后，组内内存占用较大的任务可能会被cancel以释放超出的内存，参考 enable_memory_overcommit。 -* enable_memory_overcommit: 可选，用于开启workload group内存软隔离，默认为false。如果设置为false，则该workload group为内存硬隔离，系统检测到workload group内存使用超出限制后将立即cancel组内内存占用最大的若干个任务，以释放超出的内存；如果设置为true，则该workload group为内存软隔离，如果系统有空闲内存资源则该workload group在超出memory_limit的限制后可继续使用系统内存，在系统总内存紧张时会cancel组内内存占用最大的若干个任务，释放部分超出的内存以缓解系统内存压力。建议在有workload group开启该配置时，所有workload group的 memory_limit 总合低于100%，剩余部分用于workload group内存超发。 +* enable_memory_overcommit: 可选，用于开启workload group内存软隔离，默认为false。如果设置为false，则该workload group为内存硬隔离，系统检测到workload group内存使用超出限制后将立即cancel组内内存占用最大的若干个任务，以释放超出的内存；如果设置为true，则该workload group为内存软隔离，如果系统有空闲内存资源则该workload group在超出memory_limit的限制后可继续使用系统内存，在系统总内存紧张时会cancel组内内存占用最大的若干个任务，释放部分超出的内存以缓解系统内存压力。建议在有workload group开启该配置时，所有workload group的 memory_limit 总和低于100%，剩余部分用于workload group内存超发。 ## workload group使用 diff --git a/docs/zh-CN/docs/advanced/cold_hot_separation.md b/docs/zh-CN/docs/advanced/cold_hot_separation.md index e952b42f2382e18..d551d3315a7745e 100644 --- a/docs/zh-CN/docs/advanced/cold_hot_separation.md +++ b/docs/zh-CN/docs/advanced/cold_hot_separation.md @@ -103,7 +103,7 @@ ALTER TABLE create_table_partition MODIFY PARTITION (*) SET("storage_policy"="te ### 一些限制 -- 单表或单partition只能关联一个storage policy，关联后不能drop掉storage policy，需要先解二者的除关联。 +- 单表或单partition只能关联一个storage policy，关联后不能drop掉storage policy，需要先解除二者的关联。 - storage policy关联的对象信息不支持修改数据存储path的信息，比如bucket、endpoint、root_path等信息 - storage policy支持创建和修改和支持删除，删除前需要先保证没有表引用此storage policy。 diff --git a/docs/zh-CN/docs/benchmark/ssb.md b/docs/zh-CN/docs/benchmark/ssb.md index 9c567a6daefe317..118c28537d3feed 100644 --- a/docs/zh-CN/docs/benchmark/ssb.md +++ b/docs/zh-CN/docs/benchmark/ssb.md @@ -71,7 +71,7 @@ under the License. | customer | 3,000,000 | 客户信息表 | | part | 1,400,000 | 零件信息表 | | supplier | 200,000 | 供应商信息表 | -| date | 2,556 | 日期表 | +| dates | 2,556 | 日期表 | | lineorder_flat | 600,037,902 | 数据展平后的宽表 | ## 4. SSB 宽表测试结果 @@ -170,7 +170,7 @@ sh gen-ssb-data.sh -s 100 -c 100 | customer | 300万（3000000） | 277M | 1 | | part | 140万（1400000） | 116M | 1 | | supplier | 20万（200000） | 17M | 1 | -| date | 2556 | 228K | 1 | +| dates | 2556 | 228K | 1 | ### 7.3 建表 @@ -284,7 +284,7 @@ sh bin/load-ssb-data.sh -c 10 select count(*) from part; select count(*) from customer; select count(*) from supplier; -select count(*) from date; +select count(*) from dates; select count(*) from lineorder; select count(*) from lineorder_flat; ``` @@ -298,7 +298,7 @@ select count(*) from lineorder_flat; | customer | 300万（3000000） | 277 MB | 138.247 MB | | part | 140万（1400000） | 116 MB | 12.759 MB | | supplier | 20万（200000） | 17 MB | 9.143 MB | -| date | 2556 | 228 KB | 34.276 KB | +| dates | 2556 | 228 KB | 34.276 KB | ### 7.6 查询测试 diff --git a/docs/zh-CN/docs/data-table/index/inverted-index.md b/docs/zh-CN/docs/data-table/index/inverted-index.md index e78ce537bced190..f3bb248955b459f 100644 --- a/docs/zh-CN/docs/data-table/index/inverted-index.md +++ b/docs/zh-CN/docs/data-table/index/inverted-index.md @@ -96,6 +96,15 @@ CREATE TABLE table_name table_properties; ``` +:::tip + +倒排索引在不同数据模型中有不同的使用限制： +- Aggregate 模型：只能为 Key 列建立倒排索引。 +- Unique 模型：需要开启 merge on write 特性，开启后，可以为任意列建立倒排索引。 +- Duplicate 模型：可以为任意列建立倒排索引。 + +::: + - 已有表增加倒排索引 **2.0-beta版本之前：** diff --git a/docs/zh-CN/docs/ecosystem/flink-doris-connector.md b/docs/zh-CN/docs/ecosystem/flink-doris-connector.md index 130c3d39cfe8d0a..a5f7b2977f29c7c 100644 --- a/docs/zh-CN/docs/ecosystem/flink-doris-connector.md +++ b/docs/zh-CN/docs/ecosystem/flink-doris-connector.md @@ -403,6 +403,8 @@ insert into doris_sink select id,name from cdc_mysql_source; - **--sink-conf** Doris Sink 的所有配置，可以在[这里](https://doris.apache.org/zh-CN/docs/dev/ecosystem/flink-doris-connector/#%E9%80%9A%E7%94%A8%E9%85%8D%E7%BD%AE%E9%A1%B9)查看完整的配置项。 - **--table-conf** Doris表的配置项，即properties中包含的内容。 例如 --table-conf replication_num=1 +>注：需要在$FLINK_HOME/lib 目录下添加flink-sql-connector-mysql-cdc-2.3.0.jar + ### 示例 ``` /bin/flink run \ @@ -465,7 +467,7 @@ CREATE TABLE DORIS_SINK( 'username' = 'root', 'password' = '', 'sink.enable-delete' = 'false', -- false表示不从RowKind获取事件类型 - 'sink.properties.columns' = 'name,age,__DORIS_DELETE_SIGN__' -- 显示指定streamload的导入列 + 'sink.properties.columns' = 'id, name, __DORIS_DELETE_SIGN__' -- 显示指定streamload的导入列 ); INSERT INTO KAFKA_SOURCE diff --git a/docs/zh-CN/docs/ecosystem/udf/native-user-defined-function.md b/docs/zh-CN/docs/ecosystem/udf/native-user-defined-function.md deleted file mode 100644 index f113b78b6136ae2..000000000000000 --- a/docs/zh-CN/docs/ecosystem/udf/native-user-defined-function.md +++ /dev/null @@ -1,278 +0,0 @@ ---- -{ - "title": "原生UDF", - "language": "zh-CN" -} ---- - - - -# UDF - - - -UDF 主要适用于，用户需要的分析能力 Doris 并不具备的场景。用户可以自行根据自己的需求，实现自定义的函数，并且通过 UDF 框架注册到 Doris 中，来扩展 Doris 的能力，并解决用户分析需求。 - -UDF 能满足的分析需求分为两种：UDF 和 UDAF。本文中的 UDF 指的是二者的统称。 - -1. UDF: 用户自定义函数，这种函数会对单行进行操作，并且输出单行结果。当用户在查询时使用 UDF ，每行数据最终都会出现在结果集中。典型的 UDF 比如字符串操作 concat() 等。 -2. UDAF: 用户自定义的聚合函数，这种函数对多行进行操作，并且输出单行结果。当用户在查询时使用 UDAF，分组后的每组数据最后会计算出一个值并展结果集中。典型的 UDAF 比如集合操作 sum() 等。一般来说 UDAF 都会结合 group by 一起使用。 - -这篇文档主要讲述了，如何编写自定义的 UDF 函数，以及如何在 Doris 中使用它。 - -如果用户使用 UDF 功能并扩展了 Doris 的函数分析，并且希望将自己实现的 UDF 函数贡献回 Doris 社区给其他用户使用，这时候请看文档 [Contribute UDF](./contribute-udf.md)。 - - - -## 编写 UDF 函数 - -在使用UDF之前，用户需要先在 Doris 的 UDF 框架下，编写自己的UDF函数。在`contrib/udf/src/udf_samples/udf_sample.h|cpp`文件中是一个简单的 UDF Demo。 - -编写一个 UDF 函数需要以下几个步骤。 - -### 编写函数 - -创建对应的头文件、CPP文件，在CPP文件中实现你需要的逻辑。CPP文件中的实现函数格式与UDF的对应关系。 - -用户可以把自己的 source code 统一放在一个文件夹下。这里以 udf_sample 为例，目录结构如下： - -``` -└── udf_samples - ├── uda_sample.cpp - ├── uda_sample.h - ├── udf_sample.cpp - └── udf_sample.h -``` - -#### 非可变参数 - -对于非可变参数的UDF，那么两者之间的对应关系很直接。 -比如`INT MyADD(INT, INT)`的UDF就会对应`IntVal AddUdf(FunctionContext* context, const IntVal& arg1, const IntVal& arg2)`。 - -1. `AddUdf`可以为任意的名字，只要创建UDF的时候指定即可。 -2. 实现函数中的第一个参数永远是`FunctionContext*`。实现者可以通过这个结构体获得一些查询相关的内容，以及申请一些需要使用的内存。具体使用的接口可以参考`udf/udf.h`中的定义。 -3. 实现函数中从第二个参数开始需要与UDF的参数一一对应，比如`IntVal`对应`INT`类型。这部分的类型都要使用`const`引用。 -4. 返回参数与UDF的参数的类型要相对应。 - -#### 可变参数 - -对于可变参数，可以参见以下例子，UDF`String md5sum(String, ...)`对应的 -实现函数是`StringVal md5sumUdf(FunctionContext* ctx, int num_args, const StringVal* args)` - -1. `md5sumUdf`这个也是可以任意改变的，创建的时候指定即可。 -2. 第一个参数与非可变参数函数一样，传入的是一个`FunctionContext*`。 -3. 可变参数部分由两部分组成，首先会传入一个整数，说明后面还有几个参数。后面传入的是一个可变参数部分的数组。 - -#### 类型对应关系 - -|UDF Type|Argument Type| -|----|---------| -|TinyInt|TinyIntVal| -|SmallInt|SmallIntVal| -|Int|IntVal| -|BigInt|BigIntVal| -|LargeInt|LargeIntVal| -|Float|FloatVal| -|Double|DoubleVal| -|Date|DateTimeVal| -|Datetime|DateTimeVal| -|Char|StringVal| -|Varchar|StringVal| -|Decimal|DecimalVal| - - -## 编译 UDF 函数 - -由于 UDF 实现中依赖了 Doris 的 UDF 框架 , 所以在编译 UDF 函数的时候首先要对 Doris 进行编译，也就是对 UDF 框架进行编译。 - -编译完成后会生成，UDF 框架的静态库文件。之后引入 UDF 框架依赖，并编译 UDF 即可。 - -### 编译Doris - -在 Doris 根目录下执行 `sh build.sh` 就会在 `output/udf/` 生成 UDF 框架的静态库文件 `headers|libs` - -``` -├── output -│   └── udf -│   ├── include -│   │   ├── uda_test_harness.h -│   │   └── udf.h -│   └── lib -│   └── libDorisUdf.a - -``` - -### 编写 UDF 编译文件 - -1. 准备 thirdparty - - `thirdparty` 文件夹主要用于存放用户 UDF 函数依赖的第三方库，包括头文件及静态库。其中必须包含依赖的 Doris UDF 框架中 `udf.h` 和 `libDorisUdf.a` 这两个文件。 - - 这里以 `udf_sample` 为例, 在 用户自己 `udf_samples` 目录用于存放 source code。在同级目录下再创建一个 `thirdparty` 文件夹用于存放静态库。目录结构如下： - - ``` - ├── thirdparty - │ │── include - │ │ └── udf.h - │ └── lib - │ └── libDorisUdf.a - └── udf_samples - - ``` - - `udf.h` 是 UDF 框架头文件。存放路径为 `doris/output/udf/include/udf.h`。 用户需要将 Doris 编译产出中的这个头文件拷贝到自己的 `thirdparty` 的 include 文件夹下。 - - `libDorisUdf.a` 是 UDF 框架的静态库。Doris 编译完成后该文件存放在 `doris/output/udf/lib/libDorisUdf.a`。用户需要将该文件拷贝到自己的 `thirdparty` 的 lib 文件夹下。 - - *注意：UDF 框架的静态库只有完成 Doris 编译后才会生成。 - -2. 准备编译 UDF 的 CMakeFiles.txt - - CMakeFiles.txt 用于声明 UDF 函数如何进行编译。存放在源码文件夹下，与用户代码平级。这里以 `udf_samples` 为例目录结构如下: - - ``` - ├── thirdparty - └── udf_samples - ├── CMakeLists.txt - ├── uda_sample.cpp - ├── uda_sample.h - ├── udf_sample.cpp - └── udf_sample.h - ``` - - + 需要显示声明引用 `libDorisUdf.a` - + 声明 `udf.h` 头文件位置 - - -以 udf_sample 为例 - -``` -# Include udf -include_directories(../thirdparty/include) - -# Set all libraries -add_library(udf STATIC IMPORTED) -set_target_properties(udf PROPERTIES IMPORTED_LOCATION ../thirdparty/lib/libDorisUdf.a) - -# where to put generated libraries -set(LIBRARY_OUTPUT_PATH "src/udf_samples") - -# where to put generated binaries -set(EXECUTABLE_OUTPUT_PATH "src/udf_samples") - -add_library(udfsample SHARED udf_sample.cpp) - target_link_libraries(udfsample - udf - -static-libstdc++ - -static-libgcc -) - -add_library(udasample SHARED uda_sample.cpp) - target_link_libraries(udasample - udf - -static-libstdc++ - -static-libgcc -) -``` - -如果用户的 UDF 函数还依赖了其他的三方库，则需要声明 include，lib，并在 `add_library` 中增加依赖。 - -所有文件准备齐后完整的目录结构如下： - -``` - ├── thirdparty - │ │── include - │ │ └── udf.h - │ └── lib - │ └── libDorisUdf.a - └── udf_samples - ├── CMakeLists.txt - ├── uda_sample.cpp - ├── uda_sample.h - ├── udf_sample.cpp - └── udf_sample.h -``` - -准备好上述文件就可以直接编译 UDF 了 - -### 执行编译 - -在 udf_samples 文件夹下创建一个 build 文件夹，用于存放编译产出。 - -在 build 文件夹下运行命令 `cmake ../` 生成Makefile，并执行 make 就会生成对应动态库。 - -``` -├── thirdparty -├── udf_samples - └── build -``` - -### 编译结果 - -编译完成后的 UDF 动态链接库就生成成功了。在 `build/src/` 下，以 udf_samples 为例，目录结构如下： - -``` - -├── thirdparty -├── udf_samples - └── build - └── src - └── udf_samples - ├── libudasample.so -   └── libudfsample.so - -``` - -## 创建 UDF 函数 - -通过上述的步骤后，你可以得到 UDF 的动态库（也就是编译结果中的 `.so` 文件）。你需要将这个动态库放到一个能够通过 HTTP 协议访问到的位置。 - -然后登录 Doris 系统，在 mysql-client 中通过 `CREATE FUNCTION` 语法创建 UDF 函数。你需要拥有ADMIN权限才能够完成这个操作。这时 Doris 系统内部就会存在刚才创建好的 UDF。 - -``` -CREATE [AGGREGATE] FUNCTION - name ([argtype][,...]) - [RETURNS] rettype - PROPERTIES (["key"="value"][,...]) -``` -说明： - -1. PROPERTIES中`symbol`表示的是，执行入口函数的对应symbol，这个参数是必须设定。你可以通过`nm`命令来获得对应的symbol，比如`nm libudfsample.so | grep AddUdf`获得到的`_ZN9doris_udf6AddUdfEPNS_15FunctionContextERKNS_6IntValES4_`就是对应的symbol。 -2. PROPERTIES中`object_file`表示的是从哪里能够下载到对应的动态库，这个参数是必须设定的。 -3. name: 一个function是要归属于某个DB的，name的形式为`dbName`.`funcName`。当`dbName`没有明确指定的时候，就是使用当前session所在的db作为`dbName`。 - -具体使用可以参见 `CREATE FUNCTION` 获取更详细信息。 - -## 使用 UDF - -用户使用 UDF 必须拥有对应数据库的 `SELECT` 权限。 - -UDF 的使用与普通的函数方式一致，唯一的区别在于，内置函数的作用域是全局的，而 UDF 的作用域是 DB内部。当链接 session 位于数据内部时，直接使用 UDF 名字会在当前DB内部查找对应的 UDF。否则用户需要显示的指定 UDF 的数据库名字，例如 `dbName`.`funcName`。 - -当前版本中，使用原生UDF时还需要将向量化关闭 -``` -set enable_vectorized_engine = false; -``` - - -## 删除 UDF函数 - -当你不再需要 UDF 函数时，你可以通过下述命令来删除一个 UDF 函数, 可以参考 `DROP FUNCTION`。 - diff --git a/docs/zh-CN/docs/install/standard-deployment.md b/docs/zh-CN/docs/install/standard-deployment.md index 7d4562dea6c5d89..d82b05d9ad332f7 100644 --- a/docs/zh-CN/docs/install/standard-deployment.md +++ b/docs/zh-CN/docs/install/standard-deployment.md @@ -88,7 +88,7 @@ ext4和xfs文件系统均支持。 > 注1： > 1. FE 的磁盘空间主要用于存储元数据，包括日志和 image。通常从几百 MB 到几个 GB 不等。 > 2. BE 的磁盘空间主要用于存放用户数据，总磁盘空间按用户总数据量 * 3（3副本）计算，然后再预留额外 40% 的空间用作后台 compaction 以及一些中间数据的存放。 -> 3. 一台机器上可以部署多个 BE 实例，但是**只能部署一个 FE**。如果需要 3 副本数据，那么至少需要 3 台机器各部署一个 BE 实例（而不是1台机器部署3个BE实例）。**多个FE所在服务器的时钟必须保持一致（允许最多5秒的时钟偏差）** +> 3. 一台机器上虽然可以部署多个 BE，**但只建议部署一个实例**，同时**只能部署一个 FE**。如果需要 3 副本数据，那么至少需要 3 台机器各部署一个 BE 实例（而不是1台机器部署3个BE实例）。**多个FE所在服务器的时钟必须保持一致（允许最多5秒的时钟偏差）** > 4. 测试环境也可以仅适用一个 BE 进行测试。实际生产环境，BE 实例数量直接决定了整体查询延迟。 > 5. 所有部署节点关闭 Swap。 @@ -192,31 +192,29 @@ doris默认为表名大小写敏感，如有表名大小写不敏感的需求需 * 修改所有 BE 的配置 - 修改 be/conf/be.conf。主要是配置 `storage_root_path`：数据存放目录。默认在be/storage下，需要**手动创建**该目录。多个路径之间使用英文状态的分号 `;` 分隔（**最后一个目录后不要加 `;`**）。 - 可以通过路径区别存储目录的介质，HDD或SSD。可以添加容量限制在每个路径的末尾，通过英文状态逗号`,`隔开。如果用户不是SSD和HDD磁盘混合使用的情况，不需要按照如下示例一和示例二的配置方法配置，只需指定存储目录即可；也不需要修改FE的默认存储介质配置 + 修改 be/conf/be.conf。主要是配置 `storage_root_path`：数据存放目录。默认在be/storage下，若需要指定目录的话，需要**预创建目录**。多个路径之间使用英文状态的分号 `;` 分隔（**最后一个目录后不要加 `;`**）。 + 可以通过路径区别节点内的冷热数据存储目录，HDD（冷数据目录）或 SSD（热数据目录）。如果不需要 BE 节点内的冷热机制，那么只需要配置路径即可，无需指定 medium 类型；也不需要修改FE的默认存储介质配置 - 示例1如下： - - **注意：如果是SSD磁盘要在目录后面加上`.SSD`,HDD磁盘在目录后面加`.HDD`** - - `storage_root_path=/home/disk1/doris.HDD;/home/disk2/doris.SSD;/home/disk2/doris` + **注意：** + 1. 如果指定存储路径的存储类型，则最少要有一个路径的存储类型为 HDD（冷数据目录）！ + 2. 如果未指定存储路径的存储类型，则默认全部为 HDD（冷数据目录）。 + 3. 这里的 HDD 和 SSD 与物理存储介质无关，只为了区分存储路径的存储类型，即可以在 HDD 介质的盘上标记某个目录为 SSD（热数据目录）。 + 4. 这里的 HDD 和 SSD **必须**要大写！ - **说明** + 示例1如下： - - /home/disk1/doris.HDD ： 表示存储介质是HDD; - - /home/disk2/doris.SSD： 表示存储介质是SSD； - - /home/disk2/doris： 表示存储介质是HDD（默认） + `storage_root_path=/home/disk1/doris;/home/disk2/doris;/home/disk2/doris` 示例2如下： - **注意：不论HDD磁盘目录还是SSD磁盘目录，都无需添加后缀，storage_root_path参数里指定medium即可** + **使用 storage_root_path 参数里指定 medium** `storage_root_path=/home/disk1/doris,medium:HDD;/home/disk2/doris,medium:SSD` **说明** - - /home/disk1/doris,medium:HDD： 表示存储介质是HDD; - - /home/disk2/doris,medium:SSD： 表示存储介质是SSD; + - /home/disk1/doris,medium:HDD： 表示该目录存储冷数据; + - /home/disk2/doris,medium:SSD： 表示该目录存储热数据; * BE webserver_port端口配置 diff --git a/docs/zh-CN/docs/lakehouse/multi-catalog/faq.md b/docs/zh-CN/docs/lakehouse/faq.md similarity index 100% rename from docs/zh-CN/docs/lakehouse/multi-catalog/faq.md rename to docs/zh-CN/docs/lakehouse/faq.md diff --git a/docs/zh-CN/docs/lakehouse/fs_benchmark_tool.md b/docs/zh-CN/docs/lakehouse/fs_benchmark_tool.md new file mode 100644 index 000000000000000..c7cad119f1a0494 --- /dev/null +++ b/docs/zh-CN/docs/lakehouse/fs_benchmark_tool.md @@ -0,0 +1,229 @@ +--- +{ + "title": "文件系统性能测试工具", + "language": "zh-CN" +} +--- + + + + +# 简介 + +`fs_benchmark_tool` 可以用于测试包括 hdfs 和对象存储在内的远端存储系统的基本服务性能，如读取、写入性能。该工具主要用于分析或排查远端存储系统的性能问题。 + +# 编译与安装 + +`fs_benchmark_tool` 是 BE 代码的一部分，默认不编译。如需编译，请执行以下命令： + +``` +cd doris +BUILD_FS_BENCHMARK=ON ./build.sh --be +``` +编译完之后会在`output/be/` 目录下生成如下相关内容： +``` +bin/run-fs-benchmark.sh +lib/fs_benchmark_tool +``` +> 注意，`fs_benchmark_tool` 需在BE运行环境目录下使用，因为其依赖 BE 相关的 jar 包、环境变量等内容。 + +# 使用 + +命令格式： + +```shell +sh run-fs-benchmark.sh \ + --conf=配置文件 \ + --fs_type= 文件系统 \ + --operation= 对文件系统的操作 \ + --file_size= 文件的大小 \ + --threads= 线程数量 \ + --iterations= 迭代次数 +``` +## 参数解析 + +`--conf`必选参数 + + +操作文件对应的配置文件。主要用于添加远端存储系统的相关连接信息。详见下文示例。 + +如连接`hdfs`，请将 `hdfs-site.xml`，`core-site.xml` 文件放置在 `be/conf` 目录下。 + +除连接信息外，还有以下额外参数： +- `file_size`：指定读取或写入文件的大小。 + +- `buffer_size`：一次读取操作读取的文件块大小。 + +- `base_dir`：指定读取或写入文件的 base 路径。 + +`--fs_type`必选参数 + +需要操作的文件系统类型。目前支持`hdfs`，`s3`。 + +`--operation` 必选参数 + +指定操作类型 + +- `create_write` ：每个线程在`base_dir(conf文件中设置)`目录下，创建文件名为`test_当前的线程号`，并写入文件，写入大小为`file_size`。 + +- `open_read`：在`create_write`创建好文件的基础下，每个线程读取文件名为`test_当前的线程号`的文件，读取大小为`file_size`。 + +- `exists` ：每个线程查询文件名为`test_当前的线程号`的文件是否存在。 + +- `rename` ：在`create_write`创建好文件的基础下，每个线程将文件名为为`test_当前的线程号`的文件更改为为`test_当前的线程号_new`。 + +- `single_read`：读取`file_path(conf文件中设置)`文件，读取大小为`file_size`。 + +- `list`：获取 `base_dir(conf文件中设置)` 目录下的文件列表。 + +`--file_size` +操作的文件大小，以字节为单位。 + +- `create_write`：默认为 10MB。 + +- `open_read`：默认为 10MB。 + +- `single_read`：默认为0，即读取完整文件。 + +`--threads` + +操作的线程数量，默认数量为1。 + +`--iterations` + +每个线程进行迭代的次数（函数执行次数），默认数量为1。 + +## 结果解析 + +除了`rename`操作外，其余操作都会重复三次，并求出平均值，中间值，标准偏差等。 +``` +-------------------------------------------------------------------------------------------------------------------------------- +Benchmark Time CPU Iterations UserCounters... +-------------------------------------------------------------------------------------------------------------------------------- +HdfsReadBenchmark/iterations:1/repeats:3/manual_time/threads:1 13642 ms 2433 ms 1 OpenReaderTime(S)=4.80734 ReadRate(B/S)=101.104M/s ReadTime(S)=13.642 ReadTotal(B)=1.37926G +HdfsReadBenchmark/iterations:1/repeats:3/manual_time/threads:1 3918 ms 1711 ms 1 OpenReaderTime(S)=22.041u ReadRate(B/S)=352.011M/s ReadTime(S)=3.91824 ReadTotal(B)=1.37926G +HdfsReadBenchmark/iterations:1/repeats:3/manual_time/threads:1 3685 ms 1697 ms 1 OpenReaderTime(S)=35.837u ReadRate(B/S)=374.313M/s ReadTime(S)=3.68479 ReadTotal(B)=1.37926G +HdfsReadBenchmark/iterations:1/repeats:3/manual_time/threads:1_mean 7082 ms 1947 ms 3 OpenReaderTime(S)=1.60247 ReadRate(B/S)=275.809M/s ReadTime(S)=7.08166 ReadTotal(B)=1.37926G +HdfsReadBenchmark/iterations:1/repeats:3/manual_time/threads:1_median 3918 ms 1711 ms 3 OpenReaderTime(S)=35.837u ReadRate(B/S)=352.011M/s ReadTime(S)=3.91824 ReadTotal(B)=1.37926G +HdfsReadBenchmark/iterations:1/repeats:3/manual_time/threads:1_stddev 5683 ms 421 ms 3 OpenReaderTime(S)=2.7755 ReadRate(B/S)=151.709M/s ReadTime(S)=5.68258 ReadTotal(B)=0 +HdfsReadBenchmark/iterations:1/repeats:3/manual_time/threads:1_cv 80.24 % 21.64 % 3 OpenReaderTime(S)=173.20% ReadRate(B/S)=55.01% ReadTime(S)=80.24% ReadTotal(B)=0.00% +HdfsReadBenchmark/iterations:1/repeats:3/manual_time/threads:1_max 13642 ms 2433 ms 3 OpenReaderTime(S)=4.80734 ReadRate(B/S)=374.313M/s ReadTime(S)=13.642 ReadTotal(B)=1.37926G +HdfsReadBenchmark/iterations:1/repeats:3/manual_time/threads:1_min 3685 ms 1697 ms 3 OpenReaderTime(S)=22.041u ReadRate(B/S)=101.104M/s ReadTime(S)=3.68479 ReadTotal(B)=1.37926G +``` + +重点关注前3行，分别代码3次重复执行的结果。其中第一次涉及到一些连接初始化等操作，所以耗时会较长。后两次通常代表正常的性能表现。 + +重点关注`UserCounters` 中的信息： +- `OpenReaderTime`：打开文件的耗时。 +- `ReadRate`：读取速率。这里记录的是总体的吞吐。如果是多线程，可以除以线程数，即代表每线程平均速率。 +- `ReadTime`：读取耗时。这里记录的是多线程累计时间。除以线程数，即代表每线程平均耗时。 +- `ReadTotal`：读取总量。这里记录的是多线程累计值。除以线程数，即代表每线程平均读取量。 +- `WriteRate`：同 `ReadRate`。代表写入速率。 +- `WriteTime`：同 `ReadTime`。代表写入耗时。 +- `WriteTotal`：同 `ReadTotal`。代表写入总量。 +- `ListCost/RenameCost/ExistsCost`：对应操作的单个操作耗时。 + +# 示例 + +## HDFS + +命令： +``` +sh run-fs-benchmark.sh \ + --conf=hdfs.conf \ + --fs_type=hdfs \ + --operation=create_write \ + --file_size=1024000 \ + --threads=3 \ + --iterations=5 +``` +使用`hdfs.conf`配置文件，对`hdfs`文件系统进行`create_write`操作，使用三个线程，每次操作写入 1MB，迭代次数为5次。 + +`hdfs.conf`配置文件： +``` +fs.defaultFS=hdfs://HDFS8000871 +hadoop.username=hadoop +dfs.nameservices=HDFS8000871 +dfs.ha.namenodes.HDFS8000871=nn1,nn2 +dfs.namenode.rpc-address.HDFS8000871.nn1=102.22.10.56:4007 +dfs.namenode.rpc-address.HDFS8000871.nn2=102.22.10.57:4007 +dfs.client.failover.proxy.provider.HDFS8000871=org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider +base_dir=hdfs://HDFS8000871/benchmarks/TestDFSIO/io_data/ +``` +运行结果： +``` +--------------------------------------------------------------------------------------------------------------------------------------- +Benchmark Time CPU Iterations UserCounters... +--------------------------------------------------------------------------------------------------------------------------------------- +HdfsCreateWriteBenchmark/iterations:5/repeats:3/manual_time/threads:3 61.7 ms 38.7 ms 15 WriteRate(B/S)=3.31902M/s WriteTime(S)=0.387954 WriteTotal(B)=3.072M +HdfsCreateWriteBenchmark/iterations:5/repeats:3/manual_time/threads:3 49.6 ms 3.09 ms 15 WriteRate(B/S)=4.12967M/s WriteTime(S)=0.427992 WriteTotal(B)=3.072M +HdfsCreateWriteBenchmark/iterations:5/repeats:3/manual_time/threads:3 45.2 ms 2.72 ms 15 WriteRate(B/S)=4.53148M/s WriteTime(S)=0.362854 WriteTotal(B)=3.072M +HdfsCreateWriteBenchmark/iterations:5/repeats:3/manual_time/threads:3_mean 52.2 ms 14.8 ms 3 WriteRate(B/S)=3.99339M/s WriteTime(S)=0.392933 WriteTotal(B)=3.072M +HdfsCreateWriteBenchmark/iterations:5/repeats:3/manual_time/threads:3_median 49.6 ms 3.09 ms 3 WriteRate(B/S)=4.12967M/s WriteTime(S)=0.387954 WriteTotal(B)=3.072M +HdfsCreateWriteBenchmark/iterations:5/repeats:3/manual_time/threads:3_stddev 8.55 ms 20.7 ms 3 WriteRate(B/S)=617.61k/s WriteTime(S)=0.0328536 WriteTotal(B)=0 +HdfsCreateWriteBenchmark/iterations:5/repeats:3/manual_time/threads:3_cv 16.39 % 139.34 % 3 WriteRate(B/S)=15.47% WriteTime(S)=8.36% WriteTotal(B)=0.00% +HdfsCreateWriteBenchmark/iterations:5/repeats:3/manual_time/threads:3_max 61.7 ms 38.7 ms 3 WriteRate(B/S)=4.53148M/s WriteTime(S)=0.427992 WriteTotal(B)=3.072M +HdfsCreateWriteBenchmark/iterations:5/repeats:3/manual_time/threads:3_min 45.2 ms 2.72 ms 3 WriteRate(B/S)=3.31902M/s WriteTime(S)=0.362854 WriteTotal(B)=3.072M +HDFS 上生成的文件： +[hadoop@172 ~]$ hadoop fs -ls -h /benchmarks/TestDFSIO/io_data/ +Found 3 items +-rw-r--r-- 3 hadoop supergroup 100 2023-06-27 11:55 /benchmarks/TestDFSIO/io_data/test_0 +-rw-r--r-- 3 hadoop supergroup 100 2023-06-27 11:55 /benchmarks/TestDFSIO/io_data/test_1 +-rw-r--r-- 3 hadoop supergroup 100 2023-06-27 11:55 /benchmarks/TestDFSIO/io_data/test_2 +``` + +## 对象存储 + +命令： +``` +sh bin/run-fs-benchmark.sh \ + --conf=s3.conf \ + --fs_type=s3 \ + --operation=single_read \ + --threads=1 \ + --iterations=1 +``` + +使用`s3.conf`配置文件，对 `s3`文件系统进行 `single_read`操作，使用1个线程，迭代次数为1次。 + +`s3.conf` 配置文件： +``` +AWS_ACCESS_KEY=ak +AWS_SECRET_KEY=sk +AWS_ENDPOINT=cos.ap-beijing.myqcloud.com +AWS_REGION=ap-beijing +file_path=s3://bucket-123/test_data/parquet/000016_0 +``` +运行结果： +``` +------------------------------------------------------------------------------------------------------------------------------ +Benchmark Time CPU Iterations UserCounters... +------------------------------------------------------------------------------------------------------------------------------ +S3ReadBenchmark/iterations:1/repeats:3/manual_time/threads:1 7534 ms 140 ms 1 ReadRate(B/S)=11.9109M/s ReadTime(S)=7.53353 ReadTotal(B)=89.7314M +S3ReadBenchmark/iterations:1/repeats:3/manual_time/threads:1 5988 ms 118 ms 1 ReadRate(B/S)=14.985M/s ReadTime(S)=5.98808 ReadTotal(B)=89.7314M +S3ReadBenchmark/iterations:1/repeats:3/manual_time/threads:1 6060 ms 124 ms 1 ReadRate(B/S)=14.8081M/s ReadTime(S)=6.05961 ReadTotal(B)=89.7314M +S3ReadBenchmark/iterations:1/repeats:3/manual_time/threads:1_mean 6527 ms 127 ms 3 ReadRate(B/S)=13.9014M/s ReadTime(S)=6.52707 ReadTotal(B)=89.7314M +S3ReadBenchmark/iterations:1/repeats:3/manual_time/threads:1_median 6060 ms 124 ms 3 ReadRate(B/S)=14.8081M/s ReadTime(S)=6.05961 ReadTotal(B)=89.7314M +S3ReadBenchmark/iterations:1/repeats:3/manual_time/threads:1_stddev 872 ms 11.4 ms 3 ReadRate(B/S)=1.72602M/s ReadTime(S)=0.87235 ReadTotal(B)=0 +S3ReadBenchmark/iterations:1/repeats:3/manual_time/threads:1_cv 13.37 % 8.94 % 3 ReadRate(B/S)=12.42% ReadTime(S)=13.37% ReadTotal(B)=0.00% +S3ReadBenchmark/iterations:1/repeats:3/manual_time/threads:1_max 7534 ms 140 ms 3 ReadRate(B/S)=14.985M/s ReadTime(S)=7.53353 ReadTotal(B)=89.7314M +S3ReadBenchmark/iterations:1/repeats:3/manual_time/threads:1_min 5988 ms 118 ms 3 ReadRate(B/S)=11.9109M/s ReadTime(S)=5.98808 ReadTotal(B)=89.7314M +``` + diff --git a/docs/zh-CN/docs/lakehouse/multi-catalog/jdbc.md b/docs/zh-CN/docs/lakehouse/multi-catalog/jdbc.md index b02180ce8231ba8..e706bf35214df1b 100644 --- a/docs/zh-CN/docs/lakehouse/multi-catalog/jdbc.md +++ b/docs/zh-CN/docs/lakehouse/multi-catalog/jdbc.md @@ -670,4 +670,16 @@ CREATE CATALOG jdbc_oceanbase PROPERTIES ( unable to find valid certification path to requested target". ClientConnectionId:a92f3817-e8e6-4311-bc21-7c66 ``` - 可在创建 Catalog 的 `jdbc_url` 把JDBC连接串最后增加 `encrypt=false` ,如 `"jdbc_url" = "jdbc:sqlserver://127.0.0.1:1433;DataBaseName=doris_test;encrypt=false"` \ No newline at end of file + 可在创建 Catalog 的 `jdbc_url` 把JDBC连接串最后增加 `encrypt=false` ,如 `"jdbc_url" = "jdbc:sqlserver://127.0.0.1:1433;DataBaseName=doris_test;encrypt=false"` + +11. 读取 MySQL datetime 类型出现异常 + + ``` + ERROR 1105 (HY000): errCode = 2, detailMessage = (10.16.10.6)[INTERNAL_ERROR]UdfRuntimeException: get next block failed: + CAUSED BY: SQLException: Zero date value prohibited + CAUSED BY: DataReadException: Zero date value prohibited + ``` + + 这是因为 JDBC 并不能处理 0000-00-00 00:00:00 这种时间格式， + 需要在创建 Catalog 的 `jdbc_url` 把JDBC连接串最后增加 `zeroDateTimeBehavior=convertToNull` ,如 `"jdbc_url" = "jdbc:mysql://127.0.0.1:3306/test?zeroDateTimeBehavior=convertToNull"` + 这种情况下，JDBC 会把 0000-00-00 00:00:00 转换成 null，然后 Doris 会把 DateTime 类型的列按照可空类型处理，这样就可以正常读取了。 \ No newline at end of file diff --git a/docs/zh-CN/docs/lakehouse/multi-catalog/multi-catalog.md b/docs/zh-CN/docs/lakehouse/multi-catalog/multi-catalog.md index f77634bcc7ff730..e8cec8d0cdd8062 100644 --- a/docs/zh-CN/docs/lakehouse/multi-catalog/multi-catalog.md +++ b/docs/zh-CN/docs/lakehouse/multi-catalog/multi-catalog.md @@ -1,6 +1,6 @@ --- { - "title": "多源数据目录", + "title": "概述", "language": "zh-CN" } --- @@ -25,7 +25,7 @@ under the License. --> -# 多源数据目录 +# 概述 多源数据目录（Multi-Catalog）功能，旨在能够更方便对接外部数据目录，以增强Doris的数据湖分析和联邦数据查询能力。 diff --git a/docs/zh-CN/docs/query-acceleration/join-optimization/doris-join-optimization.md b/docs/zh-CN/docs/query-acceleration/join-optimization/doris-join-optimization.md index 98ad05f5ce985b6..8f2b28560fc33ca 100644 --- a/docs/zh-CN/docs/query-acceleration/join-optimization/doris-join-optimization.md +++ b/docs/zh-CN/docs/query-acceleration/join-optimization/doris-join-optimization.md @@ -55,7 +55,7 @@ Doris 支持 4 种 Shuffle 方式 当进行 Hash Join 时候，可以通过 Join 列计算对应的 Hash 值，并进行 Hash 分桶。 - 它的网络开销则是：T（R） + T（N），但它只能支持 Hash Join，因为它是根据 Join 的条件也去做计算分桶的。 + 它的网络开销则是：T（S） + T（R），但它只能支持 Hash Join，因为它是根据 Join 的条件也去做计算分桶的。 ![image-20220523151902368](/images/join/image-20220523151902368.png) diff --git a/docs/zh-CN/docs/query-acceleration/nereids.md b/docs/zh-CN/docs/query-acceleration/nereids.md index 6868d10274329bb..a49052cd54aa7ed 100644 --- a/docs/zh-CN/docs/query-acceleration/nereids.md +++ b/docs/zh-CN/docs/query-acceleration/nereids.md @@ -30,13 +30,13 @@ under the License. ## 研发背景 -现代查询优化器面临更加复杂的查询语句、更加多样的查询场景等挑战。同时，用户也越来越迫切的希望尽快获得查询结果。旧优化器的陈旧架构，难以满足今后快速迭代的需要。基于此，我们开始着手研发现代架构的全新查询优化器。在更高效的处理当前Doris场景的查询请求同时，提供更好的扩展性，为处理今后 Doris 所需面临的更复杂的需求打下良好的基础。 +现代查询优化器面临更加复杂的查询语句、更加多样的查询场景等挑战。同时，用户也越来越迫切的希望尽快获得查询结果。旧优化器的陈旧架构，难以满足今后快速迭代的需要。基于此，我们开始着手研发现代架构的全新查询优化器。在更高效的处理当前Doris场景的查询请求的同时，提供更好的扩展性，为处理今后 Doris 所需面临的更复杂的需求打下良好的基础。 ## 新优化器的优势 ### 更智能 -新优化器将每个 RBO 和 CBO 的优化点以规则的形式呈现。对于每一个规则，新优化器都提供了一组用于描述查询计划形状的模式，可以精确的匹配可优化的查询计划。基于此，新优化器更好的可以支持诸如多层子查询嵌套等更为复杂的查询语句。 +新优化器将每个 RBO 和 CBO 的优化点以规则的形式呈现。对于每一个规则，新优化器都提供了一组用于描述查询计划形状的模式，可以精确的匹配可优化的查询计划。基于此，新优化器可以更好的支持诸如多层子查询嵌套等更为复杂的查询语句。 同时新优化器的 CBO 基于先进的 Cascades 框架，使用了更为丰富的数据统计信息，并应用了维度更科学的代价模型。这使得新优化器在面对多表 Join 的查询时，更加得心应手。 @@ -46,7 +46,7 @@ TPC-H SF100 查询速度比较。环境为 3BE，新优化器使用原始 SQL ### 更健壮 -新优化器的所有优化规则，均在逻辑执行计划树上完成。当查询语法语义解析完成后，变转换为一颗树状结构。相比于旧优化器的内部数据结构更为合理、统一。以子查询处理为例，新优化器基于新的数据结构，避免了旧优化器中众多规则对子查询的单独处理。进而减少了优化规则逻辑错误的可能。 +新优化器的所有优化规则，均在逻辑执行计划树上完成。当查询语法语义解析完成后，变转换为一棵树状结构。相比于旧优化器的内部数据结构更为合理、统一。以子查询处理为例，新优化器基于新的数据结构，避免了旧优化器中众多规则对子查询的单独处理。进而减少了优化规则出现逻辑错误的可能。 ### 更灵活 diff --git a/docs/zh-CN/docs/query-acceleration/statistics.md b/docs/zh-CN/docs/query-acceleration/statistics.md index a232d87eb0c581f..3e64b08bd53820d 100644 --- a/docs/zh-CN/docs/query-acceleration/statistics.md +++ b/docs/zh-CN/docs/query-acceleration/statistics.md @@ -671,11 +671,12 @@ mysql> SHOW TABLE STATS stats_test.example_tbl PARTITION (p_201701); 语法如下： ```SQL -SHOW COLUMN STATS table_name [ (column_name [, ...]) ] [ PARTITION (partition_name) ]; +SHOW COLUMN [cached] STATS table_name [ (column_name [, ...]) ] [ PARTITION (partition_name) ]; ``` 其中： +- cached: 展示当前FE内存缓存中的统计信息。 - table_name: 收集统计信息的目标表。可以是  `db_name.table_name`  形式。 - column_name: 指定的目标列，必须是  `table_name`  中存在的列，多个列名称用逗号分隔。 - partition_name: 指定的目标分区，必须是  `table_name`  中存在的分区，只能指定一个分区。 diff --git a/docs/zh-CN/docs/releasenotes/release-1.2.0.md b/docs/zh-CN/docs/releasenotes/release-1.2.0.md index 3b7c6c89079d62c..d57b0c931500669 100644 --- a/docs/zh-CN/docs/releasenotes/release-1.2.0.md +++ b/docs/zh-CN/docs/releasenotes/release-1.2.0.md @@ -183,9 +183,9 @@ Multi-Catalog 多源数据目录功能的目标在于能够帮助用户更方便 支持 DataV2 日期类型和 DatatimeV2 日期时间类型，相较于原有的 Data 和 Datatime 效率更高且支持最多到微秒的时间精度，建议使用新版日期类型。 -文档：[https://doris.apache.org/zh-CN/docs/dev/sql-manual/sql-reference/Data-Types/DATETIMEV2/](https://doris.apache.org/zh-CN/docs/dev/sql-manual/sql-reference/Data-Types/DATETIMEV2/) +文档：[https://doris.apache.org/zh-CN/docs/1.2/sql-manual/sql-reference/Data-Types/DATETIMEV2](https://doris.apache.org/zh-CN/docs/1.2/sql-manual/sql-reference/Data-Types/DATETIMEV2) - [https://doris.apache.org/zh-CN/docs/dev/sql-manual/sql-reference/Data-Types/DATEV2](https://doris.apache.org/zh-CN/docs/dev/sql-manual/sql-reference/Data-Types/DATEV2) + [https://doris.apache.org/zh-CN/docs/1.2/sql-manual/sql-reference/Data-Types/DATEV2](https://doris.apache.org/zh-CN/docs/1.2/sql-manual/sql-reference/Data-Types/DATEV2) 影响范围： 1. 用户需要在建表时指定 DateV2 和 DatetimeV2，原有表的 Date 以及 Datetime 不受影响。 diff --git a/docs/en/docs/sql-manual/sql-reference/Database-Administration-Statements/ENABLE-FEATURE.md b/docs/zh-CN/docs/sql-manual/sql-functions/combinators/merge.md similarity index 59% rename from docs/en/docs/sql-manual/sql-reference/Database-Administration-Statements/ENABLE-FEATURE.md rename to docs/zh-CN/docs/sql-manual/sql-functions/combinators/merge.md index e5b97a04bb6ccc0..fa263a2e3982946 100644 --- a/docs/en/docs/sql-manual/sql-reference/Database-Administration-Statements/ENABLE-FEATURE.md +++ b/docs/zh-CN/docs/sql-manual/sql-functions/combinators/merge.md @@ -1,11 +1,11 @@ --- { - "title": "ENABLE-FEATURE", - "language": "en" + "title": "MERGE", + "language": "zh-CN" } --- - -## ENABLE-FEATURE +## MERGE -### Description + + -### Example -### Keywords +### description +#### Syntax - ENABLE, FEATURE - -### Best Practice +`AGGREGATE_FUNCTION_MERGE(agg_state)` +将聚合中间结果进行聚合并计算获得实际结果。 +结果的类型与`AGGREGATE_FUNCTION`一致。 +### example +``` +mysql [test]>select avg_merge(avg_state(1)) from d_table; ++-------------------------+ +| avg_merge(avg_state(1)) | ++-------------------------+ +| 1 | ++-------------------------+ +``` +### keywords +AGG_STATE, MERGE diff --git a/docs/zh-CN/docs/sql-manual/sql-functions/combinators/state.md b/docs/zh-CN/docs/sql-manual/sql-functions/combinators/state.md new file mode 100644 index 000000000000000..8c8ddaa26e9d9ba --- /dev/null +++ b/docs/zh-CN/docs/sql-manual/sql-functions/combinators/state.md @@ -0,0 +1,50 @@ +--- +{ + "title": "STATE", + "language": "zh-CN" +} +--- + + + +## STATE + + + + + +### description +#### Syntax + +`AGGREGATE_FUNCTION_STATE(arg...)` +返回聚合函数的中间结果，可以用于后续的聚合或者通过merge组合器获得实际计算结果，也可以直接写入agg_state类型的表保存下来。 +结果的类型为agg_state，agg_state中的函数签名为`AGGREGATE_FUNCTION(arg...)`。 + +### example +``` +mysql [test]>select avg_merge(t) from (select avg_union(avg_state(1)) as t from d_table group by k1)p; ++----------------+ +| avg_merge(`t`) | ++----------------+ +| 1 | ++----------------+ +``` +### keywords +AGG_STATE,STATE diff --git a/docs/zh-CN/docs/sql-manual/sql-reference/Database-Administration-Statements/ENABLE-FEATURE.md b/docs/zh-CN/docs/sql-manual/sql-functions/combinators/union.md similarity index 61% rename from docs/zh-CN/docs/sql-manual/sql-reference/Database-Administration-Statements/ENABLE-FEATURE.md rename to docs/zh-CN/docs/sql-manual/sql-functions/combinators/union.md index d9e0b55e35b815f..8957e6c2f8d7c1c 100644 --- a/docs/zh-CN/docs/sql-manual/sql-reference/Database-Administration-Statements/ENABLE-FEATURE.md +++ b/docs/zh-CN/docs/sql-manual/sql-functions/combinators/union.md @@ -1,11 +1,11 @@ --- { - "title": "ENABLE-FEATURE", + "title": "UNION", "language": "zh-CN" } --- - -## ENABLE-FEATURE +## UNION -### Description + + -### Example -### Keywords +### description +#### Syntax - ENABLE, FEATURE - -### Best Practice +`AGGREGATE_FUNCTION_UNION(agg_state)` +将多个聚合中间结果聚合为一个。 +结果的类型为agg_state，函数签名与入参一致。 +### example +``` +mysql [test]>select avg_merge(t) from (select avg_union(avg_state(1)) as t from d_table group by k1)p; ++----------------+ +| avg_merge(`t`) | ++----------------+ +| 1 | ++----------------+ +``` +### keywords +AGG_STATE, UNION diff --git a/docs/zh-CN/docs/sql-manual/sql-functions/table-functions/hdfs.md b/docs/zh-CN/docs/sql-manual/sql-functions/table-functions/hdfs.md index 65cb7f3cde43fb1..f0f22831b08ae53 100644 --- a/docs/zh-CN/docs/sql-manual/sql-functions/table-functions/hdfs.md +++ b/docs/zh-CN/docs/sql-manual/sql-functions/table-functions/hdfs.md @@ -63,6 +63,12 @@ hdfs( - `dfs.client.read.shortcircuit`：（选填） - `dfs.domain.socket.path`：（选填） +访问 HA 模式 HDFS 相关参数： +- `dfs.nameservices`：（选填） +- `dfs.ha.namenodes.your-nameservices`：（选填） +- `dfs.namenode.rpc-address.your-nameservices.your-namenode`：（选填） +- `dfs.client.failover.proxy.provider.your-nameservices`：（选填） + 文件格式相关参数 - `format`：(必填) 目前支持 `csv/csv_with_names/csv_with_names_and_types/json/parquet/orc` - `column_separator`：(选填) 列分割符, 默认为`,`。 @@ -103,6 +109,29 @@ MySQL [(none)]> select * from hdfs( +------+---------+------+ ``` +读取并访问 HA 模式的 HDFS 存储上的csv格式文件 +```sql +MySQL [(none)]> select * from hdfs( + "uri" = "hdfs://127.0.0.1:842/user/doris/csv_format_test/student.csv", + "fs.defaultFS" = "hdfs://127.0.0.1:8424", + "hadoop.username" = "doris", + "format" = "csv", + "dfs.nameservices" = "my_hdfs", + "dfs.ha.namenodes.my_hdfs" = "nn1,nn2", + "dfs.namenode.rpc-address.my_hdfs.nn1" = "nanmenode01:8020", + "dfs.namenode.rpc-address.my_hdfs.nn2" = "nanmenode02:8020", + "dfs.client.failover.proxy.provider.my_hdfs" = "org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider"); ++------+---------+------+ +| c1 | c2 | c3 | ++------+---------+------+ +| 1 | alice | 18 | +| 2 | bob | 20 | +| 3 | jack | 24 | +| 4 | jackson | 19 | +| 5 | liming | 18 | ++------+---------+------+ +``` + 可以配合`desc function`使用 ```sql diff --git a/docs/zh-CN/docs/sql-manual/sql-functions/table-functions/s3.md b/docs/zh-CN/docs/sql-manual/sql-functions/table-functions/s3.md index 4ac6c8fa550a812..66708b0d983b5ce 100644 --- a/docs/zh-CN/docs/sql-manual/sql-functions/table-functions/s3.md +++ b/docs/zh-CN/docs/sql-manual/sql-functions/table-functions/s3.md @@ -169,7 +169,7 @@ select * from s3( ``` -**csv foramt** +**csv format** 由于S3 table-valued-function事先并不知道table schema，所以会先读一遍文件来解析出table schema。 `csv` 格式: S3 table-valued-function 读取S3上的文件并当作csv文件来处理，读取文件中的第一行用于解析table schema。文件第一行的列个数`n`将作为table schema的列个数，table schema的列名则自动取名为`c1, c2, ..., cn` ，列类型都设置为 `String`, 举例: @@ -216,7 +216,7 @@ MySQL [(none)]> Desc function s3("uri" = "http://127.0.0.1:9312/test2/student1.c +-------+------+------+-------+---------+-------+ ``` -**csv_with_names foramt** +**csv_with_names format** `csv_with_names`格式：解析文件的第一行作为table schema的列个数和列名，列类型则都设置为 `String`, 举例： student_with_names.csv文件内容为 @@ -310,7 +310,7 @@ MySQL [(none)]> Desc function s3("uri" = "http://127.0.0.1:9312/test2/student_wi +-------+------+------+-------+---------+-------+ ``` -**json foramt** +**json format** `json` 格式：json格式涉及到较多的可选参数，各个参数的意义可以参考：[Json Load](../../../data-operate/import/import-way/load-json-format.md)。 S3 tvf查询json格式文件时根据 `json_root` 和 `jsonpaths` 参数定位到一个json对象，将该对象的中的`key` 作为table schema的列名，列类型都设置为String。举例： @@ -358,7 +358,7 @@ MySQL [(none)]> select * from s3( +------+------+ ``` -**parquet foramt** +**parquet format** `parquet` 格式：S3 tvf支持从parquet文件中解析出table schema的列名、列类型。举例： @@ -402,7 +402,7 @@ MySQL [(none)]> desc function s3( +---------------+--------------+------+-------+---------+-------+ ``` -**orc foramt** +**orc format** `orc` 格式：和`parquet` format使用方法一致, 将`format`参数设置为orc。 diff --git a/docs/zh-CN/docs/sql-manual/sql-functions/table-functions/workload-group.md b/docs/zh-CN/docs/sql-manual/sql-functions/table-functions/workload-group.md index 9a4802559b24935..a3ededbfd4acb2c 100644 --- a/docs/zh-CN/docs/sql-manual/sql-functions/table-functions/workload-group.md +++ b/docs/zh-CN/docs/sql-manual/sql-functions/table-functions/workload-group.md @@ -36,7 +36,7 @@ workload_groups ### description -表函数，生成 workload_groups 临时表，可以查看当前资源组信息。 +表函数，生成 workload_groups 临时表，可以查看当前用户具有权限的资源组信息。 该函数用于from子句中。 diff --git a/docs/zh-CN/docs/sql-manual/sql-reference/Data-Definition-Statements/Create/CREATE-WORKLOAD-GROUP.md b/docs/zh-CN/docs/sql-manual/sql-reference/Data-Definition-Statements/Create/CREATE-WORKLOAD-GROUP.md index b3cfb004ae98c47..9ab0f26dfab9db3 100644 --- a/docs/zh-CN/docs/sql-manual/sql-reference/Data-Definition-Statements/Create/CREATE-WORKLOAD-GROUP.md +++ b/docs/zh-CN/docs/sql-manual/sql-reference/Data-Definition-Statements/Create/CREATE-WORKLOAD-GROUP.md @@ -49,11 +49,11 @@ PROPERTIES ( property_list 支持的属性： -* cpu_share：必选，用于设置资源组获取cpu时间的多少，可以实现cpu资源软隔离。cpu_share 是相对值，表示正在运行的资源组可获取cpu资源的权重。例如，用户创建了3个资源组 rg-a、rg-b和rg-c，cpu_share 分别为 10、30、40，某一时刻rg-a和rg-b正在跑任务，而rg-c没有任务，此时rg-a可获得 25% (10 / (10 + 30))的cpu资源，而资源组rg-b可获得75%的cpu资源。如果系统只有一个资源组正在运行，则不管其cpu_share的值为多少，它都可以获取全部的cpu资源。 +* cpu_share: 必选，用于设置资源组获取cpu时间的多少，可以实现cpu资源软隔离。cpu_share 是相对值，表示正在运行的资源组可获取cpu资源的权重。例如，用户创建了3个资源组 rg-a、rg-b和rg-c，cpu_share 分别为 10、30、40，某一时刻rg-a和rg-b正在跑任务，而rg-c没有任务，此时rg-a可获得 25% (10 / (10 + 30))的cpu资源，而资源组rg-b可获得75%的cpu资源。如果系统只有一个资源组正在运行，则不管其cpu_share的值为多少，它都可以获取全部的cpu资源。 -* memory_limit: 必选，用于设置资源组可以使用be内存的百分比。资源组内存限制的绝对值为： 物理内存 * mem_limit * memory_limit，其中 mem_limit 为be配置项。系统所有资源组的 memory_limit总合不可超过100%。资源组在绝大多数情况下保证组内任务可使用memory_limit的内存，当资源组内存使用超出该限制后，组内内存占用较大的任务可能会被cancel以释放超出的内存，参考 enable_memory_overcommit。 +* memory_limit: 必选，用于设置资源组可以使用be内存的百分比。资源组内存限制的绝对值为：`物理内存 * mem_limit * memory_limit`，其中 mem_limit 为be配置项。系统所有资源组的 memory_limit总合不可超过100%。资源组在绝大多数情况下保证组内任务可使用memory_limit的内存，当资源组内存使用超出该限制后，组内内存占用较大的任务可能会被cancel以释放超出的内存，参考 enable_memory_overcommit。 -* enable_memory_overcommit: 可选，用于开启资源组内存软隔离，默认为false。如果设置为false，则该资源组为内存硬隔离，系统检测到资源组内存使用超出限制后将立即cancel组内内存占用最大的若干个任务，以释放超出的内存；如果设置为true，则该资源组为内存软隔离，如果系统有空闲内存资源则该资源组在超出memory_limit的限制后可继续使用系统内存，在系统总内存紧张时会cancel组内内存占用最大的若干个任务，释放部分超出的内存以缓解系统内存压力。建议在有资源组开启该配置时，所有资源组的 memory_limit 总合低于100%，剩余部分用于资源组内存超发。 +* enable_memory_overcommit: 可选，用于开启资源组内存软隔离，默认为false。如果设置为false，则该资源组为内存硬隔离，系统检测到资源组内存使用超出限制后将立即cancel组内内存占用最大的若干个任务，以释放超出的内存；如果设置为true，则该资源组为内存软隔离，如果系统有空闲内存资源则该资源组在超出memory_limit的限制后可继续使用系统内存，在系统总内存紧张时会cancel组内内存占用最大的若干个任务，释放部分超出的内存以缓解系统内存压力。建议在有资源组开启该配置时，所有资源组的 memory_limit 总和低于100%，剩余部分用于资源组内存超发。 ### Example diff --git a/docs/zh-CN/docs/sql-manual/sql-reference/Data-Manipulation-Statements/Load/STREAM-LOAD.md b/docs/zh-CN/docs/sql-manual/sql-reference/Data-Manipulation-Statements/Load/STREAM-LOAD.md index 639393b4ebada9c..223bb8b46244f06 100644 --- a/docs/zh-CN/docs/sql-manual/sql-reference/Data-Manipulation-Statements/Load/STREAM-LOAD.md +++ b/docs/zh-CN/docs/sql-manual/sql-reference/Data-Manipulation-Statements/Load/STREAM-LOAD.md @@ -124,63 +124,34 @@ curl --location-trusted -u user:passwd [-H ""...] -T data.file -XPUT http://fe_h 16. merge_type: 数据的合并类型，一共支持三种类型APPEND、DELETE、MERGE 其中，APPEND是默认值，表示这批数据全部需要追加到现有数据中，DELETE 表示删除与这批数据key相同的所有行，MERGE 语义 需要与delete 条件联合使用，表示满足delete 条件的数据按照DELETE 语义处理其余的按照APPEND 语义处理， 示例：`-H "merge_type: MERGE" -H "delete: flag=1"` 17. delete: 仅在 MERGE下有意义， 表示数据的删除条件 - function_column.sequence_col: 只适用于UNIQUE_KEYS,相同key列下，保证value列按照source_sequence列进行REPLACE, source_sequence可以是数据源中的列，也可以是表结构中的一列。 + +18. function_column.sequence_col: 只适用于UNIQUE_KEYS,相同key列下，保证value列按照source_sequence列进行REPLACE, source_sequence可以是数据源中的列，也可以是表结构中的一列。 -18. fuzzy_parse: 布尔类型，为true表示json将以第一行为schema 进行解析，开启这个选项可以提高 json 导入效率，但是要求所有json 对象的key的顺序和第一行一致， 默认为false，仅用于json 格式 +19. fuzzy_parse: 布尔类型，为true表示json将以第一行为schema 进行解析，开启这个选项可以提高 json 导入效率，但是要求所有json 对象的key的顺序和第一行一致， 默认为false，仅用于json 格式 -19. num_as_string: 布尔类型，为true表示在解析json数据时会将数字类型转为字符串，然后在确保不会出现精度丢失的情况下进行导入。 +20. num_as_string: 布尔类型，为true表示在解析json数据时会将数字类型转为字符串，然后在确保不会出现精度丢失的情况下进行导入。 -20. read_json_by_line: 布尔类型，为true表示支持每行读取一个json对象，默认值为false。 +21. read_json_by_line: 布尔类型，为true表示支持每行读取一个json对象，默认值为false。 -21. send_batch_parallelism: 整型，用于设置发送批处理数据的并行度，如果并行度的值超过 BE 配置中的 `max_send_batch_parallelism_per_job`，那么作为协调点的 BE 将使用 `max_send_batch_parallelism_per_job` 的值。 +22. send_batch_parallelism: 整型，用于设置发送批处理数据的并行度，如果并行度的值超过 BE 配置中的 `max_send_batch_parallelism_per_job`，那么作为协调点的 BE 将使用 `max_send_batch_parallelism_per_job` 的值。 -22. hidden_columns: 用于指定导入数据中包含的隐藏列，在Header中不包含columns时生效，多个hidden column用逗号分割。 +23. hidden_columns: 用于指定导入数据中包含的隐藏列，在Header中不包含columns时生效，多个hidden column用逗号分割。 ``` hidden_columns: __DORIS_DELETE_SIGN__,__DORIS_SEQUENCE_COL__ 系统会使用用户指定的数据导入数据。在上述用例中，导入数据中最后一列数据为__DORIS_SEQUENCE_COL__。 ``` -23. load_to_single_tablet: 布尔类型，为true表示支持一个任务只导入数据到对应分区的一个 tablet，默认值为 false，该参数只允许在对带有 random 分区的 olap 表导数的时候设置。 - - RETURN VALUES - 导入完成后，会以Json格式返回这次导入的相关内容。当前包括以下字段 - Status: 导入最后的状态。 - Success：表示导入成功，数据已经可见； - Publish Timeout：表述导入作业已经成功Commit，但是由于某种原因并不能立即可见。用户可以视作已经成功不必重试导入 - Label Already Exists: 表明该Label已经被其他作业占用，可能是导入成功，也可能是正在导入。 - 用户需要通过get label state命令来确定后续的操作 - 其他：此次导入失败，用户可以指定Label重试此次作业 - Message: 导入状态详细的说明。失败时会返回具体的失败原因。 - NumberTotalRows: 从数据流中读取到的总行数 - NumberLoadedRows: 此次导入的数据行数，只有在Success时有效 - NumberFilteredRows: 此次导入过滤掉的行数，即数据质量不合格的行数 - NumberUnselectedRows: 此次导入，通过 where 条件被过滤掉的行数 - LoadBytes: 此次导入的源文件数据量大小 - LoadTimeMs: 此次导入所用的时间 - BeginTxnTimeMs: 向Fe请求开始一个事务所花费的时间，单位毫秒。 - StreamLoadPutTimeMs: 向Fe请求获取导入数据执行计划所花费的时间，单位毫秒。 - ReadDataTimeMs: 读取数据所花费的时间，单位毫秒。 - WriteDataTimeMs: 执行写入数据操作所花费的时间，单位毫秒。 - CommitAndPublishTimeMs: 向Fe请求提交并且发布事务所花费的时间，单位毫秒。 - ErrorURL: 被过滤数据的具体内容，仅保留前1000条 - -ERRORS: - 可以通过以下语句查看导入错误详细信息： +24. load_to_single_tablet: 布尔类型，为true表示支持一个任务只导入数据到对应分区的一个 tablet，默认值为 false，该参数只允许在对带有 random 分区的 olap 表导数的时候设置。 - ``` - SHOW LOAD WARNINGS ON 'url' - ``` - 其中 url 为 ErrorURL 给出的 url。 +25. compress_type: 指定文件的压缩格式。目前只支持 csv 文件的压缩。支持 gz, lzo, bz2, lz4, lzop, deflate 压缩格式。 -24. compress_type +26. trim_double_quotes: 布尔类型，默认值为 false，为 true 时表示裁剪掉 csv 文件每个字段最外层的双引号。 - 指定文件的压缩格式。目前只支持 csv 文件的压缩。支持 gz, lzo, bz2, lz4, lzop, deflate 压缩格式。 +27. skip_lines: 整数类型, 默认值为0, 含义为跳过csv文件的前几行. 当设置format设置为 `csv_with_names` 或、`csv_with_names_and_types` 时, 该参数会失效. -25. trim_double_quotes: 布尔类型，默认值为 false，为 true 时表示裁剪掉 csv 文件每个字段最外层的双引号。 +28. comment: 字符串类型, 默认值为空. 给任务增加额外的信息. -26. skip_lines: 整数类型, 默认值为0, 含义为跳过csv文件的前几行. 当设置format设置为 `csv_with_names` 或、`csv_with_names_and_types` 时, 该参数会失效. -27. comment: 字符串类型, 默认值为空. 给任务增加额外的信息. ### Example 1. 将本地文件'testData'中的数据导入到数据库'testDb'中'testTbl'的表，使用Label用于去重。指定超时时间为 100 秒 @@ -375,52 +346,53 @@ ERRORS: } ``` - 字段释义如下： + 下面主要解释了 Stream load 导入结果参数： + + - TxnId：导入的事务ID。用户可不感知。 + + - Label：导入 Label。由用户指定或系统自动生成。 - - TxnId：导入事务ID，由系统自动生成，全局唯一。 + - Status：导入完成状态。 - - Label：导入Label，如果没有指定，则系统会生成一个 UUID。 + "Success"：表示导入成功。 - - Status： + "Publish Timeout"：该状态也表示导入已经完成，只是数据可能会延迟可见，无需重试。 - 导入结果。有如下取值： + "Label Already Exists"：Label 重复，需更换 Label。 - - Success：表示导入成功，并且数据已经可见。 - - Publish Timeout：该状态也表示导入已经完成，只是数据可能会延迟可见。 - - Label Already Exists：Label 重复，需更换 Label。 - - Fail：导入失败。 + "Fail"：导入失败。 - - ExistingJobStatus： + - ExistingJobStatus：已存在的 Label 对应的导入作业的状态。 - 已存在的 Label 对应的导入作业的状态。 + 这个字段只有在当 Status 为 "Label Already Exists" 时才会显示。用户可以通过这个状态，知晓已存在 Label 对应的导入作业的状态。"RUNNING" 表示作业还在执行，"FINISHED" 表示作业成功。 - 这个字段只有在当 Status 为 "Label Already Exists" 是才会显示。用户可以通过这个状态，知晓已存在 Label 对应的导入作业的状态。"RUNNING" 表示作业还在执行，"FINISHED" 表示作业成功。 + - Message：导入错误信息。 - - Message：导入错误信息。 + - NumberTotalRows：导入总处理的行数。 - - NumberTotalRows：导入总处理的行数。 + - NumberLoadedRows：成功导入的行数。 - - NumberLoadedRows：成功导入的行数。 + - NumberFilteredRows：数据质量不合格的行数。 - - NumberFilteredRows：数据质量不合格的行数。 + - NumberUnselectedRows：被 where 条件过滤的行数。 - - NumberUnselectedRows：被 where 条件过滤的行数。 + - LoadBytes：导入的字节数。 - - LoadBytes：导入的字节数。 + - LoadTimeMs：导入完成时间。单位毫秒。 - - LoadTimeMs：导入完成时间。单位毫秒。 + - BeginTxnTimeMs：向Fe请求开始一个事务所花费的时间，单位毫秒。 - - BeginTxnTimeMs：向 FE 请求开始一个事务所花费的时间，单位毫秒。 + - StreamLoadPutTimeMs：向Fe请求获取导入数据执行计划所花费的时间，单位毫秒。 - - StreamLoadPutTimeMs：向 FE 请求获取导入数据执行计划所花费的时间，单位毫秒。 + - ReadDataTimeMs：读取数据所花费的时间，单位毫秒。 - - ReadDataTimeMs：读取数据所花费的时间，单位毫秒。 + - WriteDataTimeMs：执行写入数据操作所花费的时间，单位毫秒。 - - WriteDataTimeMs：执行写入数据操作所花费的时间，单位毫秒。 + - CommitAndPublishTimeMs：向Fe请求提交并且发布事务所花费的时间，单位毫秒。 - - CommitAndPublishTimeMs：向Fe请求提交并且发布事务所花费的时间，单位毫秒。 + - ErrorURL：如果有数据质量问题，通过访问这个 URL 查看具体错误行。 - - ErrorURL：如果有数据质量问题，通过访问这个 URL 查看具体错误行。 + > 注意：由于 Stream load 是同步的导入方式，所以并不会在 Doris 系统中记录导入信息，用户无法异步的通过查看导入命令看到 Stream load。使用时需监听创建导入请求的返回值获取导入结果。 2. 如何正确提交 Stream Load 作业和处理返回结果。 diff --git a/docs/zh-CN/docs/sql-manual/sql-reference/Data-Manipulation-Statements/OUTFILE.md b/docs/zh-CN/docs/sql-manual/sql-reference/Data-Manipulation-Statements/OUTFILE.md index c69c596b851272e..ff650047b2b2596 100644 --- a/docs/zh-CN/docs/sql-manual/sql-reference/Data-Manipulation-Statements/OUTFILE.md +++ b/docs/zh-CN/docs/sql-manual/sql-reference/Data-Manipulation-Statements/OUTFILE.md @@ -80,8 +80,8 @@ INTO OUTFILE "file_path" 支持如下属性： 文件相关的属性 - column_separator: 列分隔符。支持多字节分隔符，如："\\x01", "abc" - line_delimiter: 行分隔符。支持多字节分隔符，如："\\x01", "abc" + column_separator: 列分隔符，只支持csv格式。支持多字节分隔符，如："\\x01", "abc" + line_delimiter: 行分隔符，只支持csv格式。支持多字节分隔符，如："\\x01", "abc" max_file_size: 单个文件大小限制，如果结果超过这个值，将切割成多个文件。 delete_existing_files: 默认为false，若指定为true,则会先删除file_path指定的目录下的所有文件，然后导出数据到该目录下。例如："file_path" = "/user/tmp", 则会删除"/user/"下所有文件及目录；"file_path" = "/user/tmp/", 则会删除"/user/tmp/"下所有文件及目录 diff --git a/docs/zh-CN/docs/sql-manual/sql-reference/Data-Types/AGG_STATE.md b/docs/zh-CN/docs/sql-manual/sql-reference/Data-Types/AGG_STATE.md new file mode 100644 index 000000000000000..291f83541138956 --- /dev/null +++ b/docs/zh-CN/docs/sql-manual/sql-reference/Data-Types/AGG_STATE.md @@ -0,0 +1,90 @@ +--- +{ + "title": "AGG_STATE", + "language": "zh-CN" +} +--- + + + +## AGG_STATE +### description + AGG_STATE不能作为key列使用，建表时需要同时声明聚合函数的签名。 + 用户不需要指定长度和默认值。实际存储的数据大小与函数实现有关。 + + AGG_STATE只能配合[state](../../sql-functions/combinators/state.md) + /[merge](../../sql-functions/combinators/merge.md)/[union](../..//sql-functions/combinators/union.md)函数组合器使用。 + + 需要注意的是，聚合函数的签名也是类型的一部分，不同签名的agg_state无法混合使用。比如如果建表声明的签名为`max_by(int,int)`,那就无法插入`max_by(bigint,int)`或者`group_concat(varchar)`。 + 此处nullable属性也是签名的一部分，如果能确定不会输入null值，可以将参数声明为not null，这样可以获得更小的存储大小和减少序列化/反序列化开销。 + +### example + +建表示例如下： + ```sql + create table a_table( + k1 int null, + k2 agg_state max_by(int not null,int), + k3 agg_state group_concat(string) + ) + aggregate key (k1) + distributed BY hash(k1) buckets 3 + properties("replication_num" = "1"); + ``` + 这里的k2和k3分别以max_by和group_concat为聚合类型。 + +插入数据示例： + ```sql + insert into a_table values(1,max_by_state(3,1),group_concat_state('a')); + insert into a_table values(1,max_by_state(2,2),group_concat_state('bb')); + insert into a_table values(2,max_by_state(1,3),group_concat_state('ccc')); + ``` + 对于agg_state列，插入语句必须用[state](../../sql-functions/combinators/state.md)函数来生成对应的agg_state数据，这里的函数和入参类型都必须跟agg_state完全对应。 + +查询数据示例： + + ```sql + mysql [test]>select k1,max_by_merge(k2),group_concat_merge(k3) from a_table group by k1 order by k1; + +------+--------------------+--------------------------+ + | k1 | max_by_merge(`k2`) | group_concat_merge(`k3`) | + +------+--------------------+--------------------------+ + | 1 | 2 | bb,a | + | 2 | 1 | ccc | + +------+--------------------+--------------------------+ + ``` + + 如果需要获取实际结果，则要用对应的[merge](../../sql-functions/combinators/merge.md)函数。 + + ```sql + mysql [test]>select max_by_merge(u2),group_concat_merge(u3) from ( + select k1,max_by_union(k2) as u2,group_concat_union(k3) u3 from a_table group by k1 order by k1 + ) t; + +--------------------+--------------------------+ + | max_by_merge(`u2`) | group_concat_merge(`u3`) | + +--------------------+--------------------------+ + | 1 | ccc,bb,a | + +--------------------+--------------------------+ + ``` + +如果想要在过程中只聚合agg_state而不获取实际结果，可以使用[union](../..//sql-functions/combinators/union.md)函数。 + +### keywords + + AGG_STATE diff --git a/docs/zh-CN/docs/sql-manual/sql-reference/Data-Types/JSON.md b/docs/zh-CN/docs/sql-manual/sql-reference/Data-Types/JSON.md index 03203752c1318fb..b822cb15528160b 100644 --- a/docs/zh-CN/docs/sql-manual/sql-reference/Data-Types/JSON.md +++ b/docs/zh-CN/docs/sql-manual/sql-reference/Data-Types/JSON.md @@ -34,7 +34,7 @@ under the License. ### description JSON类型 - 二进制JSON类型，采用二进制JSON格式存储，通过json函数访问JSON内部字段。 + 二进制JSON类型，采用二进制JSON格式存储，通过json函数访问JSON内部字段。最大（默认）支持1048576 字节（1MB），JSONB类型还受be配置`jsonb_type_length_soft_limit_bytes`限制 ### note 与普通STRING类型存储的JSON字符串相比，JSON类型有两点优势 diff --git a/fe/be-java-extensions/avro-scanner/pom.xml b/fe/be-java-extensions/avro-scanner/pom.xml new file mode 100644 index 000000000000000..f4137696c480231 --- /dev/null +++ b/fe/be-java-extensions/avro-scanner/pom.xml @@ -0,0 +1,105 @@ + + + + + be-java-extensions + org.apache.doris + ${revision} + + 4.0.0 + + avro-scanner + + + 8 + 8 + UTF-8 + + + + + org.apache.doris + java-common + ${project.version} + + + org.apache.avro + avro + + + org.apache.avro + avro-tools + + + + + org.apache.hadoop + hadoop-client + + + org.apache.hadoop + hadoop-common + + + org.apache.hadoop + hadoop-hdfs + + + com.amazonaws + aws-java-sdk-s3 + + + org.apache.doris + hive-catalog-shade + + + + + + avro-scanner + + + org.apache.maven.plugins + maven-assembly-plugin + + + src/main/resources/package.xml + + + + + + + + + + make-assembly + package + + single + + + + + + + \ No newline at end of file diff --git a/fe/be-java-extensions/avro-scanner/src/main/java/org/apache/doris/avro/AvroColumnValue.java b/fe/be-java-extensions/avro-scanner/src/main/java/org/apache/doris/avro/AvroColumnValue.java new file mode 100644 index 000000000000000..3c796f1fc7d1919 --- /dev/null +++ b/fe/be-java-extensions/avro-scanner/src/main/java/org/apache/doris/avro/AvroColumnValue.java @@ -0,0 +1,162 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.avro; + +import org.apache.doris.common.jni.vec.ColumnValue; + +import org.apache.hadoop.hive.serde2.objectinspector.ListObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.MapObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector; + +import java.math.BigDecimal; +import java.math.BigInteger; +import java.time.LocalDate; +import java.time.LocalDateTime; +import java.util.List; +import java.util.Map.Entry; + +public class AvroColumnValue implements ColumnValue { + + private final Object fieldData; + private final ObjectInspector fieldInspector; + + public AvroColumnValue(ObjectInspector fieldInspector, Object fieldData) { + this.fieldInspector = fieldInspector; + this.fieldData = fieldData; + } + + private Object inspectObject() { + return ((PrimitiveObjectInspector) fieldInspector).getPrimitiveJavaObject(fieldData); + } + + @Override + public boolean canGetStringAsBytes() { + return false; + } + + @Override + public boolean isNull() { + return false; + } + + @Override + public boolean getBoolean() { + return (boolean) inspectObject(); + } + + @Override + public byte getByte() { + return (byte) inspectObject(); + } + + @Override + public short getShort() { + return (short) inspectObject(); + } + + @Override + public int getInt() { + return (int) inspectObject(); + } + + @Override + public float getFloat() { + return (float) inspectObject(); + } + + @Override + public long getLong() { + return (long) inspectObject(); + } + + @Override + public double getDouble() { + return (double) inspectObject(); + } + + @Override + public BigInteger getBigInteger() { + return null; + } + + @Override + public BigDecimal getDecimal() { + return (BigDecimal) inspectObject(); + } + + @Override + public String getString() { + return inspectObject().toString(); + } + + @Override + public LocalDate getDate() { + // avro has no date type + return null; + } + + @Override + public LocalDateTime getDateTime() { + // avro has no dateTime type + return null; + } + + @Override + public byte[] getBytes() { + return (byte[]) inspectObject(); + } + + @Override + public void unpackArray(List values) { + ListObjectInspector inspector = (ListObjectInspector) fieldInspector; + List items = inspector.getList(fieldData); + ObjectInspector itemInspector = inspector.getListElementObjectInspector(); + for (Object item : items) { + AvroColumnValue avroColumnValue = null; + if (item != null) { + avroColumnValue = new AvroColumnValue(itemInspector, item); + } + values.add(avroColumnValue); + } + } + + @Override + public void unpackMap(List keys, List values) { + MapObjectInspector inspector = (MapObjectInspector) fieldInspector; + ObjectInspector keyObjectInspector = inspector.getMapKeyObjectInspector(); + ObjectInspector valueObjectInspector = inspector.getMapValueObjectInspector(); + for (Entry kv : inspector.getMap(fieldData).entrySet()) { + AvroColumnValue avroKey = null; + AvroColumnValue avroValue = null; + if (kv.getKey() != null) { + avroKey = new AvroColumnValue(keyObjectInspector, kv.getKey()); + } + if (kv.getValue() != null) { + avroValue = new AvroColumnValue(valueObjectInspector, kv.getValue()); + } + keys.add(avroKey); + values.add(avroValue); + } + } + + @Override + public void unpackStruct(List structFieldIndex, List values) { + + } +} diff --git a/fe/be-java-extensions/avro-scanner/src/main/java/org/apache/doris/avro/AvroJNIScanner.java b/fe/be-java-extensions/avro-scanner/src/main/java/org/apache/doris/avro/AvroJNIScanner.java new file mode 100644 index 000000000000000..72b9e414169ec74 --- /dev/null +++ b/fe/be-java-extensions/avro-scanner/src/main/java/org/apache/doris/avro/AvroJNIScanner.java @@ -0,0 +1,247 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.avro; + +import org.apache.doris.common.jni.JniScanner; +import org.apache.doris.common.jni.vec.ColumnType; +import org.apache.doris.common.jni.vec.ScanPredicate; +import org.apache.doris.common.jni.vec.TableSchema; +import org.apache.doris.common.jni.vec.TableSchema.SchemaColumn; +import org.apache.doris.thrift.TFileType; +import org.apache.doris.thrift.TPrimitiveType; + +import org.apache.avro.Schema; +import org.apache.avro.Schema.Field; +import org.apache.avro.generic.GenericRecord; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hive.common.JavaUtils; +import org.apache.hadoop.hive.serde.serdeConstants; +import org.apache.hadoop.hive.serde2.ColumnProjectionUtils; +import org.apache.hadoop.hive.serde2.Deserializer; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.StructField; +import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector; +import org.apache.log4j.LogManager; +import org.apache.log4j.Logger; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Objects; +import java.util.Properties; +import java.util.stream.Collectors; + +public class AvroJNIScanner extends JniScanner { + + private static final Logger LOG = LogManager.getLogger(AvroJNIScanner.class); + private final TFileType fileType; + private final String uri; + private final Map requiredParams; + private final Integer fetchSize; + private int[] requiredColumnIds; + private String[] columnTypes; + private String[] requiredFields; + private ColumnType[] requiredTypes; + private AvroReader avroReader; + private final boolean isGetTableSchema; + private StructObjectInspector rowInspector; + private Deserializer deserializer; + private StructField[] structFields; + private ObjectInspector[] fieldInspectors; + private String serde; + + /** + * Call by JNI for get table data or get table schema + * + * @param fetchSize The size of data fetched each time + * @param requiredParams required params + */ + public AvroJNIScanner(int fetchSize, Map requiredParams) { + this.requiredParams = requiredParams; + this.fetchSize = fetchSize; + this.isGetTableSchema = Boolean.parseBoolean(requiredParams.get(AvroProperties.IS_GET_TABLE_SCHEMA)); + this.fileType = TFileType.findByValue(Integer.parseInt(requiredParams.get(AvroProperties.FILE_TYPE))); + this.uri = requiredParams.get(AvroProperties.URI); + if (!isGetTableSchema) { + this.columnTypes = requiredParams.get(AvroProperties.COLUMNS_TYPES) + .split(AvroProperties.COLUMNS_TYPE_DELIMITER); + this.requiredFields = requiredParams.get(AvroProperties.REQUIRED_FIELDS) + .split(AvroProperties.FIELDS_DELIMITER); + this.requiredTypes = new ColumnType[requiredFields.length]; + this.serde = requiredParams.get(AvroProperties.HIVE_SERDE); + this.structFields = new StructField[requiredFields.length]; + this.fieldInspectors = new ObjectInspector[requiredFields.length]; + } + } + + private void init() throws Exception { + requiredColumnIds = new int[requiredFields.length]; + for (int i = 0; i < requiredFields.length; i++) { + ColumnType columnType = ColumnType.parseType(requiredFields[i], columnTypes[i]); + requiredTypes[i] = columnType; + requiredColumnIds[i] = i; + } + + Properties properties = createProperties(); + deserializer = getDeserializer(new Configuration(), properties, this.serde); + rowInspector = (StructObjectInspector) deserializer.getObjectInspector(); + + for (int i = 0; i < requiredFields.length; i++) { + StructField field = rowInspector.getStructFieldRef(requiredFields[i]); + structFields[i] = field; + fieldInspectors[i] = field.getFieldObjectInspector(); + } + } + + public Properties createProperties() { + Properties properties = new Properties(); + properties.setProperty(ColumnProjectionUtils.READ_COLUMN_IDS_CONF_STR, + Arrays.stream(this.requiredColumnIds).mapToObj(String::valueOf).collect(Collectors.joining(","))); + properties.setProperty(ColumnProjectionUtils.READ_COLUMN_NAMES_CONF_STR, String.join(",", requiredFields)); + properties.setProperty(AvroProperties.COLUMNS, String.join(",", requiredFields)); + properties.setProperty(AvroProperties.COLUMNS2TYPES, String.join(",", columnTypes)); + properties.setProperty(serdeConstants.SERIALIZATION_LIB, this.serde); + return properties; + } + + private Deserializer getDeserializer(Configuration configuration, Properties properties, String name) + throws Exception { + Class deserializerClass = Class.forName(name, true, JavaUtils.getClassLoader()) + .asSubclass(Deserializer.class); + Deserializer deserializer = deserializerClass.getConstructor().newInstance(); + deserializer.initialize(configuration, properties); + return deserializer; + } + + @Override + public void open() throws IOException { + try { + if (!isGetTableSchema) { + init(); + } + } catch (Exception e) { + LOG.warn("Failed to init avro scanner. ", e); + throw new IOException(e); + } + switch (fileType) { + case FILE_HDFS: + this.avroReader = new HDFSFileReader(uri); + break; + case FILE_S3: + String bucketName = requiredParams.get(AvroProperties.S3_BUCKET); + String key = requiredParams.get(AvroProperties.S3_KEY); + String accessKey = requiredParams.get(AvroProperties.S3_ACCESS_KEY); + String secretKey = requiredParams.get(AvroProperties.S3_SECRET_KEY); + String endpoint = requiredParams.get(AvroProperties.S3_ENDPOINT); + String region = requiredParams.get(AvroProperties.S3_REGION); + this.avroReader = new S3FileReader(accessKey, secretKey, endpoint, region, bucketName, key); + break; + default: + LOG.warn("Unsupported " + fileType.name() + " file type."); + throw new IOException("Unsupported " + fileType.name() + " file type."); + } + this.avroReader.open(new Configuration()); + if (!isGetTableSchema) { + initTableInfo(requiredTypes, requiredFields, new ScanPredicate[0], fetchSize); + } + } + + @Override + public void close() throws IOException { + if (Objects.nonNull(avroReader)) { + avroReader.close(); + } + } + + @Override + protected int getNext() throws IOException { + int numRows = 0; + for (; numRows < getBatchSize(); numRows++) { + if (!avroReader.hasNext()) { + break; + } + GenericRecord rowRecord = (GenericRecord) avroReader.getNext(); + for (int i = 0; i < requiredFields.length; i++) { + Object fieldData = rowRecord.get(requiredFields[i]); + if (fieldData == null) { + appendData(i, null); + } else { + AvroColumnValue fieldValue = new AvroColumnValue(fieldInspectors[i], fieldData); + appendData(i, fieldValue); + } + } + } + return numRows; + } + + protected TableSchema parseTableSchema() throws UnsupportedOperationException { + Schema schema = avroReader.getSchema(); + List schemaFields = schema.getFields(); + List schemaColumns = new ArrayList<>(); + for (Field schemaField : schemaFields) { + Schema avroSchema = schemaField.schema(); + String columnName = schemaField.name().toLowerCase(Locale.ROOT); + + SchemaColumn schemaColumn = new SchemaColumn(); + TPrimitiveType tPrimitiveType = serializeSchemaType(avroSchema, schemaColumn); + schemaColumn.setName(columnName); + schemaColumn.setType(tPrimitiveType); + schemaColumns.add(schemaColumn); + } + return new TableSchema(schemaColumns); + } + + private TPrimitiveType serializeSchemaType(Schema avroSchema, SchemaColumn schemaColumn) + throws UnsupportedOperationException { + Schema.Type type = avroSchema.getType(); + switch (type) { + case NULL: + return TPrimitiveType.NULL_TYPE; + case STRING: + return TPrimitiveType.STRING; + case INT: + return TPrimitiveType.INT; + case BOOLEAN: + return TPrimitiveType.BOOLEAN; + case LONG: + return TPrimitiveType.BIGINT; + case FLOAT: + return TPrimitiveType.FLOAT; + case BYTES: + return TPrimitiveType.BINARY; + case DOUBLE: + return TPrimitiveType.DOUBLE; + case ARRAY: + SchemaColumn arrayChildColumn = new SchemaColumn(); + schemaColumn.addChildColumn(arrayChildColumn); + arrayChildColumn.setType(serializeSchemaType(avroSchema.getElementType(), arrayChildColumn)); + return TPrimitiveType.ARRAY; + case MAP: + SchemaColumn mapChildColumn = new SchemaColumn(); + schemaColumn.addChildColumn(mapChildColumn); + mapChildColumn.setType(serializeSchemaType(avroSchema.getValueType(), mapChildColumn)); + return TPrimitiveType.MAP; + default: + throw new UnsupportedOperationException("avro format: " + type.getName() + " is not supported."); + } + } + +} diff --git a/fe/be-java-extensions/avro-scanner/src/main/java/org/apache/doris/avro/AvroProperties.java b/fe/be-java-extensions/avro-scanner/src/main/java/org/apache/doris/avro/AvroProperties.java new file mode 100644 index 000000000000000..11066b5e08c11df --- /dev/null +++ b/fe/be-java-extensions/avro-scanner/src/main/java/org/apache/doris/avro/AvroProperties.java @@ -0,0 +1,40 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.avro; + +public class AvroProperties { + + protected static final String COLUMNS_TYPE_DELIMITER = "#"; + protected static final String FIELDS_DELIMITER = ","; + + protected static final String IS_GET_TABLE_SCHEMA = "is_get_table_schema"; + protected static final String COLUMNS_TYPES = "columns_types"; + protected static final String REQUIRED_FIELDS = "required_fields"; + protected static final String FILE_TYPE = "file_type"; + protected static final String URI = "uri"; + protected static final String S3_BUCKET = "s3.virtual.bucket"; + protected static final String S3_KEY = "s3.virtual.key"; + protected static final String S3_ACCESS_KEY = "s3.access_key"; + protected static final String S3_SECRET_KEY = "s3.secret_key"; + protected static final String S3_ENDPOINT = "s3.endpoint"; + protected static final String S3_REGION = "s3.region"; + protected static final String HIVE_SERDE = "hive.serde"; + protected static final String COLUMNS = "columns"; + protected static final String COLUMNS2TYPES = "columns.types"; + +} diff --git a/be/src/runtime/memory/chunk.h b/fe/be-java-extensions/avro-scanner/src/main/java/org/apache/doris/avro/AvroReader.java similarity index 66% rename from be/src/runtime/memory/chunk.h rename to fe/be-java-extensions/avro-scanner/src/main/java/org/apache/doris/avro/AvroReader.java index 332631d3fba1aaf..eb012e402be416a 100644 --- a/be/src/runtime/memory/chunk.h +++ b/fe/be-java-extensions/avro-scanner/src/main/java/org/apache/doris/avro/AvroReader.java @@ -15,21 +15,23 @@ // specific language governing permissions and limitations // under the License. -#pragma once +package org.apache.doris.avro; -#include -#include +import org.apache.avro.Schema; +import org.apache.hadoop.conf.Configuration; -namespace doris { +import java.io.IOException; -// A chunk of continuous memory. -// Almost all files depend on this struct, and each modification -// will result in recompilation of all files. So, we put it in a -// file to keep this file simple and infrequently changed. -struct Chunk { - uint8_t* data = nullptr; - size_t size = 0; - int core_id = -1; -}; +public interface AvroReader { -} // namespace doris + void open(Configuration conf) throws IOException; + + Schema getSchema(); + + boolean hasNext(); + + Object getNext() throws IOException; + + void close() throws IOException; + +} diff --git a/fe/be-java-extensions/avro-scanner/src/main/java/org/apache/doris/avro/HDFSFileReader.java b/fe/be-java-extensions/avro-scanner/src/main/java/org/apache/doris/avro/HDFSFileReader.java new file mode 100644 index 000000000000000..7200b4dde635b32 --- /dev/null +++ b/fe/be-java-extensions/avro-scanner/src/main/java/org/apache/doris/avro/HDFSFileReader.java @@ -0,0 +1,73 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.avro; + +import org.apache.avro.Schema; +import org.apache.avro.file.DataFileStream; +import org.apache.avro.generic.GenericDatumReader; +import org.apache.avro.generic.GenericRecord; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.log4j.LogManager; +import org.apache.log4j.Logger; + +import java.io.BufferedInputStream; +import java.io.IOException; +import java.net.URI; + +public class HDFSFileReader implements AvroReader { + private static final Logger LOG = LogManager.getLogger(HDFSFileReader.class); + private final Path filePath; + private final String url; + private DataFileStream reader; + private BufferedInputStream inputStream; + + public HDFSFileReader(String url) { + this.url = url; + this.filePath = new Path(url); + } + + @Override + public void open(Configuration conf) throws IOException { + FileSystem fs = FileSystem.get(URI.create(url), conf); + inputStream = new BufferedInputStream(fs.open(filePath)); + reader = new DataFileStream<>(inputStream, new GenericDatumReader<>()); + } + + @Override + public Schema getSchema() { + return reader.getSchema(); + } + + @Override + public boolean hasNext() { + return reader.hasNext(); + } + + @Override + public Object getNext() throws IOException { + return reader.next(); + } + + @Override + public void close() throws IOException { + inputStream.close(); + reader.close(); + } +} diff --git a/fe/be-java-extensions/avro-scanner/src/main/java/org/apache/doris/avro/S3FileReader.java b/fe/be-java-extensions/avro-scanner/src/main/java/org/apache/doris/avro/S3FileReader.java new file mode 100644 index 000000000000000..bd966eb31c66cbd --- /dev/null +++ b/fe/be-java-extensions/avro-scanner/src/main/java/org/apache/doris/avro/S3FileReader.java @@ -0,0 +1,91 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.avro; + +import com.amazonaws.auth.AWSCredentials; +import com.amazonaws.auth.AWSStaticCredentialsProvider; +import com.amazonaws.auth.BasicAWSCredentials; +import com.amazonaws.client.builder.AwsClientBuilder; +import com.amazonaws.services.s3.AmazonS3; +import com.amazonaws.services.s3.AmazonS3ClientBuilder; +import com.amazonaws.services.s3.model.GetObjectRequest; +import com.amazonaws.services.s3.model.S3Object; +import org.apache.avro.Schema; +import org.apache.avro.file.DataFileStream; +import org.apache.avro.generic.GenericDatumReader; +import org.apache.avro.generic.GenericRecord; +import org.apache.hadoop.conf.Configuration; +import org.apache.log4j.LogManager; +import org.apache.log4j.Logger; + +import java.io.IOException; +import java.io.InputStream; + +public class S3FileReader implements AvroReader { + + private static final Logger LOG = LogManager.getLogger(S3FileReader.class); + private final String bucketName; + private final String key; + private AmazonS3 s3Client; + private DataFileStream reader; + private InputStream s3ObjectInputStream; + private final AWSCredentials credentials; + private final String endpoint; + private final String region; + + public S3FileReader(String accessKey, String secretKey, String endpoint, String region, + String bucketName, String key) { + this.bucketName = bucketName; + this.key = key; + this.endpoint = endpoint; + this.region = region; + credentials = new BasicAWSCredentials(accessKey, secretKey); + } + + @Override + public void open(Configuration conf) throws IOException { + s3Client = AmazonS3ClientBuilder.standard() + .withCredentials(new AWSStaticCredentialsProvider(credentials)) + .withEndpointConfiguration(new AwsClientBuilder.EndpointConfiguration(endpoint, region)) + .build(); + S3Object object = s3Client.getObject(new GetObjectRequest(bucketName, key)); + s3ObjectInputStream = object.getObjectContent(); + reader = new DataFileStream<>(s3ObjectInputStream, new GenericDatumReader<>()); + } + + @Override + public Schema getSchema() { + return reader.getSchema(); + } + + @Override + public boolean hasNext() { + return reader.hasNext(); + } + + @Override + public Object getNext() throws IOException { + return reader.next(); + } + + @Override + public void close() throws IOException { + s3ObjectInputStream.close(); + reader.close(); + } +} diff --git a/fe/be-java-extensions/avro-scanner/src/main/resources/package.xml b/fe/be-java-extensions/avro-scanner/src/main/resources/package.xml new file mode 100644 index 000000000000000..4bbb26106033637 --- /dev/null +++ b/fe/be-java-extensions/avro-scanner/src/main/resources/package.xml @@ -0,0 +1,41 @@ + + + + jar-with-dependencies + + jar + + false + + + / + true + true + runtime + + + **/Log4j2Plugins.dat + + + + + diff --git a/fe/be-java-extensions/hudi-scanner/pom.xml b/fe/be-java-extensions/hudi-scanner/pom.xml index 71564e4be385315..098073504ead918 100644 --- a/fe/be-java-extensions/hudi-scanner/pom.xml +++ b/fe/be-java-extensions/hudi-scanner/pom.xml @@ -88,15 +88,16 @@ under the License. - com.facebook.presto.hadoop - hadoop-apache2 - ${presto.hadoop.version} - - - org.slf4j - slf4j-log4j12 - - + org.apache.hadoop + hadoop-client + + + org.apache.hadoop + hadoop-common + + + org.apache.hadoop + hadoop-hdfs commons-io diff --git a/fe/be-java-extensions/hudi-scanner/src/main/java/org/apache/doris/hudi/HudiJniScanner.java b/fe/be-java-extensions/hudi-scanner/src/main/java/org/apache/doris/hudi/HudiJniScanner.java index 556c6b2e7bf8665..bdc4960ec153b12 100644 --- a/fe/be-java-extensions/hudi-scanner/src/main/java/org/apache/doris/hudi/HudiJniScanner.java +++ b/fe/be-java-extensions/hudi-scanner/src/main/java/org/apache/doris/hudi/HudiJniScanner.java @@ -20,6 +20,7 @@ import org.apache.doris.common.jni.JniScanner; import org.apache.doris.common.jni.vec.ColumnValue; +import org.apache.doris.common.jni.vec.TableSchema; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hive.serde2.Deserializer; @@ -135,6 +136,11 @@ public int getNext() throws IOException { } } + @Override + protected TableSchema parseTableSchema() throws UnsupportedOperationException { + // do nothing + return null; + } private void init(JobConf jobConf, Properties properties) throws Exception { String basePath = hudiScanParam.getBasePath(); diff --git a/fe/be-java-extensions/java-common/pom.xml b/fe/be-java-extensions/java-common/pom.xml index eedd7c00adaa905..20ed0104fa59432 100644 --- a/fe/be-java-extensions/java-common/pom.xml +++ b/fe/be-java-extensions/java-common/pom.xml @@ -55,6 +55,10 @@ under the License. org.apache.httpcomponents httpclient + + com.fasterxml.jackson.core + jackson-databind + diff --git a/fe/be-java-extensions/java-common/src/main/java/org/apache/doris/common/jni/JniScanner.java b/fe/be-java-extensions/java-common/src/main/java/org/apache/doris/common/jni/JniScanner.java index 89c960bddcbd5df..c45b2ac8e5973bd 100644 --- a/fe/be-java-extensions/java-common/src/main/java/org/apache/doris/common/jni/JniScanner.java +++ b/fe/be-java-extensions/java-common/src/main/java/org/apache/doris/common/jni/JniScanner.java @@ -21,6 +21,7 @@ import org.apache.doris.common.jni.vec.ColumnType; import org.apache.doris.common.jni.vec.ColumnValue; import org.apache.doris.common.jni.vec.ScanPredicate; +import org.apache.doris.common.jni.vec.TableSchema; import org.apache.doris.common.jni.vec.VectorTable; import java.io.IOException; @@ -43,6 +44,9 @@ public abstract class JniScanner { // Scan data and save as vector table protected abstract int getNext() throws IOException; + // parse table schema + protected abstract TableSchema parseTableSchema() throws UnsupportedOperationException; + protected void initTableInfo(ColumnType[] requiredTypes, String[] requiredFields, ScanPredicate[] predicates, int batchSize) { this.types = requiredTypes; @@ -63,6 +67,11 @@ public VectorTable getTable() { return vectorTable; } + public String getTableSchema() throws IOException { + TableSchema tableSchema = parseTableSchema(); + return tableSchema.getTableSchema(); + } + public long getNextBatchMeta() throws IOException { if (vectorTable == null) { vectorTable = new VectorTable(types, fields, predicates, batchSize); @@ -95,7 +104,7 @@ private long getMetaAddress(int numRows) { return vectorTable.getMetaAddress(); } - protected void resetTable() { + public void resetTable() { if (vectorTable != null) { vectorTable.reset(); } @@ -105,7 +114,7 @@ protected void releaseColumn(int fieldId) { vectorTable.releaseColumn(fieldId); } - protected void releaseTable() { + public void releaseTable() { if (vectorTable != null) { vectorTable.close(); } diff --git a/fe/be-java-extensions/java-common/src/main/java/org/apache/doris/common/jni/MockJniScanner.java b/fe/be-java-extensions/java-common/src/main/java/org/apache/doris/common/jni/MockJniScanner.java index 14a412dccbb3fc1..fc2928f8ed362a3 100644 --- a/fe/be-java-extensions/java-common/src/main/java/org/apache/doris/common/jni/MockJniScanner.java +++ b/fe/be-java-extensions/java-common/src/main/java/org/apache/doris/common/jni/MockJniScanner.java @@ -21,6 +21,7 @@ import org.apache.doris.common.jni.vec.ColumnType; import org.apache.doris.common.jni.vec.ColumnValue; import org.apache.doris.common.jni.vec.ScanPredicate; +import org.apache.doris.common.jni.vec.TableSchema; import org.apache.log4j.Logger; @@ -143,7 +144,7 @@ public void unpackStruct(List structFieldIndex, List value private static final Logger LOG = Logger.getLogger(MockJniScanner.class); - private final int mockRows; + private int mockRows; private int readRows = 0; private final MockColumnValue columnValue = new MockColumnValue(); @@ -195,4 +196,9 @@ protected int getNext() throws IOException { readRows += rows; return rows; } + + @Override + protected TableSchema parseTableSchema() throws UnsupportedOperationException { + return null; + } } diff --git a/fe/be-java-extensions/java-common/src/main/java/org/apache/doris/common/jni/vec/TableSchema.java b/fe/be-java-extensions/java-common/src/main/java/org/apache/doris/common/jni/vec/TableSchema.java new file mode 100644 index 000000000000000..421feb55a3fdd04 --- /dev/null +++ b/fe/be-java-extensions/java-common/src/main/java/org/apache/doris/common/jni/vec/TableSchema.java @@ -0,0 +1,83 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.common.jni.vec; + +import org.apache.doris.thrift.TPrimitiveType; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; + +import java.io.IOException; +import java.util.List; + +/** + * Used to parse the file structure of table-value-function type. + * like avro file. + */ +public class TableSchema { + private final List schemaColumns; + private final ObjectMapper objectMapper; + + public TableSchema(List schemaColumns) { + this.schemaColumns = schemaColumns; + this.objectMapper = new ObjectMapper(); + } + + public String getTableSchema() throws IOException { + try { + return objectMapper.writeValueAsString(schemaColumns); + } catch (JsonProcessingException e) { + throw new IOException(e); + } + } + + public static class SchemaColumn { + private String name; + private int type; + private SchemaColumn childColumn; + + public SchemaColumn() { + + } + + public String getName() { + return name; + } + + public SchemaColumn getChildColumn() { + return childColumn; + } + + public int getType() { + return type; + } + + public void setName(String name) { + this.name = name; + } + + public void setType(TPrimitiveType type) { + this.type = type.getValue(); + } + + public void addChildColumn(SchemaColumn childColumn) { + this.childColumn = childColumn; + } + } + +} diff --git a/fe/be-java-extensions/max-compute-scanner/src/main/java/org/apache/doris/maxcompute/MaxComputeJniScanner.java b/fe/be-java-extensions/max-compute-scanner/src/main/java/org/apache/doris/maxcompute/MaxComputeJniScanner.java index 8f9b903afdc7160..6a3d519670d51c5 100644 --- a/fe/be-java-extensions/max-compute-scanner/src/main/java/org/apache/doris/maxcompute/MaxComputeJniScanner.java +++ b/fe/be-java-extensions/max-compute-scanner/src/main/java/org/apache/doris/maxcompute/MaxComputeJniScanner.java @@ -20,6 +20,7 @@ import org.apache.doris.common.jni.JniScanner; import org.apache.doris.common.jni.vec.ColumnType; import org.apache.doris.common.jni.vec.ScanPredicate; +import org.apache.doris.common.jni.vec.TableSchema; import com.aliyun.odps.Column; import com.aliyun.odps.OdpsType; @@ -238,6 +239,12 @@ protected int getNext() throws IOException { return realRows; } + @Override + protected TableSchema parseTableSchema() throws UnsupportedOperationException { + // do nothing + return null; + } + private int readVectors(int expectedRows) throws IOException { VectorSchemaRoot batch; int curReadRows = 0; diff --git a/fe/be-java-extensions/paimon-scanner/src/main/java/org/apache/doris/paimon/PaimonJniScanner.java b/fe/be-java-extensions/paimon-scanner/src/main/java/org/apache/doris/paimon/PaimonJniScanner.java index 71965344a4cdfe8..f6cdc436b78a1da 100644 --- a/fe/be-java-extensions/paimon-scanner/src/main/java/org/apache/doris/paimon/PaimonJniScanner.java +++ b/fe/be-java-extensions/paimon-scanner/src/main/java/org/apache/doris/paimon/PaimonJniScanner.java @@ -21,6 +21,7 @@ import org.apache.doris.common.jni.utils.OffHeap; import org.apache.doris.common.jni.vec.ColumnType; import org.apache.doris.common.jni.vec.ScanPredicate; +import org.apache.doris.common.jni.vec.TableSchema; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.log4j.Logger; @@ -140,6 +141,12 @@ protected int getNext() throws IOException { return rows; } + @Override + protected TableSchema parseTableSchema() throws UnsupportedOperationException { + // do nothing + return null; + } + private Catalog create(CatalogContext context) throws IOException { Path warehousePath = new Path(context.options().get(CatalogOptions.WAREHOUSE)); FileIO fileIO; diff --git a/fe/be-java-extensions/pom.xml b/fe/be-java-extensions/pom.xml index aee63210938cb24..f1ffe1527f92480 100644 --- a/fe/be-java-extensions/pom.xml +++ b/fe/be-java-extensions/pom.xml @@ -27,6 +27,7 @@ under the License. jdbc-scanner paimon-scanner max-compute-scanner + avro-scanner diff --git a/fe/fe-core/pom.xml b/fe/fe-core/pom.xml index 3d4e64b0879bee6..1b77adb24d748cd 100644 --- a/fe/fe-core/pom.xml +++ b/fe/fe-core/pom.xml @@ -36,7 +36,6 @@ under the License. 2.17.257 3.1.1-hw-45 3.3.5 - 3.2.2 @@ -413,7 +412,6 @@ under the License. org.apache.hadoop hadoop-aliyun - ${aliyunoss.version} org.apache.hadoop diff --git a/fe/fe-core/src/main/cup/sql_parser.cup b/fe/fe-core/src/main/cup/sql_parser.cup index 597bb706967d188..08d41b378c75aa7 100644 --- a/fe/fe-core/src/main/cup/sql_parser.cup +++ b/fe/fe-core/src/main/cup/sql_parser.cup @@ -280,6 +280,7 @@ terminal String KW_BUILD, KW_BUILTIN, KW_BY, + KW_CACHED, KW_CANCEL, KW_CASE, KW_CAST, @@ -910,6 +911,7 @@ nonterminal ColumnDef.DefaultValue opt_default_value; nonterminal Boolean opt_if_exists, opt_if_not_exists; nonterminal Boolean opt_external; nonterminal Boolean opt_force; +nonterminal Boolean opt_cached; nonterminal IndexDef.IndexType opt_index_type; nonterminal ShowAlterStmt.AlterType opt_alter_type; @@ -3653,6 +3655,17 @@ opt_external ::= :} ; +opt_cached ::= + /* empty */ + {: + RESULT = false; + :} + | KW_CACHED + {: + RESULT = true; + :} + ; + opt_force ::= /* empty */ {: @@ -4069,9 +4082,9 @@ show_param ::= RESULT = new ShowTableStatsStmt(tbl, partitionNames); :} /* show column stats */ - | KW_COLUMN KW_STATS table_name:tbl opt_col_list:cols opt_partition_names:partitionNames + | KW_COLUMN opt_cached:cached KW_STATS table_name:tbl opt_col_list:cols opt_partition_names:partitionNames {: - RESULT = new ShowColumnStatsStmt(tbl, cols, partitionNames); + RESULT = new ShowColumnStatsStmt(tbl, cols, partitionNames, cached); :} /* show column histogram */ | KW_COLUMN KW_HISTOGRAM table_name:tbl opt_col_list:cols @@ -7261,6 +7274,8 @@ keyword ::= {: RESULT = id; :} | KW_BUILD:id {: RESULT = id; :} + | KW_CACHED:id + {: RESULT = id; :} | KW_CHAIN:id {: RESULT = id; :} | KW_CHAR:id @@ -7609,6 +7624,8 @@ keyword ::= {: RESULT = id; :} | KW_FEATURE:id {: RESULT = id; :} + | KW_FRONTENDS:id + {: RESULT = id; :} | KW_MAP:id {: RESULT = id; :} | KW_VARCHAR:id diff --git a/fe/fe-core/src/main/java/org/apache/doris/alter/IndexChangeJob.java b/fe/fe-core/src/main/java/org/apache/doris/alter/IndexChangeJob.java index fecfe4b99b7c8d0..6ba3e68ffa582da 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/alter/IndexChangeJob.java +++ b/fe/fe-core/src/main/java/org/apache/doris/alter/IndexChangeJob.java @@ -290,7 +290,7 @@ protected void runWaitingTxnJob() throws AlterCancelException { partitionId, originIndexId, originTabletId, originSchemaHash, olapTable.getIndexes(), alterInvertedIndexes, originSchemaColumns, - isDropOp, taskSignature); + isDropOp, taskSignature, jobId); invertedIndexBatchTask.addTask(alterInvertedIndexTask); } } // end for tablet diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/ArithmeticExpr.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/ArithmeticExpr.java index de437e154c7602a..176a55fb0212b66 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/ArithmeticExpr.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/ArithmeticExpr.java @@ -260,7 +260,7 @@ public String toSqlImpl() { if (children.size() == 1) { return op.toString() + " " + getChild(0).toSql(); } else { - return getChild(0).toSql() + " " + op.toString() + " " + getChild(1).toSql(); + return "(" + getChild(0).toSql() + " " + op.toString() + " " + getChild(1).toSql() + ")"; } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/CastExpr.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/CastExpr.java index be93975efb441a2..6ceb8e3d84c760d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/CastExpr.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/CastExpr.java @@ -446,8 +446,11 @@ private Expr castTo(LiteralExpr value) throws AnalysisException { return new LargeIntLiteral(value.getStringValue()); } else if (type.isDecimalV2() || type.isDecimalV3()) { if (targetTypeDef != null) { - return new DecimalLiteral(value.getStringValue(), + DecimalLiteral literal = new DecimalLiteral(value.getStringValue(), ((ScalarType) targetTypeDef.getType()).getScalarScale()); + literal.checkPrecisionAndScale(targetTypeDef.getType().getPrecision(), + ((ScalarType) targetTypeDef.getType()).getScalarScale()); + return literal; } else { return new DecimalLiteral(value.getStringValue()); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateTableStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateTableStmt.java index 5ec43781eb4abc7..c737fa2d4c7f893 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateTableStmt.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateTableStmt.java @@ -411,7 +411,7 @@ public void analyze(Analyzer analyzer) throws UserException, AnalysisException { PropertyAnalyzer.ENABLE_UNIQUE_KEY_MERGE_ON_WRITE + " property only support unique key table"); } if (keysDesc.getKeysType() == KeysType.UNIQUE_KEYS) { - enableUniqueKeyMergeOnWrite = true; + enableUniqueKeyMergeOnWrite = false; if (properties != null) { // `analyzeXXX` would modify `properties`, which will be used later, // so we just clone a properties map here. diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/DateLiteral.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/DateLiteral.java index f9a4cd15cafe25b..1eda2f2c9fb5932 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/DateLiteral.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/DateLiteral.java @@ -56,7 +56,6 @@ import java.util.Collections; import java.util.List; import java.util.Map; -import java.util.Objects; import java.util.TimeZone; import java.util.regex.Pattern; @@ -1086,7 +1085,10 @@ public long getMicrosecond() { @Override public int hashCode() { - return 31 * super.hashCode() + Objects.hashCode(unixTimestamp(TimeZone.getDefault())); + // do not invoke the super.hashCode(), just use the return value of getLongValue() + // a DateV2 DateLiteral obj may be equal to a Date DateLiteral + // if the return value of getLongValue() is the same + return Long.hashCode(getLongValue()); } // parse the date string value in 'value' by 'format' pattern. diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/DeleteStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/DeleteStmt.java index 1d733d1e2f407ec..40162db22746e6d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/DeleteStmt.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/DeleteStmt.java @@ -118,7 +118,14 @@ public void analyze(Analyzer analyzer) throws UserException { if (fromClause == null) { ExprRewriter exprRewriter = new ExprRewriter(EXPR_NORMALIZE_RULES); wherePredicate = exprRewriter.rewrite(wherePredicate, analyzer); - analyzePredicate(wherePredicate); + try { + analyzePredicate(wherePredicate); + } catch (Exception e) { + if (!(((OlapTable) targetTable).getKeysType() == KeysType.UNIQUE_KEYS)) { + throw new AnalysisException(e.getMessage(), e.getCause()); + } + constructInsertStmt(); + } } else { constructInsertStmt(); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/DescribeStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/DescribeStmt.java index 5ef86bf4ea164b1..a2ca22a2136fea9 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/DescribeStmt.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/DescribeStmt.java @@ -132,7 +132,11 @@ public void analyze(Analyzer analyzer) throws UserException { "NONE" ); if (column.getOriginType().isDatetimeV2()) { - row.set(1, "DATETIME"); + StringBuilder typeStr = new StringBuilder("DATETIME"); + if (((ScalarType) column.getOriginType()).getScalarScale() > 0) { + typeStr.append("(").append(((ScalarType) column.getOriginType()).getScalarScale()).append(")"); + } + row.set(1, typeStr.toString()); } else if (column.getOriginType().isDateV2()) { row.set(1, "DATE"); } @@ -223,7 +227,12 @@ public void analyze(Analyzer analyzer) throws UserException { ""); if (column.getOriginType().isDatetimeV2()) { - row.set(3, "DATETIME"); + StringBuilder typeStr = new StringBuilder("DATETIME"); + if (((ScalarType) column.getOriginType()).getScalarScale() > 0) { + typeStr.append("(").append(((ScalarType) column.getOriginType()).getScalarScale()) + .append(")"); + } + row.set(3, typeStr.toString()); } else if (column.getOriginType().isDateV2()) { row.set(3, "DATE"); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/DropStatsStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/DropStatsStmt.java index 1e426c586f84fa0..e821fecaed48259 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/DropStatsStmt.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/DropStatsStmt.java @@ -26,7 +26,6 @@ import org.apache.doris.common.ErrorCode; import org.apache.doris.common.ErrorReport; import org.apache.doris.common.UserException; -import org.apache.doris.common.util.Util; import org.apache.doris.datasource.CatalogIf; import org.apache.doris.mysql.privilege.PrivPredicate; import org.apache.doris.qe.ConnectContext; @@ -89,9 +88,6 @@ public void analyze(Analyzer analyzer) throws UserException { DatabaseIf db = catalog.getDbOrAnalysisException(dbName); TableIf table = db.getTableOrAnalysisException(tblName); tblId = table.getId(); - // disallow external catalog - Util.prohibitExternalCatalog(tableName.getCtl(), - this.getClass().getSimpleName()); // check permission checkAnalyzePriv(db.getFullName(), table.getName()); // check columnNames diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/LateralViewRef.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/LateralViewRef.java index 522b661a6293dd5..f5536f9117acf23 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/LateralViewRef.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/LateralViewRef.java @@ -18,6 +18,7 @@ package org.apache.doris.analysis; import org.apache.doris.catalog.Column; +import org.apache.doris.catalog.Function.NullableMode; import org.apache.doris.catalog.InlineView; import org.apache.doris.common.AnalysisException; import org.apache.doris.common.Config; @@ -99,8 +100,8 @@ private void analyzeFunctionExpr(Analyzer analyzer) throws AnalysisException { public TupleDescriptor createTupleDescriptor(Analyzer analyzer) throws AnalysisException { // Create a fake catalog table for the lateral view List columnList = Lists.newArrayList(); - columnList.add(new Column(columnName, fnExpr.getFn().getReturnType(), - false, null, true, null, "")); + columnList.add(new Column(columnName, fnExpr.getFn().getReturnType(), false, null, + fnExpr.getFn().getNullableMode() == NullableMode.ALWAYS_NULLABLE, null, "")); view = new InlineView(viewName, columnList); // Create the non-materialized tuple and set the fake table in it. @@ -119,6 +120,12 @@ public void materializeRequiredSlots(ExprSubstitutionMap baseTblSmap, Analyzer a originSlotRef.getDesc().setIsMaterialized(true); } explodeSlotRef.getDesc().setIsMaterialized(true); + + for (Expr expr : baseTblSmap.getLhs()) { + if (expr instanceof SlotRef && ((SlotRef) expr).getDesc().getIsNullable()) { + explodeSlotRef.getDesc().setIsNullable(true); + } + } } // The default table name must be origin table name diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/MultiPartitionDesc.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/MultiPartitionDesc.java index 8423acc83d87d8d..dadfd69ba78f014 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/MultiPartitionDesc.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/MultiPartitionDesc.java @@ -286,7 +286,7 @@ private void intervalTrans() throws AnalysisException { } if (this.interval <= 0) { - throw new AnalysisException("Multi partition time interval mush be larger than zero."); + throw new AnalysisException("Multi partition time interval must be larger than zero."); } if (!timeType.equals("")) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/NativeInsertStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/NativeInsertStmt.java index fcd85fd80bff0be..408fcc09528c1e3 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/NativeInsertStmt.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/NativeInsertStmt.java @@ -437,9 +437,7 @@ private void analyzeSubquery(Analyzer analyzer) throws UserException { mentionedColumns.add(col.getName()); targetColumns.add(col); } - realTargetColumnNames = targetColumns.stream().map(column -> column.getName()).collect(Collectors.toList()); } else { - realTargetColumnNames = targetColumnNames; for (String colName : targetColumnNames) { Column col = targetTable.getColumn(colName); if (col == null) { @@ -453,8 +451,8 @@ private void analyzeSubquery(Analyzer analyzer) throws UserException { // hll column mush in mentionedColumns for (Column col : targetTable.getBaseSchema()) { if (col.getType().isObjectStored() && !mentionedColumns.contains(col.getName())) { - throw new AnalysisException(" object-stored column " + col.getName() - + " mush in insert into columns"); + throw new AnalysisException( + " object-stored column " + col.getName() + " mush in insert into columns"); } } } @@ -535,20 +533,30 @@ private void analyzeSubquery(Analyzer analyzer) throws UserException { } // check if size of select item equal with columns mentioned in statement - if (mentionedColumns.size() != queryStmt.getResultExprs().size() - || realTargetColumnNames.size() != queryStmt.getResultExprs().size()) { + if (mentionedColumns.size() != queryStmt.getResultExprs().size()) { ErrorReport.reportAnalysisException(ErrorCode.ERR_WRONG_VALUE_COUNT); } // Check if all columns mentioned is enough checkColumnCoverage(mentionedColumns, targetTable.getBaseSchema()); + realTargetColumnNames = targetColumns.stream().map(column -> column.getName()).collect(Collectors.toList()); Map slotToIndex = Maps.newTreeMap(String.CASE_INSENSITIVE_ORDER); - for (int i = 0; i < realTargetColumnNames.size(); i++) { + for (int i = 0; i < queryStmt.getResultExprs().size(); i++) { slotToIndex.put(realTargetColumnNames.get(i), queryStmt.getResultExprs().get(i) .checkTypeCompatibility(targetTable.getColumn(realTargetColumnNames.get(i)).getType())); } + for (Column column : targetTable.getBaseSchema()) { + if (!slotToIndex.containsKey(column.getName())) { + if (column.getDefaultValue() == null) { + slotToIndex.put(column.getName(), new NullLiteral()); + } else { + slotToIndex.put(column.getName(), new StringLiteral(column.getDefaultValue())); + } + } + } + // handle VALUES() or SELECT constant list if (isValuesOrConstantSelect) { SelectStmt selectStmt = (SelectStmt) queryStmt; diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowColumnStatsStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowColumnStatsStmt.java index 8e783c19612c290..0ebf2050ed71b13 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowColumnStatsStmt.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowColumnStatsStmt.java @@ -27,7 +27,6 @@ import org.apache.doris.common.ErrorReport; import org.apache.doris.common.Pair; import org.apache.doris.common.UserException; -import org.apache.doris.common.util.Util; import org.apache.doris.datasource.CatalogIf; import org.apache.doris.mysql.privilege.PrivPredicate; import org.apache.doris.qe.ConnectContext; @@ -63,13 +62,16 @@ public class ShowColumnStatsStmt extends ShowStmt { private final List columnNames; private final PartitionNames partitionNames; + private final boolean cached; private TableIf table; - public ShowColumnStatsStmt(TableName tableName, List columnNames, PartitionNames partitionNames) { + public ShowColumnStatsStmt(TableName tableName, List columnNames, + PartitionNames partitionNames, boolean cached) { this.tableName = tableName; this.columnNames = columnNames; this.partitionNames = partitionNames; + this.cached = cached; } public TableName getTableName() { @@ -86,8 +88,6 @@ public void analyze(Analyzer analyzer) throws UserException { throw new AnalysisException("Only one partition name could be specified"); } } - // disallow external catalog - Util.prohibitExternalCatalog(tableName.getCtl(), this.getClass().getSimpleName()); CatalogIf catalog = Env.getCurrentEnv().getCatalogMgr().getCatalog(tableName.getCtl()); if (catalog == null) { ErrorReport.reportAnalysisException("Catalog: {} not exists", tableName.getCtl()); @@ -164,4 +164,8 @@ public Set getColumnNames() { return table.getColumns().stream() .map(Column::getName).collect(Collectors.toSet()); } + + public boolean isCached() { + return cached; + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/SlotDescriptor.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/SlotDescriptor.java index 9d7825883ad26c4..e934ba45acc03eb 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/SlotDescriptor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/SlotDescriptor.java @@ -59,8 +59,6 @@ public class SlotDescriptor { // physical layout parameters private int byteSize; private int byteOffset; // within tuple - private int nullIndicatorByte; // index into byte array - private int nullIndicatorBit; // index within byte private int slotIdx; // index within tuple struct private int slotOffset; // index within slot array list @@ -85,8 +83,6 @@ public SlotDescriptor(SlotId id, TupleDescriptor parent, SlotDescriptor src) { this.id = id; this.parent = parent; this.byteOffset = src.byteOffset; - this.nullIndicatorBit = src.nullIndicatorBit; - this.nullIndicatorByte = src.nullIndicatorByte; this.slotIdx = src.slotIdx; this.isMaterialized = src.isMaterialized; this.column = src.column; @@ -121,22 +117,6 @@ public void setIsAgg(boolean agg) { isAgg = agg; } - public int getNullIndicatorByte() { - return nullIndicatorByte; - } - - public void setNullIndicatorByte(int nullIndicatorByte) { - this.nullIndicatorByte = nullIndicatorByte; - } - - public int getNullIndicatorBit() { - return nullIndicatorBit; - } - - public void setNullIndicatorBit(int nullIndicatorBit) { - this.nullIndicatorBit = nullIndicatorBit; - } - public SlotId getId() { return id; } @@ -300,19 +280,14 @@ public boolean layoutEquals(SlotDescriptor other) { if (getByteOffset() != other.getByteOffset()) { return false; } - if (getNullIndicatorByte() != other.getNullIndicatorByte()) { - return false; - } - if (getNullIndicatorBit() != other.getNullIndicatorBit()) { - return false; - } return true; } public TSlotDescriptor toThrift() { - TSlotDescriptor tSlotDescriptor = new TSlotDescriptor(id.asInt(), parent.getId().asInt(), - type.toThrift(), -1, byteOffset, nullIndicatorByte, - nullIndicatorBit, ((column != null) ? column.getName() : ""), slotIdx, isMaterialized); + // Non-nullable slots will have 0 for the byte offset and -1 for the bit mask + TSlotDescriptor tSlotDescriptor = new TSlotDescriptor(id.asInt(), parent.getId().asInt(), type.toThrift(), -1, + byteOffset, 0, getIsNullable() ? 0 : -1, ((column != null) ? column.getName() : ""), slotIdx, + isMaterialized); tSlotDescriptor.setNeedMaterialize(needMaterialize); if (column != null) { LOG.debug("column name:{}, column unique id:{}", column.getName(), column.getUniqueId()); @@ -328,8 +303,7 @@ public String debugString() { String parentTupleId = (parent == null) ? "null" : parent.getId().toString(); return MoreObjects.toStringHelper(this).add("id", id.asInt()).add("parent", parentTupleId).add("col", colStr) .add("type", typeStr).add("materialized", isMaterialized).add("byteSize", byteSize) - .add("byteOffset", byteOffset).add("nullIndicatorByte", nullIndicatorByte) - .add("nullIndicatorBit", nullIndicatorBit).add("slotIdx", slotIdx).toString(); + .add("byteOffset", byteOffset).add("slotIdx", slotIdx).add("nullable", getIsNullable()).toString(); } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/TupleDescriptor.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/TupleDescriptor.java index 4a2d60c9b272963..0bc2d28fe2f0c6e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/TupleDescriptor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/TupleDescriptor.java @@ -290,8 +290,7 @@ public void computeMemLayout() { // assign offsets to slots in order of ascending size numNullBytes = (numNullableSlots + 7) / 8; int offset = numNullBytes; - int nullIndicatorByte = 0; - int nullIndicatorBit = 0; + // slotIdx is the index into the resulting tuple struct. The first (smallest) field // is 0, next is 1, etc. int slotIdx = 0; @@ -310,20 +309,6 @@ public void computeMemLayout() { d.setByteOffset(offset); d.setSlotIdx(slotIdx++); offset += slotSize; - - // assign null indicator - if (d.getIsNullable()) { - d.setNullIndicatorByte(nullIndicatorByte); - d.setNullIndicatorBit(nullIndicatorBit); - nullIndicatorBit = (nullIndicatorBit + 1) % 8; - if (nullIndicatorBit == 0) { - ++nullIndicatorByte; - } - } else { - // Non-nullable slots will have 0 for the byte offset and -1 for the bit mask - d.setNullIndicatorBit(-1); - d.setNullIndicatorByte(0); - } } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/backup/BackupJob.java b/fe/fe-core/src/main/java/org/apache/doris/backup/BackupJob.java index 61eda11a7999f04..565efda2d04fcd5 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/backup/BackupJob.java +++ b/fe/fe-core/src/main/java/org/apache/doris/backup/BackupJob.java @@ -75,6 +75,7 @@ public class BackupJob extends AbstractJob { private static final Logger LOG = LogManager.getLogger(BackupJob.class); + private static final String TABLE_COMMIT_SEQ_PREFIX = "table_commit_seq:"; public enum BackupJobState { PENDING, // Job is newly created. Send snapshot tasks and save copied meta info, then transfer to SNAPSHOTING @@ -110,7 +111,7 @@ public enum BackupJobState { // save the local file path of meta info and job info file private String localMetaInfoFilePath = null; private String localJobInfoFilePath = null; - // backup properties + // backup properties && table commit seq with table id private Map properties = Maps.newHashMap(); private byte[] metaInfoBytes = null; @@ -431,6 +432,12 @@ private void checkOlapTable(OlapTable olapTable, TableRef backupTableRef) { private void prepareSnapshotTaskForOlapTableWithoutLock(OlapTable olapTable, TableRef backupTableRef, AgentBatchTask batchTask) { + // Add barrier editolog for barrier commit seq + long commitSeq = env.getEditLog().logBarrier(); + // format as "table:{tableId}" + String tableKey = String.format("%s%d", TABLE_COMMIT_SEQ_PREFIX, olapTable.getId()); + properties.put(tableKey, String.valueOf(commitSeq)); + // check backup table again if (backupTableRef.getPartitionNames() != null) { for (String partName : backupTableRef.getPartitionNames().getPartitionNames()) { @@ -680,8 +687,20 @@ private void saveMetaInfo() { metaInfoBytes = Files.readAllBytes(metaInfoFile.toPath()); // 3. save job info file + Map tableCommitSeqMap = Maps.newHashMap(); + // iterate properties, convert key, value from string to long + // key is "${TABLE_COMMIT_SEQ_PREFIX}{tableId}", only need tableId to long + for (Map.Entry entry : properties.entrySet()) { + String key = entry.getKey(); + String value = entry.getValue(); + if (key.startsWith(TABLE_COMMIT_SEQ_PREFIX)) { + long tableId = Long.parseLong(key.substring(TABLE_COMMIT_SEQ_PREFIX.length())); + long commitSeq = Long.parseLong(value); + tableCommitSeqMap.put(tableId, commitSeq); + } + } jobInfo = BackupJobInfo.fromCatalog(createTime, label, dbName, dbId, - getContent(), backupMeta, snapshotInfos); + getContent(), backupMeta, snapshotInfos, tableCommitSeqMap); LOG.debug("job info: {}. {}", jobInfo, this); File jobInfoFile = new File(jobDir, Repository.PREFIX_JOB_INFO + createTimeStr); if (!jobInfoFile.createNewFile()) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/backup/BackupJobInfo.java b/fe/fe-core/src/main/java/org/apache/doris/backup/BackupJobInfo.java index 4457740440c5959..fff2b755c10b53b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/backup/BackupJobInfo.java +++ b/fe/fe-core/src/main/java/org/apache/doris/backup/BackupJobInfo.java @@ -99,6 +99,9 @@ public class BackupJobInfo implements Writable { @SerializedName("tablet_snapshot_path_map") public Map tabletSnapshotPathMap = Maps.newHashMap(); + @SerializedName("table_commit_seq_map") + public Map tableCommitSeqMap; + public static class ExtraInfo { public static class NetworkAddrss { @SerializedName("ip") @@ -575,7 +578,7 @@ public Long getSchemaHash(long tableId, long partitionId, long indexId) { public static BackupJobInfo fromCatalog(long backupTime, String label, String dbName, long dbId, BackupContent content, BackupMeta backupMeta, - Map snapshotInfos) { + Map snapshotInfos, Map tableCommitSeqMap) { BackupJobInfo jobInfo = new BackupJobInfo(); jobInfo.backupTime = backupTime; @@ -584,6 +587,7 @@ public static BackupJobInfo fromCatalog(long backupTime, String label, String db jobInfo.dbId = dbId; jobInfo.metaVersion = FeConstants.meta_version; jobInfo.content = content; + jobInfo.tableCommitSeqMap = tableCommitSeqMap; Collection tbls = backupMeta.getTables().values(); // tbls diff --git a/fe/fe-core/src/main/java/org/apache/doris/binlog/AddPartitionRecord.java b/fe/fe-core/src/main/java/org/apache/doris/binlog/AddPartitionRecord.java index 5dc2f6274921438..9bc5ff7da0fd0e5 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/binlog/AddPartitionRecord.java +++ b/fe/fe-core/src/main/java/org/apache/doris/binlog/AddPartitionRecord.java @@ -18,6 +18,7 @@ package org.apache.doris.binlog; import org.apache.doris.catalog.DataProperty; +import org.apache.doris.catalog.ListPartitionItem; import org.apache.doris.catalog.Partition; import org.apache.doris.catalog.PartitionItem; import org.apache.doris.catalog.PartitionKey; @@ -51,6 +52,8 @@ public class AddPartitionRecord { private boolean isTempPartition = false; @SerializedName(value = "isMutable") private boolean isMutable = true; + @SerializedName(value = "sql") + private String sql; public AddPartitionRecord(long commitSeq, PartitionPersistInfo partitionPersistInfo) { this.commitSeq = commitSeq; @@ -64,6 +67,26 @@ public AddPartitionRecord(long commitSeq, PartitionPersistInfo partitionPersistI this.isInMemory = partitionPersistInfo.isInMemory(); this.isTempPartition = partitionPersistInfo.isTempPartition(); this.isMutable = partitionPersistInfo.isMutable(); + + StringBuilder sb = new StringBuilder(); + sb.append("ADD PARTITION ").append("`").append(partition.getName()).append("`").append(" VALUES "); + if (this.listPartitionItem.equals(ListPartitionItem.DUMMY_ITEM)) { + // range + sb.append("["); + sb.append(range.lowerEndpoint().toSql()); + sb.append(", "); + sb.append(range.upperEndpoint().toSql()); + sb.append(")"); + } else { + // list + sb.append("IN ("); + sb.append(((ListPartitionItem) listPartitionItem).toSql()); + sb.append(")"); + } + sb.append("(\"version_info\" = \""); + sb.append(partition.getVisibleVersion()).append("\""); + sb.append(");"); + this.sql = sb.toString(); } public long getCommitSeq() { diff --git a/fe/fe-core/src/main/java/org/apache/doris/binlog/BinlogManager.java b/fe/fe-core/src/main/java/org/apache/doris/binlog/BinlogManager.java index 59ba59615242abd..bccc5dfc4830685 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/binlog/BinlogManager.java +++ b/fe/fe-core/src/main/java/org/apache/doris/binlog/BinlogManager.java @@ -22,6 +22,7 @@ import org.apache.doris.common.Config; import org.apache.doris.common.Pair; import org.apache.doris.persist.BinlogGcInfo; +import org.apache.doris.persist.DropPartitionInfo; import org.apache.doris.thrift.TBinlog; import org.apache.doris.thrift.TBinlogType; import org.apache.doris.thrift.TStatus; @@ -128,6 +129,17 @@ public void addAddPartitionRecord(AddPartitionRecord addPartitionRecord) { addBinlog(dbId, tableIds, commitSeq, timestamp, type, data); } + public void addDropPartitionRecord(DropPartitionInfo dropPartitionInfo, long commitSeq) { + long dbId = dropPartitionInfo.getDbId(); + List tableIds = new ArrayList(); + tableIds.add(dropPartitionInfo.getTableId()); + long timestamp = -1; + TBinlogType type = TBinlogType.DROP_PARTITION; + String data = dropPartitionInfo.toJson(); + + addBinlog(dbId, tableIds, commitSeq, timestamp, type, data); + } + // get binlog by dbId, return first binlog.version > version public Pair getBinlog(long dbId, long tableId, long commitSeq) { TStatus status = new TStatus(TStatusCode.OK); diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/CatalogRecycleBin.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/CatalogRecycleBin.java index d05e61ff014e619..e6e8850a9a0945a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/CatalogRecycleBin.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/CatalogRecycleBin.java @@ -697,7 +697,10 @@ private synchronized boolean innerRecoverTable(Database db, Table table, String RecoverInfo recoverInfo = new RecoverInfo(db.getId(), table.getId(), -1L, "", newTableName, ""); Env.getCurrentEnv().getEditLog().logRecoverTable(recoverInfo); } - DynamicPartitionUtil.registerOrRemoveDynamicPartitionTable(db.getId(), (OlapTable) table, isReplay); + // Only olap table need recover dynamic partition, other table like jdbc odbc view.. do not need it + if (table.getType() == TableType.OLAP) { + DynamicPartitionUtil.registerOrRemoveDynamicPartitionTable(db.getId(), (OlapTable) table, isReplay); + } } finally { table.writeUnlock(); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Column.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/Column.java index e013b113a26f668..d323ae4cc3b529b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Column.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Column.java @@ -723,6 +723,9 @@ public String toSql(boolean isUniqueTable, boolean isCompatible) { if (isCompatible) { if (type.isDatetimeV2()) { sb.append("datetime"); + if (((ScalarType) type).getScalarScale() > 0) { + sb.append("(").append(((ScalarType) type).getScalarScale()).append(")"); + } } else if (type.isDateV2()) { sb.append("date"); } else { diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/FunctionSet.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/FunctionSet.java index f86bba50068a695..7269c5af11b72cc 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/FunctionSet.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/FunctionSet.java @@ -1348,6 +1348,23 @@ private void initAggregateBuiltins() { addBuiltin( AggregateFunction.createBuiltin(GROUP_ARRAY, Lists.newArrayList(t, Type.INT), new ArrayType(t), t, "", "", "", "", "", true, false, true, true)); + + //first_value/last_value for array + addBuiltin(AggregateFunction.createAnalyticBuiltin("first_value", + Lists.newArrayList(new ArrayType(t)), new ArrayType(t), Type.ARRAY, + "", + "", + null, + "", + "", true)); + + addBuiltin(AggregateFunction.createAnalyticBuiltin("last_value", + Lists.newArrayList(new ArrayType(t)), new ArrayType(t), Type.ARRAY, + "", + "", + null, + "", + "", true)); } // Avg diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/HdfsResource.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/HdfsResource.java index d812d8ec535dc2d..cdfb169590d7052 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/HdfsResource.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/HdfsResource.java @@ -48,6 +48,7 @@ public class HdfsResource extends Resource { public static String HADOOP_USER_NAME = "hadoop.username"; public static String HADOOP_SECURITY_AUTHENTICATION = "hadoop.security.authentication"; public static String HADOOP_KERBEROS_PRINCIPAL = "hadoop.kerberos.principal"; + public static String HADOOP_KERBEROS_AUTHORIZATION = "hadoop.security.authorization"; public static String HADOOP_KERBEROS_KEYTAB = "hadoop.kerberos.keytab"; public static String HADOOP_SHORT_CIRCUIT = "dfs.client.read.shortcircuit"; public static String HADOOP_SOCKET_PATH = "dfs.domain.socket.path"; diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/MaterializedIndexMeta.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/MaterializedIndexMeta.java index 07c33b85bcd11ad..12cab9f7ecbcacd 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/MaterializedIndexMeta.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/MaterializedIndexMeta.java @@ -193,9 +193,6 @@ private void setColumnsDefineExpr(Map columnNameToDefineExpr) thro // mv_count_sale_amt -> mva_SUM__CASE WHEN `sale_amt` IS NULL THEN 0 ELSE 1 END List slots = new ArrayList<>(); entry.getValue().collect(SlotRef.class, slots); - if (slots.size() > 1) { - throw new IOException("DefineExpr have multiple slot in MaterializedIndex, Expr=" + entry.getKey()); - } String name = MaterializedIndexMeta.normalizeName(slots.get(0).toSqlWithoutTbl()); Column matchedColumn = null; diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/PartitionKey.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/PartitionKey.java index b9e8f11b9cff394..70d667a519759d4 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/PartitionKey.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/PartitionKey.java @@ -434,11 +434,15 @@ public boolean equals(Object obj) { @Override public int hashCode() { + int hashCode = 1; + for (LiteralExpr key : keys) { + hashCode = 31 * hashCode + (key == null ? 0 : key.hashCode()); + } int ret = types.size() * 1000; for (PrimitiveType type : types) { ret += type.ordinal(); } - return ret; + return hashCode + ret; } public static class PartitionKeySerializer implements JsonSerializer { diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Table.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/Table.java index 17e87a8e8d01dbe..d3d72f03e30580f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Table.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Table.java @@ -551,7 +551,7 @@ public DatabaseIf getDatabase() { } @Override - public Optional getColumnStatistic() { + public Optional getColumnStatistic(String colName) { return Optional.empty(); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/TableIf.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/TableIf.java index 0b5e07441b230c2..0f12cc25509bd04 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/TableIf.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/TableIf.java @@ -132,7 +132,7 @@ default int getBaseColumnIdxByName(String colName) { DatabaseIf getDatabase(); - Optional getColumnStatistic(); + Optional getColumnStatistic(String colName); /** * Doris table type. diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/external/ExternalTable.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/external/ExternalTable.java index d04e757b15110e4..88ba9bc94e2804a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/external/ExternalTable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/external/ExternalTable.java @@ -326,8 +326,12 @@ public DatabaseIf getDatabase() { } @Override - public Optional getColumnStatistic() { - // TODO: Implement this interface for all kinds of external table. + public List getColumns() { + return getFullSchema(); + } + + @Override + public Optional getColumnStatistic(String colName) { return Optional.empty(); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/external/HMSExternalTable.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/external/HMSExternalTable.java index ebd9da942582d80..3e10f6636dea8c9 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/external/HMSExternalTable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/external/HMSExternalTable.java @@ -22,13 +22,17 @@ import org.apache.doris.catalog.HiveMetaStoreClientHelper; import org.apache.doris.catalog.HudiUtils; import org.apache.doris.catalog.Type; +import org.apache.doris.common.AnalysisException; import org.apache.doris.datasource.HMSExternalCatalog; import org.apache.doris.datasource.hive.PooledHiveMetaStoreClient; import org.apache.doris.statistics.AnalysisInfo; import org.apache.doris.statistics.BaseAnalysisTask; +import org.apache.doris.statistics.ColumnStatistic; +import org.apache.doris.statistics.ColumnStatisticBuilder; import org.apache.doris.statistics.HiveAnalysisTask; import org.apache.doris.statistics.IcebergAnalysisTask; import org.apache.doris.statistics.TableStatistic; +import org.apache.doris.statistics.util.StatisticsUtil; import org.apache.doris.thrift.THiveTable; import org.apache.doris.thrift.TTableDescriptor; import org.apache.doris.thrift.TTableType; @@ -36,15 +40,25 @@ import com.google.common.collect.Lists; import com.google.common.collect.Sets; import org.apache.commons.lang3.StringUtils; +import org.apache.hadoop.hive.metastore.api.ColumnStatisticsData; import org.apache.hadoop.hive.metastore.api.ColumnStatisticsObj; +import org.apache.hadoop.hive.metastore.api.DateColumnStatsData; +import org.apache.hadoop.hive.metastore.api.Decimal; +import org.apache.hadoop.hive.metastore.api.DecimalColumnStatsData; +import org.apache.hadoop.hive.metastore.api.DoubleColumnStatsData; import org.apache.hadoop.hive.metastore.api.FieldSchema; +import org.apache.hadoop.hive.metastore.api.LongColumnStatsData; import org.apache.hadoop.hive.metastore.api.Partition; +import org.apache.hadoop.hive.metastore.api.StringColumnStatsData; import org.apache.hadoop.hive.ql.io.AcidUtils; import org.apache.iceberg.Schema; import org.apache.iceberg.Table; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import java.math.BigDecimal; +import java.math.BigInteger; +import java.time.LocalDate; import java.util.HashMap; import java.util.HashSet; import java.util.List; @@ -66,6 +80,9 @@ public class HMSExternalTable extends ExternalTable { private static final String TBL_PROP_TXN_PROPERTIES = "transactional_properties"; private static final String TBL_PROP_INSERT_ONLY = "insert_only"; + public static final String NUM_ROWS = "numRows"; + public static final String NUM_FILES = "numFiles"; + static { SUPPORTED_HIVE_FILE_FORMATS = Sets.newHashSet(); SUPPORTED_HIVE_FILE_FORMATS.add("org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat"); @@ -433,5 +450,128 @@ private void initPartitionColumns(List schema) { LOG.debug("get {} partition columns for table: {}", partitionColumns.size(), name); } + @Override + public Optional getColumnStatistic(String colName) { + List tableStats = getHiveTableColumnStats(Lists.newArrayList(colName)); + if (tableStats == null || tableStats.isEmpty()) { + LOG.debug(String.format("No table stats found in Hive metastore for column %s in table %s.", + colName, name)); + return Optional.empty(); + } + + Column column = getColumn(colName); + if (column == null) { + LOG.warn(String.format("No column %s in table %s.", colName, name)); + return Optional.empty(); + } + Map parameters = remoteTable.getParameters(); + ColumnStatisticBuilder columnStatisticBuilder = new ColumnStatisticBuilder(); + double count = parameters.containsKey(NUM_ROWS) ? Double.parseDouble(parameters.get(NUM_ROWS)) : 0; + columnStatisticBuilder.setCount(count); + // The tableStats length is at most 1. + for (ColumnStatisticsObj tableStat : tableStats) { + if (!tableStat.isSetStatsData()) { + continue; + } + ColumnStatisticsData data = tableStat.getStatsData(); + try { + setStatData(column, data, columnStatisticBuilder, count); + } catch (AnalysisException e) { + LOG.debug(e); + return Optional.empty(); + } + } + + return Optional.of(columnStatisticBuilder.build()); + } + + private void setStatData(Column col, ColumnStatisticsData data, ColumnStatisticBuilder builder, double count) + throws AnalysisException { + long ndv = 0; + long nulls = 0; + String min = ""; + String max = ""; + double colSize = 0; + if (!data.isSetStringStats()) { + colSize = count * col.getType().getSlotSize(); + } + // Collect ndv, nulls, min and max for different data type. + if (data.isSetLongStats()) { + LongColumnStatsData longStats = data.getLongStats(); + ndv = longStats.getNumDVs(); + nulls = longStats.getNumNulls(); + min = String.valueOf(longStats.getLowValue()); + max = String.valueOf(longStats.getHighValue()); + } else if (data.isSetStringStats()) { + StringColumnStatsData stringStats = data.getStringStats(); + ndv = stringStats.getNumDVs(); + nulls = stringStats.getNumNulls(); + double avgColLen = stringStats.getAvgColLen(); + colSize = Math.round(avgColLen * count); + } else if (data.isSetDecimalStats()) { + DecimalColumnStatsData decimalStats = data.getDecimalStats(); + ndv = decimalStats.getNumDVs(); + nulls = decimalStats.getNumNulls(); + if (decimalStats.isSetLowValue()) { + Decimal lowValue = decimalStats.getLowValue(); + if (lowValue != null) { + BigDecimal lowDecimal = new BigDecimal(new BigInteger(lowValue.getUnscaled()), lowValue.getScale()); + min = lowDecimal.toString(); + } + } + if (decimalStats.isSetHighValue()) { + Decimal highValue = decimalStats.getHighValue(); + if (highValue != null) { + BigDecimal highDecimal = + new BigDecimal(new BigInteger(highValue.getUnscaled()), highValue.getScale()); + max = highDecimal.toString(); + } + } + } else if (data.isSetDoubleStats()) { + DoubleColumnStatsData doubleStats = data.getDoubleStats(); + ndv = doubleStats.getNumDVs(); + nulls = doubleStats.getNumNulls(); + min = String.valueOf(doubleStats.getLowValue()); + max = String.valueOf(doubleStats.getHighValue()); + } else if (data.isSetDateStats()) { + DateColumnStatsData dateStats = data.getDateStats(); + ndv = dateStats.getNumDVs(); + nulls = dateStats.getNumNulls(); + if (dateStats.isSetLowValue()) { + org.apache.hadoop.hive.metastore.api.Date lowValue = dateStats.getLowValue(); + if (lowValue != null) { + LocalDate lowDate = LocalDate.ofEpochDay(lowValue.getDaysSinceEpoch()); + min = lowDate.toString(); + } + } + if (dateStats.isSetHighValue()) { + org.apache.hadoop.hive.metastore.api.Date highValue = dateStats.getHighValue(); + if (highValue != null) { + LocalDate highDate = LocalDate.ofEpochDay(highValue.getDaysSinceEpoch()); + max = highDate.toString(); + } + } + } else { + LOG.debug(String.format("Not suitable data type for column %s", col.getName())); + throw new RuntimeException("Not supported data type."); + } + builder.setNdv(ndv); + builder.setNumNulls(nulls); + builder.setDataSize(colSize); + builder.setAvgSizeByte(colSize / count); + if (!min.equals("")) { + builder.setMinValue(StatisticsUtil.convertToDouble(col.getType(), min)); + builder.setMinExpr(StatisticsUtil.readableValue(col.getType(), min)); + } else { + builder.setMinValue(Double.MIN_VALUE); + } + if (!max.equals("")) { + builder.setMaxValue(StatisticsUtil.convertToDouble(col.getType(), max)); + builder.setMaxExpr(StatisticsUtil.readableValue(col.getType(), max)); + } else { + builder.setMaxValue(Double.MAX_VALUE); + } + } + } diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/proc/IndexSchemaProcNode.java b/fe/fe-core/src/main/java/org/apache/doris/common/proc/IndexSchemaProcNode.java index 7ede055f1da75e1..70b31db1e460959 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/proc/IndexSchemaProcNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/proc/IndexSchemaProcNode.java @@ -18,6 +18,7 @@ package org.apache.doris.common.proc; import org.apache.doris.catalog.Column; +import org.apache.doris.catalog.ScalarType; import org.apache.doris.common.AnalysisException; import org.apache.doris.common.FeConstants; @@ -81,7 +82,11 @@ public ProcResult fetchResult() throws AnalysisException { rowList.set(1, "DATE"); } if (column.getOriginType().isDatetimeV2()) { - rowList.set(1, "DATETIME"); + StringBuilder typeStr = new StringBuilder("DATETIME"); + if (((ScalarType) column.getOriginType()).getScalarScale() > 0) { + typeStr.append("(").append(((ScalarType) column.getOriginType()).getScalarScale()).append(")"); + } + rowList.set(1, typeStr.toString()); } result.addRow(rowList); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/profile/SummaryProfile.java b/fe/fe-core/src/main/java/org/apache/doris/common/profile/SummaryProfile.java index e3e2586307a8d4b..d395af219034cc4 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/profile/SummaryProfile.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/profile/SummaryProfile.java @@ -42,6 +42,8 @@ public class SummaryProfile { public static final String DEFAULT_DB = "Default Db"; public static final String SQL_STATEMENT = "Sql Statement"; public static final String IS_CACHED = "Is Cached"; + public static final String IS_NEREIDS = "Is Nereids"; + public static final String IS_PIPELINE = "Is Pipeline"; public static final String TOTAL_INSTANCES_NUM = "Total Instances Num"; public static final String INSTANCES_NUM_PER_BE = "Instances Num Per BE"; public static final String PARALLEL_FRAGMENT_EXEC_INSTANCE = "Parallel Fragment Exec Instance Num"; @@ -56,8 +58,8 @@ public class SummaryProfile { public static final String WAIT_FETCH_RESULT_TIME = "Wait and Fetch Result Time"; public static final ImmutableList SUMMARY_KEYS = ImmutableList.of(PROFILE_ID, TASK_TYPE, - START_TIME, END_TIME, TOTAL_TIME, TASK_STATE, USER, DEFAULT_DB, SQL_STATEMENT, IS_CACHED, - TOTAL_INSTANCES_NUM, INSTANCES_NUM_PER_BE, PARALLEL_FRAGMENT_EXEC_INSTANCE, TRACE_ID); + START_TIME, END_TIME, TOTAL_TIME, TASK_STATE, USER, DEFAULT_DB, SQL_STATEMENT, IS_NEREIDS, IS_PIPELINE, + IS_CACHED, TOTAL_INSTANCES_NUM, INSTANCES_NUM_PER_BE, PARALLEL_FRAGMENT_EXEC_INSTANCE, TRACE_ID); public static final ImmutableList EXECUTION_SUMMARY_KEYS = ImmutableList.of(ANALYSIS_TIME, PLAN_TIME, SCHEDULE_TIME, FETCH_RESULT_TIME, WRITE_RESULT_TIME, WAIT_FETCH_RESULT_TIME); @@ -229,6 +231,16 @@ public SummaryBuilder traceId(String val) { return this; } + public SummaryBuilder isNereids(String isNereids) { + map.put(IS_NEREIDS, isNereids); + return this; + } + + public SummaryBuilder isPipeline(String isPipeline) { + map.put(IS_PIPELINE, isPipeline); + return this; + } + public Map build() { return map; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetaStoreCache.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetaStoreCache.java index 5777cf49a0c5d2c..d64cdf477b21641 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetaStoreCache.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetaStoreCache.java @@ -78,6 +78,7 @@ import java.io.FileNotFoundException; import java.io.UnsupportedEncodingException; +import java.net.URI; import java.net.URLDecoder; import java.nio.charset.StandardCharsets; import java.security.PrivilegedExceptionAction; @@ -328,6 +329,17 @@ private FileCacheValue loadFiles(FileCacheKey key) { try { Thread.currentThread().setContextClassLoader(ClassLoader.getSystemClassLoader()); String finalLocation = S3Util.convertToS3IfNecessary(key.location); + // disable the fs cache in FileSystem, or it will always from new FileSystem + // and save it in cache when calling FileInputFormat.setInputPaths(). + try { + Path path = new Path(finalLocation); + URI uri = path.toUri(); + if (uri.getScheme() != null) { + updateJobConf("fs." + uri.getScheme() + ".impl.disable.cache", "true"); + } + } catch (Exception e) { + LOG.warn("unknown scheme in path: " + finalLocation, e); + } FileInputFormat.setInputPaths(jobConf, finalLocation); try { FileCacheValue result; @@ -381,6 +393,13 @@ private synchronized void setJobConf() { // Otherwise, getSplits() may throw exception: "Not a file xxx" // https://blog.actorsfit.com/a?ID=00550-ce56ec63-1bff-4b0c-a6f7-447b93efaa31 jobConf.set("mapreduce.input.fileinputformat.input.dir.recursive", "true"); + // disable FileSystem's cache + jobConf.set("fs.hdfs.impl.disable.cache", "true"); + jobConf.set("fs.file.impl.disable.cache", "true"); + } + + private synchronized void updateJobConf(String key, String value) { + jobConf.set(key, value); } public HivePartitionValues getPartitionValues(String dbName, String tblName, List types) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/constants/S3Properties.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/constants/S3Properties.java index 4b9d7a1023c6ec1..945cfc386fe8343 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/constants/S3Properties.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/constants/S3Properties.java @@ -54,6 +54,8 @@ public class S3Properties extends BaseProperties { // required by storage policy public static final String ROOT_PATH = "s3.root.path"; public static final String BUCKET = "s3.bucket"; + public static final String VIRTUAL_BUCKET = "s3.virtual.bucket"; + public static final String VIRTUAL_KEY = "s3.virtual.key"; public static final String VALIDITY_CHECK = "s3_validity_check"; public static final List REQUIRED_FIELDS = Arrays.asList(ENDPOINT, ACCESS_KEY, SECRET_KEY); public static final List TVF_REQUIRED_FIELDS = Arrays.asList(ACCESS_KEY, SECRET_KEY); diff --git a/fe/fe-core/src/main/java/org/apache/doris/external/jdbc/JdbcMySQLClient.java b/fe/fe-core/src/main/java/org/apache/doris/external/jdbc/JdbcMySQLClient.java index c5ceefb89d1180c..a6c0ce29a21e6c5 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/external/jdbc/JdbcMySQLClient.java +++ b/fe/fe-core/src/main/java/org/apache/doris/external/jdbc/JdbcMySQLClient.java @@ -36,8 +36,12 @@ import java.util.function.Consumer; public class JdbcMySQLClient extends JdbcClient { + + private static boolean convertDateToNull = false; + protected JdbcMySQLClient(JdbcClientConfig jdbcClientConfig) { super(jdbcClientConfig); + convertDateToNull = isConvertDatetimeToNull(jdbcClientConfig); } @Override @@ -281,6 +285,9 @@ protected Type jdbcTypeToDoris(JdbcFieldSchema fieldSchema) { if (scale > 6) { scale = 6; } + if (convertDateToNull) { + fieldSchema.setAllowNull(true); + } return ScalarType.createDatetimeV2Type(scale); } case "FLOAT": @@ -331,4 +338,9 @@ protected Type jdbcTypeToDoris(JdbcFieldSchema fieldSchema) { return Type.UNSUPPORTED; } } + + private boolean isConvertDatetimeToNull(JdbcClientConfig jdbcClientConfig) { + // Check if the JDBC URL contains "zeroDateTimeBehavior=convertToNull". + return jdbcClientConfig.getJdbcUrl().contains("zeroDateTimeBehavior=convertToNull"); + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/external/jdbc/JdbcOracleClient.java b/fe/fe-core/src/main/java/org/apache/doris/external/jdbc/JdbcOracleClient.java index 5adfa63b9dcc73b..52700dd3f38b17b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/external/jdbc/JdbcOracleClient.java +++ b/fe/fe-core/src/main/java/org/apache/doris/external/jdbc/JdbcOracleClient.java @@ -47,11 +47,15 @@ protected Type jdbcTypeToDoris(JdbcFieldSchema fieldSchema) { if (oracleType.startsWith("INTERVAL")) { oracleType = oracleType.substring(0, 8); } else if (oracleType.startsWith("TIMESTAMP")) { - if (oracleType.equals("TIMESTAMPTZ") || oracleType.equals("TIMESTAMPLTZ")) { + if (oracleType.contains("TIME ZONE") || oracleType.contains("LOCAL TIME ZONE")) { return Type.UNSUPPORTED; } // oracle can support nanosecond, will lose precision - return ScalarType.createDatetimeV2Type(JDBC_DATETIME_SCALE); + int scale = fieldSchema.getDecimalDigits(); + if (scale > 6) { + scale = 6; + } + return ScalarType.createDatetimeV2Type(scale); } switch (oracleType) { /** diff --git a/fe/fe-core/src/main/java/org/apache/doris/fs/remote/RemoteFileSystem.java b/fe/fe-core/src/main/java/org/apache/doris/fs/remote/RemoteFileSystem.java index 7d8799373324533..ffe63f20ac76840 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/fs/remote/RemoteFileSystem.java +++ b/fe/fe-core/src/main/java/org/apache/doris/fs/remote/RemoteFileSystem.java @@ -31,7 +31,8 @@ import java.util.List; public abstract class RemoteFileSystem extends PersistentFileSystem { - protected org.apache.hadoop.fs.FileSystem dfsFileSystem = null; + // this field will be visited by multi-threads, better use volatile qualifier + protected volatile org.apache.hadoop.fs.FileSystem dfsFileSystem = null; public RemoteFileSystem(String name, StorageBackend.StorageType type) { super(name, type); diff --git a/fe/fe-core/src/main/java/org/apache/doris/fs/remote/dfs/DFSFileSystem.java b/fe/fe-core/src/main/java/org/apache/doris/fs/remote/dfs/DFSFileSystem.java index 9f72595ad8d545f..ce297ce92068f05 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/fs/remote/dfs/DFSFileSystem.java +++ b/fe/fe-core/src/main/java/org/apache/doris/fs/remote/dfs/DFSFileSystem.java @@ -29,6 +29,7 @@ import org.apache.doris.fs.remote.RemoteFile; import org.apache.doris.fs.remote.RemoteFileSystem; +import com.google.common.base.Preconditions; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FSDataInputStream; import org.apache.hadoop.fs.FSDataOutputStream; @@ -54,7 +55,6 @@ import java.util.Comparator; import java.util.List; import java.util.Map; -import java.util.concurrent.atomic.AtomicReference; public class DFSFileSystem extends RemoteFileSystem { @@ -82,42 +82,61 @@ protected FileSystem nativeFileSystem(String remotePath) throws UserException { conf.set(propEntry.getKey(), propEntry.getValue()); } - UserGroupInformation ugi = getUgi(conf); - AtomicReference exception = new AtomicReference<>(); + boolean hasRelogin = false; + UserGroupInformation ugi; + try { + // try use current ugi first to avoid relogin + // because it may be a time-consuming task + ugi = UserGroupInformation.getCurrentUser(); + } catch (IOException e) { + LOG.warn("An IOException occurs when invoke " + + "UserGroupInformation.getCurrentUser(), relogin immediately.", e); + ugi = doLogin(conf); + hasRelogin = true; + } - dfsFileSystem = ugi.doAs((PrivilegedAction) () -> { + do { try { - String username = properties.get(HdfsResource.HADOOP_USER_NAME); - if (username == null) { - return FileSystem.get(new Path(remotePath).toUri(), conf); - } else { - return FileSystem.get(new Path(remotePath).toUri(), conf, username); + dfsFileSystem = ugi.doAs((PrivilegedAction) () -> { + try { + String username = properties.get(HdfsResource.HADOOP_USER_NAME); + return username == null + ? FileSystem.get(new Path(remotePath).toUri(), conf) + : FileSystem.get(new Path(remotePath).toUri(), conf, username); + } catch (IOException | InterruptedException e) { + throw new RuntimeException(e); + } + }); + LOG.debug("Reuse current ugi for dfs, remote path: {}", remotePath); + break; + } catch (SecurityException e) { + LOG.warn("A SecurityException occurs when invoke ugi.doAs(), " + + "relogin and retry immediately.", e); + if (hasRelogin) { + throw new UserException(e); } - } catch (Exception e) { - exception.set(e); - return null; + ugi = doLogin(conf); + hasRelogin = true; } - }); + } while (true); - if (dfsFileSystem == null) { - LOG.error("errors while connect to " + remotePath, exception.get()); - throw new UserException("errors while connect to " + remotePath, exception.get()); - } + Preconditions.checkNotNull(dfsFileSystem); operations = new HDFSFileOperations(dfsFileSystem); return dfsFileSystem; } - private UserGroupInformation getUgi(Configuration conf) throws UserException { - String authentication = conf.get(HdfsResource.HADOOP_SECURITY_AUTHENTICATION, null); - if (AuthType.KERBEROS.getDesc().equals(authentication)) { - conf.set("hadoop.security.authorization", "true"); - UserGroupInformation.setConfiguration(conf); + private UserGroupInformation doLogin(Configuration conf) throws UserException { + if (AuthType.KERBEROS.getDesc().equals( + conf.get(HdfsResource.HADOOP_SECURITY_AUTHENTICATION, null))) { + conf.set(HdfsResource.HADOOP_KERBEROS_AUTHORIZATION, "true"); String principal = conf.get(HdfsResource.HADOOP_KERBEROS_PRINCIPAL); String keytab = conf.get(HdfsResource.HADOOP_KERBEROS_KEYTAB); + + UserGroupInformation.setConfiguration(conf); try { UserGroupInformation ugi = UserGroupInformation.loginUserFromKeytabAndReturnUGI(principal, keytab); UserGroupInformation.setLoginUser(ugi); - LOG.info("kerberos authentication successful"); + LOG.info("Login by kerberos authentication with principal: {}", principal); return ugi; } catch (IOException e) { throw new UserException(e); @@ -128,7 +147,10 @@ private UserGroupInformation getUgi(Configuration conf) throws UserException { hadoopUserName = "hadoop"; LOG.debug(HdfsResource.HADOOP_USER_NAME + " is unset, use default user: hadoop"); } - return UserGroupInformation.createRemoteUser(hadoopUserName); + UserGroupInformation ugi = UserGroupInformation.createRemoteUser(hadoopUserName); + UserGroupInformation.setLoginUser(ugi); + LOG.info("Login by proxy user, hadoop.username: {}", hadoopUserName); + return ugi; } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/journal/JournalEntity.java b/fe/fe-core/src/main/java/org/apache/doris/journal/JournalEntity.java index 39b804eb40274c8..2a29b18c85c32e0 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/journal/JournalEntity.java +++ b/fe/fe-core/src/main/java/org/apache/doris/journal/JournalEntity.java @@ -69,6 +69,7 @@ import org.apache.doris.persist.AnalyzeDeletionLog; import org.apache.doris.persist.BackendReplicasInfo; import org.apache.doris.persist.BackendTabletsInfo; +import org.apache.doris.persist.BarrierLog; import org.apache.doris.persist.BatchDropInfo; import org.apache.doris.persist.BatchModifyPartitionsInfo; import org.apache.doris.persist.BatchRemoveTransactionsOperation; @@ -829,6 +830,11 @@ public void readFields(DataInput in) throws IOException { isRead = true; break; } + case OperationType.OP_BARRIER: { + data = new BarrierLog(); + isRead = true; + break; + } default: { IOException e = new IOException(); LOG.error("UNKNOWN Operation Type {}", opCode, e); diff --git a/fe/fe-core/src/main/java/org/apache/doris/load/Load.java b/fe/fe-core/src/main/java/org/apache/doris/load/Load.java index ae9ff3bed71be91..a1aab0e075cf288 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/load/Load.java +++ b/fe/fe-core/src/main/java/org/apache/doris/load/Load.java @@ -736,9 +736,6 @@ private static void initColumns(Table tbl, List columnExprs, slotDesc.setType(Type.VARIANT); slotDesc.setColumn(col); slotDesc.setIsNullable(false); - // Non-nullable slots will have 0 for the byte offset and -1 for the bit mask - slotDesc.setNullIndicatorBit(-1); - slotDesc.setNullIndicatorByte(0); slotDesc.setIsMaterialized(true); srcSlotIds.add(slotDesc.getId().asInt()); slotDescByName.put(name, slotDesc); diff --git a/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/LoadingTaskPlanner.java b/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/LoadingTaskPlanner.java index cd5f13cf973d165..7b8ca0371e595bf 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/LoadingTaskPlanner.java +++ b/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/LoadingTaskPlanner.java @@ -150,9 +150,6 @@ public void plan(TUniqueId loadId, List> fileStatusesLis Column col = new Column(Column.DYNAMIC_COLUMN_NAME, Type.VARIANT, false, null, false, "", "stream load auto dynamic column"); slotDesc.setIsMaterialized(true); - // Non-nullable slots will have 0 for the byte offset and -1 for the bit mask - slotDesc.setNullIndicatorBit(-1); - slotDesc.setNullIndicatorByte(0); slotDesc.setColumn(col); slotDesc.setIsNullable(false); LOG.debug("plan scanTupleDesc{}", scanTupleDesc.toString()); diff --git a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVJobFactory.java b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVJobFactory.java index ac771c426494d6e..3ac142d0639a6a6 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVJobFactory.java +++ b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVJobFactory.java @@ -22,7 +22,9 @@ import org.apache.doris.analysis.MVRefreshInfo.RefreshTrigger; import org.apache.doris.analysis.MVRefreshIntervalTriggerInfo; import org.apache.doris.analysis.MVRefreshTriggerInfo; +import org.apache.doris.catalog.Env; import org.apache.doris.catalog.MaterializedView; +import org.apache.doris.common.Config; import org.apache.doris.common.FeConstants; import org.apache.doris.common.util.TimeUtils; import org.apache.doris.mtmv.MTMVUtils.TriggerMode; @@ -76,6 +78,7 @@ public static List buildJob(MaterializedView materializedView, String d private static MTMVJob genPeriodicalJob(MaterializedView materializedView, String dbName) { String uid = UUID.randomUUID().toString(); MTMVJob job = new MTMVJob(materializedView.getName() + "_" + uid); + job.setId(Env.getCurrentEnv().getNextId()); job.setTriggerMode(TriggerMode.PERIODICAL); job.setSchedule(genJobSchedule(materializedView)); job.setDBName(dbName); @@ -88,11 +91,14 @@ private static MTMVJob genPeriodicalJob(MaterializedView materializedView, Strin public static MTMVJob genOnceJob(MaterializedView materializedView, String dbName) { String uid = UUID.randomUUID().toString(); MTMVJob job = new MTMVJob(materializedView.getName() + "_" + uid); + job.setId(Env.getCurrentEnv().getNextId()); job.setTriggerMode(TriggerMode.ONCE); job.setDBName(dbName); job.setMVName(materializedView.getName()); job.setQuery(materializedView.getQuery()); - job.setCreateTime(MTMVUtils.getNowTimeStamp()); + long nowTimeStamp = MTMVUtils.getNowTimeStamp(); + job.setCreateTime(nowTimeStamp); + job.setExpireTime(nowTimeStamp + Config.scheduler_mtmv_job_expired); return job; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVJobManager.java b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVJobManager.java index cae36ab3da0a6d2..7295f40b6029e2a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVJobManager.java +++ b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVJobManager.java @@ -21,7 +21,6 @@ import org.apache.doris.catalog.Env; import org.apache.doris.catalog.MaterializedView; import org.apache.doris.catalog.TableIf.TableType; -import org.apache.doris.common.Config; import org.apache.doris.common.DdlException; import org.apache.doris.common.MetaNotFoundException; import org.apache.doris.common.io.Text; @@ -30,16 +29,12 @@ import org.apache.doris.metric.MetricLabel; import org.apache.doris.metric.MetricRepo; import org.apache.doris.mtmv.MTMVUtils.JobState; -import org.apache.doris.mtmv.MTMVUtils.TaskSubmitStatus; -import org.apache.doris.mtmv.MTMVUtils.TriggerMode; import org.apache.doris.mtmv.metadata.ChangeMTMVJob; import org.apache.doris.mtmv.metadata.MTMVCheckpointData; import org.apache.doris.mtmv.metadata.MTMVJob; -import org.apache.doris.mtmv.metadata.MTMVJob.JobSchedule; import org.apache.doris.mtmv.metadata.MTMVTask; import org.apache.doris.persist.gson.GsonUtils; -import com.google.common.base.Preconditions; import com.google.common.base.Strings; import com.google.common.collect.Lists; import com.google.common.collect.Maps; @@ -55,10 +50,9 @@ import java.util.Map; import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.ScheduledFuture; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; -import java.util.concurrent.locks.ReentrantLock; +import java.util.concurrent.locks.ReentrantReadWriteLock; import java.util.stream.Collectors; public class MTMVJobManager { @@ -69,7 +63,6 @@ public class MTMVJobManager { private final Map idToJobMap; private final Map nameToJobMap; - private final Map> periodFutureMap; private final MTMVTaskManager taskManager; @@ -77,16 +70,15 @@ public class MTMVJobManager { private ScheduledExecutorService cleanerScheduler = Executors.newScheduledThreadPool(1); - private final ReentrantLock reentrantLock; + private final ReentrantReadWriteLock rwLock; private final AtomicBoolean isStarted = new AtomicBoolean(false); public MTMVJobManager() { idToJobMap = Maps.newConcurrentMap(); nameToJobMap = Maps.newConcurrentMap(); - periodFutureMap = Maps.newConcurrentMap(); - reentrantLock = new ReentrantLock(true); - taskManager = new MTMVTaskManager(this); + rwLock = new ReentrantReadWriteLock(true); + taskManager = new MTMVTaskManager(); } public void start() { @@ -104,19 +96,11 @@ public void start() { } cleanerScheduler.scheduleAtFixedRate(() -> { if (!Env.getCurrentEnv().isMaster()) { + LOG.warn("only master can run MTMVJob"); return; } - if (!tryLock()) { - return; - } - try { - removeExpiredJobs(); - taskManager.removeExpiredTasks(); - } catch (Exception ex) { - LOG.warn("failed remove expired jobs and tasks.", ex); - } finally { - unlock(); - } + removeExpiredJobs(); + taskManager.removeExpiredTasks(); }, 0, 1, TimeUnit.MINUTES); taskManager.startTaskScheduler(); @@ -146,7 +130,13 @@ public Integer getValue() { Metric.MetricUnit.NOUNIT, "Active job number of mtmv.") { @Override public Integer getValue() { - return periodFutureMap.size(); + int result = 0; + for (MTMVJob job : getAllJobsWithLock()) { + if (job.getState() == JobState.ACTIVE) { + result++; + } + } + return result; } }; activeJob.addLabel(new MetricLabel("type", "ACTIVE-JOB")); @@ -206,126 +196,35 @@ public void stop() { } private void registerJobs() { - int num = nameToJobMap.size(); - int periodNum = 0; - int onceNum = 0; - for (MTMVJob job : nameToJobMap.values()) { - if (!job.getState().equals(JobState.ACTIVE)) { - continue; - } - if (job.getTriggerMode() == TriggerMode.PERIODICAL) { - JobSchedule schedule = job.getSchedule(); - ScheduledFuture future = periodScheduler.scheduleAtFixedRate(() -> submitJobTask(job.getName()), - MTMVUtils.getDelaySeconds(job), schedule.getSecondPeriod(), TimeUnit.SECONDS); - periodFutureMap.put(job.getId(), future); - periodNum++; - } else if (job.getTriggerMode() == TriggerMode.ONCE) { - MTMVTaskExecuteParams executeOption = new MTMVTaskExecuteParams(); - submitJobTask(job.getName(), executeOption); - onceNum++; - } + for (MTMVJob job : getAllJobsWithLock()) { + job.start(); } - LOG.info("Register {} period jobs and {} once jobs in the total {} jobs.", periodNum, onceNum, num); } public void createJob(MTMVJob job, boolean isReplay) throws DdlException { - if (!tryLock()) { - throw new DdlException("Failed to get job manager lock when create Job [" + job.getName() + "]"); + createJobWithLock(job, isReplay); + if (!isReplay) { + job.start(); } + } + + private void createJobWithLock(MTMVJob job, boolean isReplay) throws DdlException { + writeLock(); try { if (nameToJobMap.containsKey(job.getName())) { throw new DdlException("Job [" + job.getName() + "] already exists"); } + nameToJobMap.put(job.getName(), job); + idToJobMap.put(job.getId(), job); if (!isReplay) { - Preconditions.checkArgument(job.getId() == 0); - job.setId(Env.getCurrentEnv().getNextId()); - } - if (job.getTriggerMode() == TriggerMode.PERIODICAL) { - JobSchedule schedule = job.getSchedule(); - if (schedule == null) { - throw new DdlException("Job [" + job.getName() + "] has no scheduling"); - } - job.setState(JobState.ACTIVE); - nameToJobMap.put(job.getName(), job); - idToJobMap.put(job.getId(), job); - if (!isReplay) { - // log job before submit any task. - Env.getCurrentEnv().getEditLog().logCreateMTMVJob(job); - ScheduledFuture future = periodScheduler.scheduleAtFixedRate(() -> submitJobTask(job.getName()), - MTMVUtils.getDelaySeconds(job), schedule.getSecondPeriod(), TimeUnit.SECONDS); - periodFutureMap.put(job.getId(), future); - } - } else if (job.getTriggerMode() == TriggerMode.ONCE) { - // only change once job state from unknown to active. if job is completed, only put it in map - if (job.getState() == JobState.UNKNOWN) { - job.setState(JobState.ACTIVE); - job.setExpireTime(MTMVUtils.getNowTimeStamp() + Config.scheduler_mtmv_job_expired); - } - nameToJobMap.put(job.getName(), job); - idToJobMap.put(job.getId(), job); - if (!isReplay) { - Env.getCurrentEnv().getEditLog().logCreateMTMVJob(job); - MTMVTaskExecuteParams executeOption = new MTMVTaskExecuteParams(); - MTMVUtils.TaskSubmitStatus status = submitJobTask(job.getName(), executeOption); - if (status != TaskSubmitStatus.SUBMITTED) { - throw new DdlException("submit job task with: " + status.toString()); - } - } - } else if (job.getTriggerMode() == TriggerMode.MANUAL) { - // only change once job state from unknown to active. if job is completed, only put it in map - if (job.getState() == JobState.UNKNOWN) { - job.setState(JobState.ACTIVE); - } - nameToJobMap.put(job.getName(), job); - idToJobMap.put(job.getId(), job); - if (!isReplay) { - Env.getCurrentEnv().getEditLog().logCreateMTMVJob(job); - } - } else { - throw new DdlException("Unsupported trigger mode for multi-table mv."); + // log job before submit any task. + Env.getCurrentEnv().getEditLog().logCreateMTMVJob(job); } } finally { - unlock(); + writeUnlock(); } } - private boolean stopScheduler(String jobName) { - MTMVJob job = nameToJobMap.get(jobName); - if (job.getTriggerMode() != TriggerMode.PERIODICAL) { - return false; - } - if (job.getState() == MTMVUtils.JobState.PAUSE) { - return true; - } - JobSchedule jobSchedule = job.getSchedule(); - // this will not happen - if (jobSchedule == null) { - LOG.warn("fail to obtain scheduled info for job [{}]", job.getName()); - return true; - } - ScheduledFuture future = periodFutureMap.get(job.getId()); - if (future == null) { - LOG.warn("fail to obtain scheduled info for job [{}]", job.getName()); - return true; - } - // MUST not set true for "mayInterruptIfRunning". - // Because this thread may doing bdbje write operation, it is interrupted, - // FE may exit due to bdbje write failure. - boolean isCancel = future.cancel(false); - if (!isCancel) { - LOG.warn("fail to cancel scheduler for job [{}]", job.getName()); - } - return isCancel; - } - - public boolean killJobTask(String jobName, boolean clearPending) { - MTMVJob job = nameToJobMap.get(jobName); - if (job == null) { - return false; - } - return taskManager.killTask(job.getId(), clearPending); - } - public void refreshMTMV(String dbName, String mvName) throws DdlException, MetaNotFoundException { Database db = Env.getCurrentInternalCatalog().getDbOrDdlException(dbName); @@ -334,85 +233,53 @@ public void refreshMTMV(String dbName, String mvName) createJob(mtmvJob, false); } - public MTMVUtils.TaskSubmitStatus submitJobTask(String jobName) { - return submitJobTask(jobName, new MTMVTaskExecuteParams()); - } - - public MTMVUtils.TaskSubmitStatus submitJobTask(String jobName, MTMVTaskExecuteParams param) { - MTMVJob job = nameToJobMap.get(jobName); - if (job == null) { - return MTMVUtils.TaskSubmitStatus.FAILED; - } - return taskManager.submitTask(MTMVUtils.buildTask(job), param); - } - - public void updateJob(ChangeMTMVJob changeJob, boolean isReplay) { - if (!tryLock()) { - return; - } - try { - MTMVJob job = idToJobMap.get(changeJob.getJobId()); - if (job == null) { - LOG.warn("change jobId {} failed because job is null", changeJob.getJobId()); - return; - } - job.setState(changeJob.getToStatus()); - job.setLastModifyTime(changeJob.getLastModifyTime()); - if (!isReplay) { - Env.getCurrentEnv().getEditLog().logChangeMTMVJob(changeJob); - } - } finally { - unlock(); - } - LOG.info("change job:{}", changeJob.getJobId()); - } - public void dropJobByName(String dbName, String mvName, boolean isReplay) { + List jobIds = Lists.newArrayList(); for (String jobName : nameToJobMap.keySet()) { MTMVJob job = nameToJobMap.get(jobName); if (job.getMVName().equals(mvName) && job.getDBName().equals(dbName)) { - dropJobs(Collections.singletonList(job.getId()), isReplay); - return; + jobIds.add(job.getId()); } } + dropJobs(jobIds, isReplay); } public void dropJobs(List jobIds, boolean isReplay) { if (jobIds.isEmpty()) { return; } - if (!tryLock()) { - return; + + for (long jobId : jobIds) { + dropJob(jobId, isReplay); } + + LOG.info("drop jobs:{}", jobIds); + } + + private void dropJob(long jobId, boolean isReplay) { + MTMVJob job = dropJobWithLock(jobId, isReplay); + if (!isReplay && job != null) { + job.stop(); + } + } + + private MTMVJob dropJobWithLock(long jobId, boolean isReplay) { + writeLock(); try { - for (long jobId : jobIds) { - MTMVJob job = idToJobMap.get(jobId); - if (job == null) { - LOG.warn("drop jobId {} failed because job is null", jobId); - continue; - } - if (job.getTriggerMode() == TriggerMode.PERIODICAL && !isReplay) { - boolean isCancel = stopScheduler(job.getName()); - if (!isCancel) { - continue; - } - periodFutureMap.remove(job.getId()); - } - killJobTask(job.getName(), true); - if (!Config.keep_scheduler_mtmv_task_when_job_deleted) { - taskManager.clearTasksByJobName(job.getName(), isReplay); - } - idToJobMap.remove(job.getId()); - nameToJobMap.remove(job.getName()); + MTMVJob job = idToJobMap.get(jobId); + if (job == null) { + LOG.warn("drop jobId {} failed because job is null", jobId); + return null; } - + idToJobMap.remove(job.getId()); + nameToJobMap.remove(job.getName()); if (!isReplay) { - Env.getCurrentEnv().getEditLog().logDropMTMVJob(jobIds); + Env.getCurrentEnv().getEditLog().logDropMTMVJob(Collections.singletonList(jobId)); } + return job; } finally { - unlock(); + writeUnlock(); } - LOG.info("drop jobs:{}", jobIds); } public List showAllJobs() { @@ -422,9 +289,9 @@ public List showAllJobs() { public List showJobs(String dbName) { List jobList = Lists.newArrayList(); if (Strings.isNullOrEmpty(dbName)) { - jobList.addAll(nameToJobMap.values()); + jobList.addAll(getAllJobsWithLock()); } else { - jobList.addAll(nameToJobMap.values().stream().filter(u -> u.getDBName().equals(dbName)) + jobList.addAll(getAllJobsWithLock().stream().filter(u -> u.getDBName().equals(dbName)) .collect(Collectors.toList())); } return jobList.stream().sorted().collect(Collectors.toList()); @@ -434,27 +301,7 @@ public List showJobs(String dbName, String mvName) { return showJobs(dbName).stream().filter(u -> u.getMVName().equals(mvName)).collect(Collectors.toList()); } - private boolean tryLock() { - try { - return reentrantLock.tryLock(5, TimeUnit.SECONDS); - } catch (InterruptedException e) { - LOG.warn("got exception while getting job manager lock", e); - } - return false; - } - - public void unlock() { - this.reentrantLock.unlock(); - } - public void replayCreateJob(MTMVJob job) { - if (job.getTriggerMode() == TriggerMode.PERIODICAL) { - JobSchedule jobSchedule = job.getSchedule(); - if (jobSchedule == null) { - LOG.warn("replay a null schedule period job [{}]", job.getName()); - return; - } - } if (job.getExpireTime() > 0 && MTMVUtils.getNowTimeStamp() > job.getExpireTime()) { return; } @@ -470,7 +317,10 @@ public void replayDropJobs(List jobIds) { } public void replayUpdateJob(ChangeMTMVJob changeJob) { - updateJob(changeJob, true); + MTMVJob mtmvJob = idToJobMap.get(changeJob.getJobId()); + if (mtmvJob != null) { + mtmvJob.updateJob(changeJob, true); + } } public void replayCreateJobTask(MTMVTask task) { @@ -478,41 +328,23 @@ public void replayCreateJobTask(MTMVTask task) { } public void replayDropJobTasks(List taskIds) { - taskManager.dropTasks(taskIds, true); + taskManager.dropHistoryTasks(taskIds, true); } - public void removeExpiredJobs() { + private void removeExpiredJobs() { long currentTimeSeconds = MTMVUtils.getNowTimeStamp(); - List jobIdsToDelete = Lists.newArrayList(); - if (!tryLock()) { - return; - } - try { - List jobs = showJobs(null); - for (MTMVJob job : jobs) { - // active job should not clean - if (job.getState() == MTMVUtils.JobState.ACTIVE) { - continue; - } - if (job.getTriggerMode() == MTMVUtils.TriggerMode.PERIODICAL) { - JobSchedule jobSchedule = job.getSchedule(); - if (jobSchedule == null) { - jobIdsToDelete.add(job.getId()); - LOG.warn("clean up a null schedule periodical Task [{}]", job.getName()); - continue; - } - - } - long expireTime = job.getExpireTime(); - if (expireTime > 0 && currentTimeSeconds > expireTime) { - jobIdsToDelete.add(job.getId()); - } + List jobs = getAllJobsWithLock(); + for (MTMVJob job : jobs) { + // active job should not clean + if (job.getState() != MTMVUtils.JobState.COMPLETE) { + continue; + } + long expireTime = job.getExpireTime(); + if (expireTime > 0 && currentTimeSeconds > expireTime) { + jobIdsToDelete.add(job.getId()); } - } finally { - unlock(); } - dropJobs(jobIdsToDelete, false); } @@ -520,6 +352,40 @@ public MTMVJob getJob(String jobName) { return nameToJobMap.get(jobName); } + private List getAllJobsWithLock() { + readLock(); + try { + return Lists.newArrayList(nameToJobMap.values()); + } finally { + readUnlock(); + } + + } + + public MTMVTaskManager getTaskManager() { + return taskManager; + } + + public ScheduledExecutorService getPeriodScheduler() { + return periodScheduler; + } + + private void readLock() { + this.rwLock.readLock().lock(); + } + + private void readUnlock() { + this.rwLock.readLock().unlock(); + } + + private void writeLock() { + this.rwLock.writeLock().lock(); + } + + private void writeUnlock() { + this.rwLock.writeLock().unlock(); + } + public long write(DataOutputStream dos, long checksum) throws IOException { MTMVCheckpointData data = new MTMVCheckpointData(); data.jobs = new ArrayList<>(nameToJobMap.values()); @@ -548,8 +414,4 @@ public static MTMVJobManager read(DataInputStream dis, long checksum) throws IOE } return mtmvJobManager; } - - public MTMVTaskManager getTaskManager() { - return taskManager; - } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVTaskExecutor.java b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVTaskExecutor.java index 55b4949c07b81d6..fb2aaf5419b211e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVTaskExecutor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVTaskExecutor.java @@ -147,6 +147,12 @@ public MTMVTask initTask(String taskId, Long createTime) { return task; } + public void stop() { + if (ctx != null) { + ctx.kill(false); + } + } + @Override public int compareTo(@NotNull MTMVTaskExecutor task) { if (this.getTask().getPriority() != task.getTask().getPriority()) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVTaskManager.java b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVTaskManager.java index 3396b30bbb7de3c..d6e370480bba732 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVTaskManager.java +++ b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVTaskManager.java @@ -20,12 +20,9 @@ import org.apache.doris.catalog.Env; import org.apache.doris.common.AnalysisException; import org.apache.doris.common.Config; -import org.apache.doris.mtmv.MTMVUtils.JobState; import org.apache.doris.mtmv.MTMVUtils.TaskState; -import org.apache.doris.mtmv.MTMVUtils.TriggerMode; -import org.apache.doris.mtmv.metadata.ChangeMTMVJob; +import org.apache.doris.mtmv.metadata.MTMVJob; import org.apache.doris.mtmv.metadata.MTMVTask; -import org.apache.doris.qe.ConnectContext; import com.google.common.base.Strings; import com.google.common.collect.Lists; @@ -33,7 +30,6 @@ import com.google.common.collect.Queues; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import org.jetbrains.annotations.Nullable; import java.util.Deque; import java.util.HashSet; @@ -49,7 +45,7 @@ import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; -import java.util.concurrent.locks.ReentrantLock; +import java.util.concurrent.locks.ReentrantReadWriteLock; import java.util.stream.Collectors; public class MTMVTaskManager { @@ -64,37 +60,23 @@ public class MTMVTaskManager { private final MTMVTaskExecutorPool taskExecutorPool = new MTMVTaskExecutorPool(); - private final ReentrantLock reentrantLock = new ReentrantLock(true); + private final ReentrantReadWriteLock rwLock = new ReentrantReadWriteLock(true); // keep track of all the completed tasks private final Deque historyTasks = Queues.newLinkedBlockingDeque(); private ScheduledExecutorService taskScheduler = Executors.newScheduledThreadPool(1); - private final MTMVJobManager mtmvJobManager; - private final AtomicInteger failedTaskCount = new AtomicInteger(0); - public MTMVTaskManager(MTMVJobManager mtmvJobManager) { - this.mtmvJobManager = mtmvJobManager; - } - public void startTaskScheduler() { if (taskScheduler.isShutdown()) { taskScheduler = Executors.newScheduledThreadPool(1); } taskScheduler.scheduleAtFixedRate(() -> { - if (!tryLock()) { - return; - } - try { - checkRunningTask(); - scheduledPendingTask(); - } catch (Exception ex) { - LOG.warn("failed to schedule task.", ex); - } finally { - unlock(); - } + checkRunningTask(); + scheduledPendingTask(); + }, 0, 1, TimeUnit.SECONDS); } @@ -102,7 +84,69 @@ public void stopTaskScheduler() { taskScheduler.shutdown(); } - public MTMVUtils.TaskSubmitStatus submitTask(MTMVTaskExecutor taskExecutor, MTMVTaskExecuteParams params) { + private void checkRunningTask() { + writeLock(); + try { + Iterator runningIterator = runningTaskMap.keySet().iterator(); + while (runningIterator.hasNext()) { + Long jobId = runningIterator.next(); + MTMVTaskExecutor taskExecutor = runningTaskMap.get(jobId); + Future future = taskExecutor.getFuture(); + if (future.isDone()) { + runningIterator.remove(); + addHistory(taskExecutor.getTask()); + MTMVUtils.TaskState finalState = taskExecutor.getTask().getState(); + if (finalState == TaskState.FAILURE) { + failedTaskCount.incrementAndGet(); + } + //task save final state only + Env.getCurrentEnv().getEditLog().logCreateMTMVTask(taskExecutor.getTask()); + taskExecutor.getJob().taskFinished(); + } + } + } finally { + writeUnlock(); + } + } + + private void scheduledPendingTask() { + writeLock(); + try { + int currentRunning = runningTaskMap.size(); + + Iterator pendingIterator = pendingTaskMap.keySet().iterator(); + while (pendingIterator.hasNext()) { + Long jobId = pendingIterator.next(); + MTMVTaskExecutor runningTaskExecutor = runningTaskMap.get(jobId); + if (runningTaskExecutor == null) { + Queue taskQueue = pendingTaskMap.get(jobId); + if (taskQueue.size() == 0) { + pendingIterator.remove(); + } else { + if (currentRunning >= Config.max_running_mtmv_scheduler_task_num) { + break; + } + MTMVTaskExecutor pendingTaskExecutor = taskQueue.poll(); + taskExecutorPool.executeTask(pendingTaskExecutor); + runningTaskMap.put(jobId, pendingTaskExecutor); + currentRunning++; + } + } + } + } finally { + writeUnlock(); + } + } + + public MTMVUtils.TaskSubmitStatus submitJobTask(MTMVJob job) { + return submitJobTask(job, new MTMVTaskExecuteParams()); + } + + private MTMVUtils.TaskSubmitStatus submitJobTask(MTMVJob job, MTMVTaskExecuteParams param) { + return submitTask(MTMVUtils.buildTask(job), param); + } + + private MTMVUtils.TaskSubmitStatus submitTask(MTMVTaskExecutor taskExecutor, MTMVTaskExecuteParams params) { // duplicate submit if (taskExecutor.getTask() != null) { return MTMVUtils.TaskSubmitStatus.FAILED; @@ -129,132 +173,39 @@ public MTMVUtils.TaskSubmitStatus submitTask(MTMVTaskExecutor taskExecutor, MTMV return MTMVUtils.TaskSubmitStatus.SUBMITTED; } - public boolean killTask(Long jobId, boolean clearPending) { - if (clearPending) { - if (!tryLock()) { - return false; - } - try { - getPendingTaskMap().remove(jobId); - } catch (Exception ex) { - LOG.warn("failed to kill task.", ex); - } finally { - unlock(); - } - } - MTMVTaskExecutor task = runningTaskMap.get(jobId); - if (task == null) { - return false; - } - ConnectContext connectContext = task.getCtx(); - if (connectContext != null) { - connectContext.kill(false); - return true; - } - return false; - } - - public void arrangeToPendingTask(MTMVTaskExecutor task) { - if (!tryLock()) { - return; - } + private void arrangeToPendingTask(MTMVTaskExecutor task) { + writeLock(); try { long jobId = task.getJobId(); PriorityBlockingQueue tasks = pendingTaskMap.computeIfAbsent(jobId, u -> Queues.newPriorityBlockingQueue()); tasks.offer(task); } finally { - unlock(); + writeUnlock(); } } - @Nullable - private MTMVTaskExecutor getTask(PriorityBlockingQueue tasks, MTMVTaskExecutor task) { - MTMVTaskExecutor oldTask = null; - for (MTMVTaskExecutor t : tasks) { - if (t.equals(task)) { - oldTask = t; - break; - } + public void dealJobRemoved(MTMVJob job) { + removePendingTask(job.getId()); + removeRunningTask(job.getId()); + if (!Config.keep_scheduler_mtmv_task_when_job_deleted) { + clearHistoryTasksByJobName(job.getName(), false); } - return oldTask; } - private void checkRunningTask() { - Iterator runningIterator = runningTaskMap.keySet().iterator(); - while (runningIterator.hasNext()) { - Long jobId = runningIterator.next(); - MTMVTaskExecutor taskExecutor = runningTaskMap.get(jobId); - if (taskExecutor == null) { - LOG.warn("failed to get running task by jobId:{}", jobId); - runningIterator.remove(); - return; - } - Future future = taskExecutor.getFuture(); - if (future.isDone()) { - runningIterator.remove(); - addHistory(taskExecutor.getTask()); - MTMVUtils.TaskState finalState = taskExecutor.getTask().getState(); - if (finalState == TaskState.FAILURE) { - failedTaskCount.incrementAndGet(); - } - Env.getCurrentEnv().getEditLog().logCreateMTMVTask(taskExecutor.getTask()); - - TriggerMode triggerMode = taskExecutor.getJob().getTriggerMode(); - if (triggerMode == TriggerMode.ONCE) { - // update the run once job status - ChangeMTMVJob changeJob = new ChangeMTMVJob(taskExecutor.getJobId(), JobState.COMPLETE); - mtmvJobManager.updateJob(changeJob, false); - } else if (triggerMode == TriggerMode.PERIODICAL) { - // just update the last modify time. - ChangeMTMVJob changeJob = new ChangeMTMVJob(taskExecutor.getJobId(), JobState.ACTIVE); - mtmvJobManager.updateJob(changeJob, false); - } - } - } + private void removePendingTask(Long jobId) { + pendingTaskMap.remove(jobId); } - public int getFailedTaskCount() { - return failedTaskCount.get(); - } - - private void scheduledPendingTask() { - int currentRunning = runningTaskMap.size(); - - Iterator pendingIterator = pendingTaskMap.keySet().iterator(); - while (pendingIterator.hasNext()) { - Long jobId = pendingIterator.next(); - MTMVTaskExecutor runningTaskExecutor = runningTaskMap.get(jobId); - if (runningTaskExecutor == null) { - Queue taskQueue = pendingTaskMap.get(jobId); - if (taskQueue.size() == 0) { - pendingIterator.remove(); - } else { - if (currentRunning >= Config.max_running_mtmv_scheduler_task_num) { - break; - } - MTMVTaskExecutor pendingTaskExecutor = taskQueue.poll(); - taskExecutorPool.executeTask(pendingTaskExecutor); - runningTaskMap.put(jobId, pendingTaskExecutor); - currentRunning++; - } - } + private void removeRunningTask(Long jobId) { + MTMVTaskExecutor task = runningTaskMap.remove(jobId); + if (task != null) { + task.stop(); } } - public boolean tryLock() { - try { - return reentrantLock.tryLock(5, TimeUnit.SECONDS); - } catch (InterruptedException e) { - LOG.warn("got exception while getting task lock", e); - Thread.currentThread().interrupt(); - } - return false; - } - - - public void unlock() { - this.reentrantLock.unlock(); + public int getFailedTaskCount() { + return failedTaskCount.get(); } public Map> getPendingTaskMap() { @@ -273,36 +224,50 @@ public Deque getHistoryTasks() { return historyTasks; } + public List getHistoryTasksByJobName(String jobName) { + return getHistoryTasks().stream().filter(u -> u.getJobName().equals(jobName)) + .collect(Collectors.toList()); + } + public List showAllTasks() { - return showTasks(null); + return showTasksWithLock(null); } - public List showTasks(String dbName) { + public List showTasksWithLock(String dbName) { List taskList = Lists.newArrayList(); - if (Strings.isNullOrEmpty(dbName)) { - for (Queue pTaskQueue : getPendingTaskMap().values()) { - taskList.addAll(pTaskQueue.stream().map(MTMVTaskExecutor::getTask).collect(Collectors.toList())); - } - taskList.addAll( - getRunningTaskMap().values().stream().map(MTMVTaskExecutor::getTask).collect(Collectors.toList())); - taskList.addAll(getHistoryTasks()); - } else { - for (Queue pTaskQueue : getPendingTaskMap().values()) { + readLock(); + try { + if (Strings.isNullOrEmpty(dbName)) { + for (Queue pTaskQueue : getPendingTaskMap().values()) { + taskList.addAll(pTaskQueue.stream().map(MTMVTaskExecutor::getTask).collect(Collectors.toList())); + } taskList.addAll( - pTaskQueue.stream().map(MTMVTaskExecutor::getTask).filter(u -> u.getDBName().equals(dbName)) + getRunningTaskMap().values().stream().map(MTMVTaskExecutor::getTask) + .collect(Collectors.toList())); + taskList.addAll(getHistoryTasks()); + } else { + for (Queue pTaskQueue : getPendingTaskMap().values()) { + taskList.addAll( + pTaskQueue.stream().map(MTMVTaskExecutor::getTask).filter(u -> u.getDBName().equals(dbName)) + .collect(Collectors.toList())); + } + taskList.addAll(getRunningTaskMap().values().stream().map(MTMVTaskExecutor::getTask) + .filter(u -> u.getDBName().equals(dbName)).collect(Collectors.toList())); + taskList.addAll( + getHistoryTasks().stream().filter(u -> u.getDBName().equals(dbName)) .collect(Collectors.toList())); - } - taskList.addAll(getRunningTaskMap().values().stream().map(MTMVTaskExecutor::getTask) - .filter(u -> u.getDBName().equals(dbName)).collect(Collectors.toList())); - taskList.addAll( - getHistoryTasks().stream().filter(u -> u.getDBName().equals(dbName)).collect(Collectors.toList())); + } + } finally { + readUnlock(); } + return taskList.stream().sorted().collect(Collectors.toList()); } public List showTasks(String dbName, String mvName) { - return showTasks(dbName).stream().filter(u -> u.getMVName().equals(mvName)).collect(Collectors.toList()); + return showTasksWithLock(dbName).stream().filter(u -> u.getMVName().equals(mvName)) + .collect(Collectors.toList()); } public MTMVTask getTask(String taskId) throws AnalysisException { @@ -321,75 +286,63 @@ public void replayCreateJobTask(MTMVTask task) { addHistory(task); } - public void clearTasksByJobName(String jobName, boolean isReplay) { + private void clearHistoryTasksByJobName(String jobName, boolean isReplay) { List clearTasks = Lists.newArrayList(); - if (!tryLock()) { - return; - } - try { - List taskHistory = showAllTasks(); - for (MTMVTask task : taskHistory) { - if (task.getJobName().equals(jobName)) { - clearTasks.add(task.getTaskId()); - } + Deque taskHistory = getHistoryTasks(); + for (MTMVTask task : taskHistory) { + if (task.getJobName().equals(jobName)) { + clearTasks.add(task.getTaskId()); } - } finally { - unlock(); } - dropTasks(clearTasks, isReplay); + + dropHistoryTasks(clearTasks, isReplay); } public void removeExpiredTasks() { long currentTime = MTMVUtils.getNowTimeStamp(); - List historyToDelete = Lists.newArrayList(); - - if (!tryLock()) { - return; - } - try { - Deque taskHistory = getHistoryTasks(); - for (MTMVTask task : taskHistory) { - long expireTime = task.getExpireTime(); - if (currentTime > expireTime) { - historyToDelete.add(task.getTaskId()); - } + Deque taskHistory = getHistoryTasks(); + for (MTMVTask task : taskHistory) { + long expireTime = task.getExpireTime(); + if (currentTime > expireTime) { + historyToDelete.add(task.getTaskId()); } - } finally { - unlock(); } - dropTasks(historyToDelete, false); + dropHistoryTasks(historyToDelete, false); } - public void dropTasks(List taskIds, boolean isReplay) { + public void dropHistoryTasks(List taskIds, boolean isReplay) { if (taskIds.isEmpty()) { return; } - if (!tryLock()) { - return; - } + writeLock(); try { Set taskSet = new HashSet<>(taskIds); - // Pending tasks will be clear directly. So we don't drop it again here. - // Check the running task since the task was killed but was not move to the history queue. - if (!isReplay) { - for (long key : runningTaskMap.keySet()) { - MTMVTaskExecutor executor = runningTaskMap.get(key); - // runningTaskMap may be removed in the runningIterator - if (executor != null && taskSet.contains(executor.getTask().getTaskId())) { - runningTaskMap.remove(key); - } - } - } // Try to remove history tasks. getHistoryTasks().removeIf(mtmvTask -> taskSet.contains(mtmvTask.getTaskId())); if (!isReplay) { Env.getCurrentEnv().getEditLog().logDropMTMVTasks(taskIds); } } finally { - unlock(); + writeUnlock(); } LOG.info("drop task history:{}", taskIds); } + + private void readLock() { + this.rwLock.readLock().lock(); + } + + private void readUnlock() { + this.rwLock.readLock().unlock(); + } + + private void writeLock() { + this.rwLock.writeLock().lock(); + } + + private void writeUnlock() { + this.rwLock.writeLock().unlock(); + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/mtmv/metadata/MTMVJob.java b/fe/fe-core/src/main/java/org/apache/doris/mtmv/metadata/MTMVJob.java index 8a348b7f9937c46..f7e2cadeb618dcc 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/mtmv/metadata/MTMVJob.java +++ b/fe/fe-core/src/main/java/org/apache/doris/mtmv/metadata/MTMVJob.java @@ -17,6 +17,7 @@ package org.apache.doris.mtmv.metadata; +import org.apache.doris.catalog.Env; import org.apache.doris.common.io.Text; import org.apache.doris.common.io.Writable; import org.apache.doris.mtmv.MTMVUtils; @@ -28,6 +29,8 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.Lists; import com.google.gson.annotations.SerializedName; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.jetbrains.annotations.NotNull; import java.io.DataInput; @@ -38,9 +41,13 @@ import java.time.ZoneId; import java.util.List; import java.util.Map; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ScheduledFuture; import java.util.concurrent.TimeUnit; public class MTMVJob implements Writable, Comparable { + private static final Logger LOG = LogManager.getLogger(MTMVJob.class); + @SerializedName("id") private long id; @@ -51,9 +58,8 @@ public class MTMVJob implements Writable, Comparable { @SerializedName("triggerMode") private MTMVUtils.TriggerMode triggerMode = MTMVUtils.TriggerMode.MANUAL; - // set default to UNKNOWN is for compatibility @SerializedName("state") - private MTMVUtils.JobState state = MTMVUtils.JobState.UNKNOWN; + private MTMVUtils.JobState state = JobState.ACTIVE; @SerializedName("schedule") private JobSchedule schedule; @@ -86,6 +92,8 @@ public class MTMVJob implements Writable, Comparable { @SerializedName("lastModifyTime") private long lastModifyTime; + private ScheduledFuture future; + public MTMVJob(String name) { this.name = name; this.createTime = MTMVUtils.getNowTimeStamp(); @@ -299,6 +307,56 @@ public List toStringRow() { return list; } + public synchronized void start() { + + if (state == JobState.COMPLETE || state == JobState.PAUSE) { + return; + } + if (getTriggerMode() == TriggerMode.PERIODICAL) { + JobSchedule schedule = getSchedule(); + ScheduledExecutorService periodScheduler = Env.getCurrentEnv().getMTMVJobManager().getPeriodScheduler(); + future = periodScheduler.scheduleAtFixedRate( + () -> Env.getCurrentEnv().getMTMVJobManager().getTaskManager().submitJobTask(this), + MTMVUtils.getDelaySeconds(this), schedule.getSecondPeriod(), TimeUnit.SECONDS); + + } else if (getTriggerMode() == TriggerMode.ONCE) { + Env.getCurrentEnv().getMTMVJobManager().getTaskManager().submitJobTask(this); + } + } + + public synchronized void stop() { + // MUST not set true for "mayInterruptIfRunning". + // Because this thread may doing bdbje write operation, it is interrupted, + // FE may exit due to bdbje write failure. + if (future != null) { + boolean isCancel = future.cancel(false); + if (!isCancel) { + LOG.warn("fail to cancel scheduler for job [{}]", name); + } + } + Env.getCurrentEnv().getMTMVJobManager().getTaskManager().dealJobRemoved(this); + } + + public void taskFinished() { + if (triggerMode == TriggerMode.ONCE) { + // update the run once job status + ChangeMTMVJob changeJob = new ChangeMTMVJob(id, JobState.COMPLETE); + updateJob(changeJob, false); + } else if (triggerMode == TriggerMode.PERIODICAL) { + // just update the last modify time. + ChangeMTMVJob changeJob = new ChangeMTMVJob(id, JobState.ACTIVE); + updateJob(changeJob, false); + } + } + + public void updateJob(ChangeMTMVJob changeJob, boolean isReplay) { + setState(changeJob.getToStatus()); + setLastModifyTime(changeJob.getLastModifyTime()); + if (!isReplay) { + Env.getCurrentEnv().getEditLog().logChangeMTMVJob(changeJob); + } + } + @Override public int compareTo(@NotNull Object o) { return (int) (getCreateTime() - ((MTMVJob) o).getCreateTime()); diff --git a/fe/fe-core/src/main/java/org/apache/doris/mysql/privilege/Auth.java b/fe/fe-core/src/main/java/org/apache/doris/mysql/privilege/Auth.java index ea39bb9f08f389b..a2ab5a7b78e1f2a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/mysql/privilege/Auth.java +++ b/fe/fe-core/src/main/java/org/apache/doris/mysql/privilege/Auth.java @@ -1035,6 +1035,15 @@ public String getWorkloadGroup(String qualifiedUser) { } } + public Pair isWorkloadGroupInUse(String groupName) { + readLock(); + try { + return propertyMgr.isWorkloadGroupInUse(groupName); + } finally { + readUnlock(); + } + } + public void getAllDomains(Set allDomains) { readLock(); try { diff --git a/fe/fe-core/src/main/java/org/apache/doris/mysql/privilege/UserProperty.java b/fe/fe-core/src/main/java/org/apache/doris/mysql/privilege/UserProperty.java index e312b2152062fae..b717d7c6fbf048a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/mysql/privilege/UserProperty.java +++ b/fe/fe-core/src/main/java/org/apache/doris/mysql/privilege/UserProperty.java @@ -305,6 +305,10 @@ public void update(List> properties) throws UserException { if (keyArr.length != 1) { throw new DdlException(PROP_WORKLOAD_GROUP + " format error"); } + boolean ret = Env.getCurrentEnv().getWorkloadGroupMgr().isWorkloadGroupExists(value); + if (!ret) { + throw new DdlException("workload group " + value + " not exists"); + } workloadGroup = value; } else { throw new DdlException("Unknown user property(" + key + ")"); diff --git a/fe/fe-core/src/main/java/org/apache/doris/mysql/privilege/UserPropertyMgr.java b/fe/fe-core/src/main/java/org/apache/doris/mysql/privilege/UserPropertyMgr.java index 4458a36a79eda68..7232417a2e8ac44 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/mysql/privilege/UserPropertyMgr.java +++ b/fe/fe-core/src/main/java/org/apache/doris/mysql/privilege/UserPropertyMgr.java @@ -37,6 +37,7 @@ import java.io.IOException; import java.util.List; import java.util.Map; +import java.util.Map.Entry; import java.util.Set; import java.util.concurrent.atomic.AtomicLong; @@ -182,6 +183,15 @@ public String getWorkloadGroup(String qualifiedUser) { return existProperty.getWorkloadGroup(); } + public Pair isWorkloadGroupInUse(String groupName) { + for (Entry entry : propertyMap.entrySet()) { + if (entry.getValue().getWorkloadGroup().equals(groupName)) { + return Pair.of(true, entry.getKey()); + } + } + return Pair.of(false, ""); + } + private UserProperty getLdapPropertyIfNull(String qualifiedUser, UserProperty existProperty) { if (existProperty == null && Env.getCurrentEnv().getAuth().getLdapManager().doesUserExist(qualifiedUser)) { return LDAP_PROPERTY; diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/CascadesContext.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/CascadesContext.java index 8a1dd686d93a424..dff58a97e5119b8 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/CascadesContext.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/CascadesContext.java @@ -41,6 +41,7 @@ import org.apache.doris.nereids.rules.RuleFactory; import org.apache.doris.nereids.rules.RuleSet; import org.apache.doris.nereids.rules.RuleType; +import org.apache.doris.nereids.rules.analysis.BindRelation.CustomTableResolver; import org.apache.doris.nereids.trees.expressions.CTEId; import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.SubqueryExpr; @@ -180,6 +181,14 @@ public Analyzer newAnalyzer() { return new Analyzer(this); } + public Analyzer newAnalyzer(Optional customTableResolver) { + return new Analyzer(this, customTableResolver); + } + + public Analyzer newCustomAnalyzer(Optional customTableResolver) { + return new Analyzer(this, customTableResolver); + } + @Override public void pushJob(Job job) { jobPool.push(job); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/NereidsPlanner.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/NereidsPlanner.java index ad3d65b8dba4fa3..bf74eceee387c55 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/NereidsPlanner.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/NereidsPlanner.java @@ -370,6 +370,7 @@ private PhysicalPlan chooseBestPlan(Group rootGroup, PhysicalProperties physical Plan plan = groupExpression.getPlan().withChildren(planChildren); if (!(plan instanceof PhysicalPlan)) { + // TODO need add some log throw new AnalysisException("Result plan must be PhysicalPlan"); } // add groupExpression to plan so that we could print group id in plan.treeString() diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/cost/CostWeight.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/cost/CostWeight.java index a0bb8c7a8778653..3c62f8857428a2f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/cost/CostWeight.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/cost/CostWeight.java @@ -34,9 +34,6 @@ * An example is tpch q15. */ public class CostWeight { - static final double CPU_WEIGHT = 1; - static final double MEMORY_WEIGHT = 1; - static final double NETWORK_WEIGHT = 1.5; static final double DELAY = 0.5; final double cpuWeight; @@ -69,7 +66,10 @@ public CostWeight(double cpuWeight, double memoryWeight, double networkWeight, d } public static CostWeight get() { - return new CostWeight(CPU_WEIGHT, MEMORY_WEIGHT, NETWORK_WEIGHT, + double cpuWeight = ConnectContext.get().getSessionVariable().getCboCpuWeight(); + double memWeight = ConnectContext.get().getSessionVariable().getCboMemWeight(); + double netWeight = ConnectContext.get().getSessionVariable().getCboNetWeight(); + return new CostWeight(cpuWeight, memWeight, netWeight, ConnectContext.get().getSessionVariable().getNereidsCboPenaltyFactor()); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/ExpressionTranslator.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/ExpressionTranslator.java index d085baf02f92a30..bdf34f43b7418bd 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/ExpressionTranslator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/ExpressionTranslator.java @@ -75,7 +75,6 @@ import org.apache.doris.nereids.trees.expressions.WhenClause; import org.apache.doris.nereids.trees.expressions.functions.AlwaysNotNullable; import org.apache.doris.nereids.trees.expressions.functions.AlwaysNullable; -import org.apache.doris.nereids.trees.expressions.functions.BoundFunction; import org.apache.doris.nereids.trees.expressions.functions.agg.AggregateFunction; import org.apache.doris.nereids.trees.expressions.functions.agg.AggregateParam; import org.apache.doris.nereids.trees.expressions.functions.agg.Count; @@ -365,14 +364,11 @@ public Expr visitWindowFunction(WindowFunction function, PlanTranslatorContext c @Override public Expr visitScalarFunction(ScalarFunction function, PlanTranslatorContext context) { - List nereidsArguments = adaptFunctionArgumentsForBackends(function); - - List arguments = nereidsArguments - .stream() + List arguments = function.getArguments().stream() .map(arg -> arg.accept(this, context)) .collect(Collectors.toList()); - List argTypes = nereidsArguments.stream() + List argTypes = function.getArguments().stream() .map(Expression::getDataType) .map(AbstractDataType::toCatalogDataType) .collect(Collectors.toList()); @@ -594,12 +590,4 @@ private static org.apache.doris.analysis.AssertNumRowsElement.Assertion translat throw new AnalysisException("UnSupported type: " + assertion); } } - - /** - * some special arguments not need exists in the nereids, and backends need it, so we must add the - * special arguments for backends, e.g. the json data type string in the json_object function. - */ - private List adaptFunctionArgumentsForBackends(BoundFunction function) { - return function.getArguments(); - } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java index 14dbb80e145e9dd..6aff68e6b449609 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java @@ -38,7 +38,6 @@ import org.apache.doris.analysis.TupleDescriptor; import org.apache.doris.analysis.TupleId; import org.apache.doris.catalog.Column; -import org.apache.doris.catalog.Env; import org.apache.doris.catalog.Function.NullableMode; import org.apache.doris.catalog.OlapTable; import org.apache.doris.catalog.Table; @@ -49,23 +48,24 @@ import org.apache.doris.catalog.external.IcebergExternalTable; import org.apache.doris.catalog.external.JdbcExternalTable; import org.apache.doris.catalog.external.PaimonExternalTable; -import org.apache.doris.common.Pair; import org.apache.doris.common.UserException; import org.apache.doris.common.util.Util; import org.apache.doris.nereids.exceptions.AnalysisException; +import org.apache.doris.nereids.properties.DistributionSpec; import org.apache.doris.nereids.properties.DistributionSpecAny; +import org.apache.doris.nereids.properties.DistributionSpecExecutionAny; import org.apache.doris.nereids.properties.DistributionSpecGather; import org.apache.doris.nereids.properties.DistributionSpecHash; -import org.apache.doris.nereids.properties.DistributionSpecHash.ShuffleType; import org.apache.doris.nereids.properties.DistributionSpecReplicated; +import org.apache.doris.nereids.properties.DistributionSpecStorageAny; +import org.apache.doris.nereids.properties.DistributionSpecStorageGather; import org.apache.doris.nereids.properties.OrderKey; -import org.apache.doris.nereids.properties.PhysicalProperties; import org.apache.doris.nereids.rules.implementation.LogicalWindowToPhysicalWindow.WindowFrameGroup; import org.apache.doris.nereids.stats.StatsErrorEstimator; +import org.apache.doris.nereids.trees.UnaryNode; import org.apache.doris.nereids.trees.expressions.AggregateExpression; import org.apache.doris.nereids.trees.expressions.Alias; import org.apache.doris.nereids.trees.expressions.CTEId; -import org.apache.doris.nereids.trees.expressions.Cast; import org.apache.doris.nereids.trees.expressions.EqualTo; import org.apache.doris.nereids.trees.expressions.ExprId; import org.apache.doris.nereids.trees.expressions.Expression; @@ -120,10 +120,8 @@ import org.apache.doris.nereids.trees.plans.physical.PhysicalWindow; import org.apache.doris.nereids.trees.plans.physical.RuntimeFilter; import org.apache.doris.nereids.trees.plans.visitor.DefaultPlanVisitor; -import org.apache.doris.nereids.types.DataType; import org.apache.doris.nereids.util.ExpressionUtils; import org.apache.doris.nereids.util.JoinUtils; -import org.apache.doris.nereids.util.TypeCoercionUtils; import org.apache.doris.nereids.util.Utils; import org.apache.doris.planner.AggregationNode; import org.apache.doris.planner.AnalyticEvalNode; @@ -161,6 +159,7 @@ import org.apache.doris.planner.external.iceberg.IcebergScanNode; import org.apache.doris.planner.external.paimon.PaimonScanNode; import org.apache.doris.qe.ConnectContext; +import org.apache.doris.system.SystemInfoService; import org.apache.doris.tablefunction.TableValuedFunctionIf; import org.apache.doris.thrift.TColumn; import org.apache.doris.thrift.TFetchOption; @@ -169,7 +168,6 @@ import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableList.Builder; import com.google.common.collect.ImmutableSet; import com.google.common.collect.Lists; import com.google.common.collect.Maps; @@ -180,12 +178,10 @@ import java.util.ArrayList; import java.util.Collection; import java.util.Collections; -import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Objects; -import java.util.Optional; import java.util.Set; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -217,54 +213,6 @@ public PhysicalPlanTranslator(PlanTranslatorContext context, StatsErrorEstimator this.statsErrorEstimator = statsErrorEstimator; } - // We use two phase read to optimize sql like: select * from tbl [where xxx = ???] [order by column1] [limit n] - // in the first phase, we add an extra column `RowId` to Block, and sort blocks in TopN nodes - // in the second phase, we have n rows, we do a fetch rpc to get all rowids data for the n rows - // and reconconstruct the final block - private void setResultSinkFetchOptionIfNeed() { - boolean needFetch = false; - // Only single olap table should be fetched - OlapTable fetchOlapTable = null; - OlapScanNode scanNode = null; - for (PlanFragment fragment : context.getPlanFragments()) { - PlanNode node = fragment.getPlanRoot(); - PlanNode parent = null; - // OlapScanNode is the last node. - // So, just get the last two node and check if they are SortNode and OlapScan. - while (node.getChildren().size() != 0) { - parent = node; - node = node.getChildren().get(0); - } - - // case1: general topn optimized query - if ((node instanceof OlapScanNode) && (parent instanceof SortNode)) { - SortNode sortNode = (SortNode) parent; - scanNode = (OlapScanNode) node; - if (sortNode.getUseTwoPhaseReadOpt()) { - needFetch = true; - fetchOlapTable = scanNode.getOlapTable(); - break; - } - } - } - for (PlanFragment fragment : context.getPlanFragments()) { - if (needFetch && fragment.getSink() instanceof ResultSink) { - TFetchOption fetchOption = new TFetchOption(); - fetchOption.setFetchRowStore(fetchOlapTable.storeRowColumn()); - fetchOption.setUseTwoPhaseFetch(true); - fetchOption.setNodesInfo(Env.getCurrentSystemInfo().createAliveNodesInfo()); - if (!fetchOlapTable.storeRowColumn()) { - // Set column desc for each column - List columnsDesc = new ArrayList(); - scanNode.getColumnDesc(columnsDesc, null, null); - fetchOption.setColumnDesc(columnsDesc); - } - ((ResultSink) fragment.getSink()).setFetchOption(fetchOption); - break; - } - } - } - /** * Translate Nereids Physical Plan tree to Stale Planner PlanFragment tree. * @@ -273,27 +221,9 @@ private void setResultSinkFetchOptionIfNeed() { */ public PlanFragment translatePlan(PhysicalPlan physicalPlan) { PlanFragment rootFragment = physicalPlan.accept(this, context); - if (physicalPlan instanceof PhysicalDistribute) { - PhysicalDistribute distribute = (PhysicalDistribute) physicalPlan; - DataPartition dataPartition; - if (distribute.getDistributionSpec().equals(PhysicalProperties.GATHER.getDistributionSpec())) { - dataPartition = DataPartition.UNPARTITIONED; - } else { - throw new AnalysisException("Unsupported PhysicalDistribute in the root plan: " + distribute); - } - ExchangeNode exchangeNode = (ExchangeNode) rootFragment.getPlanRoot(); - PlanFragment currentFragment = new PlanFragment(context.nextFragmentId(), exchangeNode, dataPartition); - rootFragment.setOutputPartition(dataPartition); - rootFragment.setPlanRoot(exchangeNode.getChild(0)); - rootFragment.setDestination(exchangeNode); - context.addPlanFragment(currentFragment); - rootFragment = currentFragment; - } - - if (!(physicalPlan instanceof PhysicalOlapTableSink) && isFragmentPartitioned(rootFragment)) { - rootFragment = exchangeToMergeFragment(rootFragment, context); - } + // TODO: why we need if? we should always set output expr? + // OlapSink? maybe OlapSink should not set output exprs by it self if (rootFragment.getOutputExprs() == null) { List outputExprs = Lists.newArrayList(); physicalPlan.getOutput().stream().map(Slot::getExprId) @@ -305,10 +235,63 @@ public PlanFragment translatePlan(PhysicalPlan physicalPlan) { } setResultSinkFetchOptionIfNeed(); Collections.reverse(context.getPlanFragments()); + // TODO: maybe we need to trans nullable directly? and then we could remove call computeMemLayout context.getDescTable().computeMemLayout(); return rootFragment; } + /* ******************************************************************************************** + * distribute node + * ******************************************************************************************** */ + + @Override + public PlanFragment visitPhysicalDistribute(PhysicalDistribute distribute, + PlanTranslatorContext context) { + PlanFragment childFragment = distribute.child().accept(this, context); + // TODO: why need set streaming here? should remove this. + if (childFragment.getPlanRoot() instanceof AggregationNode + && distribute.child() instanceof PhysicalHashAggregate + && context.getFirstAggregateInFragment(childFragment) == distribute.child()) { + PhysicalHashAggregate hashAggregate = (PhysicalHashAggregate) distribute.child(); + if (hashAggregate.getAggPhase() == AggPhase.LOCAL + && hashAggregate.getAggMode() == AggMode.INPUT_TO_BUFFER) { + AggregationNode aggregationNode = (AggregationNode) childFragment.getPlanRoot(); + aggregationNode.setUseStreamingPreagg(hashAggregate.isMaybeUsingStream()); + } + } + + ExchangeNode exchangeNode = new ExchangeNode(context.nextPlanNodeId(), childFragment.getPlanRoot()); + updateLegacyPlanIdToPhysicalPlan(exchangeNode, distribute); + exchangeNode.setNumInstances(childFragment.getPlanRoot().getNumInstances()); + if (distribute.getDistributionSpec() instanceof DistributionSpecGather) { + // gather to one instance + exchangeNode.setNumInstances(1); + } + + List validOutputIds = distribute.getOutputExprIds(); + if (distribute.child() instanceof PhysicalHashAggregate) { + // we must add group by keys to output list, + // otherwise we could not process aggregate's output without group by keys + List keys = ((PhysicalHashAggregate) distribute.child()).getGroupByExpressions().stream() + .filter(SlotReference.class::isInstance) + .map(SlotReference.class::cast) + .map(SlotReference::getExprId) + .collect(Collectors.toList()); + keys.addAll(validOutputIds); + validOutputIds = keys; + } + DataPartition dataPartition = toDataPartition(distribute.getDistributionSpec(), validOutputIds, context); + PlanFragment parentFragment = new PlanFragment(context.nextFragmentId(), exchangeNode, dataPartition); + childFragment.setDestination(exchangeNode); + childFragment.setOutputPartition(dataPartition); + context.addPlanFragment(parentFragment); + return parentFragment; + } + + /* ******************************************************************************************** + * sink Node, in lexicographical order + * ******************************************************************************************** */ + @Override public PlanFragment visitPhysicalOlapTableSink(PhysicalOlapTableSink olapTableSink, PlanTranslatorContext context) { @@ -331,206 +314,75 @@ public PlanFragment visitPhysicalOlapTableSink(PhysicalOlapTableSink aggregate, - PlanTranslatorContext context) { - - PlanFragment inputPlanFragment = aggregate.child(0).accept(this, context); - - List groupByExpressionList = aggregate.getGroupByExpressions(); - List outputExpressionList = aggregate.getOutputExpressions(); + /* ******************************************************************************************** + * scan Node, in lexicographical order + * ******************************************************************************************** */ - // 1. generate slot reference for each group expression - List groupSlotList = collectGroupBySlots(groupByExpressionList, outputExpressionList); - ArrayList execGroupingExpressions = groupByExpressionList.stream() - .map(e -> ExpressionTranslator.translate(e, context)) - .collect(Collectors.toCollection(ArrayList::new)); - // 2. collect agg expressions and generate agg function to slot reference map - List aggFunctionOutput = Lists.newArrayList(); - List aggregateExpressionList = outputExpressionList.stream() - .filter(o -> o.anyMatch(AggregateExpression.class::isInstance)) - .peek(o -> aggFunctionOutput.add(o.toSlot())) - .map(o -> o.>collect(AggregateExpression.class::isInstance)) - .flatMap(Set::stream) - .collect(Collectors.toList()); - ArrayList execAggregateFunctions = aggregateExpressionList.stream() - .map(aggregateFunction -> (FunctionCallExpr) ExpressionTranslator.translate(aggregateFunction, context)) - .collect(Collectors.toCollection(ArrayList::new)); + @Override + public PlanFragment visitPhysicalFileScan(PhysicalFileScan fileScan, PlanTranslatorContext context) { + List slots = fileScan.getOutput(); + ExternalTable table = fileScan.getTable(); + TupleDescriptor tupleDescriptor = generateTupleDesc(slots, table, context); - PlanFragment currentFragment; - if (inputPlanFragment.getPlanRoot() instanceof ExchangeNode - && aggregate.child() instanceof PhysicalDistribute) { - //the exchange node is generated in two cases: - // 1. some nodes (e.g. sort node) need to gather data from multiple instances, and hence their gather phase - // need an exchange node. For this type of exchange, their data partition is un_partitioned, do not - // create a new plan fragment. - // 2. PhysicalDistribute node is translated to exchange node. PhysicalDistribute node means we need to - // shuffle data, and we have to create a new plan fragment. - ExchangeNode exchangeNode = (ExchangeNode) inputPlanFragment.getPlanRoot(); - Optional> partitionExpressions = aggregate.getPartitionExpressions(); - PhysicalDistribute physicalDistribute = (PhysicalDistribute) aggregate.child(); - DataPartition dataPartition = toDataPartition(physicalDistribute, partitionExpressions, context).get(); - currentFragment = new PlanFragment(context.nextFragmentId(), exchangeNode, dataPartition); - inputPlanFragment.setOutputPartition(dataPartition); - inputPlanFragment.setPlanRoot(exchangeNode.getChild(0)); - inputPlanFragment.setDestination(exchangeNode); - context.addPlanFragment(currentFragment); + // TODO(cmy): determine the needCheckColumnPriv param + ScanNode scanNode; + if (table instanceof HMSExternalTable) { + switch (((HMSExternalTable) table).getDlaType()) { + case HUDI: + scanNode = new HudiScanNode(context.nextPlanNodeId(), tupleDescriptor, false); + break; + case ICEBERG: + scanNode = new IcebergScanNode(context.nextPlanNodeId(), tupleDescriptor, false); + break; + case HIVE: + scanNode = new HiveScanNode(context.nextPlanNodeId(), tupleDescriptor, false); + break; + default: + throw new RuntimeException("do not support DLA type " + ((HMSExternalTable) table).getDlaType()); + } + } else if (table instanceof IcebergExternalTable) { + scanNode = new IcebergScanNode(context.nextPlanNodeId(), tupleDescriptor, false); + } else if (table instanceof PaimonExternalTable) { + scanNode = new PaimonScanNode(context.nextPlanNodeId(), tupleDescriptor, false); } else { - currentFragment = inputPlanFragment; - } - - // 3. generate output tuple - List slotList = Lists.newArrayList(); - TupleDescriptor outputTupleDesc; - slotList.addAll(groupSlotList); - slotList.addAll(aggFunctionOutput); - outputTupleDesc = generateTupleDesc(slotList, null, context); - - List aggFunOutputIds = ImmutableList.of(); - if (!aggFunctionOutput.isEmpty()) { - aggFunOutputIds = outputTupleDesc - .getSlots() - .subList(groupSlotList.size(), outputTupleDesc.getSlots().size()) - .stream() - .map(slot -> slot.getId().asInt()) - .collect(ImmutableList.toImmutableList()); - } - boolean isPartial = aggregate.getAggregateParam().aggMode.productAggregateBuffer; - AggregateInfo aggInfo = AggregateInfo.create(execGroupingExpressions, execAggregateFunctions, - aggFunOutputIds, isPartial, outputTupleDesc, outputTupleDesc, aggregate.getAggPhase().toExec()); - AggregationNode aggregationNode = new AggregationNode(context.nextPlanNodeId(), - currentFragment.getPlanRoot(), aggInfo); - if (!aggregate.getAggMode().isFinalPhase) { - aggregationNode.unsetNeedsFinalize(); - } - PhysicalHashAggregate firstAggregateInFragment = context.getFirstAggregateInFragment(currentFragment); - - switch (aggregate.getAggPhase()) { - case LOCAL: - // we should set is useStreamingAgg when has exchange, - // so the `aggregationNode.setUseStreamingPreagg()` in the visitPhysicalDistribute - break; - case DISTINCT_LOCAL: - aggregationNode.setIntermediateTuple(); - break; - case GLOBAL: - case DISTINCT_GLOBAL: - break; - default: - throw new RuntimeException("Unsupported yet"); + throw new RuntimeException("do not support table type " + table.getType()); } - if (firstAggregateInFragment == null) { - context.setFirstAggregateInFragment(currentFragment, aggregate); - } - // in pipeline engine, we use parallel scan by default, but it broke the rule of data distribution - // so, if we do final phase or merge without exchange. - // we need turn of parallel scan to ensure to get correct result. - PlanNode leftMostNode = currentFragment.getPlanRoot(); - while (leftMostNode.getChildren().size() != 0 && !(leftMostNode instanceof ExchangeNode)) { - leftMostNode = leftMostNode.getChild(0); - } - // TODO: nereids forbid all parallel scan under aggregate temporary, because nereids could generate - // so complex aggregate plan than legacy planner, and should add forbid parallel scan hint when - // generate physical aggregate plan. - if (leftMostNode instanceof OlapScanNode - && currentFragment.getDataPartition().getType() != TPartitionType.RANDOM) { - currentFragment.setHasColocatePlanNode(true); - } - setPlanRoot(currentFragment, aggregationNode, aggregate); - if (aggregate.getStats() != null) { - aggregationNode.setCardinality((long) aggregate.getStats().getRowCount()); - } - updateLegacyPlanIdToPhysicalPlan(currentFragment.getPlanRoot(), aggregate); - return currentFragment; - } - - @Override - public PlanFragment visitPhysicalRepeat(PhysicalRepeat repeat, PlanTranslatorContext context) { - PlanFragment inputPlanFragment = repeat.child(0).accept(this, context); - - Set sortedVirtualSlots = repeat.getSortedVirtualSlots(); - TupleDescriptor virtualSlotsTuple = - generateTupleDesc(ImmutableList.copyOf(sortedVirtualSlots), null, context); - - ImmutableSet flattenGroupingSetExprs = ImmutableSet.copyOf( - ExpressionUtils.flatExpressions(repeat.getGroupingSets())); - - List aggregateFunctionUsedSlots = repeat.getOutputExpressions() - .stream() - .filter(output -> !(output instanceof VirtualSlotReference)) - .filter(output -> !flattenGroupingSetExprs.contains(output)) - .distinct() - .map(NamedExpression::toSlot) - .collect(ImmutableList.toImmutableList()); - - Set usedSlotInRepeat = ImmutableSet.builder() - .addAll(flattenGroupingSetExprs) - .addAll(aggregateFunctionUsedSlots) - .build(); - - List preRepeatExprs = usedSlotInRepeat.stream() - .map(expr -> ExpressionTranslator.translate(expr, context)) - .collect(ImmutableList.toImmutableList()); - - List outputSlots = repeat.getOutputExpressions() - .stream() - .map(NamedExpression::toSlot) - .collect(ImmutableList.toImmutableList()); - - // NOTE: we should first translate preRepeatExprs, then generate output tuple, - // or else the preRepeatExprs can not find the bottom slotRef and throw - // exception: invalid slot id - TupleDescriptor outputTuple = generateTupleDesc(outputSlots, null, context); - - // cube and rollup already convert to grouping sets in LogicalPlanBuilder.withAggregate() - GroupingInfo groupingInfo = new GroupingInfo( - GroupingType.GROUPING_SETS, virtualSlotsTuple, outputTuple, preRepeatExprs); - - List> repeatSlotIdList = repeat.computeRepeatSlotIdList(getSlotIdList(outputTuple)); - Set allSlotId = repeatSlotIdList.stream() - .flatMap(Set::stream) - .collect(ImmutableSet.toImmutableSet()); + // TODO: should not translate conjunct here. need a new attr in FileScanNode to save push down conjuncts. + fileScan.getConjuncts().stream() + .map(e -> ExpressionTranslator.translate(e, context)) + .forEach(scanNode::addConjunct); + TableName tableName = new TableName(null, "", ""); + TableRef ref = new TableRef(tableName, null, null); + BaseTableRef tableRef = new BaseTableRef(ref, table, tableName); + tupleDescriptor.setRef(tableRef); - RepeatNode repeatNode = new RepeatNode(context.nextPlanNodeId(), - inputPlanFragment.getPlanRoot(), groupingInfo, repeatSlotIdList, - allSlotId, repeat.computeVirtualSlotValues(sortedVirtualSlots)); - repeatNode.setNumInstances(inputPlanFragment.getPlanRoot().getNumInstances()); - addPlanRoot(inputPlanFragment, repeatNode, repeat); - inputPlanFragment.updateDataPartition(DataPartition.RANDOM); - updateLegacyPlanIdToPhysicalPlan(inputPlanFragment.getPlanRoot(), repeat); - return inputPlanFragment; + Utils.execWithUncheckedException(scanNode::init); + context.addScanNode(scanNode); + ScanNode finalScanNode = scanNode; + context.getRuntimeTranslator().ifPresent( + runtimeFilterGenerator -> runtimeFilterGenerator.getTargetOnScanNode(fileScan.getId()).forEach( + expr -> runtimeFilterGenerator.translateRuntimeFilterTarget(expr, finalScanNode, context) + ) + ); + Utils.execWithUncheckedException(scanNode::finalizeForNereids); + // Create PlanFragment + DataPartition dataPartition = DataPartition.RANDOM; + PlanFragment planFragment = createPlanFragment(scanNode, dataPartition, fileScan); + context.addPlanFragment(planFragment); + updateLegacyPlanIdToPhysicalPlan(planFragment.getPlanRoot(), fileScan); + return planFragment; } @Override public PlanFragment visitPhysicalEmptyRelation(PhysicalEmptyRelation emptyRelation, PlanTranslatorContext context) { List output = emptyRelation.getOutput(); TupleDescriptor tupleDescriptor = generateTupleDesc(output, null, context); - for (int i = 0; i < output.size(); i++) { - Slot slot = output.get(i); + for (Slot slot : output) { SlotRef slotRef = context.findSlotRef(slot.getExprId()); slotRef.setLabel(slot.getName()); } @@ -547,89 +399,72 @@ public PlanFragment visitPhysicalEmptyRelation(PhysicalEmptyRelation emptyRelati } @Override - public PlanFragment visitPhysicalOneRowRelation(PhysicalOneRowRelation oneRowRelation, - PlanTranslatorContext context) { - if (oneRowRelation.notBuildUnionNode()) { - return null; - } - - List slots = oneRowRelation.getLogicalProperties().getOutput(); - TupleDescriptor oneRowTuple = generateTupleDesc(slots, null, context); - - List legacyExprs = oneRowRelation.getProjects() - .stream() - .map(expr -> ExpressionTranslator.translate(expr, context)) - .collect(Collectors.toList()); - - for (int i = 0; i < legacyExprs.size(); i++) { - SlotDescriptor slotDescriptor = oneRowTuple.getSlots().get(i); - Expr expr = legacyExprs.get(i); - slotDescriptor.setSourceExpr(expr); - slotDescriptor.setIsNullable(expr.isNullable()); - } - - UnionNode unionNode = new UnionNode(context.nextPlanNodeId(), oneRowTuple.getId()); - unionNode.setCardinality(1L); - unionNode.addConstExprList(legacyExprs); - unionNode.finalizeForNereids(oneRowTuple.getSlots(), new ArrayList<>()); - - PlanFragment planFragment = createPlanFragment(unionNode, DataPartition.UNPARTITIONED, oneRowRelation); + public PlanFragment visitPhysicalEsScan(PhysicalEsScan esScan, PlanTranslatorContext context) { + List slots = esScan.getOutput(); + ExternalTable table = esScan.getTable(); + TupleDescriptor tupleDescriptor = generateTupleDesc(slots, table, context); + EsScanNode esScanNode = new EsScanNode(context.nextPlanNodeId(), tupleDescriptor, true); + Utils.execWithUncheckedException(esScanNode::init); + context.addScanNode(esScanNode); + context.getRuntimeTranslator().ifPresent( + runtimeFilterGenerator -> runtimeFilterGenerator.getTargetOnScanNode(esScan.getId()).forEach( + expr -> runtimeFilterGenerator.translateRuntimeFilterTarget(expr, esScanNode, context) + ) + ); + Utils.execWithUncheckedException(esScanNode::finalizeForNereids); + DataPartition dataPartition = DataPartition.RANDOM; + PlanFragment planFragment = new PlanFragment(context.nextFragmentId(), esScanNode, dataPartition); context.addPlanFragment(planFragment); - updateLegacyPlanIdToPhysicalPlan(planFragment.getPlanRoot(), oneRowRelation); + updateLegacyPlanIdToPhysicalPlan(planFragment.getPlanRoot(), esScan); return planFragment; } @Override - public PlanFragment visitPhysicalStorageLayerAggregate( - PhysicalStorageLayerAggregate storageLayerAggregate, PlanTranslatorContext context) { - Preconditions.checkState(storageLayerAggregate.getRelation() instanceof PhysicalOlapScan, - "PhysicalStorageLayerAggregate only support PhysicalOlapScan: " - + storageLayerAggregate.getRelation().getClass().getName()); - PlanFragment planFragment = storageLayerAggregate.getRelation().accept(this, context); - - OlapScanNode olapScanNode = (OlapScanNode) planFragment.getPlanRoot(); - TPushAggOp pushAggOp; - switch (storageLayerAggregate.getAggOp()) { - case COUNT: - pushAggOp = TPushAggOp.COUNT; - break; - case MIN_MAX: - pushAggOp = TPushAggOp.MINMAX; - break; - case MIX: - pushAggOp = TPushAggOp.MIX; - break; - default: - throw new AnalysisException("Unsupported storage layer aggregate: " - + storageLayerAggregate.getAggOp()); - } - olapScanNode.setPushDownAggNoGrouping(pushAggOp); - updateLegacyPlanIdToPhysicalPlan(planFragment.getPlanRoot(), storageLayerAggregate); + public PlanFragment visitPhysicalJdbcScan(PhysicalJdbcScan jdbcScan, PlanTranslatorContext context) { + List slots = jdbcScan.getOutput(); + TableIf table = jdbcScan.getTable(); + TupleDescriptor tupleDescriptor = generateTupleDesc(slots, table, context); + JdbcScanNode jdbcScanNode = new JdbcScanNode(context.nextPlanNodeId(), tupleDescriptor, + table instanceof JdbcExternalTable); + Utils.execWithUncheckedException(jdbcScanNode::init); + context.addScanNode(jdbcScanNode); + context.getRuntimeTranslator().ifPresent( + runtimeFilterGenerator -> runtimeFilterGenerator.getTargetOnScanNode(jdbcScan.getId()).forEach( + expr -> runtimeFilterGenerator.translateRuntimeFilterTarget(expr, jdbcScanNode, context) + ) + ); + Utils.execWithUncheckedException(jdbcScanNode::finalizeForNereids); + DataPartition dataPartition = DataPartition.RANDOM; + PlanFragment planFragment = new PlanFragment(context.nextFragmentId(), jdbcScanNode, dataPartition); + context.addPlanFragment(planFragment); + updateLegacyPlanIdToPhysicalPlan(planFragment.getPlanRoot(), jdbcScan); return planFragment; } @Override public PlanFragment visitPhysicalOlapScan(PhysicalOlapScan olapScan, PlanTranslatorContext context) { - // Create OlapScanNode - List slotList = olapScan.getOutput(); - Set deferredMaterializedExprIds = Collections.emptySet(); - if (olapScan.getMutableState(PhysicalOlapScan.DEFERRED_MATERIALIZED_SLOTS).isPresent()) { - deferredMaterializedExprIds = (Set) (olapScan - .getMutableState(PhysicalOlapScan.DEFERRED_MATERIALIZED_SLOTS).get()); - } - OlapTable olapTable = olapScan.getTable(); - TupleDescriptor tupleDescriptor = generateTupleDesc(slotList, olapTable, deferredMaterializedExprIds, context); + // deferred materialized slots used for topn opt. + Set deferredMaterializedExprIds = olapScan + .getMutableState(PhysicalOlapScan.DEFERRED_MATERIALIZED_SLOTS) + .map(s -> (Set) s) + .orElse(Collections.emptySet()); + List slots = olapScan.getOutput(); + OlapTable olapTable = olapScan.getTable(); + // generate real output tuple + TupleDescriptor tupleDescriptor = generateTupleDesc(slots, olapTable, deferredMaterializedExprIds, context); + // generate base index tuple because this fragment partitioned expr relay on slots of based index if (olapScan.getSelectedIndexId() != olapScan.getTable().getBaseIndexId()) { generateTupleDesc(olapScan.getBaseOutputs(), olapTable, deferredMaterializedExprIds, context); } + // TODO: remove this, we should add this column in Nereids if (olapScan.getMutableState(PhysicalOlapScan.DEFERRED_MATERIALIZED_SLOTS).isPresent()) { injectRowIdColumnSlot(tupleDescriptor); } - tupleDescriptor.setTable(olapTable); OlapScanNode olapScanNode = new OlapScanNode(context.nextPlanNodeId(), tupleDescriptor, "OlapScanNode"); + // TODO: move all node set cardinality into one place if (olapScan.getStats() != null) { olapScanNode.setCardinality((long) olapScan.getStats().getRowCount()); } @@ -641,6 +476,7 @@ public PlanFragment visitPhysicalOlapScan(PhysicalOlapScan olapScan, PlanTransla olapScanNode.setSelectedPartitionIds(olapScan.getSelectedPartitionIds()); olapScanNode.setSampleTabletIds(olapScan.getSelectedTabletIds()); + // TODO: remove this switch? switch (olapScan.getTable().getKeysType()) { case AGG_KEYS: case UNIQUE_KEYS: @@ -652,16 +488,21 @@ public PlanFragment visitPhysicalOlapScan(PhysicalOlapScan olapScan, PlanTransla throw new RuntimeException("Not supported key type: " + olapScan.getTable().getKeysType()); } + // create scan range Utils.execWithUncheckedException(olapScanNode::init); + // TODO: process collect scan node in one place context.addScanNode(olapScanNode); - // translate runtime filter + // TODO: process translate runtime filter in one place + // use real plan node to present rf apply and rf generator context.getRuntimeTranslator().ifPresent( runtimeFilterTranslator -> runtimeFilterTranslator.getTargetOnScanNode(olapScan.getId()).forEach( expr -> runtimeFilterTranslator.translateRuntimeFilterTarget(expr, olapScanNode, context) ) ); + // TODO: we need to remove all finalizeForNereids olapScanNode.finalizeForNereids(); // Create PlanFragment + // TODO: use a util function to convert distribution to DataPartition DataPartition dataPartition = DataPartition.RANDOM; if (olapScan.getDistributionSpec() instanceof DistributionSpecHash) { DistributionSpecHash distributionSpecHash = (DistributionSpecHash) olapScan.getDistributionSpec(); @@ -669,6 +510,7 @@ public PlanFragment visitPhysicalOlapScan(PhysicalOlapScan olapScan, PlanTransla .map(context::findSlotRef).collect(Collectors.toList()); dataPartition = new DataPartition(TPartitionType.HASH_PARTITIONED, partitionExprs); } + // TODO: maybe we could have a better way to create fragment PlanFragment planFragment = createPlanFragment(olapScanNode, dataPartition, olapScan); context.addPlanFragment(planFragment); updateLegacyPlanIdToPhysicalPlan(planFragment.getPlanRoot(), olapScan); @@ -676,78 +518,50 @@ public PlanFragment visitPhysicalOlapScan(PhysicalOlapScan olapScan, PlanTransla } @Override - public PlanFragment visitPhysicalSchemaScan(PhysicalSchemaScan schemaScan, PlanTranslatorContext context) { - Table table = schemaScan.getTable(); + public PlanFragment visitPhysicalOneRowRelation(PhysicalOneRowRelation oneRowRelation, + PlanTranslatorContext context) { + List slots = oneRowRelation.getLogicalProperties().getOutput(); + TupleDescriptor oneRowTuple = generateTupleDesc(slots, null, context); - List slotList = new ImmutableList.Builder() - .addAll(schemaScan.getOutput()) - .build(); - TupleDescriptor tupleDescriptor = generateTupleDesc(slotList, table, context); - tupleDescriptor.setTable(table); - SchemaScanNode scanNode = new SchemaScanNode(context.nextPlanNodeId(), tupleDescriptor); - context.getRuntimeTranslator().ifPresent( - runtimeFilterGenerator -> runtimeFilterGenerator.getTargetOnScanNode(schemaScan.getId()).forEach( - expr -> runtimeFilterGenerator.translateRuntimeFilterTarget(expr, scanNode, context) - ) - ); - scanNode.finalizeForNereids(); - context.getScanNodes().add(scanNode); - PlanFragment planFragment = createPlanFragment(scanNode, DataPartition.RANDOM, schemaScan); + List legacyExprs = oneRowRelation.getProjects() + .stream() + .map(expr -> ExpressionTranslator.translate(expr, context)) + .collect(Collectors.toList()); + + for (int i = 0; i < legacyExprs.size(); i++) { + SlotDescriptor slotDescriptor = oneRowTuple.getSlots().get(i); + Expr expr = legacyExprs.get(i); + slotDescriptor.setSourceExpr(expr); + slotDescriptor.setIsNullable(expr.isNullable()); + } + + UnionNode unionNode = new UnionNode(context.nextPlanNodeId(), oneRowTuple.getId()); + unionNode.setCardinality(1L); + unionNode.addConstExprList(legacyExprs); + unionNode.finalizeForNereids(oneRowTuple.getSlots(), new ArrayList<>()); + + PlanFragment planFragment = createPlanFragment(unionNode, DataPartition.UNPARTITIONED, oneRowRelation); context.addPlanFragment(planFragment); + updateLegacyPlanIdToPhysicalPlan(planFragment.getPlanRoot(), oneRowRelation); return planFragment; } @Override - public PlanFragment visitPhysicalFileScan(PhysicalFileScan fileScan, PlanTranslatorContext context) { - List slotList = fileScan.getOutput(); - ExternalTable table = fileScan.getTable(); - TupleDescriptor tupleDescriptor = generateTupleDesc(slotList, table, context); - tupleDescriptor.setTable(table); - - // TODO(cmy): determine the needCheckColumnPriv param - ScanNode scanNode = null; - if (table instanceof HMSExternalTable) { - switch (((HMSExternalTable) table).getDlaType()) { - case HUDI: - scanNode = new HudiScanNode(context.nextPlanNodeId(), tupleDescriptor, false); - break; - case ICEBERG: - scanNode = new IcebergScanNode(context.nextPlanNodeId(), tupleDescriptor, false); - break; - case HIVE: - scanNode = new HiveScanNode(context.nextPlanNodeId(), tupleDescriptor, false); - break; - default: - break; - } - } else if (table instanceof IcebergExternalTable) { - scanNode = new IcebergScanNode(context.nextPlanNodeId(), tupleDescriptor, false); - } else if (table instanceof PaimonExternalTable) { - scanNode = new PaimonScanNode(context.nextPlanNodeId(), tupleDescriptor, false); - } - Preconditions.checkNotNull(scanNode); - fileScan.getConjuncts().stream() - .map(e -> ExpressionTranslator.translate(e, context)) - .forEach(scanNode::addConjunct); - TableName tableName = new TableName(null, "", ""); - TableRef ref = new TableRef(tableName, null, null); - BaseTableRef tableRef = new BaseTableRef(ref, table, tableName); - tupleDescriptor.setRef(tableRef); - - Utils.execWithUncheckedException(scanNode::init); - context.addScanNode(scanNode); - ScanNode finalScanNode = scanNode; + public PlanFragment visitPhysicalSchemaScan(PhysicalSchemaScan schemaScan, PlanTranslatorContext context) { + Table table = schemaScan.getTable(); + List slots = ImmutableList.copyOf(schemaScan.getOutput()); + TupleDescriptor tupleDescriptor = generateTupleDesc(slots, table, context); + SchemaScanNode scanNode = new SchemaScanNode(context.nextPlanNodeId(), tupleDescriptor); context.getRuntimeTranslator().ifPresent( - runtimeFilterGenerator -> runtimeFilterGenerator.getTargetOnScanNode(fileScan.getId()).forEach( - expr -> runtimeFilterGenerator.translateRuntimeFilterTarget(expr, finalScanNode, context) - ) + runtimeFilterGenerator -> runtimeFilterGenerator.getTargetOnScanNode(schemaScan.getId()).forEach( + expr -> runtimeFilterGenerator.translateRuntimeFilterTarget(expr, scanNode, context) + ) ); - Utils.execWithUncheckedException(scanNode::finalizeForNereids); - // Create PlanFragment - DataPartition dataPartition = DataPartition.RANDOM; - PlanFragment planFragment = createPlanFragment(scanNode, dataPartition, fileScan); + scanNode.finalizeForNereids(); + context.addScanNode(scanNode); + PlanFragment planFragment = createPlanFragment(scanNode, DataPartition.RANDOM, schemaScan); context.addPlanFragment(planFragment); - updateLegacyPlanIdToPhysicalPlan(planFragment.getPlanRoot(), fileScan); + updateLegacyPlanIdToPhysicalPlan(planFragment.getPlanRoot(), schemaScan); return planFragment; } @@ -761,12 +575,13 @@ public PlanFragment visitPhysicalTVFRelation(PhysicalTVFRelation tvfRelation, Pl Utils.execWithUncheckedException(scanNode::init); context.getRuntimeTranslator().ifPresent( runtimeFilterGenerator -> runtimeFilterGenerator.getTargetOnScanNode(tvfRelation.getId()).forEach( - expr -> runtimeFilterGenerator.translateRuntimeFilterTarget(expr, scanNode, context) - ) + expr -> runtimeFilterGenerator.translateRuntimeFilterTarget(expr, scanNode, context) + ) ); Utils.execWithUncheckedException(scanNode::finalizeForNereids); context.addScanNode(scanNode); + // TODO: it is weird update label in this way // set label for explain for (Slot slot : slots) { String tableColumnName = "_table_valued_function_" + tvfRelation.getFunction().getName() @@ -776,433 +591,315 @@ public PlanFragment visitPhysicalTVFRelation(PhysicalTVFRelation tvfRelation, Pl PlanFragment planFragment = createPlanFragment(scanNode, DataPartition.RANDOM, tvfRelation); context.addPlanFragment(planFragment); + updateLegacyPlanIdToPhysicalPlan(planFragment.getPlanRoot(), tvfRelation); return planFragment; } - @Override - public PlanFragment visitPhysicalJdbcScan(PhysicalJdbcScan jdbcScan, PlanTranslatorContext context) { - List slotList = jdbcScan.getOutput(); - TableIf table = jdbcScan.getTable(); - TupleDescriptor tupleDescriptor = generateTupleDesc(slotList, table, context); - tupleDescriptor.setTable(table); - JdbcScanNode jdbcScanNode = new JdbcScanNode(context.nextPlanNodeId(), tupleDescriptor, - table instanceof JdbcExternalTable); - Utils.execWithUncheckedException(jdbcScanNode::init); - context.addScanNode(jdbcScanNode); - context.getRuntimeTranslator().ifPresent( - runtimeFilterGenerator -> runtimeFilterGenerator.getTargetOnScanNode(jdbcScan.getId()).forEach( - expr -> runtimeFilterGenerator.translateRuntimeFilterTarget(expr, jdbcScanNode, context) - ) - ); - Utils.execWithUncheckedException(jdbcScanNode::finalizeForNereids); - DataPartition dataPartition = DataPartition.RANDOM; - PlanFragment planFragment = new PlanFragment(context.nextFragmentId(), jdbcScanNode, dataPartition); - context.addPlanFragment(planFragment); - updateLegacyPlanIdToPhysicalPlan(planFragment.getPlanRoot(), jdbcScan); - return planFragment; - } - @Override - public PlanFragment visitPhysicalEsScan(PhysicalEsScan esScan, PlanTranslatorContext context) { - List slotList = esScan.getOutput(); - ExternalTable table = esScan.getTable(); - TupleDescriptor tupleDescriptor = generateTupleDesc(slotList, table, context); - tupleDescriptor.setTable(table); - EsScanNode esScanNode = new EsScanNode(context.nextPlanNodeId(), tupleDescriptor, "EsScanNode", true); - Utils.execWithUncheckedException(esScanNode::init); - context.addScanNode(esScanNode); - context.getRuntimeTranslator().ifPresent( - runtimeFilterGenerator -> runtimeFilterGenerator.getTargetOnScanNode(esScan.getId()).forEach( - expr -> runtimeFilterGenerator.translateRuntimeFilterTarget(expr, esScanNode, context) - ) - ); - Utils.execWithUncheckedException(esScanNode::finalizeForNereids); - DataPartition dataPartition = DataPartition.RANDOM; - PlanFragment planFragment = new PlanFragment(context.nextFragmentId(), esScanNode, dataPartition); - context.addPlanFragment(planFragment); - updateLegacyPlanIdToPhysicalPlan(planFragment.getPlanRoot(), esScan); - return planFragment; - } + /* ******************************************************************************************** + * other Node, in lexicographical order, ignore algorithm name. for example, HashAggregate -> Aggregate + * ******************************************************************************************** */ - /*- - * Physical sort: - * 1. Build sortInfo - * There are two types of slotRef: - * one is generated by the previous node, collectively called old. - * the other is newly generated by the sort node, collectively called new. - * Filling of sortInfo related data structures, - * a. ordering use newSlotRef. - * b. sortTupleSlotExprs use oldSlotRef. - * 2. Create sortNode - * 3. Create mergeFragment - * TODO: When the slotRef of sort is currently generated, - * it will be based on the expression in select and orderBy expression in to ensure the uniqueness of slotRef. - * But eg: - * select a+1 from table order by a+1; - * the expressions of the two are inconsistent. - * The former will perform an additional Alias. - * Currently we cannot test whether this will have any effect. - * After a+1 can be parsed , reprocessing. + /** + * Translate Agg. */ @Override - public PlanFragment visitPhysicalQuickSort(PhysicalQuickSort sort, + public PlanFragment visitPhysicalHashAggregate( + PhysicalHashAggregate aggregate, PlanTranslatorContext context) { - PlanFragment inputFragment = sort.child(0).accept(this, context); - PlanFragment currentFragment = inputFragment; - - //1. generate new fragment for sort when the child is exchangeNode - if (inputFragment.getPlanRoot() instanceof ExchangeNode - && sort.child(0) instanceof PhysicalDistribute) { - DataPartition outputPartition = DataPartition.UNPARTITIONED; - if (sort.getSortPhase().isLocal()) { - // The window function like over (partition by xx order by) can generate plan - // shuffle -> localSort -> windowFunction - // In this case, the child's partition need to keep - outputPartition = hashSpecToDataPartition((PhysicalDistribute) sort.child(0), context); - } - ExchangeNode exchangeNode = (ExchangeNode) inputFragment.getPlanRoot(); - inputFragment.setOutputPartition(outputPartition); - inputFragment.setPlanRoot(exchangeNode.getChild(0)); - currentFragment = new PlanFragment(context.nextFragmentId(), exchangeNode, outputPartition); - inputFragment.setDestination(exchangeNode); - context.addPlanFragment(currentFragment); - } - - // 2. According to the type of sort, generate physical plan - if (!sort.getSortPhase().isMerge()) { - // For localSort or Gather->Sort, we just need to add sortNode - SortNode sortNode = translateSortNode(sort, inputFragment.getPlanRoot(), context); - addPlanRoot(currentFragment, sortNode, sort); - } else { - // For mergeSort, we need to push sortInfo to exchangeNode - if (!(currentFragment.getPlanRoot() instanceof ExchangeNode)) { - // if there is no exchange node for mergeSort - // e.g., localSort -> mergeSort - // It means the local has satisfied the Gather property. We can just ignore mergeSort - return currentFragment; - } - Preconditions.checkArgument(inputFragment.getPlanRoot() instanceof SortNode); - SortNode sortNode = (SortNode) inputFragment.getPlanRoot(); - ((ExchangeNode) currentFragment.getPlanRoot()).setMergeInfo(sortNode.getSortInfo()); - } - return currentFragment; - } - @Override - public PlanFragment visitPhysicalWindow(PhysicalWindow physicalWindow, - PlanTranslatorContext context) { - PlanFragment inputPlanFragment = physicalWindow.child(0).accept(this, context); + PlanFragment inputPlanFragment = aggregate.child(0).accept(this, context); - // 1. translate to old optimizer variable - // variable in Nereids - WindowFrameGroup windowFrameGroup = physicalWindow.getWindowFrameGroup(); - List partitionKeyList = Lists.newArrayList(windowFrameGroup.getPartitionKeys()); - List orderKeyList = windowFrameGroup.getOrderKeys(); - List windowFunctionList = windowFrameGroup.getGroups(); - WindowFrame windowFrame = windowFrameGroup.getWindowFrame(); + List groupByExpressions = aggregate.getGroupByExpressions(); + List outputExpressions = aggregate.getOutputExpressions(); - // partition by clause - List partitionExprs = partitionKeyList.stream() + // 1. generate slot reference for each group expression + List groupSlots = collectGroupBySlots(groupByExpressions, outputExpressions); + ArrayList execGroupingExpressions = groupByExpressions.stream() .map(e -> ExpressionTranslator.translate(e, context)) + .collect(Collectors.toCollection(ArrayList::new)); + // 2. collect agg expressions and generate agg function to slot reference map + List aggFunctionOutput = Lists.newArrayList(); + List aggregateExpressionList = outputExpressions.stream() + .filter(o -> o.anyMatch(AggregateExpression.class::isInstance)) + .peek(o -> aggFunctionOutput.add(o.toSlot())) + .map(o -> o.>collect(AggregateExpression.class::isInstance)) + .flatMap(Set::stream) .collect(Collectors.toList()); + ArrayList execAggregateFunctions = aggregateExpressionList.stream() + .map(aggregateFunction -> (FunctionCallExpr) ExpressionTranslator.translate(aggregateFunction, context)) + .collect(Collectors.toCollection(ArrayList::new)); - // order by clause - List orderByElements = orderKeyList.stream() - .map(orderKey -> new OrderByElement( - ExpressionTranslator.translate(orderKey.child(), context), - orderKey.isAsc(), orderKey.isNullFirst())) - .collect(Collectors.toList()); + // 3. generate output tuple + List slotList = Lists.newArrayList(); + TupleDescriptor outputTupleDesc; + slotList.addAll(groupSlots); + slotList.addAll(aggFunctionOutput); + outputTupleDesc = generateTupleDesc(slotList, null, context); - // function calls - List analyticFnCalls = windowFunctionList.stream() - .map(e -> { - Expression function = e.child(0).child(0); - if (function instanceof AggregateFunction) { - AggregateParam param = AggregateParam.localResult(); - function = new AggregateExpression((AggregateFunction) function, param); - } - return ExpressionTranslator.translate(function, context); - }) - .map(FunctionCallExpr.class::cast) - .map(fnCall -> { - fnCall.setIsAnalyticFnCall(true); - ((org.apache.doris.catalog.AggregateFunction) fnCall.getFn()).setIsAnalyticFn(true); - return fnCall; - }) - .collect(Collectors.toList()); + List aggFunOutputIds = ImmutableList.of(); + if (!aggFunctionOutput.isEmpty()) { + aggFunOutputIds = outputTupleDesc + .getSlots() + .subList(groupSlots.size(), outputTupleDesc.getSlots().size()) + .stream() + .map(slot -> slot.getId().asInt()) + .collect(ImmutableList.toImmutableList()); + } + boolean isPartial = aggregate.getAggregateParam().aggMode.productAggregateBuffer; + AggregateInfo aggInfo = AggregateInfo.create(execGroupingExpressions, execAggregateFunctions, + aggFunOutputIds, isPartial, outputTupleDesc, outputTupleDesc, aggregate.getAggPhase().toExec()); + AggregationNode aggregationNode = new AggregationNode(context.nextPlanNodeId(), + inputPlanFragment.getPlanRoot(), aggInfo); + if (!aggregate.getAggMode().isFinalPhase) { + aggregationNode.unsetNeedsFinalize(); + } - // analytic window - AnalyticWindow analyticWindow = physicalWindow.translateWindowFrame(windowFrame, context); + switch (aggregate.getAggPhase()) { + case LOCAL: + // we should set is useStreamingAgg when has exchange, + // so the `aggregationNode.setUseStreamingPreagg()` in the visitPhysicalDistribute + break; + case DISTINCT_LOCAL: + aggregationNode.setIntermediateTuple(); + break; + case GLOBAL: + case DISTINCT_GLOBAL: + break; + default: + throw new RuntimeException("Unsupported agg phase: " + aggregate.getAggPhase()); + } + // TODO: use to set useStreamingAgg, we should remove it by set it in Nereids + PhysicalHashAggregate firstAggregateInFragment = context.getFirstAggregateInFragment(inputPlanFragment); + if (firstAggregateInFragment == null) { + context.setFirstAggregateInFragment(inputPlanFragment, aggregate); + } - // 2. get bufferedTupleDesc from SortNode and compute isNullableMatched - Map bufferedSlotRefForWindow = getBufferedSlotRefForWindow(windowFrameGroup, context); - TupleDescriptor bufferedTupleDesc = context.getBufferedTupleForWindow(); + // in pipeline engine, we use parallel scan by default, but it broke the rule of data distribution + // so, if we do final phase or merge without exchange. + // we need turn of parallel scan to ensure to get correct result. + PlanNode leftMostNode = inputPlanFragment.getPlanRoot(); + while (leftMostNode.getChildren().size() != 0 && !(leftMostNode instanceof ExchangeNode)) { + leftMostNode = leftMostNode.getChild(0); + } + // TODO: nereids forbid all parallel scan under aggregate temporary, because nereids could generate + // so complex aggregate plan than legacy planner, and should add forbid parallel scan hint when + // generate physical aggregate plan. + if (leftMostNode instanceof OlapScanNode + && inputPlanFragment.getDataPartition().getType() != TPartitionType.RANDOM) { + inputPlanFragment.setHasColocatePlanNode(true); + } + setPlanRoot(inputPlanFragment, aggregationNode, aggregate); + if (aggregate.getStats() != null) { + aggregationNode.setCardinality((long) aggregate.getStats().getRowCount()); + } + updateLegacyPlanIdToPhysicalPlan(inputPlanFragment.getPlanRoot(), aggregate); + return inputPlanFragment; + } - // generate predicates to check if the exprs of partitionKeys and orderKeys have matched isNullable between - // sortNode and analyticNode - Expr partitionExprsIsNullableMatched = partitionExprs.isEmpty() ? null : windowExprsHaveMatchedNullable( - partitionKeyList, partitionExprs, bufferedSlotRefForWindow); + @Override + public PlanFragment visitPhysicalStorageLayerAggregate( + PhysicalStorageLayerAggregate storageLayerAggregate, PlanTranslatorContext context) { + Preconditions.checkState(storageLayerAggregate.getRelation() instanceof PhysicalOlapScan, + "PhysicalStorageLayerAggregate only support PhysicalOlapScan: " + + storageLayerAggregate.getRelation().getClass().getName()); + PlanFragment planFragment = storageLayerAggregate.getRelation().accept(this, context); - Expr orderElementsIsNullableMatched = orderByElements.isEmpty() ? null : windowExprsHaveMatchedNullable( - orderKeyList.stream().map(order -> order.child()).collect(Collectors.toList()), - orderByElements.stream().map(order -> order.getExpr()).collect(Collectors.toList()), - bufferedSlotRefForWindow); + OlapScanNode olapScanNode = (OlapScanNode) planFragment.getPlanRoot(); + TPushAggOp pushAggOp; + switch (storageLayerAggregate.getAggOp()) { + case COUNT: + pushAggOp = TPushAggOp.COUNT; + break; + case MIN_MAX: + pushAggOp = TPushAggOp.MINMAX; + break; + case MIX: + pushAggOp = TPushAggOp.MIX; + break; + default: + throw new AnalysisException("Unsupported storage layer aggregate: " + + storageLayerAggregate.getAggOp()); + } + olapScanNode.setPushDownAggNoGrouping(pushAggOp); + updateLegacyPlanIdToPhysicalPlan(planFragment.getPlanRoot(), storageLayerAggregate); + return planFragment; + } - // 3. generate tupleDesc - List windowSlotList = windowFunctionList.stream() - .map(NamedExpression::toSlot) - .collect(Collectors.toList()); - TupleDescriptor outputTupleDesc = generateTupleDesc(windowSlotList, null, context); - TupleDescriptor intermediateTupleDesc = outputTupleDesc; + @Override + public PlanFragment visitPhysicalAssertNumRows(PhysicalAssertNumRows assertNumRows, + PlanTranslatorContext context) { + PlanFragment currentFragment = assertNumRows.child().accept(this, context); + // create assertNode + AssertNumRowsNode assertNumRowsNode = new AssertNumRowsNode(context.nextPlanNodeId(), + currentFragment.getPlanRoot(), + ExpressionTranslator.translateAssert(assertNumRows.getAssertNumRowsElement())); + addPlanRoot(currentFragment, assertNumRowsNode, assertNumRows); + return currentFragment; + } - // 4. generate AnalyticEvalNode - AnalyticEvalNode analyticEvalNode = new AnalyticEvalNode( - context.nextPlanNodeId(), - inputPlanFragment.getPlanRoot(), - analyticFnCalls, - partitionExprs, - orderByElements, - analyticWindow, - intermediateTupleDesc, - outputTupleDesc, - partitionExprsIsNullableMatched, - orderElementsIsNullableMatched, - bufferedTupleDesc - ); + /** + * NOTICE: Must translate left, which it's the producer of consumer. + */ + @Override + public PlanFragment visitPhysicalCTEAnchor(PhysicalCTEAnchor cteAnchor, + PlanTranslatorContext context) { + cteAnchor.child(0).accept(this, context); + return cteAnchor.child(1).accept(this, context); + } + + @Override + public PlanFragment visitPhysicalCTEConsumer(PhysicalCTEConsumer cteConsumer, + PlanTranslatorContext context) { + CTEId cteId = cteConsumer.getCteId(); + + MultiCastPlanFragment multiCastFragment = (MultiCastPlanFragment) context.getCteProduceFragments().get(cteId); + Preconditions.checkState(multiCastFragment.getSink() instanceof MultiCastDataSink, + "invalid multiCastFragment"); + + MultiCastDataSink multiCastDataSink = (MultiCastDataSink) multiCastFragment.getSink(); + Preconditions.checkState(multiCastDataSink != null, "invalid multiCastDataSink"); + + PhysicalCTEProducer cteProducer = context.getCteProduceMap().get(cteId); + Preconditions.checkState(cteProducer != null, "invalid cteProducer"); - if (partitionExprs.isEmpty() && orderByElements.isEmpty()) { - if (inputPlanFragment.isPartitioned()) { - PlanFragment parentFragment = new PlanFragment(context.nextFragmentId(), analyticEvalNode, - DataPartition.UNPARTITIONED); - context.addPlanFragment(parentFragment); - connectChildFragment(analyticEvalNode, 0, parentFragment, inputPlanFragment, context); - inputPlanFragment = parentFragment; + ExchangeNode exchangeNode = new ExchangeNode(context.nextPlanNodeId(), multiCastFragment.getPlanRoot()); + + DataStreamSink streamSink = new DataStreamSink(exchangeNode.getId()); + streamSink.setPartition(DataPartition.RANDOM); + streamSink.setFragment(multiCastFragment); + + multiCastDataSink.getDataStreamSinks().add(streamSink); + multiCastDataSink.getDestinations().add(Lists.newArrayList()); + + exchangeNode.setNumInstances(multiCastFragment.getPlanRoot().getNumInstances()); + + PlanFragment consumeFragment = new PlanFragment(context.nextFragmentId(), exchangeNode, + multiCastFragment.getDataPartition()); + + Map projectMap = Maps.newHashMap(); + projectMap.putAll(cteConsumer.getProducerToConsumerSlotMap()); + + List execList = new ArrayList<>(); + PlanNode inputPlanNode = consumeFragment.getPlanRoot(); + List cteProjects = cteProducer.getProjects(); + for (Slot slot : cteProjects) { + if (projectMap.containsKey(slot)) { + execList.add(projectMap.get(slot)); } else { - inputPlanFragment.addPlanRoot(analyticEvalNode); + throw new RuntimeException("could not find slot in cte producer consumer projectMap"); } - } else { - analyticEvalNode.setNumInstances(inputPlanFragment.getPlanRoot().getNumInstances()); - inputPlanFragment.addPlanRoot(analyticEvalNode); } - return inputPlanFragment; - } - @Override - public PlanFragment visitPhysicalPartitionTopN(PhysicalPartitionTopN partitionTopN, - PlanTranslatorContext context) { - PlanFragment inputFragment = partitionTopN.child(0).accept(this, context); + List slotList = execList + .stream() + .map(NamedExpression::toSlot) + .collect(Collectors.toList()); - Preconditions.checkArgument(!(partitionTopN.child(0) instanceof ExchangeNode)); - PartitionSortNode partitionSortNode = translatePartitionSortNode(partitionTopN, - inputFragment.getPlanRoot(), context); - addPlanRoot(inputFragment, partitionSortNode, partitionTopN); + TupleDescriptor tupleDescriptor = generateTupleDesc(slotList, null, context); - return inputFragment; - } + // update tuple list and tblTupleList + inputPlanNode.getTupleIds().clear(); + inputPlanNode.getTupleIds().add(tupleDescriptor.getId()); + inputPlanNode.getTblRefIds().clear(); + inputPlanNode.getTblRefIds().add(tupleDescriptor.getId()); + inputPlanNode.getNullableTupleIds().clear(); + inputPlanNode.getNullableTupleIds().add(tupleDescriptor.getId()); - private PartitionSortNode translatePartitionSortNode(PhysicalPartitionTopN partitionTopN, - PlanNode childNode, PlanTranslatorContext context) { - // Generate the SortInfo, similar to 'translateSortNode'. - List oldOrderingExprList = Lists.newArrayList(); - List ascOrderList = Lists.newArrayList(); - List nullsFirstParamList = Lists.newArrayList(); - List orderKeyList = partitionTopN.getOrderKeys(); - // 1. Get previous slotRef - orderKeyList.forEach(k -> { - oldOrderingExprList.add(ExpressionTranslator.translate(k.getExpr(), context)); - ascOrderList.add(k.isAsc()); - nullsFirstParamList.add(k.isNullFirst()); - }); - List sortTupleOutputList = new ArrayList<>(); - List outputList = partitionTopN.getOutput(); - outputList.forEach(k -> { - sortTupleOutputList.add(ExpressionTranslator.translate(k, context)); - }); - List partitionExprs = partitionTopN.getPartitionKeys().stream() + List execExprList = execList + .stream() .map(e -> ExpressionTranslator.translate(e, context)) .collect(Collectors.toList()); - // 2. Generate new Tuple and get current slotRef for newOrderingExprList - List newOrderingExprList = Lists.newArrayList(); - TupleDescriptor tupleDesc = generateTupleDesc(outputList, orderKeyList, newOrderingExprList, context, null); - // 3. fill in SortInfo members - SortInfo sortInfo = new SortInfo(newOrderingExprList, ascOrderList, nullsFirstParamList, tupleDesc); - PartitionSortNode partitionSortNode = new PartitionSortNode(context.nextPlanNodeId(), childNode, - partitionTopN.getFunction(), partitionExprs, sortInfo, partitionTopN.hasGlobalLimit(), - partitionTopN.getPartitionLimit(), sortTupleOutputList, oldOrderingExprList); + inputPlanNode.setProjectList(execExprList); + inputPlanNode.setOutputTupleDesc(tupleDescriptor); - if (partitionTopN.getStats() != null) { - partitionSortNode.setCardinality((long) partitionTopN.getStats().getRowCount()); - } - updateLegacyPlanIdToPhysicalPlan(partitionSortNode, partitionTopN); - return partitionSortNode; + // update data partition + consumeFragment.setDataPartition(DataPartition.RANDOM); + + SelectNode projectNode = new SelectNode(context.nextPlanNodeId(), inputPlanNode); + consumeFragment.setPlanRoot(projectNode); + + multiCastFragment.getDestNodeList().add(exchangeNode); + consumeFragment.addChild(multiCastFragment); + context.getPlanFragments().add(consumeFragment); + + return consumeFragment; } @Override - public PlanFragment visitPhysicalTopN(PhysicalTopN topN, PlanTranslatorContext context) { - PlanFragment inputFragment = topN.child(0).accept(this, context); - PlanFragment currentFragment = inputFragment; - - //1. Generate new fragment for sort when the child is exchangeNode, If the child is - // mergingExchange, it means we have always generated a new fragment when processing mergeSort - if (inputFragment.getPlanRoot() instanceof ExchangeNode - && !((ExchangeNode) inputFragment.getPlanRoot()).isMergingExchange()) { - // Except LocalTopN->MergeTopN, we don't allow localTopN's child is Exchange Node - Preconditions.checkArgument(!topN.getSortPhase().isLocal(), - "local sort requires any property but child is" + inputFragment.getPlanRoot()); - DataPartition outputPartition = DataPartition.UNPARTITIONED; - ExchangeNode exchangeNode = (ExchangeNode) inputFragment.getPlanRoot(); - inputFragment.setOutputPartition(outputPartition); - inputFragment.setPlanRoot(exchangeNode.getChild(0)); - inputFragment.setDestination(exchangeNode); - currentFragment = new PlanFragment(context.nextFragmentId(), exchangeNode, DataPartition.UNPARTITIONED); - context.addPlanFragment(currentFragment); - } + public PlanFragment visitPhysicalCTEProducer(PhysicalCTEProducer cteProducer, + PlanTranslatorContext context) { + PlanFragment child = cteProducer.child().accept(this, context); + CTEId cteId = cteProducer.getCteId(); + context.getPlanFragments().remove(child); + MultiCastPlanFragment cteProduce = new MultiCastPlanFragment(child); + MultiCastDataSink multiCastDataSink = new MultiCastDataSink(); + cteProduce.setSink(multiCastDataSink); - // 2. According to the type of sort, generate physical plan - if (!topN.getSortPhase().isMerge()) { - // For localSort or Gather->Sort, we just need to add TopNNode - SortNode sortNode = translateSortNode(topN, inputFragment.getPlanRoot(), context); - sortNode.setOffset(topN.getOffset()); - sortNode.setLimit(topN.getLimit()); - if (topN.getMutableState(PhysicalTopN.TOPN_RUNTIME_FILTER).isPresent()) { - sortNode.setUseTopnOpt(true); - PlanNode child = sortNode.getChild(0); - Preconditions.checkArgument(child instanceof OlapScanNode, - "topN opt expect OlapScanNode, but we get " + child); - OlapScanNode scanNode = ((OlapScanNode) child); - scanNode.setUseTopnOpt(true); - } - // push sort to scan opt - if (sortNode.getChild(0) instanceof OlapScanNode) { - OlapScanNode scanNode = ((OlapScanNode) sortNode.getChild(0)); - if (checkPushSort(sortNode, scanNode.getOlapTable())) { - SortInfo sortInfo = sortNode.getSortInfo(); - scanNode.setSortInfo(sortInfo); - scanNode.getSortInfo().setSortTupleSlotExprs(sortNode.getResolvedTupleExprs()); - for (Expr expr : sortInfo.getOrderingExprs()) { - scanNode.getSortInfo().addMaterializedOrderingExpr(expr); - } - if (sortNode.getOffset() > 0) { - scanNode.setSortLimit(sortNode.getLimit() + sortNode.getOffset()); - } else { - scanNode.setSortLimit(sortNode.getLimit()); - } - } - } - addPlanRoot(currentFragment, sortNode, topN); - } else { - // For mergeSort, we need to push sortInfo to exchangeNode - if (!(currentFragment.getPlanRoot() instanceof ExchangeNode)) { - // if there is no exchange node for mergeSort - // e.g., mergeTopN -> localTopN - // It means the local has satisfied the Gather property. We can just ignore mergeSort - currentFragment.getPlanRoot().setOffset(topN.getOffset()); - currentFragment.getPlanRoot().setLimit(topN.getLimit()); - return currentFragment; - } - Preconditions.checkArgument(inputFragment.getPlanRoot() instanceof SortNode, - "mergeSort' child must be sortNode"); - SortNode sortNode = (SortNode) inputFragment.getPlanRoot(); - ExchangeNode exchangeNode = (ExchangeNode) currentFragment.getPlanRoot(); - exchangeNode.setMergeInfo(sortNode.getSortInfo()); - exchangeNode.setLimit(topN.getLimit()); - exchangeNode.setOffset(topN.getOffset()); - } - updateLegacyPlanIdToPhysicalPlan(currentFragment.getPlanRoot(), topN); - return currentFragment; - } + List outputs = cteProducer.getProjects().stream() + .map(e -> ExpressionTranslator.translate(e, context)) + .collect(Collectors.toList()); - /** - * topN opt: using storage data ordering to accelerate topn operation. - * refer pr: optimize topn query if order by columns is prefix of sort keys of table (#10694) - */ - public boolean checkPushSort(SortNode sortNode, OlapTable olapTable) { - // Ensure limit is less then threshold - if (sortNode.getLimit() <= 0 - || sortNode.getLimit() > ConnectContext.get().getSessionVariable().topnOptLimitThreshold) { - return false; + cteProduce.setOutputExprs(outputs); + context.getCteProduceFragments().put(cteId, cteProduce); + context.getCteProduceMap().put(cteId, cteProducer); + if (context.getRuntimeTranslator().isPresent()) { + context.getRuntimeTranslator().get().getContext().getCteProduceMap().put(cteId, cteProducer); } + context.getPlanFragments().add(cteProduce); + return child; + } - // Ensure all isAscOrder is same, ande length != 0. - // Can't be zorder. - if (sortNode.getSortInfo().getIsAscOrder().stream().distinct().count() != 1 - || olapTable.isZOrderSort()) { - return false; + @Override + public PlanFragment visitPhysicalFilter(PhysicalFilter filter, PlanTranslatorContext context) { + if (filter.child(0) instanceof AbstractPhysicalJoin) { + AbstractPhysicalJoin join = (AbstractPhysicalJoin) filter.child(); + join.addFilterConjuncts(filter.getConjuncts()); } + PlanFragment inputFragment = filter.child(0).accept(this, context); - // Tablet's order by key only can be the front part of schema. - // Like: schema: a.b.c.d.e.f.g order by key: a.b.c (no a,b,d) - // Do **prefix match** to check if order by key can be pushed down. - // olap order by key: a.b.c.d - // sort key: (a) (a,b) (a,b,c) (a,b,c,d) is ok - // (a,c) (a,c,d), (a,c,b) (a,c,f) (a,b,c,d,e)is NOT ok - List sortExprs = sortNode.getSortInfo().getOrderingExprs(); - List nullsFirsts = sortNode.getSortInfo().getNullsFirst(); - List isAscOrders = sortNode.getSortInfo().getIsAscOrder(); - if (sortExprs.size() > olapTable.getDataSortInfo().getColNum()) { - return false; - } - for (int i = 0; i < sortExprs.size(); i++) { - // table key. - Column tableKey = olapTable.getFullSchema().get(i); - // sort slot. - Expr sortExpr = sortExprs.get(i); - if (sortExpr instanceof SlotRef) { - SlotRef slotRef = (SlotRef) sortExpr; - if (tableKey.equals(slotRef.getColumn())) { - // ORDER BY DESC NULLS FIRST can not be optimized to only read file tail, - // since NULLS is at file head but data is at tail - if (tableKey.isAllowNull() && nullsFirsts.get(i) && !isAscOrders.get(i)) { - return false; - } - } else { - return false; - } - } else { - return false; + PlanNode planNode = inputFragment.getPlanRoot(); + if (planNode instanceof ExchangeNode || planNode instanceof SortNode || planNode instanceof UnionNode) { + // the three nodes don't support conjuncts, need create a SelectNode to filter data + SelectNode selectNode = new SelectNode(context.nextPlanNodeId(), planNode); + addConjunctsToPlanNode(filter, selectNode, context); + addPlanRoot(inputFragment, selectNode, filter); + } else { + if (!(filter.child(0) instanceof AbstractPhysicalJoin)) { + addConjunctsToPlanNode(filter, planNode, context); + updateLegacyPlanIdToPhysicalPlan(inputFragment.getPlanRoot(), filter); } } - - return true; + //in ut, filter.stats may be null + if (filter.getStats() != null) { + inputFragment.getPlanRoot().setCardinalityAfterFilter((long) filter.getStats().getRowCount()); + } + return inputFragment; } - private SortNode translateSortNode(AbstractPhysicalSort sort, PlanNode childNode, + @Override + public PlanFragment visitPhysicalGenerate(PhysicalGenerate generate, PlanTranslatorContext context) { - List oldOrderingExprList = Lists.newArrayList(); - List ascOrderList = Lists.newArrayList(); - List nullsFirstParamList = Lists.newArrayList(); - List orderKeyList = sort.getOrderKeys(); - // 1. Get previous slotRef - orderKeyList.forEach(k -> { - oldOrderingExprList.add(ExpressionTranslator.translate(k.getExpr(), context)); - ascOrderList.add(k.isAsc()); - nullsFirstParamList.add(k.isNullFirst()); - }); - List sortTupleOutputList = new ArrayList<>(); - List outputList = sort.getOutput(); - outputList.forEach(k -> sortTupleOutputList.add(ExpressionTranslator.translate(k, context))); - // 2. Generate new Tuple and get current slotRef for newOrderingExprList - List newOrderingExprList = Lists.newArrayList(); - TupleDescriptor tupleDesc = generateTupleDesc(outputList, orderKeyList, newOrderingExprList, context, null); - // 3. fill in SortInfo members - SortInfo sortInfo = new SortInfo(newOrderingExprList, ascOrderList, nullsFirstParamList, tupleDesc); - SortNode sortNode = new SortNode(context.nextPlanNodeId(), childNode, sortInfo, true); - sortNode.finalizeForNereids(tupleDesc, sortTupleOutputList, oldOrderingExprList); - if (sort.getMutableState(PhysicalTopN.TWO_PHASE_READ_OPT).isPresent()) { - sortNode.setUseTwoPhaseReadOpt(true); - sortNode.getSortInfo().setUseTwoPhaseRead(); - injectRowIdColumnSlot(sortNode.getSortInfo().getSortTupleDescriptor()); - TupleDescriptor childTuple = childNode.getOutputTupleDesc() != null - ? childNode.getOutputTupleDesc() : context.getTupleDesc(childNode.getTupleIds().get(0)); - SlotDescriptor childRowIdDesc = childTuple.getSlots().get(childTuple.getSlots().size() - 1); - sortNode.getResolvedTupleExprs().add(new SlotRef(childRowIdDesc)); - } - if (sort.getStats() != null) { - sortNode.setCardinality((long) sort.getStats().getRowCount()); + PlanFragment currentFragment = generate.child().accept(this, context); + ArrayList functionCalls = generate.getGenerators().stream() + .map(e -> ExpressionTranslator.translate(e, context)) + .collect(Collectors.toCollection(ArrayList::new)); + TupleDescriptor tupleDescriptor = generateTupleDesc(generate.getGeneratorOutput(), null, context); + List childOutputTupleIds = currentFragment.getPlanRoot().getOutputTupleIds(); + if (childOutputTupleIds == null || childOutputTupleIds.isEmpty()) { + childOutputTupleIds = currentFragment.getPlanRoot().getTupleIds(); } - updateLegacyPlanIdToPhysicalPlan(sortNode, sort); - return sortNode; + List outputSlotIds = Stream.concat(childOutputTupleIds.stream(), + Stream.of(tupleDescriptor.getId())) + .map(id -> context.getTupleDesc(id).getSlots()) + .flatMap(List::stream) + .map(SlotDescriptor::getId) + .collect(Collectors.toList()); + TableFunctionNode tableFunctionNode = new TableFunctionNode(context.nextPlanNodeId(), + currentFragment.getPlanRoot(), tupleDescriptor.getId(), functionCalls, outputSlotIds); + addPlanRoot(currentFragment, tableFunctionNode, generate); + return currentFragment; } /** @@ -1211,44 +908,44 @@ private SortNode translateSortNode(AbstractPhysicalSort sort, Pl * a. equal join conjuncts * b. other join conjuncts * c. other predicates (denoted by filter conjuncts in the rest of comments) - * + *

* 2. hash join contains 3 tuple descriptors * a. input tuple descriptors, corresponding to the left child output and right child output. * If its column is selected, it will be displayed in explain by `tuple ids`. * for example, select L.* from L join R on ..., because no column from R are selected, tuple ids only * contains output tuple of L. * equal join conjuncts is bound on input tuple descriptors. - * + *

* b.intermediate tuple. * This tuple describes schema of the output block after evaluating equal join conjuncts * and other join conjuncts. - * + *

* Other join conjuncts currently is bound on intermediate tuple. There are some historical reason, and it * should be bound on input tuple in the future. - * + *

* filter conjuncts will be evaluated on the intermediate tuple. That means the input block of filter is * described by intermediate tuple, and hence filter conjuncts should be bound on intermediate tuple. - * + *

* In order to be compatible with old version, intermediate tuple is not pruned. For example, intermediate * tuple contains all slots from both sides of children. After probing hash-table, BE does not need to * materialize all slots in intermediate tuple. The slots in HashJoinNode.hashOutputSlotIds will be * materialized by BE. If `hashOutputSlotIds` is empty, all slots will be materialized. - * + *

* In case of outer join, the slots in intermediate should be set nullable. * For example, * select L.*, R.* from L left outer join R on ... * All slots from R in intermediate tuple should be nullable. - * + *

* c. output tuple * This describes the schema of hash join output block. * 3. Intermediate tuple - * for BE performance reason, the slots in intermediate tuple depends on the join type and other join conjucts. + * for BE performance reason, the slots in intermediate tuple + * depends on the join type and other join conjuncts. * In general, intermediate tuple contains all slots of both children, except one case. * For left-semi/left-ant (right-semi/right-semi) join without other join conjuncts, intermediate tuple * only contains left (right) children output slots. * */ - // TODO: 1. support shuffle join / co-locate / bucket shuffle join later @Override public PlanFragment visitPhysicalHashJoin( PhysicalHashJoin hashJoin, @@ -1281,17 +978,22 @@ public PlanFragment visitPhysicalHashJoin( rightPlanRoot, JoinType.toJoinOperator(joinType), execEqConjuncts, Lists.newArrayList(), null, null, null, hashJoin.isMarkJoin()); - PlanFragment currentFragment; + PlanFragment currentFragment = connectJoinNode(hashJoinNode, leftFragment, rightFragment, context, hashJoin); if (JoinUtils.shouldColocateJoin(physicalHashJoin)) { - currentFragment = constructColocateJoin(hashJoinNode, leftFragment, rightFragment, context, hashJoin); - } else if (JoinUtils.shouldBucketShuffleJoin(physicalHashJoin)) { - currentFragment = constructBucketShuffleJoin( - physicalHashJoin, hashJoinNode, leftFragment, rightFragment, context); + // TODO: add reason + hashJoinNode.setColocate(true, ""); + leftFragment.setHasColocatePlanNode(true); } else if (JoinUtils.shouldBroadcastJoin(physicalHashJoin)) { - currentFragment = constructBroadcastJoin(hashJoinNode, leftFragment, rightFragment, context, hashJoin); + Preconditions.checkState(rightFragment.getPlanRoot() instanceof ExchangeNode, + "right child of broadcast join must be ExchangeNode but it is " + rightFragment.getPlanRoot()); + Preconditions.checkState(rightFragment.getChildren().size() == 1, + "right child of broadcast join must have 1 child, but meet " + rightFragment.getChildren().size()); + rightFragment.getChild(0).setRightChildOfBroadcastHashJoin(true); + hashJoinNode.setDistributionMode(DistributionMode.BROADCAST); + } else if (JoinUtils.shouldBucketShuffleJoin(physicalHashJoin)) { + hashJoinNode.setDistributionMode(DistributionMode.BUCKET_SHUFFLE); } else { - currentFragment = constructShuffleJoin( - physicalHashJoin, hashJoinNode, leftFragment, rightFragment, context); + hashJoinNode.setDistributionMode(DistributionMode.PARTITIONED); } // Nereids does not care about output order of join, // but BE need left child's output must be before right child's output. @@ -1368,6 +1070,10 @@ public PlanFragment visitPhysicalHashJoin( sd = context.getDescTable().copySlotDescriptor(intermediateDescriptor, leftSlotDescriptor); } else { sd = context.createSlotDesc(intermediateDescriptor, sf); + if (hashOutputSlotReferenceMap.get(sf.getExprId()) != null) { + hashJoinNode.addSlotIdToHashOutputSlotIds(leftSlotDescriptor.getId()); + hashJoinNode.getHashOutputExprSlotIdMap().put(sf.getExprId(), leftSlotDescriptor.getId()); + } } leftIntermediateSlotDescriptor.add(sd); } @@ -1384,6 +1090,10 @@ public PlanFragment visitPhysicalHashJoin( sd = context.getDescTable().copySlotDescriptor(intermediateDescriptor, rightSlotDescriptor); } else { sd = context.createSlotDesc(intermediateDescriptor, sf); + if (hashOutputSlotReferenceMap.get(sf.getExprId()) != null) { + hashJoinNode.addSlotIdToHashOutputSlotIds(rightSlotDescriptor.getId()); + hashJoinNode.getHashOutputExprSlotIdMap().put(sf.getExprId(), rightSlotDescriptor.getId()); + } } rightIntermediateSlotDescriptor.add(sd); } @@ -1401,6 +1111,7 @@ public PlanFragment visitPhysicalHashJoin( sd = context.createSlotDesc(intermediateDescriptor, sf); if (hashOutputSlotReferenceMap.get(sf.getExprId()) != null) { hashJoinNode.addSlotIdToHashOutputSlotIds(leftSlotDescriptor.getId()); + hashJoinNode.getHashOutputExprSlotIdMap().put(sf.getExprId(), leftSlotDescriptor.getId()); } } leftIntermediateSlotDescriptor.add(sd); @@ -1418,6 +1129,7 @@ public PlanFragment visitPhysicalHashJoin( sd = context.createSlotDesc(intermediateDescriptor, sf); if (hashOutputSlotReferenceMap.get(sf.getExprId()) != null) { hashJoinNode.addSlotIdToHashOutputSlotIds(rightSlotDescriptor.getId()); + hashJoinNode.getHashOutputExprSlotIdMap().put(sf.getExprId(), rightSlotDescriptor.getId()); } } rightIntermediateSlotDescriptor.add(sd); @@ -1425,8 +1137,15 @@ public PlanFragment visitPhysicalHashJoin( } if (hashJoin.getMarkJoinSlotReference().isPresent()) { - outputSlotReferences.add(hashJoin.getMarkJoinSlotReference().get()); - context.createSlotDesc(intermediateDescriptor, hashJoin.getMarkJoinSlotReference().get()); + SlotReference sf = hashJoin.getMarkJoinSlotReference().get(); + outputSlotReferences.add(sf); + context.createSlotDesc(intermediateDescriptor, sf); + if (hashOutputSlotReferenceMap.get(sf.getExprId()) != null) { + SlotRef markJoinSlotId = context.findSlotRef(sf.getExprId()); + Preconditions.checkState(markJoinSlotId != null); + hashJoinNode.addSlotIdToHashOutputSlotIds(markJoinSlotId.getSlotId()); + hashJoinNode.getHashOutputExprSlotIdMap().put(sf.getExprId(), markJoinSlotId.getSlotId()); + } } // set slots as nullable for outer join @@ -1500,17 +1219,14 @@ public PlanFragment visitPhysicalNestedLoopJoin( if (nestedLoopJoin.getStats() != null) { nestedLoopJoinNode.setCardinality((long) nestedLoopJoin.getStats().getRowCount()); } - boolean needNewRootFragment = nestedLoopJoin.child(0) instanceof PhysicalDistribute; - PlanFragment joinFragment; - if (needNewRootFragment) { - joinFragment = createPlanFragment(nestedLoopJoinNode, - DataPartition.UNPARTITIONED, nestedLoopJoin); - context.addPlanFragment(joinFragment); - connectChildFragment(nestedLoopJoinNode, 0, joinFragment, leftFragment, context); - } else { - joinFragment = leftFragment; - nestedLoopJoinNode.setChild(0, leftFragment.getPlanRoot()); - setPlanRoot(joinFragment, nestedLoopJoinNode, nestedLoopJoin); + nestedLoopJoinNode.setChild(0, leftFragment.getPlanRoot()); + nestedLoopJoinNode.setChild(1, rightFragment.getPlanRoot()); + setPlanRoot(leftFragment, nestedLoopJoinNode, nestedLoopJoin); + // TODO: what's this? do we really need to set this? + rightFragment.getPlanRoot().setCompactData(false); + context.mergePlanFragment(rightFragment, leftFragment); + for (PlanFragment rightChild : rightFragment.getChildren()) { + leftFragment.addChild(rightChild); } // translate runtime filter context.getRuntimeTranslator().ifPresent(runtimeFilterTranslator -> { @@ -1612,9 +1328,6 @@ public PlanFragment visitPhysicalNestedLoopJoin( nestedLoopJoinNode.setvIntermediateTupleDescList(Lists.newArrayList(intermediateDescriptor)); - rightFragment.getPlanRoot().setCompactData(false); - - connectChildFragment(nestedLoopJoinNode, 1, joinFragment, rightFragment, context); List joinConjuncts = nestedLoopJoin.getOtherJoinConjuncts().stream() .filter(e -> !nestedLoopJoin.isBitmapRuntimeFilterCondition(e)) .map(e -> ExpressionTranslator.translate(e, context)).collect(Collectors.toList()); @@ -1646,28 +1359,42 @@ public PlanFragment visitPhysicalNestedLoopJoin( if (nestedLoopJoin.getStats() != null) { nestedLoopJoinNode.setCardinality((long) nestedLoopJoin.getStats().getRowCount()); } - updateLegacyPlanIdToPhysicalPlan(joinFragment.getPlanRoot(), nestedLoopJoin); - return joinFragment; + updateLegacyPlanIdToPhysicalPlan(leftFragment.getPlanRoot(), nestedLoopJoin); + return leftFragment; } else { throw new RuntimeException("Physical nested loop join could not execute with equal join condition."); } } + @Override + public PlanFragment visitPhysicalLimit(PhysicalLimit physicalLimit, PlanTranslatorContext context) { + PlanFragment inputFragment = physicalLimit.child(0).accept(this, context); + PlanNode child = inputFragment.getPlanRoot(); + child.setOffset(physicalLimit.getOffset()); + child.setLimit(physicalLimit.getLimit()); + updateLegacyPlanIdToPhysicalPlan(child, physicalLimit); + return inputFragment; + } + + @Override + public PlanFragment visitPhysicalPartitionTopN(PhysicalPartitionTopN partitionTopN, + PlanTranslatorContext context) { + PlanFragment inputFragment = partitionTopN.child(0).accept(this, context); + PartitionSortNode partitionSortNode = translatePartitionSortNode( + partitionTopN, inputFragment.getPlanRoot(), context); + addPlanRoot(inputFragment, partitionSortNode, partitionTopN); + return inputFragment; + } + // TODO: generate expression mapping when be project could do in ExecNode. @Override public PlanFragment visitPhysicalProject(PhysicalProject project, PlanTranslatorContext context) { - if (project.child(0) instanceof PhysicalHashJoin) { - ((PhysicalHashJoin) project.child(0)).setShouldTranslateOutput(false); - } - if (project.child(0) instanceof PhysicalNestedLoopJoin) { - ((PhysicalNestedLoopJoin) project.child(0)).setShouldTranslateOutput(false); + if (project.child(0) instanceof AbstractPhysicalJoin) { + ((AbstractPhysicalJoin) project.child(0)).setShouldTranslateOutput(false); } if (project.child(0) instanceof PhysicalFilter) { - if (project.child(0).child(0) instanceof PhysicalHashJoin) { - ((PhysicalHashJoin) project.child(0).child(0)).setShouldTranslateOutput(false); - } - if (project.child(0).child(0) instanceof PhysicalNestedLoopJoin) { - ((PhysicalNestedLoopJoin) project.child(0).child(0)).setShouldTranslateOutput(false); + if (project.child(0).child(0) instanceof AbstractPhysicalJoin) { + ((AbstractPhysicalJoin) project.child(0).child(0)).setShouldTranslateOutput(false); } } PlanFragment inputFragment = project.child(0).accept(this, context); @@ -1680,9 +1407,18 @@ public PlanFragment visitPhysicalProject(PhysicalProject project PlanNode inputPlanNode = inputFragment.getPlanRoot(); List slotList = project.getProjects() .stream() - .map(e -> e.toSlot()) + .map(NamedExpression::toSlot) .collect(Collectors.toList()); + List predicateList = inputPlanNode.getConjuncts(); + Set requiredSlotIdSet = Sets.newHashSet(); + for (Expr expr : execExprList) { + extractExecSlot(expr, requiredSlotIdSet); + } + Set requiredByProjectSlotIdSet = Sets.newHashSet(requiredSlotIdSet); + for (Expr expr : predicateList) { + extractExecSlot(expr, requiredSlotIdSet); + } // For hash join node, use vSrcToOutputSMap to describe the expression calculation, use // vIntermediateTupleDescList as input, and set vOutputTupleDesc as the final output. // TODO: HashJoinNode's be implementation is not support projection yet, remove this after when supported. @@ -1691,17 +1427,26 @@ public PlanFragment visitPhysicalProject(PhysicalProject project JoinNodeBase hashJoinNode = (JoinNodeBase) inputPlanNode; hashJoinNode.setvOutputTupleDesc(tupleDescriptor); hashJoinNode.setvSrcToOutputSMap(execExprList); + // prune the hashOutputSlotIds + if (hashJoinNode instanceof HashJoinNode) { + ((HashJoinNode) hashJoinNode).getHashOutputSlotIds().clear(); + Set requiredExprIds = Sets.newHashSet(); + Set requiredOtherConjunctsSlotIdSet = Sets.newHashSet(); + List otherConjuncts = ((HashJoinNode) hashJoinNode).getOtherJoinConjuncts(); + for (Expr expr : otherConjuncts) { + extractExecSlot(expr, requiredOtherConjunctsSlotIdSet); + } + requiredOtherConjunctsSlotIdSet.forEach(e -> requiredExprIds.add(context.findExprId(e))); + requiredSlotIdSet.forEach(e -> requiredExprIds.add(context.findExprId(e))); + for (ExprId exprId : requiredExprIds) { + SlotId slotId = ((HashJoinNode) hashJoinNode).getHashOutputExprSlotIdMap().get(exprId); + Preconditions.checkState(slotId != null); + ((HashJoinNode) hashJoinNode).addSlotIdToHashOutputSlotIds(slotId); + } + } return inputFragment; } - List predicateList = inputPlanNode.getConjuncts(); - Set requiredSlotIdSet = Sets.newHashSet(); - for (Expr expr : execExprList) { - extractExecSlot(expr, requiredSlotIdSet); - } - Set requiredByProjectSlotIdSet = Sets.newHashSet(requiredSlotIdSet); - for (Expr expr : predicateList) { - extractExecSlot(expr, requiredSlotIdSet); - } + if (inputPlanNode instanceof TableFunctionNode) { TableFunctionNode tableFunctionNode = (TableFunctionNode) inputPlanNode; tableFunctionNode.setOutputSlotIds(Lists.newArrayList(requiredSlotIdSet)); @@ -1710,7 +1455,7 @@ public PlanFragment visitPhysicalProject(PhysicalProject project if (inputPlanNode instanceof ScanNode) { TupleDescriptor tupleDescriptor = null; if (requiredByProjectSlotIdSet.size() != requiredSlotIdSet.size() - || execExprList.stream().collect(Collectors.toSet()).size() != execExprList.size() + || new HashSet<>(execExprList).size() != execExprList.size() || execExprList.stream().anyMatch(expr -> !(expr instanceof SlotRef))) { tupleDescriptor = generateTupleDesc(slotList, null, context); inputPlanNode.setProjectList(execExprList); @@ -1749,149 +1494,6 @@ public PlanFragment visitPhysicalProject(PhysicalProject project return inputFragment; } - private void updateChildSlotsMaterialization(PlanNode execPlan, - Set requiredSlotIdSet, Set requiredByProjectSlotIdSet, - PlanTranslatorContext context) { - Set slotRefSet = new HashSet<>(); - for (Expr expr : execPlan.getConjuncts()) { - expr.collect(SlotRef.class, slotRefSet); - } - Set slotIdSet = slotRefSet.stream() - .map(SlotRef::getSlotId).collect(Collectors.toSet()); - slotIdSet.addAll(requiredSlotIdSet); - boolean noneMaterialized = execPlan.getTupleIds().stream() - .map(context::getTupleDesc) - .map(TupleDescriptor::getSlots) - .flatMap(List::stream) - .peek(s -> s.setIsMaterialized(slotIdSet.contains(s.getId()))) - .filter(SlotDescriptor::isMaterialized) - .count() == 0; - if (noneMaterialized) { - context.getDescTable() - .getTupleDesc(execPlan.getTupleIds().get(0)).getSlots().get(0).setIsMaterialized(true); - } - if (execPlan instanceof ScanNode) { - try { - ((ScanNode) execPlan).updateRequiredSlots(context, requiredByProjectSlotIdSet); - } catch (UserException e) { - Util.logAndThrowRuntimeException(LOG, - "User Exception while reset external file scan node contexts.", e); - } - } - } - - @Override - public PlanFragment visitPhysicalFilter(PhysicalFilter filter, PlanTranslatorContext context) { - if (filter.child(0) instanceof AbstractPhysicalJoin) { - AbstractPhysicalJoin join = (AbstractPhysicalJoin) filter.child(0); - join.addFilterConjuncts(filter.getConjuncts()); - } - PlanFragment inputFragment = filter.child(0).accept(this, context); - - // Union contains oneRowRelation --> inputFragment = null - if (inputFragment == null) { - return inputFragment; - } - - PlanNode planNode = inputFragment.getPlanRoot(); - if (planNode instanceof ExchangeNode || planNode instanceof SortNode || planNode instanceof UnionNode) { - // the three nodes don't support conjuncts, need create a SelectNode to filter data - SelectNode selectNode = new SelectNode(context.nextPlanNodeId(), planNode); - addConjunctsToPlanNode(filter, selectNode, context); - addPlanRoot(inputFragment, selectNode, filter); - } else { - if (!(filter.child(0) instanceof AbstractPhysicalJoin)) { - addConjunctsToPlanNode(filter, planNode, context); - updateLegacyPlanIdToPhysicalPlan(inputFragment.getPlanRoot(), filter); - } - } - //in ut, filter.stats may be null - if (filter.getStats() != null) { - inputFragment.getPlanRoot().setCardinalityAfterFilter((long) filter.getStats().getRowCount()); - } - return inputFragment; - } - - private void addConjunctsToPlanNode(PhysicalFilter filter, - PlanNode planNode, - PlanTranslatorContext context) { - filter.getConjuncts().stream() - .map(e -> ExpressionTranslator.translate(e, context)) - .forEach(planNode::addConjunct); - updateLegacyPlanIdToPhysicalPlan(planNode, filter); - } - - @Override - public PlanFragment visitPhysicalLimit(PhysicalLimit physicalLimit, PlanTranslatorContext context) { - PlanFragment inputFragment = physicalLimit.child(0).accept(this, context); - - // Union contains oneRowRelation - if (inputFragment == null) { - return null; - } - - PlanNode child = inputFragment.getPlanRoot(); - if (physicalLimit.isGlobal()) { - if (child instanceof ExchangeNode) { - DataPartition outputPartition = DataPartition.UNPARTITIONED; - ExchangeNode exchangeNode = (ExchangeNode) inputFragment.getPlanRoot(); - inputFragment.setOutputPartition(outputPartition); - inputFragment.setPlanRoot(exchangeNode.getChild(0)); - inputFragment.setDestination(exchangeNode); - inputFragment = new PlanFragment(context.nextFragmentId(), exchangeNode, DataPartition.UNPARTITIONED); - context.addPlanFragment(inputFragment); - } else if (physicalLimit.hasValidOffset()) { - // This case means GlobalLimit's child isn't gatherNode, which suggests the child is UNPARTITIONED - // When there is valid offset, exchangeNode should be added because other node don't support offset - inputFragment = createParentFragment(inputFragment, DataPartition.UNPARTITIONED, context); - child = inputFragment.getPlanRoot(); - } - } - child.setOffset(physicalLimit.getOffset()); - child.setLimit(physicalLimit.getLimit()); - updateLegacyPlanIdToPhysicalPlan(child, physicalLimit); - return inputFragment; - } - - @Override - public PlanFragment visitPhysicalDistribute(PhysicalDistribute distribute, - PlanTranslatorContext context) { - PlanFragment childFragment = distribute.child().accept(this, context); - - if (childFragment.getPlanRoot() instanceof AggregationNode - && distribute.child() instanceof PhysicalHashAggregate - && context.getFirstAggregateInFragment(childFragment) == distribute.child()) { - PhysicalHashAggregate hashAggregate = (PhysicalHashAggregate) distribute.child(); - if (hashAggregate.getAggPhase() == AggPhase.LOCAL - && hashAggregate.getAggMode() == AggMode.INPUT_TO_BUFFER) { - AggregationNode aggregationNode = (AggregationNode) childFragment.getPlanRoot(); - aggregationNode.setUseStreamingPreagg(hashAggregate.isMaybeUsingStream()); - } - } - - ExchangeNode exchange = new ExchangeNode(context.nextPlanNodeId(), childFragment.getPlanRoot()); - exchange.setNumInstances(childFragment.getPlanRoot().getNumInstances()); - childFragment.setPlanRoot(exchange); - updateLegacyPlanIdToPhysicalPlan(childFragment.getPlanRoot(), distribute); - return childFragment; - } - - @Override - public PlanFragment visitPhysicalAssertNumRows(PhysicalAssertNumRows assertNumRows, - PlanTranslatorContext context) { - PlanFragment currentFragment = assertNumRows.child().accept(this, context); - // create assertNode - AssertNumRowsNode assertNumRowsNode = new AssertNumRowsNode(context.nextPlanNodeId(), - currentFragment.getPlanRoot(), - ExpressionTranslator.translateAssert(assertNumRows.getAssertNumRowsElement())); - if (currentFragment.getPlanRoot() instanceof ExchangeNode) { - currentFragment.setPlanRoot(currentFragment.getPlanRoot().getChild(0)); - currentFragment = createParentFragment(currentFragment, DataPartition.UNPARTITIONED, context); - } - addPlanRoot(currentFragment, assertNumRowsNode, assertNumRows); - return currentFragment; - } - /** * Returns a new fragment with a UnionNode as its root. The data partition of the * returned fragment and how the data of the child fragments is consumed depends on the @@ -1908,280 +1510,419 @@ public PlanFragment visitPhysicalAssertNumRows(PhysicalAssertNumRows childrenFragments = new ArrayList<>(); - Map childNodeToFragment = new HashMap<>(); for (Plan plan : setOperation.children()) { - PlanFragment planFragment = plan.accept(this, context); - if (planFragment != null) { - childrenFragments.add(planFragment); - } - childNodeToFragment.put(plan, planFragment); + childrenFragments.add(plan.accept(this, context)); } - PlanFragment setOperationFragment; - SetOperationNode setOperationNode; - - List allSlots = new Builder() - .addAll(setOperation.getOutput()) - .build(); - TupleDescriptor setTuple = generateTupleDesc(allSlots, null, context); - List outputSLotDescs = new ArrayList<>(setTuple.getSlots()); + TupleDescriptor setTuple = generateTupleDesc(setOperation.getOutput(), null, context); + List outputSlotDescs = new ArrayList<>(setTuple.getSlots()); + SetOperationNode setOperationNode; // create setOperationNode if (setOperation instanceof PhysicalUnion) { - setOperationNode = new UnionNode( - context.nextPlanNodeId(), setTuple.getId()); + setOperationNode = new UnionNode(context.nextPlanNodeId(), setTuple.getId()); } else if (setOperation instanceof PhysicalExcept) { - setOperationNode = new ExceptNode( - context.nextPlanNodeId(), setTuple.getId()); + setOperationNode = new ExceptNode(context.nextPlanNodeId(), setTuple.getId()); } else if (setOperation instanceof PhysicalIntersect) { - setOperationNode = new IntersectNode( - context.nextPlanNodeId(), setTuple.getId()); + setOperationNode = new IntersectNode(context.nextPlanNodeId(), setTuple.getId()); } else { - throw new RuntimeException("not support"); + throw new RuntimeException("not support set operation type " + setOperation); } - SetOperationResult setOperationResult = collectSetOperationResult(setOperation, childNodeToFragment); - for (List expressions : setOperationResult.getResultExpressions()) { - List resultExprs = expressions - .stream() - .map(expr -> ExpressionTranslator.translate(expr, context)) - .collect(ImmutableList.toImmutableList()); - setOperationNode.addResultExprLists(resultExprs); + setOperation.children().stream() + .map(Plan::getOutput) + .map(l -> l.stream() + .map(e -> ExpressionTranslator.translate(e, context)) + .collect(ImmutableList.toImmutableList())) + .forEach(setOperationNode::addResultExprLists); + if (setOperation instanceof PhysicalUnion) { + ((PhysicalUnion) setOperation).getConstantExprsList().stream() + .map(l -> l.stream() + .map(e -> ExpressionTranslator.translate(e, context)) + .collect(ImmutableList.toImmutableList())) + .forEach(setOperationNode::addConstExprList); } - for (List expressions : setOperationResult.getConstExpressions()) { - List constExprs = expressions - .stream() - .map(expr -> ExpressionTranslator.translate(expr, context)) - .collect(ImmutableList.toImmutableList()); - setOperationNode.addConstExprList(constExprs); + for (PlanFragment childFragment : childrenFragments) { + setOperationNode.addChild(childFragment.getPlanRoot()); } + setOperationNode.finalizeForNereids(outputSlotDescs, outputSlotDescs); - for (PlanFragment childFragment : childrenFragments) { - if (childFragment != null) { - setOperationNode.addChild(childFragment.getPlanRoot()); + PlanFragment setOperationFragment; + if (childrenFragments.isEmpty()) { + setOperationFragment = createPlanFragment(setOperationNode, + DataPartition.UNPARTITIONED, setOperation); + context.addPlanFragment(setOperationFragment); + } else { + int childrenSize = childrenFragments.size(); + setOperationFragment = childrenFragments.get(childrenSize - 1); + for (int i = childrenSize - 2; i >= 0; i--) { + context.mergePlanFragment(childrenFragments.get(i), setOperationFragment); + for (PlanFragment child : childrenFragments.get(i).getChildren()) { + setOperationFragment.addChild(child); + } + } + setPlanRoot(setOperationFragment, setOperationNode, setOperation); + } + + return setOperationFragment; + } + + /*- + * Physical sort: + * 1. Build sortInfo + * There are two types of slotRef: + * one is generated by the previous node, collectively called old. + * the other is newly generated by the sort node, collectively called new. + * Filling of sortInfo related data structures, + * a. ordering use newSlotRef. + * b. sortTupleSlotExprs use oldSlotRef. + * 2. Create sortNode + * 3. Create mergeFragment + * TODO: When the slotRef of sort is currently generated, + * it will be based on the expression in select and orderBy expression in to ensure the uniqueness of slotRef. + * But eg: + * select a+1 from table order by a+1; + * the expressions of the two are inconsistent. + * The former will perform an additional Alias. + * Currently we cannot test whether this will have any effect. + * After a+1 can be parsed , reprocessing. + */ + @Override + public PlanFragment visitPhysicalQuickSort(PhysicalQuickSort sort, + PlanTranslatorContext context) { + PlanFragment inputFragment = sort.child(0).accept(this, context); + + // 2. According to the type of sort, generate physical plan + if (!sort.getSortPhase().isMerge()) { + // For localSort or Gather->Sort, we just need to add sortNode + SortNode sortNode = translateSortNode(sort, inputFragment.getPlanRoot(), context); + addPlanRoot(inputFragment, sortNode, sort); + } else { + // For mergeSort, we need to push sortInfo to exchangeNode + if (!(inputFragment.getPlanRoot() instanceof ExchangeNode)) { + // if there is no exchange node for mergeSort + // e.g., localSort -> mergeSort + // It means the local has satisfied the Gather property. We can just ignore mergeSort + return inputFragment; } + SortNode sortNode = (SortNode) inputFragment.getPlanRoot().getChild(0); + ((ExchangeNode) inputFragment.getPlanRoot()).setMergeInfo(sortNode.getSortInfo()); } - setOperationNode.finalizeForNereids(outputSLotDescs, outputSLotDescs); - - // create setOperationFragment - // If all child fragments are unpartitioned, return a single unpartitioned fragment - // with a UnionNode that merges all child fragments. - if (allChildFragmentsUnPartitioned(childrenFragments)) { - setOperationFragment = createPlanFragment(setOperationNode, DataPartition.UNPARTITIONED, setOperation); - // Absorb the plan trees of all childFragments into unionNode - // and fix up the fragment tree in the process. - for (int i = 0; i < childrenFragments.size(); ++i) { - connectChildFragmentNotCheckExchangeNode(setOperationNode, i, setOperationFragment, - childrenFragments.get(i), - context); + return inputFragment; + } + + @Override + public PlanFragment visitPhysicalTopN(PhysicalTopN topN, PlanTranslatorContext context) { + PlanFragment inputFragment = topN.child(0).accept(this, context); + + // 2. According to the type of sort, generate physical plan + if (!topN.getSortPhase().isMerge()) { + // For localSort or Gather->Sort, we just need to add TopNNode + SortNode sortNode = translateSortNode(topN, inputFragment.getPlanRoot(), context); + sortNode.setOffset(topN.getOffset()); + sortNode.setLimit(topN.getLimit()); + if (topN.getMutableState(PhysicalTopN.TOPN_RUNTIME_FILTER).isPresent()) { + sortNode.setUseTopnOpt(true); + PlanNode child = sortNode.getChild(0); + Preconditions.checkArgument(child instanceof OlapScanNode, + "topN opt expect OlapScanNode, but we get " + child); + OlapScanNode scanNode = ((OlapScanNode) child); + scanNode.setUseTopnOpt(true); + } + // push sort to scan opt + if (sortNode.getChild(0) instanceof OlapScanNode) { + OlapScanNode scanNode = ((OlapScanNode) sortNode.getChild(0)); + if (checkPushSort(sortNode, scanNode.getOlapTable())) { + SortInfo sortInfo = sortNode.getSortInfo(); + scanNode.setSortInfo(sortInfo); + scanNode.getSortInfo().setSortTupleSlotExprs(sortNode.getResolvedTupleExprs()); + for (Expr expr : sortInfo.getOrderingExprs()) { + scanNode.getSortInfo().addMaterializedOrderingExpr(expr); + } + if (sortNode.getOffset() > 0) { + scanNode.setSortLimit(sortNode.getLimit() + sortNode.getOffset()); + } else { + scanNode.setSortLimit(sortNode.getLimit()); + } + } } + addPlanRoot(inputFragment, sortNode, topN); } else { - setOperationFragment = createPlanFragment(setOperationNode, - new DataPartition(TPartitionType.HASH_PARTITIONED, - setOperationNode.getMaterializedResultExprLists().get(0)), setOperation); - for (int i = 0; i < childrenFragments.size(); ++i) { - PlanFragment childFragment = childrenFragments.get(i); - // Connect the unpartitioned child fragments to SetOperationNode via a random exchange. - connectChildFragmentNotCheckExchangeNode( - setOperationNode, i, setOperationFragment, childFragment, context); - childFragment.setOutputPartition( - DataPartition.hashPartitioned(setOperationNode.getMaterializedResultExprLists().get(i))); + // For mergeSort, we need to push sortInfo to exchangeNode + if (!(inputFragment.getPlanRoot() instanceof ExchangeNode)) { + // if there is no exchange node for mergeSort + // e.g., mergeTopN -> localTopN + // It means the local has satisfied the Gather property. We can just ignore mergeSort + inputFragment.getPlanRoot().setOffset(topN.getOffset()); + inputFragment.getPlanRoot().setLimit(topN.getLimit()); + return inputFragment; } + ExchangeNode exchangeNode = (ExchangeNode) inputFragment.getPlanRoot(); + exchangeNode.setMergeInfo(((SortNode) exchangeNode.getChild(0)).getSortInfo()); + exchangeNode.setLimit(topN.getLimit()); + exchangeNode.setOffset(topN.getOffset()); } - context.addPlanFragment(setOperationFragment); - return setOperationFragment; + updateLegacyPlanIdToPhysicalPlan(inputFragment.getPlanRoot(), topN); + return inputFragment; } @Override - public PlanFragment visitPhysicalGenerate(PhysicalGenerate generate, - PlanTranslatorContext context) { - PlanFragment currentFragment = generate.child().accept(this, context); - ArrayList functionCalls = generate.getGenerators().stream() - .map(e -> ExpressionTranslator.translate(e, context)) - .collect(Collectors.toCollection(ArrayList::new)); - TupleDescriptor tupleDescriptor = generateTupleDesc(generate.getGeneratorOutput(), null, context); - List childOutputTupleIds = currentFragment.getPlanRoot().getOutputTupleIds(); - if (childOutputTupleIds == null || childOutputTupleIds.isEmpty()) { - childOutputTupleIds = currentFragment.getPlanRoot().getTupleIds(); - } - List outputSlotIds = Stream.concat(childOutputTupleIds.stream(), - Stream.of(tupleDescriptor.getId())) - .map(id -> context.getTupleDesc(id).getSlots()) - .flatMap(List::stream) - .map(SlotDescriptor::getId) - .collect(Collectors.toList()); - TableFunctionNode tableFunctionNode = new TableFunctionNode(context.nextPlanNodeId(), - currentFragment.getPlanRoot(), tupleDescriptor.getId(), functionCalls, outputSlotIds); - addPlanRoot(currentFragment, tableFunctionNode, generate); - return currentFragment; - } + public PlanFragment visitPhysicalRepeat(PhysicalRepeat repeat, PlanTranslatorContext context) { + PlanFragment inputPlanFragment = repeat.child(0).accept(this, context); - @Override - public PlanFragment visitPhysicalCTEConsumer(PhysicalCTEConsumer cteConsumer, - PlanTranslatorContext context) { - CTEId cteId = cteConsumer.getCteId(); + Set sortedVirtualSlots = repeat.getSortedVirtualSlots(); + TupleDescriptor virtualSlotsTuple = + generateTupleDesc(ImmutableList.copyOf(sortedVirtualSlots), null, context); - MultiCastPlanFragment multCastFragment = (MultiCastPlanFragment) context.getCteProduceFragments().get(cteId); - Preconditions.checkState(multCastFragment.getSink() instanceof MultiCastDataSink, - "invalid multCastFragment"); + ImmutableSet flattenGroupingSetExprs = ImmutableSet.copyOf( + ExpressionUtils.flatExpressions(repeat.getGroupingSets())); - MultiCastDataSink multiCastDataSink = (MultiCastDataSink) multCastFragment.getSink(); - Preconditions.checkState(multiCastDataSink != null, "invalid multiCastDataSink"); + List aggregateFunctionUsedSlots = repeat.getOutputExpressions() + .stream() + .filter(output -> !(output instanceof VirtualSlotReference)) + .filter(output -> !flattenGroupingSetExprs.contains(output)) + .distinct() + .map(NamedExpression::toSlot) + .collect(ImmutableList.toImmutableList()); - PhysicalCTEProducer cteProducer = context.getCteProduceMap().get(cteId); - Preconditions.checkState(cteProducer != null, "invalid cteProducer"); + Set usedSlotInRepeat = ImmutableSet.builder() + .addAll(flattenGroupingSetExprs) + .addAll(aggregateFunctionUsedSlots) + .build(); - ExchangeNode exchangeNode = new ExchangeNode(context.nextPlanNodeId(), multCastFragment.getPlanRoot()); + List preRepeatExprs = usedSlotInRepeat.stream() + .map(expr -> ExpressionTranslator.translate(expr, context)) + .collect(ImmutableList.toImmutableList()); - DataStreamSink streamSink = new DataStreamSink(exchangeNode.getId()); - streamSink.setPartition(DataPartition.RANDOM); - streamSink.setFragment(multCastFragment); + List outputSlots = repeat.getOutputExpressions() + .stream() + .map(NamedExpression::toSlot) + .collect(ImmutableList.toImmutableList()); - multiCastDataSink.getDataStreamSinks().add(streamSink); - multiCastDataSink.getDestinations().add(Lists.newArrayList()); + // NOTE: we should first translate preRepeatExprs, then generate output tuple, + // or else the preRepeatExprs can not find the bottom slotRef and throw + // exception: invalid slot id + TupleDescriptor outputTuple = generateTupleDesc(outputSlots, null, context); - exchangeNode.setNumInstances(multCastFragment.getPlanRoot().getNumInstances()); + // cube and rollup already convert to grouping sets in LogicalPlanBuilder.withAggregate() + GroupingInfo groupingInfo = new GroupingInfo( + GroupingType.GROUPING_SETS, virtualSlotsTuple, outputTuple, preRepeatExprs); - PlanFragment consumeFragment = new PlanFragment(context.nextFragmentId(), exchangeNode, - multCastFragment.getDataPartition()); + List> repeatSlotIdList = repeat.computeRepeatSlotIdList(getSlotIds(outputTuple)); + Set allSlotId = repeatSlotIdList.stream() + .flatMap(Set::stream) + .collect(ImmutableSet.toImmutableSet()); - Map projectMap = Maps.newHashMap(); - projectMap.putAll(cteConsumer.getProducerToConsumerSlotMap()); + RepeatNode repeatNode = new RepeatNode(context.nextPlanNodeId(), + inputPlanFragment.getPlanRoot(), groupingInfo, repeatSlotIdList, + allSlotId, repeat.computeVirtualSlotValues(sortedVirtualSlots)); + repeatNode.setNumInstances(inputPlanFragment.getPlanRoot().getNumInstances()); + addPlanRoot(inputPlanFragment, repeatNode, repeat); + updateLegacyPlanIdToPhysicalPlan(inputPlanFragment.getPlanRoot(), repeat); + return inputPlanFragment; + } - List execList = new ArrayList<>(); - PlanNode inputPlanNode = consumeFragment.getPlanRoot(); - List cteProjects = cteProducer.getProjects(); - for (Slot slot : cteProjects) { - if (projectMap.containsKey(slot)) { - execList.add(projectMap.get(slot)); - } else { - throw new RuntimeException("could not find slot in cte producer consumer projectMap"); - } - } + @Override + public PlanFragment visitPhysicalWindow(PhysicalWindow physicalWindow, + PlanTranslatorContext context) { + PlanFragment inputPlanFragment = physicalWindow.child(0).accept(this, context); - List slotList = execList - .stream() - .map(e -> e.toSlot()) - .collect(Collectors.toList()); + // 1. translate to old optimizer variable + // variable in Nereids + WindowFrameGroup windowFrameGroup = physicalWindow.getWindowFrameGroup(); + List partitionKeyList = Lists.newArrayList(windowFrameGroup.getPartitionKeys()); + List orderKeyList = windowFrameGroup.getOrderKeys(); + List windowFunctionList = windowFrameGroup.getGroups(); + WindowFrame windowFrame = windowFrameGroup.getWindowFrame(); - TupleDescriptor tupleDescriptor = generateTupleDesc(slotList, null, context); + // partition by clause + List partitionExprs = partitionKeyList.stream() + .map(e -> ExpressionTranslator.translate(e, context)) + .collect(Collectors.toList()); - // update tuple list and tblTupleList - inputPlanNode.getTupleIds().clear(); - inputPlanNode.getTupleIds().add(tupleDescriptor.getId()); - inputPlanNode.getTblRefIds().clear(); - inputPlanNode.getTblRefIds().add(tupleDescriptor.getId()); - inputPlanNode.getNullableTupleIds().clear(); - inputPlanNode.getNullableTupleIds().add(tupleDescriptor.getId()); + // order by clause + List orderByElements = orderKeyList.stream() + .map(orderKey -> new OrderByElement( + ExpressionTranslator.translate(orderKey.child(), context), + orderKey.isAsc(), orderKey.isNullFirst())) + .collect(Collectors.toList()); - List execExprList = execList - .stream() - .map(e -> ExpressionTranslator.translate(e, context)) + // function calls + List analyticFnCalls = windowFunctionList.stream() + .map(e -> { + Expression function = e.child(0).child(0); + if (function instanceof AggregateFunction) { + AggregateParam param = AggregateParam.localResult(); + function = new AggregateExpression((AggregateFunction) function, param); + } + return ExpressionTranslator.translate(function, context); + }) + .map(FunctionCallExpr.class::cast) + .peek(fnCall -> { + fnCall.setIsAnalyticFnCall(true); + ((org.apache.doris.catalog.AggregateFunction) fnCall.getFn()).setIsAnalyticFn(true); + }) .collect(Collectors.toList()); - inputPlanNode.setProjectList(execExprList); - inputPlanNode.setOutputTupleDesc(tupleDescriptor); + // analytic window + AnalyticWindow analyticWindow = physicalWindow.translateWindowFrame(windowFrame, context); - // update data partition - DataPartition dataPartition = new DataPartition(TPartitionType.HASH_PARTITIONED, execExprList); - consumeFragment.setDataPartition(dataPartition); + // 2. get bufferedTupleDesc from SortNode and compute isNullableMatched + Map bufferedSlotRefForWindow = getBufferedSlotRefForWindow(windowFrameGroup, context); + TupleDescriptor bufferedTupleDesc = context.getBufferedTupleForWindow(); - SelectNode projectNode = new SelectNode(context.nextPlanNodeId(), inputPlanNode); - consumeFragment.setPlanRoot(projectNode); + // generate predicates to check if the exprs of partitionKeys and orderKeys have matched isNullable between + // sortNode and analyticNode + Expr partitionExprsIsNullableMatched = partitionExprs.isEmpty() ? null : windowExprsHaveMatchedNullable( + partitionKeyList, partitionExprs, bufferedSlotRefForWindow); - multCastFragment.getDestNodeList().add(exchangeNode); - consumeFragment.addChild(multCastFragment); - context.getPlanFragments().add(consumeFragment); + Expr orderElementsIsNullableMatched = orderByElements.isEmpty() ? null : windowExprsHaveMatchedNullable( + orderKeyList.stream().map(UnaryNode::child).collect(Collectors.toList()), + orderByElements.stream().map(OrderByElement::getExpr).collect(Collectors.toList()), + bufferedSlotRefForWindow); - return consumeFragment; + // 3. generate tupleDesc + List windowSlotList = windowFunctionList.stream() + .map(NamedExpression::toSlot) + .collect(Collectors.toList()); + TupleDescriptor outputTupleDesc = generateTupleDesc(windowSlotList, null, context); + + // 4. generate AnalyticEvalNode + AnalyticEvalNode analyticEvalNode = new AnalyticEvalNode( + context.nextPlanNodeId(), + inputPlanFragment.getPlanRoot(), + analyticFnCalls, + partitionExprs, + orderByElements, + analyticWindow, + outputTupleDesc, + outputTupleDesc, + partitionExprsIsNullableMatched, + orderElementsIsNullableMatched, + bufferedTupleDesc + ); + analyticEvalNode.setNumInstances(inputPlanFragment.getPlanRoot().getNumInstances()); + inputPlanFragment.addPlanRoot(analyticEvalNode); + return inputPlanFragment; } - @Override - public PlanFragment visitPhysicalCTEProducer(PhysicalCTEProducer cteProducer, - PlanTranslatorContext context) { - PlanFragment child = cteProducer.child().accept(this, context); - CTEId cteId = cteProducer.getCteId(); - context.getPlanFragments().remove(child); - MultiCastPlanFragment cteProduce = new MultiCastPlanFragment(child); - MultiCastDataSink multiCastDataSink = new MultiCastDataSink(); - cteProduce.setSink(multiCastDataSink); + /* ******************************************************************************************** + * private functions + * ******************************************************************************************** */ - List outputs = cteProducer.getProjects().stream() + private PartitionSortNode translatePartitionSortNode(PhysicalPartitionTopN partitionTopN, + PlanNode childNode, PlanTranslatorContext context) { + // Generate the SortInfo, similar to 'translateSortNode'. + List oldOrderingExprList = Lists.newArrayList(); + List ascOrderList = Lists.newArrayList(); + List nullsFirstParamList = Lists.newArrayList(); + List orderKeyList = partitionTopN.getOrderKeys(); + // 1. Get previous slotRef + orderKeyList.forEach(k -> { + oldOrderingExprList.add(ExpressionTranslator.translate(k.getExpr(), context)); + ascOrderList.add(k.isAsc()); + nullsFirstParamList.add(k.isNullFirst()); + }); + List sortTupleOutputList = new ArrayList<>(); + List outputList = partitionTopN.getOutput(); + outputList.forEach(k -> sortTupleOutputList.add(ExpressionTranslator.translate(k, context))); + List partitionExprs = partitionTopN.getPartitionKeys().stream() .map(e -> ExpressionTranslator.translate(e, context)) .collect(Collectors.toList()); + // 2. Generate new Tuple and get current slotRef for newOrderingExprList + List newOrderingExprList = Lists.newArrayList(); + TupleDescriptor tupleDesc = generateTupleDesc(outputList, orderKeyList, newOrderingExprList, context, null); + // 3. fill in SortInfo members + SortInfo sortInfo = new SortInfo(newOrderingExprList, ascOrderList, nullsFirstParamList, tupleDesc); - cteProduce.setOutputExprs(outputs); - context.getCteProduceFragments().put(cteId, cteProduce); - context.getCteProduceMap().put(cteId, cteProducer); - context.getPlanFragments().add(cteProduce); - return child; - } + PartitionSortNode partitionSortNode = new PartitionSortNode(context.nextPlanNodeId(), childNode, + partitionTopN.getFunction(), partitionExprs, sortInfo, partitionTopN.hasGlobalLimit(), + partitionTopN.getPartitionLimit(), sortTupleOutputList, oldOrderingExprList); - /** - * NOTICE: Must translate left, which it's the producer of consumer. - */ - @Override - public PlanFragment visitPhysicalCTEAnchor(PhysicalCTEAnchor cteAnchor, - PlanTranslatorContext context) { - cteAnchor.child(0).accept(this, context); - return cteAnchor.child(1).accept(this, context); + if (partitionTopN.getStats() != null) { + partitionSortNode.setCardinality((long) partitionTopN.getStats().getRowCount()); + } + updateLegacyPlanIdToPhysicalPlan(partitionSortNode, partitionTopN); + return partitionSortNode; } - private List castCommonDataTypeOutputs(List outputs, List childOutputs) { - List newChildOutputs = new ArrayList<>(); - for (int i = 0; i < outputs.size(); ++i) { - Slot right = childOutputs.get(i); - DataType tightestCommonType = outputs.get(i).getDataType(); - Expression newRight = TypeCoercionUtils.castIfNotMatchType(right, tightestCommonType); - newChildOutputs.add(newRight); + private SortNode translateSortNode(AbstractPhysicalSort sort, PlanNode childNode, + PlanTranslatorContext context) { + List oldOrderingExprList = Lists.newArrayList(); + List ascOrderList = Lists.newArrayList(); + List nullsFirstParamList = Lists.newArrayList(); + List orderKeyList = sort.getOrderKeys(); + // 1. Get previous slotRef + orderKeyList.forEach(k -> { + oldOrderingExprList.add(ExpressionTranslator.translate(k.getExpr(), context)); + ascOrderList.add(k.isAsc()); + nullsFirstParamList.add(k.isNullFirst()); + }); + List sortTupleOutputList = new ArrayList<>(); + List outputList = sort.getOutput(); + outputList.forEach(k -> sortTupleOutputList.add(ExpressionTranslator.translate(k, context))); + // 2. Generate new Tuple and get current slotRef for newOrderingExprList + List newOrderingExprList = Lists.newArrayList(); + TupleDescriptor tupleDesc = generateTupleDesc(outputList, orderKeyList, newOrderingExprList, context, null); + // 3. fill in SortInfo members + SortInfo sortInfo = new SortInfo(newOrderingExprList, ascOrderList, nullsFirstParamList, tupleDesc); + SortNode sortNode = new SortNode(context.nextPlanNodeId(), childNode, sortInfo, true); + sortNode.finalizeForNereids(tupleDesc, sortTupleOutputList, oldOrderingExprList); + if (sort.getMutableState(PhysicalTopN.TWO_PHASE_READ_OPT).isPresent()) { + sortNode.setUseTwoPhaseReadOpt(true); + sortNode.getSortInfo().setUseTwoPhaseRead(); + injectRowIdColumnSlot(sortNode.getSortInfo().getSortTupleDescriptor()); + TupleDescriptor childTuple = childNode.getOutputTupleDesc() != null + ? childNode.getOutputTupleDesc() : context.getTupleDesc(childNode.getTupleIds().get(0)); + SlotDescriptor childRowIdDesc = childTuple.getSlots().get(childTuple.getSlots().size() - 1); + sortNode.getResolvedTupleExprs().add(new SlotRef(childRowIdDesc)); } - return ImmutableList.copyOf(newChildOutputs); - } - - private SetOperationResult collectSetOperationResult( - PhysicalSetOperation setOperation, Map childPlanToFragment) { - List> resultExprs = new ArrayList<>(); - List> constExprs = new ArrayList<>(); - List outputs = setOperation.getOutput(); - for (Plan child : setOperation.children()) { - List castCommonDataTypeOutputs = castCommonDataTypeOutputs(outputs, child.getOutput()); - if (child.anyMatch(PhysicalOneRowRelation.class::isInstance) && childPlanToFragment.get(child) == null) { - constExprs.add(collectConstExpressions(castCommonDataTypeOutputs, child)); - } else { - resultExprs.add(castCommonDataTypeOutputs); - } + if (sort.getStats() != null) { + sortNode.setCardinality((long) sort.getStats().getRowCount()); } - return new SetOperationResult(resultExprs, constExprs); + updateLegacyPlanIdToPhysicalPlan(sortNode, sort); + return sortNode; } - private List collectConstExpressions( - List castExpressions, Plan child) { - List newCastExpressions = new ArrayList<>(); - for (int i = 0; i < castExpressions.size(); ++i) { - Expression expression = castExpressions.get(i); - if (expression instanceof Cast) { - newCastExpressions.add(expression.withChildren( - (collectPhysicalOneRowRelation(child).getProjects().get(i).children()))); - } else { - newCastExpressions.add( - (collectPhysicalOneRowRelation(child).getProjects().get(i))); + private void updateChildSlotsMaterialization(PlanNode execPlan, + Set requiredSlotIdSet, Set requiredByProjectSlotIdSet, + PlanTranslatorContext context) { + Set slotRefSet = new HashSet<>(); + for (Expr expr : execPlan.getConjuncts()) { + expr.collect(SlotRef.class, slotRefSet); + } + Set slotIdSet = slotRefSet.stream() + .map(SlotRef::getSlotId).collect(Collectors.toSet()); + slotIdSet.addAll(requiredSlotIdSet); + boolean noneMaterialized = execPlan.getTupleIds().stream() + .map(context::getTupleDesc) + .map(TupleDescriptor::getSlots) + .flatMap(List::stream) + .peek(s -> s.setIsMaterialized(slotIdSet.contains(s.getId()))) + .filter(SlotDescriptor::isMaterialized) + .count() == 0; + if (noneMaterialized) { + context.getDescTable() + .getTupleDesc(execPlan.getTupleIds().get(0)).getSlots().get(0).setIsMaterialized(true); + } + if (execPlan instanceof ScanNode) { + try { + ((ScanNode) execPlan).updateRequiredSlots(context, requiredByProjectSlotIdSet); + } catch (UserException e) { + Util.logAndThrowRuntimeException(LOG, + "User Exception while reset external file scan node contexts.", e); } } - return newCastExpressions; - } - - private PhysicalOneRowRelation collectPhysicalOneRowRelation(Plan child) { - return (PhysicalOneRowRelation) - ((ImmutableSet) child.collect(PhysicalOneRowRelation.class::isInstance)).asList().get(0); } - private boolean allChildFragmentsUnPartitioned(List childrenFragments) { - boolean allChildFragmentsUnPartitioned = true; - for (PlanFragment child : childrenFragments) { - allChildFragmentsUnPartitioned = allChildFragmentsUnPartitioned && !child.isPartitioned(); - } - return allChildFragmentsUnPartitioned; + private void addConjunctsToPlanNode(PhysicalFilter filter, + PlanNode planNode, + PlanTranslatorContext context) { + filter.getConjuncts().stream() + .map(e -> ExpressionTranslator.translate(e, context)) + .forEach(planNode::addConjunct); + updateLegacyPlanIdToPhysicalPlan(planNode, filter); } private void extractExecSlot(Expr root, Set slotIdList) { @@ -2249,260 +1990,46 @@ private TupleDescriptor generateTupleDesc(List slotList, List or return tupleDescriptor; } - private PlanFragment createParentFragment(PlanFragment childFragment, DataPartition parentPartition, - PlanTranslatorContext context) { - ExchangeNode exchangeNode = new ExchangeNode(context.nextPlanNodeId(), childFragment.getPlanRoot()); - exchangeNode.setNumInstances(childFragment.getPlanRoot().getNumInstances()); - PlanFragment parentFragment = new PlanFragment(context.nextFragmentId(), exchangeNode, parentPartition); - childFragment.setDestination(exchangeNode); - childFragment.setOutputPartition(parentPartition); - context.addPlanFragment(parentFragment); - return parentFragment; - } - - private void connectChildFragment(PlanNode parent, int childIdx, - PlanFragment parentFragment, PlanFragment childFragment, - PlanTranslatorContext context) { - PlanNode exchange = parent.getChild(childIdx); - if (!(exchange instanceof ExchangeNode)) { - exchange = new ExchangeNode(context.nextPlanNodeId(), childFragment.getPlanRoot()); - exchange.setNumInstances(childFragment.getPlanRoot().getNumInstances()); - } - childFragment.setPlanRoot(exchange.getChild(0)); - exchange.setFragment(parentFragment); - parent.setChild(childIdx, exchange); - childFragment.setDestination((ExchangeNode) exchange); - } - - private void connectChildFragmentNotCheckExchangeNode(PlanNode parent, int childIdx, - PlanFragment parentFragment, PlanFragment childFragment, - PlanTranslatorContext context) { - PlanNode exchange = new ExchangeNode(context.nextPlanNodeId(), childFragment.getPlanRoot()); - exchange.setNumInstances(childFragment.getPlanRoot().getNumInstances()); - childFragment.setPlanRoot(exchange.getChild(0)); - exchange.setFragment(parentFragment); - parent.setChild(childIdx, exchange); - childFragment.setDestination((ExchangeNode) exchange); - } - - /** - * Return unpartitioned fragment that merges the input fragment's output via - * an ExchangeNode. - * Requires that input fragment be partitioned. - */ - private PlanFragment exchangeToMergeFragment(PlanFragment inputFragment, PlanTranslatorContext context) { - if (!inputFragment.isPartitioned()) { - return inputFragment; - } - - // exchange node clones the behavior of its input, aside from the conjuncts - ExchangeNode mergePlan = new ExchangeNode(context.nextPlanNodeId(), inputFragment.getPlanRoot()); - DataPartition dataPartition = DataPartition.UNPARTITIONED; - mergePlan.setNumInstances(inputFragment.getPlanRoot().getNumInstances()); - PlanFragment fragment = new PlanFragment(context.nextFragmentId(), mergePlan, dataPartition); - inputFragment.setDestination(mergePlan); - context.addPlanFragment(fragment); - return fragment; - } - - private PlanFragment constructColocateJoin(HashJoinNode hashJoinNode, PlanFragment leftFragment, + private PlanFragment connectJoinNode(HashJoinNode hashJoinNode, PlanFragment leftFragment, PlanFragment rightFragment, PlanTranslatorContext context, AbstractPlan join) { - // TODO: add reason - hashJoinNode.setColocate(true, ""); hashJoinNode.setChild(0, leftFragment.getPlanRoot()); hashJoinNode.setChild(1, rightFragment.getPlanRoot()); setPlanRoot(leftFragment, hashJoinNode, join); - rightFragment.getTargetRuntimeFilterIds().forEach(leftFragment::setTargetRuntimeFilterIds); - rightFragment.getBuilderRuntimeFilterIds().forEach(leftFragment::setBuilderRuntimeFilterIds); - context.removePlanFragment(rightFragment); - leftFragment.setHasColocatePlanNode(true); - return leftFragment; - } - - private PlanFragment constructBucketShuffleJoin(PhysicalHashJoin physicalHashJoin, - HashJoinNode hashJoinNode, PlanFragment leftFragment, - PlanFragment rightFragment, PlanTranslatorContext context) { - // according to left partition to generate right partition expr list - DistributionSpecHash leftDistributionSpec - = (DistributionSpecHash) physicalHashJoin.left().getPhysicalProperties().getDistributionSpec(); - - Pair, List> onClauseUsedSlots = physicalHashJoin.getHashConjunctsExprIds(); - List rightPartitionExprIds = Lists.newArrayList(leftDistributionSpec.getOrderedShuffledColumns()); - for (int i = 0; i < leftDistributionSpec.getOrderedShuffledColumns().size(); i++) { - int idx = leftDistributionSpec.getExprIdToEquivalenceSet() - .get(leftDistributionSpec.getOrderedShuffledColumns().get(i)); - ExprId leftShuffleColumnId = leftDistributionSpec.getOrderedShuffledColumns().get(i); - Set equivalIds = leftDistributionSpec.getEquivalenceExprIdsOf(leftShuffleColumnId); - int index = -1; - for (ExprId id : equivalIds) { - index = onClauseUsedSlots.first.indexOf(id); - if (index != -1) { - break; - } - } - Preconditions.checkArgument(index != -1); - rightPartitionExprIds.set(idx, onClauseUsedSlots.second.get(index)); - } - // assemble fragment - hashJoinNode.setDistributionMode(HashJoinNode.DistributionMode.BUCKET_SHUFFLE); - if (leftDistributionSpec.getShuffleType() != ShuffleType.NATURAL - && leftDistributionSpec.getShuffleType() != ShuffleType.BUCKETED) { - hashJoinNode.setDistributionMode(DistributionMode.PARTITIONED); - } - connectChildFragment(hashJoinNode, 1, leftFragment, rightFragment, context); - setPlanRoot(leftFragment, hashJoinNode, physicalHashJoin); - // HASH_PARTITIONED and BUCKET_SHFFULE_HASH_PARTITIONED are two type of hash algorithm - // And the nature left child means it use BUCKET_SHFFULE_HASH_PARTITIONED in storage layer - TPartitionType partitionType = TPartitionType.BUCKET_SHFFULE_HASH_PARTITIONED; - if (leftDistributionSpec.getShuffleType() != ShuffleType.NATURAL) { - partitionType = TPartitionType.HASH_PARTITIONED; + context.mergePlanFragment(rightFragment, leftFragment); + for (PlanFragment rightChild : rightFragment.getChildren()) { + leftFragment.addChild(rightChild); } - DataPartition rhsJoinPartition = new DataPartition(partitionType, - rightPartitionExprIds.stream().map(context::findSlotRef).collect(Collectors.toList())); - rightFragment.setOutputPartition(rhsJoinPartition); - - return leftFragment; - } - - private PlanFragment constructBroadcastJoin(HashJoinNode hashJoinNode, PlanFragment leftFragment, - PlanFragment rightFragment, PlanTranslatorContext context, AbstractPlan join) { - hashJoinNode.setDistributionMode(DistributionMode.BROADCAST); - setPlanRoot(leftFragment, hashJoinNode, join); - rightFragment.setRightChildOfBroadcastHashJoin(true); - connectChildFragment(hashJoinNode, 1, leftFragment, rightFragment, context); return leftFragment; } - private PlanFragment constructShuffleJoin(AbstractPhysicalJoin physicalHashJoin, - HashJoinNode hashJoinNode, PlanFragment leftFragment, - PlanFragment rightFragment, PlanTranslatorContext context) { - hashJoinNode.setDistributionMode(HashJoinNode.DistributionMode.PARTITIONED); - // TODO should according nereids distribute indicate - // first, extract join exprs - List eqJoinConjuncts = hashJoinNode.getEqJoinConjuncts(); - List lhsJoinExprs = Lists.newArrayList(); - List rhsJoinExprs = Lists.newArrayList(); - for (BinaryPredicate eqJoinPredicate : eqJoinConjuncts) { - // no remapping necessary - lhsJoinExprs.add(eqJoinPredicate.getChild(0).clone(null)); - rhsJoinExprs.add(eqJoinPredicate.getChild(1).clone(null)); - } - - // create the parent fragment containing the HashJoin node - DataPartition lhsJoinPartition = new DataPartition(TPartitionType.HASH_PARTITIONED, - Expr.cloneList(lhsJoinExprs, null)); - DataPartition rhsJoinPartition = - new DataPartition(TPartitionType.HASH_PARTITIONED, rhsJoinExprs); - PlanFragment joinFragment = createPlanFragment(hashJoinNode, lhsJoinPartition, physicalHashJoin); - context.addPlanFragment(joinFragment); - - connectChildFragment(hashJoinNode, 0, joinFragment, leftFragment, context); - connectChildFragment(hashJoinNode, 1, joinFragment, rightFragment, context); - - leftFragment.setOutputPartition(lhsJoinPartition); - rightFragment.setOutputPartition(rhsJoinPartition); - - return joinFragment; - } - - private List collectGroupBySlots(List groupByExpressionList, - List outputExpressionList) { - List groupSlotList = Lists.newArrayList(); - Set virtualSlotReferences = groupByExpressionList.stream() + private List collectGroupBySlots(List groupByExpressions, + List outputExpressions) { + List groupSlots = Lists.newArrayList(); + Set virtualSlotReferences = groupByExpressions.stream() .filter(VirtualSlotReference.class::isInstance) .map(VirtualSlotReference.class::cast) .collect(Collectors.toSet()); - for (Expression e : groupByExpressionList) { - if (e instanceof SlotReference && outputExpressionList.stream().anyMatch(o -> o.anyMatch(e::equals))) { - groupSlotList.add((SlotReference) e); + for (Expression e : groupByExpressions) { + if (e instanceof SlotReference && outputExpressions.stream().anyMatch(o -> o.anyMatch(e::equals))) { + groupSlots.add((SlotReference) e); } else if (e instanceof SlotReference && !virtualSlotReferences.isEmpty()) { // When there is a virtualSlot, it is a groupingSets scenario, // and the original exprId should be retained at this time. - groupSlotList.add((SlotReference) e); + groupSlots.add((SlotReference) e); } else { - groupSlotList.add(new SlotReference(e.toSql(), e.getDataType(), e.nullable(), ImmutableList.of())); + groupSlots.add(new SlotReference(e.toSql(), e.getDataType(), e.nullable(), ImmutableList.of())); } } - return groupSlotList; + return groupSlots; } - private List getSlotIdList(TupleDescriptor tupleDescriptor) { + private List getSlotIds(TupleDescriptor tupleDescriptor) { return tupleDescriptor.getSlots() .stream() .map(slot -> slot.getId().asInt()) .collect(ImmutableList.toImmutableList()); } - private boolean isUnnecessaryProject(PhysicalProject project) { - // The project list for agg is always needed,since tuple of agg contains the slots used by group by expr - return !hasPrune(project) && !hasExprCalc(project); - } - - private boolean hasPrune(PhysicalProject project) { - PhysicalPlan child = (PhysicalPlan) project.child(0); - - return project.getProjects().size() != child.getOutput().size(); - } - - private boolean isFragmentPartitioned(PlanFragment fragment) { - return fragment.isPartitioned() && fragment.getPlanRoot().getNumInstances() > 1; - } - - private boolean hasExprCalc(PhysicalProject project) { - for (NamedExpression p : project.getProjects()) { - if (p.children().size() > 1) { - return true; - } - for (Expression e : p.children()) { - if (!(e instanceof SlotReference)) { - return true; - } - } - } - return false; - } - - private List removeAlias(PhysicalProject project) { - List namedExpressions = project.getProjects(); - List slotReferences = new ArrayList<>(); - for (NamedExpression n : namedExpressions) { - if (n instanceof Alias) { - slotReferences.add((SlotReference) n.child(0)); - } else { - slotReferences.add((SlotReference) n); - } - } - return slotReferences; - } - - private Optional toDataPartition(PhysicalDistribute distribute, - Optional> partitionExpressions, PlanTranslatorContext context) { - if (distribute.getDistributionSpec() == DistributionSpecGather.INSTANCE) { - return Optional.of(DataPartition.UNPARTITIONED); - } else if (distribute.getDistributionSpec() == DistributionSpecReplicated.INSTANCE) { - // the data partition should be left child of join - return Optional.empty(); - } else if (distribute.getDistributionSpec() instanceof DistributionSpecHash - || distribute.getDistributionSpec() == DistributionSpecAny.INSTANCE) { - if (!partitionExpressions.isPresent()) { - throw new AnalysisException("Missing partition expressions"); - } - Preconditions.checkState( - partitionExpressions.get().stream().allMatch(expr -> expr instanceof SlotReference), - "All partition expression should be slot: " + partitionExpressions.get()); - if (!partitionExpressions.isPresent() || partitionExpressions.get().isEmpty()) { - return Optional.of(DataPartition.UNPARTITIONED); - } - List partitionExprs = partitionExpressions.get() - .stream() - .map(p -> ExpressionTranslator.translate(p, context)) - .collect(ImmutableList.toImmutableList()); - return Optional.of(new DataPartition(TPartitionType.HASH_PARTITIONED, partitionExprs)); - } else { - return Optional.empty(); - } - } - private Map getBufferedSlotRefForWindow(WindowFrameGroup windowFrameGroup, PlanTranslatorContext context) { Map bufferedSlotRefForWindow = context.getBufferedSlotRefForWindow(); @@ -2515,7 +2042,7 @@ private Map getBufferedSlotRefForWindow(WindowFrameGroup window bufferedSlotRefForWindow.putIfAbsent(exprId, context.findSlotRef(exprId)); }); windowFrameGroup.getOrderKeys().stream() - .map(ok -> ok.child()) + .map(UnaryNode::child) .map(NamedExpression.class::cast) .forEach(expression -> { ExprId exprId = expression.getExprId(); @@ -2557,56 +2084,78 @@ private Expr windowExprsHaveMatchedNullable(Map exprIdToExpr, Map< new CompoundPredicate(CompoundPredicate.Operator.OR, bothNull, lhsEqRhsNotNull), remainder); } - private DataPartition hashSpecToDataPartition(PhysicalDistribute distribute, PlanTranslatorContext context) { - Preconditions.checkState(distribute.getDistributionSpec() instanceof DistributionSpecHash); - DistributionSpecHash hashSpec = (DistributionSpecHash) distribute.getDistributionSpec(); - List partitions = hashSpec.getOrderedShuffledColumns().stream() - .map(exprId -> context.findSlotRef(exprId)) - .collect(Collectors.toList()); - return new DataPartition(TPartitionType.HASH_PARTITIONED, partitions); - } - - private static class SetOperationResult { - private final List> resultExpressions; - private final List> constExpressions; - - public SetOperationResult(List> resultExpressions, List> constExpressions) { - this.resultExpressions = ImmutableList.copyOf(resultExpressions); - this.constExpressions = ImmutableList.copyOf(constExpressions); - } - - public List> getConstExpressions() { - return constExpressions; - } - - public List> getResultExpressions() { - return resultExpressions; - } - } - private PlanFragment createPlanFragment(PlanNode planNode, DataPartition dataPartition, AbstractPlan physicalPlan) { PlanFragment planFragment = new PlanFragment(context.nextFragmentId(), planNode, dataPartition); updateLegacyPlanIdToPhysicalPlan(planNode, physicalPlan); return planFragment; } + // TODO: refactor this, call it every where is not a good way private void setPlanRoot(PlanFragment fragment, PlanNode planNode, AbstractPlan physicalPlan) { fragment.setPlanRoot(planNode); updateLegacyPlanIdToPhysicalPlan(planNode, physicalPlan); } + // TODO: refactor this, call it every where is not a good way private void addPlanRoot(PlanFragment fragment, PlanNode planNode, AbstractPlan physicalPlan) { fragment.addPlanRoot(planNode); updateLegacyPlanIdToPhysicalPlan(planNode, physicalPlan); } + private DataPartition toDataPartition(DistributionSpec distributionSpec, + List childOutputIds, PlanTranslatorContext context) { + if (distributionSpec instanceof DistributionSpecAny + || distributionSpec instanceof DistributionSpecStorageAny + || distributionSpec instanceof DistributionSpecExecutionAny) { + return DataPartition.RANDOM; + } else if (distributionSpec instanceof DistributionSpecGather + || distributionSpec instanceof DistributionSpecStorageGather + || distributionSpec instanceof DistributionSpecReplicated) { + return DataPartition.UNPARTITIONED; + } else if (distributionSpec instanceof DistributionSpecHash) { + DistributionSpecHash distributionSpecHash = (DistributionSpecHash) distributionSpec; + List partitionExprs = Lists.newArrayList(); + for (int i = 0; i < distributionSpecHash.getEquivalenceExprIds().size(); i++) { + Set equivalenceExprId = distributionSpecHash.getEquivalenceExprIds().get(i); + for (ExprId exprId : equivalenceExprId) { + if (childOutputIds.contains(exprId)) { + partitionExprs.add(context.findSlotRef(exprId)); + break; + } + } + if (partitionExprs.size() != i + 1) { + throw new RuntimeException("Cannot translate DistributionSpec to DataPartition," + + " DistributionSpec: " + distributionSpec + + ", child output: " + childOutputIds); + } + } + TPartitionType partitionType; + switch (distributionSpecHash.getShuffleType()) { + case STORAGE_BUCKETED: + partitionType = TPartitionType.BUCKET_SHFFULE_HASH_PARTITIONED; + break; + case EXECUTION_BUCKETED: + partitionType = TPartitionType.HASH_PARTITIONED; + break; + case NATURAL: + default: + throw new RuntimeException("Do not support shuffle type: " + + distributionSpecHash.getShuffleType()); + } + return new DataPartition(partitionType, partitionExprs); + } else { + throw new RuntimeException("Unknown DistributionSpec: " + distributionSpec); + } + } + + // TODO: refactor this, call it every where is not a good way private void updateLegacyPlanIdToPhysicalPlan(PlanNode planNode, AbstractPlan physicalPlan) { if (statsErrorEstimator != null) { statsErrorEstimator.updateLegacyPlanIdToPhysicalPlan(planNode, physicalPlan); } } - private SlotDescriptor injectRowIdColumnSlot(TupleDescriptor tupleDesc) { + private void injectRowIdColumnSlot(TupleDescriptor tupleDesc) { SlotDescriptor slotDesc = context.addSlotDesc(tupleDesc); LOG.debug("inject slot {}", slotDesc); String name = Column.ROWID_COL; @@ -2615,6 +2164,109 @@ private SlotDescriptor injectRowIdColumnSlot(TupleDescriptor tupleDesc) { slotDesc.setColumn(col); slotDesc.setIsNullable(false); slotDesc.setIsMaterialized(true); - return slotDesc; + } + + /** + * We use two phase read to optimize sql like: select * from tbl [where xxx = ???] [order by column1] [limit n] + * in the first phase, we add an extra column `RowId` to Block, and sort blocks in TopN nodes + * in the second phase, we have n rows, we do a fetch rpc to get all rowids data for the n rows + * and reconstruct the final block + */ + private void setResultSinkFetchOptionIfNeed() { + boolean needFetch = false; + // Only single olap table should be fetched + OlapTable fetchOlapTable = null; + OlapScanNode scanNode = null; + for (PlanFragment fragment : context.getPlanFragments()) { + PlanNode node = fragment.getPlanRoot(); + PlanNode parent = null; + // OlapScanNode is the last node. + // So, just get the last two node and check if they are SortNode and OlapScan. + while (node.getChildren().size() != 0) { + parent = node; + node = node.getChildren().get(0); + } + + // case1: general topn optimized query + if ((node instanceof OlapScanNode) && (parent instanceof SortNode)) { + SortNode sortNode = (SortNode) parent; + scanNode = (OlapScanNode) node; + if (sortNode.getUseTwoPhaseReadOpt()) { + needFetch = true; + fetchOlapTable = scanNode.getOlapTable(); + break; + } + } + } + for (PlanFragment fragment : context.getPlanFragments()) { + if (needFetch && fragment.getSink() instanceof ResultSink) { + TFetchOption fetchOption = new TFetchOption(); + fetchOption.setFetchRowStore(fetchOlapTable.storeRowColumn()); + fetchOption.setUseTwoPhaseFetch(true); + fetchOption.setNodesInfo(SystemInfoService.createAliveNodesInfo()); + if (!fetchOlapTable.storeRowColumn()) { + // Set column desc for each column + List columnsDesc = new ArrayList<>(); + scanNode.getColumnDesc(columnsDesc, null, null); + fetchOption.setColumnDesc(columnsDesc); + } + ((ResultSink) fragment.getSink()).setFetchOption(fetchOption); + break; + } + } + } + + /** + * topN opt: using storage data ordering to accelerate topn operation. + * refer pr: optimize topn query if order by columns is prefix of sort keys of table (#10694) + */ + private boolean checkPushSort(SortNode sortNode, OlapTable olapTable) { + // Ensure limit is less then threshold + if (sortNode.getLimit() <= 0 + || sortNode.getLimit() > ConnectContext.get().getSessionVariable().topnOptLimitThreshold) { + return false; + } + + // Ensure all isAscOrder is same, ande length != 0. + // Can't be zorder. + if (sortNode.getSortInfo().getIsAscOrder().stream().distinct().count() != 1 + || olapTable.isZOrderSort()) { + return false; + } + + // Tablet's order by key only can be the front part of schema. + // Like: schema: a.b.c.d.e.f.g order by key: a.b.c (no a,b,d) + // Do **prefix match** to check if order by key can be pushed down. + // olap order by key: a.b.c.d + // sort key: (a) (a,b) (a,b,c) (a,b,c,d) is ok + // (a,c) (a,c,d), (a,c,b) (a,c,f) (a,b,c,d,e)is NOT ok + List sortExprs = sortNode.getSortInfo().getOrderingExprs(); + List nullsFirsts = sortNode.getSortInfo().getNullsFirst(); + List isAscOrders = sortNode.getSortInfo().getIsAscOrder(); + if (sortExprs.size() > olapTable.getDataSortInfo().getColNum()) { + return false; + } + for (int i = 0; i < sortExprs.size(); i++) { + // table key. + Column tableKey = olapTable.getFullSchema().get(i); + // sort slot. + Expr sortExpr = sortExprs.get(i); + if (sortExpr instanceof SlotRef) { + SlotRef slotRef = (SlotRef) sortExpr; + if (tableKey.equals(slotRef.getColumn())) { + // ORDER BY DESC NULLS FIRST can not be optimized to only read file tail, + // since NULLS is at file head but data is at tail + if (tableKey.isAllowNull() && nullsFirsts.get(i) && !isAscOrders.get(i)) { + return false; + } + } else { + return false; + } + } else { + return false; + } + } + + return true; } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PlanTranslatorContext.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PlanTranslatorContext.java index 5a474b549276326..24d6d6e1f76bad6 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PlanTranslatorContext.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PlanTranslatorContext.java @@ -137,8 +137,10 @@ public void addExprIdSlotRefPair(ExprId exprId, SlotRef slotRef) { slotIdToExprId.put(slotRef.getDesc().getId(), exprId); } - public void removePlanFragment(PlanFragment planFragment) { - this.planFragments.remove(planFragment); + public void mergePlanFragment(PlanFragment srcFragment, PlanFragment targetFragment) { + srcFragment.getTargetRuntimeFilterIds().forEach(targetFragment::setTargetRuntimeFilterIds); + srcFragment.getBuilderRuntimeFilterIds().forEach(targetFragment::setBuilderRuntimeFilterIds); + this.planFragments.remove(srcFragment); } public SlotRef findSlotRef(ExprId exprId) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/RuntimeFilterTranslator.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/RuntimeFilterTranslator.java index da4bb2e13176ba4..3308ed35802128f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/RuntimeFilterTranslator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/RuntimeFilterTranslator.java @@ -64,6 +64,10 @@ public Set getRuntimeFilterOfHashJoinNode(AbstractPhysicalJoin jo return context.getRuntimeFilterOnHashJoinNode(join); } + public RuntimeFilterContext getContext() { + return context; + } + public List getTargetOnScanNode(ObjectId id) { return context.getTargetOnOlapScanNodeMap().getOrDefault(id, Collections.emptyList()); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/cascades/CostAndEnforcerJob.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/cascades/CostAndEnforcerJob.java index b673bd6e875631b..f6ae1d6879bd31f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/cascades/CostAndEnforcerJob.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/cascades/CostAndEnforcerJob.java @@ -222,8 +222,8 @@ private boolean calculateEnforce(List requestChildrenPropert // it's certain that lowestCostChildren is equals to arity(). ChildrenPropertiesRegulator regulator = new ChildrenPropertiesRegulator(groupExpression, lowestCostChildren, outputChildrenProperties, requestChildrenProperties, context); - double enforceCost = regulator.adjustChildrenProperties(); - if (enforceCost < 0) { + boolean success = regulator.adjustChildrenProperties(); + if (!success) { // invalid enforce, return. return false; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Analyzer.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Analyzer.java index 8fb6296fd43c225..7f089d14b1801c3 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Analyzer.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Analyzer.java @@ -23,6 +23,7 @@ import org.apache.doris.nereids.rules.analysis.BindExpression; import org.apache.doris.nereids.rules.analysis.BindInsertTargetTable; import org.apache.doris.nereids.rules.analysis.BindRelation; +import org.apache.doris.nereids.rules.analysis.BindRelation.CustomTableResolver; import org.apache.doris.nereids.rules.analysis.CheckAnalysis; import org.apache.doris.nereids.rules.analysis.CheckBound; import org.apache.doris.nereids.rules.analysis.CheckPolicy; @@ -35,9 +36,10 @@ import org.apache.doris.nereids.rules.analysis.ResolveOrdinalInOrderByAndGroupBy; import org.apache.doris.nereids.rules.analysis.SubqueryToApply; import org.apache.doris.nereids.rules.analysis.UserAuthentication; -import org.apache.doris.nereids.rules.rewrite.HideOneRowRelationUnderUnion; import java.util.List; +import java.util.Objects; +import java.util.Optional; /** * Bind symbols according to metadata in the catalog, perform semantic analysis, etc. @@ -45,12 +47,45 @@ */ public class Analyzer extends AbstractBatchJobExecutor { - public static final List ANALYZE_JOBS = jobs( + public static final List DEFAULT_ANALYZE_JOBS = buildAnalyzeJobs(Optional.empty()); + + private Optional customTableResolver; + + private List jobs; + + /** + * Execute the analysis job with scope. + * @param cascadesContext planner context for execute job + */ + public Analyzer(CascadesContext cascadesContext) { + this(cascadesContext, Optional.empty()); + } + + public Analyzer(CascadesContext cascadesContext, Optional customTableResolver) { + super(cascadesContext); + this.customTableResolver = Objects.requireNonNull(customTableResolver, "customTableResolver cannot be null"); + this.jobs = !customTableResolver.isPresent() ? DEFAULT_ANALYZE_JOBS : buildAnalyzeJobs(customTableResolver); + } + + @Override + public List getJobs() { + return jobs; + } + + /** + * nereids analyze sql. + */ + public void analyze() { + execute(); + } + + private static List buildAnalyzeJobs(Optional customTableResolver) { + return jobs( topDown( new RegisterCTE() ), bottomUp( - new BindRelation(), + new BindRelation(customTableResolver.orElse(null)), new CheckPolicy(), new UserAuthentication(), new BindExpression() @@ -70,8 +105,7 @@ public class Analyzer extends AbstractBatchJobExecutor { // please see rule BindSlotReference or BindFunction for example new ProjectWithDistinctToAggregate(), new ResolveOrdinalInOrderByAndGroupBy(), - new ReplaceExpressionByChildOutput(), - new HideOneRowRelationUnderUnion() + new ReplaceExpressionByChildOutput() ), topDown( new FillUpMissingSlots(), @@ -83,25 +117,6 @@ public class Analyzer extends AbstractBatchJobExecutor { bottomUp(new SubqueryToApply()), bottomUp(new AdjustAggregateNullableForEmptySet()), bottomUp(new CheckAnalysis()) - ); - - /** - * Execute the analysis job with scope. - * @param cascadesContext planner context for execute job - */ - public Analyzer(CascadesContext cascadesContext) { - super(cascadesContext); - } - - @Override - public List getJobs() { - return ANALYZE_JOBS; - } - - /** - * nereids analyze sql. - */ - public void analyze() { - execute(); + ); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Rewriter.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Rewriter.java index 70d7ed2e8e48b87..ba3d71d89662c46 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Rewriter.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Rewriter.java @@ -61,8 +61,10 @@ import org.apache.doris.nereids.rules.rewrite.InferFilterNotNull; import org.apache.doris.nereids.rules.rewrite.InferJoinNotNull; import org.apache.doris.nereids.rules.rewrite.InferPredicates; +import org.apache.doris.nereids.rules.rewrite.InferSetOperatorDistinct; import org.apache.doris.nereids.rules.rewrite.InlineCTE; import org.apache.doris.nereids.rules.rewrite.MergeFilters; +import org.apache.doris.nereids.rules.rewrite.MergeOneRowRelationIntoUnion; import org.apache.doris.nereids.rules.rewrite.MergeProjects; import org.apache.doris.nereids.rules.rewrite.MergeSetOperations; import org.apache.doris.nereids.rules.rewrite.NormalizeAggregate; @@ -71,6 +73,8 @@ import org.apache.doris.nereids.rules.rewrite.PruneOlapScanPartition; import org.apache.doris.nereids.rules.rewrite.PruneOlapScanTablet; import org.apache.doris.nereids.rules.rewrite.PushFilterInsideJoin; +import org.apache.doris.nereids.rules.rewrite.PushProjectIntoOneRowRelation; +import org.apache.doris.nereids.rules.rewrite.PushProjectThroughUnion; import org.apache.doris.nereids.rules.rewrite.PushdownFilterThroughProject; import org.apache.doris.nereids.rules.rewrite.PushdownFilterThroughWindow; import org.apache.doris.nereids.rules.rewrite.PushdownLimit; @@ -223,10 +227,14 @@ public class Rewriter extends AbstractBatchJobExecutor { // this rule should invoke after ColumnPruning custom(RuleType.ELIMINATE_UNNECESSARY_PROJECT, EliminateUnnecessaryProject::new), - topic("Intersection optimization", + topic("Set operation optimization", // Do MergeSetOperation first because we hope to match pattern of Distinct SetOperator. + topDown(new PushProjectThroughUnion(), new MergeProjects()), bottomUp(new MergeSetOperations()), - bottomUp(new BuildAggForUnion()) + bottomUp(new PushProjectIntoOneRowRelation()), + topDown(new MergeOneRowRelationIntoUnion()), + costBased(topDown(new InferSetOperatorDistinct())), + topDown(new BuildAggForUnion()) ), topic("Window optimization", diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/memo/GroupExpression.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/memo/GroupExpression.java index 008ed0e8173a86b..6264dd2ce667286 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/memo/GroupExpression.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/memo/GroupExpression.java @@ -248,9 +248,14 @@ public Cost getCostValueByProperties(PhysicalProperties property) { return lowestCostTable.get(property).first; } - public void putOutputPropertiesMap(PhysicalProperties outputPropertySet, - PhysicalProperties requiredPropertySet) { - this.requestPropertiesMap.put(requiredPropertySet, outputPropertySet); + public void putOutputPropertiesMap(PhysicalProperties outputProperties, + PhysicalProperties requiredProperties) { + this.requestPropertiesMap.put(requiredProperties, outputProperties); + } + + public void putOutputPropertiesMapIfAbsent(PhysicalProperties outputProperties, + PhysicalProperties requiredProperties) { + this.requestPropertiesMap.putIfAbsent(requiredProperties, outputProperties); } /** diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/RuntimeFilterContext.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/RuntimeFilterContext.java index b9d2ac301efae09..76fb3311fe5fa68 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/RuntimeFilterContext.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/RuntimeFilterContext.java @@ -20,6 +20,8 @@ import org.apache.doris.analysis.SlotRef; import org.apache.doris.common.IdGenerator; import org.apache.doris.common.Pair; +import org.apache.doris.nereids.trees.expressions.CTEId; +import org.apache.doris.nereids.trees.expressions.EqualTo; import org.apache.doris.nereids.trees.expressions.ExprId; import org.apache.doris.nereids.trees.expressions.NamedExpression; import org.apache.doris.nereids.trees.expressions.Slot; @@ -27,6 +29,7 @@ import org.apache.doris.nereids.trees.plans.ObjectId; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.physical.AbstractPhysicalJoin; +import org.apache.doris.nereids.trees.plans.physical.PhysicalCTEProducer; import org.apache.doris.nereids.trees.plans.physical.PhysicalHashJoin; import org.apache.doris.nereids.trees.plans.physical.PhysicalRelation; import org.apache.doris.nereids.trees.plans.physical.RuntimeFilter; @@ -77,6 +80,21 @@ public class RuntimeFilterContext { private final Map scanNodeOfLegacyRuntimeFilterTarget = Maps.newHashMap(); private final Set effectiveSrcNodes = Sets.newHashSet(); + + // cte to related joins map which can extract common runtime filter to cte inside + private final Map> cteToJoinsMap = Maps.newHashMap(); + + // cte candidates which can be pushed into common runtime filter into from outside + private final Map> cteRFPushDownMap = Maps.newHashMap(); + + private final Map cteProducerMap = Maps.newHashMap(); + + // cte whose runtime filter has been extracted + private final Set processedCTE = Sets.newHashSet(); + + // cte whose outer runtime filter has been pushed down into + private final Set pushedDownCTE = Sets.newHashSet(); + private final SessionVariable sessionVariable; private final FilterSizeLimits limits; @@ -96,6 +114,26 @@ public FilterSizeLimits getLimits() { return limits; } + public Map getCteProduceMap() { + return cteProducerMap; + } + + public Map> getCteRFPushDownMap() { + return cteRFPushDownMap; + } + + public Map> getCteToJoinsMap() { + return cteToJoinsMap; + } + + public Set getProcessedCTE() { + return processedCTE; + } + + public Set getPushedDownCTE() { + return pushedDownCTE; + } + public void setTargetExprIdToFilter(ExprId id, RuntimeFilter filter) { Preconditions.checkArgument(filter.getTargetExprs().stream().anyMatch(expr -> expr.getExprId() == id)); this.targetExprIdToFilter.computeIfAbsent(id, k -> Lists.newArrayList()).add(filter); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/RuntimeFilterGenerator.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/RuntimeFilterGenerator.java index 9ec1a34c9e0838f..524ba17ba00eac2 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/RuntimeFilterGenerator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/RuntimeFilterGenerator.java @@ -22,7 +22,9 @@ import org.apache.doris.nereids.CascadesContext; import org.apache.doris.nereids.stats.ExpressionEstimation; import org.apache.doris.nereids.trees.expressions.Alias; +import org.apache.doris.nereids.trees.expressions.CTEId; import org.apache.doris.nereids.trees.expressions.EqualTo; +import org.apache.doris.nereids.trees.expressions.ExprId; import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.NamedExpression; import org.apache.doris.nereids.trees.expressions.Not; @@ -32,10 +34,17 @@ import org.apache.doris.nereids.trees.plans.AbstractPlan; import org.apache.doris.nereids.trees.plans.JoinType; import org.apache.doris.nereids.trees.plans.Plan; +import org.apache.doris.nereids.trees.plans.logical.LogicalCTEConsumer; +import org.apache.doris.nereids.trees.plans.physical.AbstractPhysicalJoin; +import org.apache.doris.nereids.trees.plans.physical.PhysicalCTEConsumer; +import org.apache.doris.nereids.trees.plans.physical.PhysicalCTEProducer; +import org.apache.doris.nereids.trees.plans.physical.PhysicalDistribute; import org.apache.doris.nereids.trees.plans.physical.PhysicalExcept; +import org.apache.doris.nereids.trees.plans.physical.PhysicalFilter; import org.apache.doris.nereids.trees.plans.physical.PhysicalHashJoin; import org.apache.doris.nereids.trees.plans.physical.PhysicalIntersect; import org.apache.doris.nereids.trees.plans.physical.PhysicalNestedLoopJoin; +import org.apache.doris.nereids.trees.plans.physical.PhysicalOlapScan; import org.apache.doris.nereids.trees.plans.physical.PhysicalPlan; import org.apache.doris.nereids.trees.plans.physical.PhysicalProject; import org.apache.doris.nereids.trees.plans.physical.PhysicalRelation; @@ -50,9 +59,13 @@ import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Lists; import java.util.ArrayList; import java.util.Arrays; +import java.util.HashMap; +import java.util.HashSet; +import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.Set; @@ -70,6 +83,14 @@ public class RuntimeFilterGenerator extends PlanPostProcessor { JoinType.NULL_AWARE_LEFT_ANTI_JOIN ); + private static final Set> SPJ_PLAN = ImmutableSet.of( + PhysicalOlapScan.class, + PhysicalProject.class, + PhysicalFilter.class, + PhysicalDistribute.class, + PhysicalHashJoin.class + ); + private final IdGenerator generator = RuntimeFilterId.createGenerator(); /** @@ -95,121 +116,45 @@ public PhysicalPlan visitPhysicalHashJoin(PhysicalHashJoin slots = join.getOutputSet(); - slots.forEach(aliasTransferMap::remove); + // aliasTransMap is also used for judging whether the slot can be as rf target. + // for denied join type, the forbidden slots will be removed from the map. + // for example: a full outer join b on a.id = b.id, all slots will be removed out. + // for left outer join, only remove the right side slots and leave the left side. + // in later visit, the additional checking for the join type will be invoked for different cases: + // case 1: a left join b on a.id = b.id, checking whether rf on b.id can be pushed to a, the answer is no, + // since current join type is left outer join which is in denied list; + // case 2: (a left join b on a.id = b.id) inner join c on a.id2 = c.id2, checking whether rf on c.id2 can + // be pushed to a, the answer is yes, since the current join is inner join which is permitted. + if (join.getJoinType() == JoinType.LEFT_OUTER_JOIN) { + Set slots = join.right().getOutputSet(); + slots.forEach(aliasTransferMap::remove); + } else { + Set slots = join.getOutputSet(); + slots.forEach(aliasTransferMap::remove); + } } else { - List legalTypes = Arrays.stream(TRuntimeFilterType.values()) - .filter(type -> (type.getValue() & ctx.getSessionVariable().getRuntimeFilterType()) > 0) - .collect(Collectors.toList()); - // TODO: some complex situation cannot be handled now, see testPushDownThroughJoin. - // we will support it in later version. - for (int i = 0; i < join.getHashJoinConjuncts().size(); i++) { - EqualTo equalTo = ((EqualTo) JoinUtils.swapEqualToForChildrenOrder( - (EqualTo) join.getHashJoinConjuncts().get(i), join.left().getOutputSet())); - for (TRuntimeFilterType type : legalTypes) { - //bitmap rf is generated by nested loop join. - if (type == TRuntimeFilterType.BITMAP) { - continue; - } - if (join.left() instanceof PhysicalUnion - || join.left() instanceof PhysicalIntersect - || join.left() instanceof PhysicalExcept) { - List targetList = new ArrayList<>(); - int projIndex = -1; - for (int j = 0; j < join.left().children().size(); j++) { - PhysicalPlan child = (PhysicalPlan) join.left().child(j); - if (child instanceof PhysicalProject) { - PhysicalProject project = (PhysicalProject) child; - Slot leftSlot = checkTargetChild(equalTo.left()); - if (leftSlot == null) { - break; - } - for (int k = 0; projIndex < 0 && k < project.getProjects().size(); k++) { - NamedExpression expr = (NamedExpression) project.getProjects().get(k); - if (expr.getName().equals(leftSlot.getName())) { - projIndex = k; - break; - } - } - Preconditions.checkState(projIndex >= 0 - && projIndex < project.getProjects().size()); - - NamedExpression targetExpr = (NamedExpression) project.getProjects().get(projIndex); - - SlotReference origSlot = null; - if (targetExpr instanceof Alias) { - origSlot = (SlotReference) targetExpr.child(0); - } else { - origSlot = (SlotReference) targetExpr; - } - Slot olapScanSlot = aliasTransferMap.get(origSlot).second; - PhysicalRelation scan = aliasTransferMap.get(origSlot).first; - if (type == TRuntimeFilterType.IN_OR_BLOOM - && ctx.getSessionVariable().enablePipelineEngine() - && hasRemoteTarget(join, scan)) { - type = TRuntimeFilterType.BLOOM; - } - targetList.add(olapScanSlot); - ctx.addJoinToTargetMap(join, olapScanSlot.getExprId()); - ctx.setTargetsOnScanNode(aliasTransferMap.get(origSlot).first.getId(), olapScanSlot); - } - } - if (!targetList.isEmpty()) { - long buildSideNdv = getBuildSideNdv(join, equalTo); - RuntimeFilter filter = new RuntimeFilter(generator.getNextId(), - equalTo.right(), targetList, type, i, join, buildSideNdv); - for (int j = 0; j < targetList.size(); j++) { - ctx.setTargetExprIdToFilter(targetList.get(j).getExprId(), filter); - } - } - } else { - // currently, we can ensure children in the two side are corresponding to the equal_to's. - // so right maybe an expression and left is a slot - Slot unwrappedSlot = checkTargetChild(equalTo.left()); - // aliasTransMap doesn't contain the key, means that the path from the olap scan to the join - // contains join with denied join type. for example: a left join b on a.id = b.id - if (unwrappedSlot == null || !aliasTransferMap.containsKey(unwrappedSlot)) { - continue; - } - Slot olapScanSlot = aliasTransferMap.get(unwrappedSlot).second; - PhysicalRelation scan = aliasTransferMap.get(unwrappedSlot).first; - // in-filter is not friendly to pipeline - if (type == TRuntimeFilterType.IN_OR_BLOOM - && ctx.getSessionVariable().enablePipelineEngine() - && hasRemoteTarget(join, scan)) { - type = TRuntimeFilterType.BLOOM; - } - long buildSideNdv = getBuildSideNdv(join, equalTo); - RuntimeFilter filter = new RuntimeFilter(generator.getNextId(), - equalTo.right(), ImmutableList.of(olapScanSlot), type, i, join, buildSideNdv); - ctx.addJoinToTargetMap(join, olapScanSlot.getExprId()); - ctx.setTargetExprIdToFilter(olapScanSlot.getExprId(), filter); - ctx.setTargetsOnScanNode(aliasTransferMap.get(unwrappedSlot).first.getId(), olapScanSlot); - } - } + collectPushDownCTEInfos(join, context); + if (!getPushDownCTECandidates(ctx).isEmpty()) { + pushDownRuntimeFilterIntoCTE(ctx); + } else { + pushDownRuntimeFilterCommon(join, context); } } return join; } - private boolean hasRemoteTarget(AbstractPlan join, AbstractPlan scan) { - Preconditions.checkArgument(join.getMutableState(AbstractPlan.FRAGMENT_ID).isPresent(), - "cannot find fragment id for Join node"); - Preconditions.checkArgument(scan.getMutableState(AbstractPlan.FRAGMENT_ID).isPresent(), - "cannot find fragment id for scan node"); - return join.getMutableState(AbstractPlan.FRAGMENT_ID).get() - != scan.getMutableState(AbstractPlan.FRAGMENT_ID).get(); + @Override + public PhysicalCTEConsumer visitPhysicalCTEConsumer(PhysicalCTEConsumer scan, CascadesContext context) { + RuntimeFilterContext ctx = context.getRuntimeFilterContext(); + scan.getOutput().forEach(slot -> ctx.getAliasTransferMap().put(slot, Pair.of(scan, slot))); + return scan; } - private long getBuildSideNdv(PhysicalHashJoin join, EqualTo equalTo) { - AbstractPlan right = (AbstractPlan) join.right(); - //make ut test friendly - if (right.getStats() == null) { - return -1L; - } - ExpressionEstimation estimator = new ExpressionEstimation(); - ColumnStatistic buildColStats = equalTo.right().accept(estimator, right.getStats()); - return buildColStats.isUnKnown ? -1 : Math.max(1, (long) buildColStats.ndv); + @Override + public PhysicalCTEProducer visitPhysicalCTEProducer(PhysicalCTEProducer producer, CascadesContext context) { + CTEId id = producer.getCteId(); + context.getRuntimeFilterContext().getCteProduceMap().put(id, producer); + return producer; } @Override @@ -249,17 +194,18 @@ public PhysicalPlan visitPhysicalNestedLoopJoin(PhysicalNestedLoopJoin targetSlots = bitmapContains.child(1).getInputSlots(); for (Slot targetSlot : targetSlots) { - if (targetSlot != null && aliasTransferMap.containsKey(targetSlot)) { - Slot olapScanSlot = aliasTransferMap.get(targetSlot).second; - RuntimeFilter filter = new RuntimeFilter(generator.getNextId(), - bitmapContains.child(0), ImmutableList.of(olapScanSlot), - ImmutableList.of(bitmapContains.child(1)), type, i, join, isNot, -1L); - ctx.addJoinToTargetMap(join, olapScanSlot.getExprId()); - ctx.setTargetExprIdToFilter(olapScanSlot.getExprId(), filter); - ctx.setTargetsOnScanNode(aliasTransferMap.get(targetSlot).first.getId(), - olapScanSlot); - join.addBitmapRuntimeFilterCondition(bitmapRuntimeFilterCondition); + if (!checkCanPushDownFromJoinType(join, ctx, targetSlot)) { + continue; } + Slot olapScanSlot = aliasTransferMap.get(targetSlot).second; + RuntimeFilter filter = new RuntimeFilter(generator.getNextId(), + bitmapContains.child(0), ImmutableList.of(olapScanSlot), + ImmutableList.of(bitmapContains.child(1)), type, i, join, isNot, -1L); + ctx.addJoinToTargetMap(join, olapScanSlot.getExprId()); + ctx.setTargetExprIdToFilter(olapScanSlot.getExprId(), filter); + ctx.setTargetsOnScanNode(aliasTransferMap.get(targetSlot).first.getId(), + olapScanSlot); + join.addBitmapRuntimeFilterCondition(bitmapRuntimeFilterCondition); } } return join; @@ -294,8 +240,465 @@ public PhysicalRelation visitPhysicalScan(PhysicalRelation scan, CascadesContext return scan; } + private long getBuildSideNdv(PhysicalHashJoin join, EqualTo equalTo) { + AbstractPlan right = (AbstractPlan) join.right(); + //make ut test friendly + if (right.getStats() == null) { + return -1L; + } + ExpressionEstimation estimator = new ExpressionEstimation(); + ColumnStatistic buildColStats = equalTo.right().accept(estimator, right.getStats()); + return buildColStats.isUnKnown ? -1 : Math.max(1, (long) buildColStats.ndv); + } + private static Slot checkTargetChild(Expression leftChild) { Expression expression = ExpressionUtils.getExpressionCoveredByCast(leftChild); return expression instanceof Slot ? ((Slot) expression) : null; } + + private void pushDownRuntimeFilterCommon(PhysicalHashJoin join, + CascadesContext context) { + RuntimeFilterContext ctx = context.getRuntimeFilterContext(); + List legalTypes = Arrays.stream(TRuntimeFilterType.values()) + .filter(type -> (type.getValue() & ctx.getSessionVariable().getRuntimeFilterType()) > 0) + .collect(Collectors.toList()); + // TODO: some complex situation cannot be handled now, see testPushDownThroughJoin. + // we will support it in later version. + for (int i = 0; i < join.getHashJoinConjuncts().size(); i++) { + EqualTo equalTo = ((EqualTo) JoinUtils.swapEqualToForChildrenOrder( + (EqualTo) join.getHashJoinConjuncts().get(i), join.left().getOutputSet())); + for (TRuntimeFilterType type : legalTypes) { + //bitmap rf is generated by nested loop join. + if (type == TRuntimeFilterType.BITMAP) { + continue; + } + if (join.left() instanceof PhysicalUnion + || join.left() instanceof PhysicalIntersect + || join.left() instanceof PhysicalExcept) { + doPushDownIntoSetOperation(join, ctx, equalTo, type, i); + } else { + doPushDownBasic(join, context, ctx, equalTo, type, i); + } + } + } + } + + private void doPushDownBasic(PhysicalHashJoin join, CascadesContext context, + RuntimeFilterContext ctx, EqualTo equalTo, TRuntimeFilterType type, int exprOrder) { + Map> aliasTransferMap = ctx.getAliasTransferMap(); + // currently, we can ensure children in the two side are corresponding to the equal_to's. + // so right maybe an expression and left is a slot + Slot unwrappedSlot = checkTargetChild(equalTo.left()); + // aliasTransMap doesn't contain the key, means that the path from the olap scan to the join + // contains join with denied join type. for example: a left join b on a.id = b.id + if (!checkCanPushDownFromJoinType(join, ctx, unwrappedSlot)) { + return; + } + Slot olapScanSlot = aliasTransferMap.get(unwrappedSlot).second; + PhysicalRelation scan = aliasTransferMap.get(unwrappedSlot).first; + + Preconditions.checkState(olapScanSlot != null && scan != null); + + if (scan instanceof PhysicalCTEConsumer) { + Set processedCTE = context.getRuntimeFilterContext().getProcessedCTE(); + CTEId cteId = ((PhysicalCTEConsumer) scan).getCteId(); + if (!processedCTE.contains(cteId)) { + PhysicalCTEProducer cteProducer = context.getRuntimeFilterContext() + .getCteProduceMap().get(cteId); + PhysicalPlan inputPlanNode = (PhysicalPlan) cteProducer.child(0); + // process cte producer self recursively + inputPlanNode.accept(this, context); + processedCTE.add(cteId); + } + } else { + // in-filter is not friendly to pipeline + if (type == TRuntimeFilterType.IN_OR_BLOOM + && ctx.getSessionVariable().enablePipelineEngine() + && hasRemoteTarget(join, scan)) { + type = TRuntimeFilterType.BLOOM; + } + long buildSideNdv = getBuildSideNdv(join, equalTo); + RuntimeFilter filter = new RuntimeFilter(generator.getNextId(), + equalTo.right(), ImmutableList.of(olapScanSlot), type, exprOrder, join, buildSideNdv); + ctx.addJoinToTargetMap(join, olapScanSlot.getExprId()); + ctx.setTargetExprIdToFilter(olapScanSlot.getExprId(), filter); + ctx.setTargetsOnScanNode(aliasTransferMap.get(unwrappedSlot).first.getId(), olapScanSlot); + } + } + + private void doPushDownIntoSetOperation(PhysicalHashJoin join, + RuntimeFilterContext ctx, EqualTo equalTo, TRuntimeFilterType type, int exprOrder) { + Map> aliasTransferMap = ctx.getAliasTransferMap(); + List targetList = new ArrayList<>(); + int projIndex = -1; + for (int j = 0; j < join.left().children().size(); j++) { + PhysicalPlan child = (PhysicalPlan) join.left().child(j); + if (child instanceof PhysicalProject) { + PhysicalProject project = (PhysicalProject) child; + Slot leftSlot = checkTargetChild(equalTo.left()); + if (leftSlot == null) { + break; + } + for (int k = 0; projIndex < 0 && k < project.getProjects().size(); k++) { + NamedExpression expr = (NamedExpression) project.getProjects().get(k); + if (expr.getName().equals(leftSlot.getName())) { + projIndex = k; + break; + } + } + Preconditions.checkState(projIndex >= 0 + && projIndex < project.getProjects().size()); + + NamedExpression targetExpr = (NamedExpression) project.getProjects().get(projIndex); + + SlotReference origSlot = null; + if (targetExpr instanceof Alias) { + origSlot = (SlotReference) targetExpr.child(0); + } else { + origSlot = (SlotReference) targetExpr; + } + Slot olapScanSlot = aliasTransferMap.get(origSlot).second; + if (!checkCanPushDownFromJoinType(join, ctx, olapScanSlot)) { + continue; + } + PhysicalRelation scan = aliasTransferMap.get(origSlot).first; + if (type == TRuntimeFilterType.IN_OR_BLOOM + && ctx.getSessionVariable().enablePipelineEngine() + && hasRemoteTarget(join, scan)) { + type = TRuntimeFilterType.BLOOM; + } + targetList.add(olapScanSlot); + ctx.addJoinToTargetMap(join, olapScanSlot.getExprId()); + ctx.setTargetsOnScanNode(aliasTransferMap.get(origSlot).first.getId(), olapScanSlot); + } + } + if (!targetList.isEmpty()) { + long buildSideNdv = getBuildSideNdv(join, equalTo); + RuntimeFilter filter = new RuntimeFilter(generator.getNextId(), + equalTo.right(), targetList, type, exprOrder, join, buildSideNdv); + for (int j = 0; j < targetList.size(); j++) { + ctx.setTargetExprIdToFilter(targetList.get(j).getExprId(), filter); + } + } + } + + private void collectPushDownCTEInfos(PhysicalHashJoin join, + CascadesContext context) { + RuntimeFilterContext ctx = context.getRuntimeFilterContext(); + Set cteIds = new HashSet<>(); + PhysicalPlan leftChild = (PhysicalPlan) join.left(); + PhysicalPlan rightChild = (PhysicalPlan) join.right(); + + Preconditions.checkState(leftChild != null && rightChild != null); + + boolean leftHasCTE = hasCTEConsumerUnderJoin(leftChild, cteIds); + boolean rightHasCTE = hasCTEConsumerUnderJoin(rightChild, cteIds); + // only support single cte in join currently + if ((leftHasCTE && !rightHasCTE) || (!leftHasCTE && rightHasCTE)) { + for (CTEId id : cteIds) { + if (ctx.getCteToJoinsMap().get(id) == null) { + Set newJoin = new HashSet<>(); + newJoin.add(join); + ctx.getCteToJoinsMap().put(id, newJoin); + } else { + ctx.getCteToJoinsMap().get(id).add(join); + } + } + } + if (!ctx.getCteToJoinsMap().isEmpty()) { + analyzeRuntimeFilterPushDownIntoCTEInfos(join, context); + } + } + + private List getPushDownCTECandidates(RuntimeFilterContext ctx) { + List candidates = new ArrayList<>(); + Map> cteRFPushDownMap = ctx.getCteRFPushDownMap(); + for (Map.Entry> entry : cteRFPushDownMap.entrySet()) { + CTEId cteId = entry.getKey().getCteId(); + if (ctx.getPushedDownCTE().contains(cteId)) { + continue; + } + candidates.add(cteId); + } + return candidates; + } + + private boolean hasCTEConsumerUnderJoin(PhysicalPlan root, Set cteIds) { + if (root instanceof PhysicalCTEConsumer) { + cteIds.add(((PhysicalCTEConsumer) root).getCteId()); + return true; + } else if (root.children().size() != 1) { + // only collect cte in one side + return false; + } else if (root instanceof PhysicalDistribute + || root instanceof PhysicalFilter + || root instanceof PhysicalProject) { + // only collect cte as single child node under join + return hasCTEConsumerUnderJoin((PhysicalPlan) root.child(0), cteIds); + } else { + return false; + } + } + + private void analyzeRuntimeFilterPushDownIntoCTEInfos(PhysicalHashJoin curJoin, + CascadesContext context) { + RuntimeFilterContext ctx = context.getRuntimeFilterContext(); + Map> cteToJoinsMap = ctx.getCteToJoinsMap(); + for (Map.Entry> entry : cteToJoinsMap.entrySet()) { + CTEId cteId = entry.getKey(); + Set joinSet = entry.getValue(); + if (joinSet.contains(curJoin)) { + // skip current join + continue; + } + Set cteSet = context.getCteIdToConsumers().get(cteId); + Preconditions.checkState(!cteSet.isEmpty()); + String cteName = cteSet.iterator().next().getName(); + // preconditions for rf pushing into cte producer: + // multiple joins whose join condition is on the same cte's column of the same cte + // the other side of these join conditions are the same column of the same table, or + // they in the same equal sets, such as under an equal join condition + // case 1: two joins with t1.c1 = cte1_consumer1.c1 and t1.c1 = cte1_consumer2.c1 conditions + // rf of t1.c1 can be pushed down into cte1 producer. + // ----------------------hashJoin(t1.c1 = cte2_consumer1.c1) + // ----------------------------CteConsumer[cteId= ( CTEId#1=] ) + // ----------------------------PhysicalOlapScan[t1] + // ----------------------hashJoin(t1.c1 = cte2_consumer2.c1) + // ----------------------------CteConsumer[cteId= ( CTEId#1=] ) + // ----------------------------PhysicalOlapScan[t1] + // case 2: two joins with t1.c1 = cte2_consumer1.c1 and t2.c2 = cte2_consumer2.c1 and another equal join + // condition t1.c1 = t2.c2, which means t1.c1 and t2.c2 are in the same equal set. + // rf of t1.c1 and t2.c2 can be pushed down into cte2 producer. + // --------------------hashJoin(t1.c1 = t2.c2) + // ----------------------hashJoin(t2.c2 = cte2_consumer1.c1) + // ----------------------------CteConsumer[cteId= ( CTEId#1=] ) + // ----------------------------PhysicalOlapScan[t2] + // ----------------------hashJoin(t1.c1 = cte2_consumer2.c1) + // ----------------------------CteConsumer[cteId= ( CTEId#1=] ) + // ----------------------------PhysicalOlapScan[t1] + if (joinSet.size() != cteSet.size()) { + continue; + } + List equalTos = new ArrayList<>(); + Map equalCondToJoinMap = new LinkedHashMap<>(); + for (PhysicalHashJoin join : joinSet) { + // precondition: + // 1. no non-equal join condition + // 2. only equalTo and slotReference both sides + // 3. only support one join condition (will be refined further) + if (join.getOtherJoinConjuncts().size() > 1 + || join.getHashJoinConjuncts().size() != 1 + || !(join.getHashJoinConjuncts().get(0) instanceof EqualTo)) { + break; + } else { + EqualTo equalTo = (EqualTo) join.getHashJoinConjuncts().get(0); + equalTos.add(equalTo); + equalCondToJoinMap.put(equalTo, join); + } + } + if (joinSet.size() == equalTos.size()) { + int matchNum = 0; + Set cteNameSet = new HashSet<>(); + Set anotherSideSlotSet = new HashSet<>(); + for (EqualTo equalTo : equalTos) { + SlotReference left = (SlotReference) equalTo.left(); + SlotReference right = (SlotReference) equalTo.right(); + if (left.getQualifier().size() == 1 && left.getQualifier().get(0).equals(cteName)) { + matchNum += 1; + anotherSideSlotSet.add(right); + cteNameSet.add(left.getQualifiedName()); + } else if (right.getQualifier().size() == 1 && right.getQualifier().get(0).equals(cteName)) { + matchNum += 1; + anotherSideSlotSet.add(left); + cteNameSet.add(right.getQualifiedName()); + } + } + if (matchNum == equalTos.size() && cteNameSet.size() == 1) { + // means all join condition points to the same cte on the same cte column. + // collect the other side columns besides cte column side. + Preconditions.checkState(equalTos.size() == equalCondToJoinMap.size(), + "equalTos.size() != equalCondToJoinMap.size()"); + + PhysicalCTEProducer cteProducer = context.getRuntimeFilterContext().getCteProduceMap().get(cteId); + if (anotherSideSlotSet.size() == 1) { + // meet requirement for pushing down into cte producer + ctx.getCteRFPushDownMap().put(cteProducer, equalCondToJoinMap); + } else { + // check further whether the join upper side can bring equal set, which + // indicating actually the same runtime filter build side + // see above case 2 for reference + List conditions = curJoin.getHashJoinConjuncts(); + boolean inSameEqualSet = false; + for (Expression e : conditions) { + if (e instanceof EqualTo) { + SlotReference oneSide = (SlotReference) ((EqualTo) e).left(); + SlotReference anotherSide = (SlotReference) ((EqualTo) e).right(); + if (anotherSideSlotSet.contains(oneSide) && anotherSideSlotSet.contains(anotherSide)) { + inSameEqualSet = true; + break; + } + } + } + if (inSameEqualSet) { + ctx.getCteRFPushDownMap().put(cteProducer, equalCondToJoinMap); + } + } + } + } + } + } + + private void pushDownRuntimeFilterIntoCTE(RuntimeFilterContext ctx) { + Map> cteRFPushDownMap = ctx.getCteRFPushDownMap(); + for (Map.Entry> entry : cteRFPushDownMap.entrySet()) { + PhysicalCTEProducer cteProducer = entry.getKey(); + Preconditions.checkState(cteProducer != null); + if (ctx.getPushedDownCTE().contains(cteProducer.getCteId())) { + continue; + } + Map equalCondToJoinMap = entry.getValue(); + int exprOrder = 0; + for (Map.Entry innerEntry : equalCondToJoinMap.entrySet()) { + EqualTo equalTo = innerEntry.getKey(); + PhysicalHashJoin join = innerEntry.getValue(); + Preconditions.checkState(join != null); + TRuntimeFilterType type = TRuntimeFilterType.IN_OR_BLOOM; + if (ctx.getSessionVariable().enablePipelineEngine()) { + type = TRuntimeFilterType.BLOOM; + } + EqualTo newEqualTo = ((EqualTo) JoinUtils.swapEqualToForChildrenOrder( + equalTo, join.child(0).getOutputSet())); + doPushDownIntoCTEProducerInternal(join, ctx, newEqualTo, type, exprOrder++, cteProducer); + } + ctx.getPushedDownCTE().add(cteProducer.getCteId()); + } + } + + private void doPushDownIntoCTEProducerInternal(PhysicalHashJoin join, + RuntimeFilterContext ctx, EqualTo equalTo, TRuntimeFilterType type, int exprOrder, + PhysicalCTEProducer cteProducer) { + Map> aliasTransferMap = ctx.getAliasTransferMap(); + PhysicalPlan inputPlanNode = (PhysicalPlan) cteProducer.child(0); + Slot unwrappedSlot = checkTargetChild(equalTo.left()); + // aliasTransMap doesn't contain the key, means that the path from the olap scan to the join + // contains join with denied join type. for example: a left join b on a.id = b.id + if (!checkCanPushDownFromJoinType(join, ctx, unwrappedSlot)) { + return; + } + Slot cteSlot = aliasTransferMap.get(unwrappedSlot).second; + PhysicalRelation cteNode = aliasTransferMap.get(unwrappedSlot).first; + long buildSideNdv = getBuildSideNdv(join, equalTo); + if (cteNode instanceof PhysicalCTEConsumer && inputPlanNode instanceof PhysicalProject) { + PhysicalProject project = (PhysicalProject) inputPlanNode; + NamedExpression targetExpr = null; + for (Object column : project.getProjects()) { + NamedExpression alias = (NamedExpression) column; + if (cteSlot.getName().equals(alias.getName())) { + targetExpr = alias; + break; + } + } + Preconditions.checkState(targetExpr != null); + if (!(targetExpr instanceof SlotReference)) { + // if not SlotReference, skip the push down + return; + } else if (!checkCanPushDownIntoBasicTable(project)) { + return; + } else { + Map pushDownBasicTableInfos = getPushDownBasicTablesInfos(project, + (SlotReference) targetExpr, aliasTransferMap); + if (!pushDownBasicTableInfos.isEmpty()) { + List targetList = new ArrayList<>(); + for (Map.Entry entry : pushDownBasicTableInfos.entrySet()) { + Slot targetSlot = entry.getKey(); + PhysicalOlapScan scan = entry.getValue(); + targetList.add(targetSlot); + ctx.addJoinToTargetMap(join, targetSlot.getExprId()); + ctx.setTargetsOnScanNode(scan.getId(), targetSlot); + } + // build multi-target runtime filter + RuntimeFilter filter = new RuntimeFilter(generator.getNextId(), + equalTo.right(), targetList, type, exprOrder, join, buildSideNdv); + for (Slot slot : targetList) { + ctx.setTargetExprIdToFilter(slot.getExprId(), filter); + } + } + } + } + } + + private boolean checkCanPushDownFromJoinType(AbstractPhysicalJoin physicalJoin, + RuntimeFilterContext ctx, Slot slot) { + Map> aliasTransferMap = ctx.getAliasTransferMap(); + if (slot == null || !aliasTransferMap.containsKey(slot)) { + return false; + } else if (DENIED_JOIN_TYPES.contains(physicalJoin.getJoinType()) || physicalJoin.isMarkJoin()) { + return false; + } else { + return true; + } + } + + private boolean checkCanPushDownIntoBasicTable(PhysicalPlan root) { + // only support spj currently + List plans = Lists.newArrayList(); + plans.addAll(root.collect(PhysicalPlan.class::isInstance)); + return plans.stream().allMatch(p -> SPJ_PLAN.stream().anyMatch(c -> c.isInstance(p))); + } + + private Map getPushDownBasicTablesInfos(PhysicalPlan root, SlotReference slot, + Map> aliasTransferMap) { + Map basicTableInfos = new HashMap<>(); + Set joins = new HashSet<>(); + ExprId exprId = slot.getExprId(); + if (aliasTransferMap.get(slot) != null && aliasTransferMap.get(slot).first instanceof PhysicalOlapScan) { + basicTableInfos.put(slot, (PhysicalOlapScan) aliasTransferMap.get(slot).first); + } + // try to find propagation condition from join + getAllJoinInfo(root, joins); + for (PhysicalHashJoin join : joins) { + List conditions = join.getHashJoinConjuncts(); + for (Expression equalTo : conditions) { + if (equalTo instanceof EqualTo) { + SlotReference leftSlot = (SlotReference) ((EqualTo) equalTo).left(); + SlotReference rightSlot = (SlotReference) ((EqualTo) equalTo).right(); + if (leftSlot.getExprId() == exprId) { + PhysicalOlapScan rightTable = (PhysicalOlapScan) aliasTransferMap.get(rightSlot).first; + if (rightTable != null) { + basicTableInfos.put(rightSlot, rightTable); + } + } else if (rightSlot.getExprId() == exprId) { + PhysicalOlapScan leftTable = (PhysicalOlapScan) aliasTransferMap.get(leftSlot).first; + if (leftTable != null) { + basicTableInfos.put(leftSlot, leftTable); + } + } + } + } + } + return basicTableInfos; + } + + private void getAllJoinInfo(PhysicalPlan root, Set joins) { + if (root instanceof PhysicalHashJoin) { + joins.add((PhysicalHashJoin) root); + } else { + for (Object child : root.children()) { + getAllJoinInfo((PhysicalPlan) child, joins); + } + } + } + + private boolean hasRemoteTarget(AbstractPlan join, AbstractPlan scan) { + if (scan instanceof PhysicalCTEConsumer) { + return true; + } else { + Preconditions.checkArgument(join.getMutableState(AbstractPlan.FRAGMENT_ID).isPresent(), + "cannot find fragment id for Join node"); + Preconditions.checkArgument(scan.getMutableState(AbstractPlan.FRAGMENT_ID).isPresent(), + "cannot find fragment id for scan node"); + return join.getMutableState(AbstractPlan.FRAGMENT_ID).get() + != scan.getMutableState(AbstractPlan.FRAGMENT_ID).get(); + } + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/ChildOutputPropertyDeriver.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/ChildOutputPropertyDeriver.java index 73fa3bca6f68ed7..2695b67a93e8930 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/ChildOutputPropertyDeriver.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/ChildOutputPropertyDeriver.java @@ -47,19 +47,24 @@ import org.apache.doris.nereids.trees.plans.physical.PhysicalNestedLoopJoin; import org.apache.doris.nereids.trees.plans.physical.PhysicalOlapScan; import org.apache.doris.nereids.trees.plans.physical.PhysicalOlapTableSink; +import org.apache.doris.nereids.trees.plans.physical.PhysicalOneRowRelation; import org.apache.doris.nereids.trees.plans.physical.PhysicalPartitionTopN; import org.apache.doris.nereids.trees.plans.physical.PhysicalProject; -import org.apache.doris.nereids.trees.plans.physical.PhysicalStorageLayerAggregate; +import org.apache.doris.nereids.trees.plans.physical.PhysicalRepeat; +import org.apache.doris.nereids.trees.plans.physical.PhysicalSetOperation; import org.apache.doris.nereids.trees.plans.physical.PhysicalTVFRelation; +import org.apache.doris.nereids.trees.plans.physical.PhysicalUnion; +import org.apache.doris.nereids.trees.plans.physical.PhysicalWindow; import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor; import org.apache.doris.nereids.types.DataType; import org.apache.doris.nereids.util.JoinUtils; -import org.apache.doris.qe.ConnectContext; import com.google.common.base.Preconditions; +import com.google.common.collect.Lists; import com.google.common.collect.Maps; import com.google.common.collect.Sets; +import java.util.Arrays; import java.util.List; import java.util.Map; import java.util.Objects; @@ -86,18 +91,25 @@ public PhysicalProperties getOutputProperties(GroupExpression groupExpression) { return groupExpression.getPlan().accept(this, new PlanContext(groupExpression)); } + /* ******************************************************************************************** + * sink Node, in lexicographical order + * ******************************************************************************************** */ + @Override - public PhysicalProperties visit(Plan plan, PlanContext context) { - return PhysicalProperties.ANY; + public PhysicalProperties visitPhysicalOlapTableSink(PhysicalOlapTableSink olapTableSink, + PlanContext context) { + return PhysicalProperties.GATHER; } @Override - public PhysicalProperties visitPhysicalCTEProducer( - PhysicalCTEProducer cteProducer, PlanContext context) { - Preconditions.checkState(childrenOutputProperties.size() == 1); - return childrenOutputProperties.get(0); + public PhysicalProperties visit(Plan plan, PlanContext context) { + return PhysicalProperties.ANY; } + /* ******************************************************************************************** + * Leaf Plan Node, in lexicographical order + * ******************************************************************************************** */ + @Override public PhysicalProperties visitPhysicalCTEConsumer( PhysicalCTEConsumer cteConsumer, PlanContext context) { @@ -106,13 +118,45 @@ public PhysicalProperties visitPhysicalCTEConsumer( } @Override - public PhysicalProperties visitPhysicalCTEAnchor( - PhysicalCTEAnchor cteAnchor, PlanContext context) { - Preconditions.checkState(childrenOutputProperties.size() == 2); - // return properties inherited from consumer side which may further be used at upper layer - return childrenOutputProperties.get(1); + public PhysicalProperties visitPhysicalEmptyRelation(PhysicalEmptyRelation emptyRelation, PlanContext context) { + return PhysicalProperties.GATHER; + } + + @Override + public PhysicalProperties visitPhysicalEsScan(PhysicalEsScan esScan, PlanContext context) { + return PhysicalProperties.ANY; + } + + @Override + public PhysicalProperties visitPhysicalFileScan(PhysicalFileScan fileScan, PlanContext context) { + return PhysicalProperties.ANY; + } + + @Override + public PhysicalProperties visitPhysicalJdbcScan(PhysicalJdbcScan jdbcScan, PlanContext context) { + return PhysicalProperties.ANY; } + @Override + public PhysicalProperties visitPhysicalOlapScan(PhysicalOlapScan olapScan, PlanContext context) { + return new PhysicalProperties(olapScan.getDistributionSpec()); + } + + @Override + public PhysicalProperties visitPhysicalOneRowRelation(PhysicalOneRowRelation oneRowRelation, PlanContext context) { + return PhysicalProperties.GATHER; + } + + @Override + public PhysicalProperties visitPhysicalTVFRelation(PhysicalTVFRelation tvfRelation, PlanContext context) { + TableValuedFunction function = tvfRelation.getFunction(); + return function.getPhysicalProperties(); + } + + /* ******************************************************************************************** + * Other Node, in lexicographical order + * ******************************************************************************************** */ + @Override public PhysicalProperties visitPhysicalHashAggregate( PhysicalHashAggregate agg, PlanContext context) { @@ -123,13 +167,6 @@ public PhysicalProperties visitPhysicalHashAggregate( case GLOBAL: case DISTINCT_LOCAL: case DISTINCT_GLOBAL: - DistributionSpec childSpec = childOutputProperty.getDistributionSpec(); - // If child's property is enforced, change it to bucketed - if (childSpec instanceof DistributionSpecHash - && ((DistributionSpecHash) childSpec).getShuffleType().equals(ShuffleType.ENFORCED)) { - DistributionSpecHash distributionSpecHash = (DistributionSpecHash) childSpec; - return new PhysicalProperties(distributionSpecHash.withShuffleType(ShuffleType.BUCKETED)); - } return new PhysicalProperties(childOutputProperty.getDistributionSpec()); default: throw new RuntimeException("Could not derive output properties for agg phase: " + agg.getAggPhase()); @@ -137,74 +174,32 @@ public PhysicalProperties visitPhysicalHashAggregate( } @Override - public PhysicalProperties visitAbstractPhysicalSort(AbstractPhysicalSort sort, + public PhysicalProperties visitPhysicalAssertNumRows(PhysicalAssertNumRows assertNumRows, PlanContext context) { Preconditions.checkState(childrenOutputProperties.size() == 1); - if (sort.getSortPhase().isLocal()) { - return new PhysicalProperties( - childrenOutputProperties.get(0).getDistributionSpec(), - new OrderSpec(sort.getOrderKeys())); - } - return new PhysicalProperties(DistributionSpecGather.INSTANCE, new OrderSpec(sort.getOrderKeys())); + PhysicalProperties childOutputProperty = childrenOutputProperties.get(0); + return new PhysicalProperties(childOutputProperty.getDistributionSpec()); } @Override - public PhysicalProperties visitPhysicalLimit(PhysicalLimit limit, PlanContext context) { - Preconditions.checkState(childrenOutputProperties.size() == 1); - PhysicalProperties childOutputProperty = childrenOutputProperties.get(0); - if (limit.getPhase().isLocal()) { - return new PhysicalProperties(childOutputProperty.getDistributionSpec(), - childOutputProperty.getOrderSpec()); - } - return new PhysicalProperties(DistributionSpecGather.INSTANCE, childOutputProperty.getOrderSpec()); + public PhysicalProperties visitPhysicalCTEAnchor( + PhysicalCTEAnchor cteAnchor, PlanContext context) { + Preconditions.checkState(childrenOutputProperties.size() == 2); + // return properties inherited from consumer side which may further be used at upper layer + return childrenOutputProperties.get(1); } @Override - public PhysicalProperties visitPhysicalPartitionTopN(PhysicalPartitionTopN partitionTopN, - PlanContext context) { + public PhysicalProperties visitPhysicalCTEProducer( + PhysicalCTEProducer cteProducer, PlanContext context) { Preconditions.checkState(childrenOutputProperties.size() == 1); - PhysicalProperties childOutputProperty = childrenOutputProperties.get(0); - return new PhysicalProperties(childOutputProperty.getDistributionSpec()); + return childrenOutputProperties.get(0); } @Override - public PhysicalProperties visitPhysicalProject(PhysicalProject project, PlanContext context) { - // TODO: order spec do not process since we do not use it. - Preconditions.checkState(childrenOutputProperties.size() == 1); - PhysicalProperties childProperties = childrenOutputProperties.get(0); - DistributionSpec childDistributionSpec = childProperties.getDistributionSpec(); - OrderSpec childOrderSpec = childProperties.getOrderSpec(); - DistributionSpec outputDistributionSpec; - if (childDistributionSpec instanceof DistributionSpecHash) { - Map projections = Maps.newHashMap(); - Set obstructions = Sets.newHashSet(); - for (NamedExpression namedExpression : project.getProjects()) { - if (namedExpression instanceof Alias) { - Alias alias = (Alias) namedExpression; - Expression child = alias.child(); - if (child instanceof SlotReference) { - projections.put(((SlotReference) child).getExprId(), alias.getExprId()); - } else if (child instanceof Cast && child.child(0) instanceof Slot - && isSameHashValue(child.child(0).getDataType(), child.getDataType())) { - // cast(slot as varchar(10)) can do projection if slot is varchar(3) - projections.put(((Slot) child.child(0)).getExprId(), alias.getExprId()); - } else { - obstructions.addAll( - child.getInputSlots().stream() - .map(NamedExpression::getExprId) - .collect(Collectors.toSet())); - } - } - } - if (projections.entrySet().stream().allMatch(kv -> kv.getKey().equals(kv.getValue()))) { - return childrenOutputProperties.get(0); - } - outputDistributionSpec = ((DistributionSpecHash) childDistributionSpec).project(projections, obstructions); - return new PhysicalProperties(outputDistributionSpec, childOrderSpec); - } else { - return childrenOutputProperties.get(0); - } - + public PhysicalProperties visitPhysicalDistribute( + PhysicalDistribute distribute, PlanContext context) { + return distribute.getPhysicalProperties(); } @Override @@ -214,9 +209,9 @@ public PhysicalProperties visitPhysicalFilter(PhysicalFilter fil } @Override - public PhysicalProperties visitPhysicalDistribute( - PhysicalDistribute distribute, PlanContext context) { - return distribute.getPhysicalProperties(); + public PhysicalProperties visitPhysicalGenerate(PhysicalGenerate generate, PlanContext context) { + Preconditions.checkState(childrenOutputProperties.size() == 1); + return childrenOutputProperties.get(0); } @Override @@ -248,87 +243,159 @@ public PhysicalProperties visitPhysicalHashJoin( // shuffle, if left child is natural mean current join is bucket shuffle join // and remain natural for colocate join on upper join. - return new PhysicalProperties(DistributionSpecHash.merge(leftHashSpec, rightHashSpec, - leftHashSpec.getShuffleType() == ShuffleType.NATURAL ? ShuffleType.NATURAL : ShuffleType.BUCKETED)); + return new PhysicalProperties(DistributionSpecHash.merge( + leftHashSpec, rightHashSpec, leftHashSpec.getShuffleType())); } throw new RuntimeException("Could not derive hash join's output properties. join: " + hashJoin); } + @Override + public PhysicalProperties visitPhysicalLimit(PhysicalLimit limit, PlanContext context) { + Preconditions.checkState(childrenOutputProperties.size() == 1); + return childrenOutputProperties.get(0); + } + @Override public PhysicalProperties visitPhysicalNestedLoopJoin( PhysicalNestedLoopJoin nestedLoopJoin, PlanContext context) { - // TODO: currently, only support cross join in BE Preconditions.checkState(childrenOutputProperties.size() == 2); PhysicalProperties leftOutputProperty = childrenOutputProperties.get(0); return new PhysicalProperties(leftOutputProperty.getDistributionSpec()); } @Override - public PhysicalProperties visitPhysicalOlapScan(PhysicalOlapScan olapScan, PlanContext context) { - // TODO: find a better way to handle both tablet num == 1 and colocate table together in future - if (!olapScan.getTable().isColocateTable() && olapScan.getScanTabletNum() == 1 - && (!ConnectContext.get().getSessionVariable().enablePipelineEngine() - || ConnectContext.get().getSessionVariable().getParallelExecInstanceNum() == 1)) { - return PhysicalProperties.GATHER; - } else if (olapScan.getDistributionSpec() instanceof DistributionSpecHash) { - return PhysicalProperties.createHash((DistributionSpecHash) olapScan.getDistributionSpec()); + public PhysicalProperties visitPhysicalProject(PhysicalProject project, PlanContext context) { + // TODO: order spec do not process since we do not use it. + Preconditions.checkState(childrenOutputProperties.size() == 1); + PhysicalProperties childProperties = childrenOutputProperties.get(0); + DistributionSpec childDistributionSpec = childProperties.getDistributionSpec(); + OrderSpec childOrderSpec = childProperties.getOrderSpec(); + if (childDistributionSpec instanceof DistributionSpecHash) { + Map projections = Maps.newHashMap(); + Set obstructions = Sets.newHashSet(); + for (NamedExpression namedExpression : project.getProjects()) { + if (namedExpression instanceof Alias) { + Alias alias = (Alias) namedExpression; + Expression child = alias.child(); + if (child instanceof SlotReference) { + projections.put(((SlotReference) child).getExprId(), alias.getExprId()); + } else if (child instanceof Cast && child.child(0) instanceof Slot + && isSameHashValue(child.child(0).getDataType(), child.getDataType())) { + // cast(slot as varchar(10)) can do projection if slot is varchar(3) + projections.put(((Slot) child.child(0)).getExprId(), alias.getExprId()); + } else { + obstructions.addAll( + child.getInputSlots().stream() + .map(NamedExpression::getExprId) + .collect(Collectors.toSet())); + } + } + } + if (projections.entrySet().stream().allMatch(kv -> kv.getKey().equals(kv.getValue()))) { + return childrenOutputProperties.get(0); + } + DistributionSpecHash childDistributionSpecHash = (DistributionSpecHash) childDistributionSpec; + DistributionSpec defaultAnySpec = childDistributionSpecHash.getShuffleType() == ShuffleType.NATURAL + ? DistributionSpecStorageAny.INSTANCE : DistributionSpecAny.INSTANCE; + DistributionSpec outputDistributionSpec = childDistributionSpecHash.project( + projections, obstructions, defaultAnySpec); + return new PhysicalProperties(outputDistributionSpec, childOrderSpec); } else { - return PhysicalProperties.ANY; + return childrenOutputProperties.get(0); } } @Override - public PhysicalProperties visitPhysicalFileScan(PhysicalFileScan fileScan, PlanContext context) { - return PhysicalProperties.ANY; - } - - @Override - public PhysicalProperties visitPhysicalEmptyRelation(PhysicalEmptyRelation emptyRelation, PlanContext context) { - return PhysicalProperties.GATHER; - } - - @Override - public PhysicalProperties visitPhysicalStorageLayerAggregate( - PhysicalStorageLayerAggregate storageLayerAggregate, PlanContext context) { - return storageLayerAggregate.getRelation().accept(this, context); + public PhysicalProperties visitPhysicalRepeat(PhysicalRepeat repeat, PlanContext context) { + Preconditions.checkState(childrenOutputProperties.size() == 1); + return PhysicalProperties.ANY.withOrderSpec(childrenOutputProperties.get(0).getOrderSpec()); } @Override - public PhysicalProperties visitPhysicalJdbcScan(PhysicalJdbcScan jdbcScan, PlanContext context) { - return PhysicalProperties.ANY; + public PhysicalProperties visitPhysicalPartitionTopN(PhysicalPartitionTopN partitionTopN, + PlanContext context) { + Preconditions.checkState(childrenOutputProperties.size() == 1); + PhysicalProperties childOutputProperty = childrenOutputProperties.get(0); + return new PhysicalProperties(childOutputProperty.getDistributionSpec()); } @Override - public PhysicalProperties visitPhysicalEsScan(PhysicalEsScan esScan, PlanContext context) { - return PhysicalProperties.ANY; + public PhysicalProperties visitPhysicalSetOperation(PhysicalSetOperation setOperation, PlanContext context) { + int[] offsetsOfFirstChild = null; + ShuffleType firstType = null; + List childrenDistribution = childrenOutputProperties.stream() + .map(PhysicalProperties::getDistributionSpec) + .collect(Collectors.toList()); + if (childrenDistribution.isEmpty()) { + // no child, mean it only has some one-row-relations + return PhysicalProperties.GATHER; + } + if (childrenDistribution.stream().allMatch(DistributionSpecGather.class::isInstance)) { + return PhysicalProperties.GATHER; + } + for (int i = 0; i < childrenDistribution.size(); i++) { + DistributionSpec childDistribution = childrenDistribution.get(i); + if (!(childDistribution instanceof DistributionSpecHash)) { + return PhysicalProperties.ANY; + } + DistributionSpecHash distributionSpecHash = (DistributionSpecHash) childDistribution; + int[] offsetsOfCurrentChild = new int[distributionSpecHash.getOrderedShuffledColumns().size()]; + for (int j = 0; j < setOperation.getChildOutput(i).size(); j++) { + int offset = distributionSpecHash.getExprIdToEquivalenceSet() + .getOrDefault(setOperation.getChildOutput(i).get(j).getExprId(), -1); + if (offset > 0) { + offsetsOfCurrentChild[offset] = j; + } else { + return PhysicalProperties.ANY; + } + } + if (offsetsOfFirstChild == null) { + firstType = ((DistributionSpecHash) childDistribution).getShuffleType(); + offsetsOfFirstChild = offsetsOfCurrentChild; + } else if (!Arrays.equals(offsetsOfFirstChild, offsetsOfCurrentChild) + || firstType != ((DistributionSpecHash) childDistribution).getShuffleType()) { + return PhysicalProperties.ANY; + } + } + // bucket + List request = Lists.newArrayList(); + for (int offset : offsetsOfFirstChild) { + request.add(setOperation.getOutput().get(offset).getExprId()); + } + return PhysicalProperties.createHash(request, firstType); } @Override - public PhysicalProperties visitPhysicalTVFRelation(PhysicalTVFRelation tvfRelation, PlanContext context) { - TableValuedFunction function = tvfRelation.getFunction(); - return function.getPhysicalProperties(); + public PhysicalProperties visitPhysicalUnion(PhysicalUnion union, PlanContext context) { + if (union.getConstantExprsList().isEmpty()) { + return visitPhysicalSetOperation(union, context); + } else { + // current be could not run const expr on appropriate node, + // so if we have constant exprs on union, the output of union always any + return PhysicalProperties.ANY; + } } @Override - public PhysicalProperties visitPhysicalAssertNumRows(PhysicalAssertNumRows assertNumRows, + public PhysicalProperties visitAbstractPhysicalSort(AbstractPhysicalSort sort, PlanContext context) { - return PhysicalProperties.GATHER; + Preconditions.checkState(childrenOutputProperties.size() == 1); + if (sort.getSortPhase().isLocal()) { + return new PhysicalProperties( + childrenOutputProperties.get(0).getDistributionSpec(), + new OrderSpec(sort.getOrderKeys())); + } + return new PhysicalProperties(DistributionSpecGather.INSTANCE, new OrderSpec(sort.getOrderKeys())); } @Override - public PhysicalProperties visitPhysicalGenerate(PhysicalGenerate generate, PlanContext context) { + public PhysicalProperties visitPhysicalWindow(PhysicalWindow window, PlanContext context) { Preconditions.checkState(childrenOutputProperties.size() == 1); return childrenOutputProperties.get(0); } - @Override - public PhysicalProperties visitPhysicalOlapTableSink(PhysicalOlapTableSink olapTableSink, - PlanContext context) { - return PhysicalProperties.GATHER; - } - private boolean isSameHashValue(DataType originType, DataType castType) { if (originType.isStringLikeType() && (castType.isVarcharType() || castType.isStringType()) && (castType.width() >= originType.width() || castType.width() < 0)) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/ChildrenPropertiesRegulator.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/ChildrenPropertiesRegulator.java index bd73a33cf1059b5..862dbb5e2b6f52d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/ChildrenPropertiesRegulator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/ChildrenPropertiesRegulator.java @@ -29,28 +29,29 @@ import org.apache.doris.nereids.trees.plans.physical.PhysicalDistribute; import org.apache.doris.nereids.trees.plans.physical.PhysicalHashAggregate; import org.apache.doris.nereids.trees.plans.physical.PhysicalHashJoin; +import org.apache.doris.nereids.trees.plans.physical.PhysicalNestedLoopJoin; +import org.apache.doris.nereids.trees.plans.physical.PhysicalSetOperation; import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor; import org.apache.doris.nereids.util.JoinUtils; -import org.apache.doris.qe.ConnectContext; import com.google.common.base.Preconditions; import com.google.common.collect.Lists; import java.util.ArrayList; import java.util.List; +import java.util.Optional; import java.util.Set; /** - * ensure child add enough distribute. + * ensure child add enough distribute. update children properties if we do regular */ -public class ChildrenPropertiesRegulator extends PlanVisitor { +public class ChildrenPropertiesRegulator extends PlanVisitor { private final GroupExpression parent; private final List children; private final List childrenProperties; private final List requiredProperties; private final JobContext jobContext; - private double enforceCost = 0.0; public ChildrenPropertiesRegulator(GroupExpression parent, List children, List childrenProperties, List requiredProperties, @@ -67,26 +68,27 @@ public ChildrenPropertiesRegulator(GroupExpression parent, List * * @return enforce cost. */ - public double adjustChildrenProperties() { + public boolean adjustChildrenProperties() { return parent.getPlan().accept(this, null); } @Override - public Double visit(Plan plan, Void context) { - return enforceCost; + public Boolean visit(Plan plan, Void context) { + return true; } @Override - public Double visitPhysicalHashAggregate(PhysicalHashAggregate agg, Void context) { + public Boolean visitPhysicalHashAggregate(PhysicalHashAggregate agg, Void context) { if (agg.getAggMode() == AggMode.INPUT_TO_RESULT && children.get(0).getPlan() instanceof PhysicalDistribute) { - return -1.0; + // this means one stage gather agg, usually bad pattern + return false; } - return 0.0; + return true; } @Override - public Double visitPhysicalHashJoin(PhysicalHashJoin hashJoin, + public Boolean visitPhysicalHashJoin(PhysicalHashJoin hashJoin, Void context) { Preconditions.checkArgument(children.size() == 2, String.format("children.size() is %d", children.size())); Preconditions.checkArgument(childrenProperties.size() == 2); @@ -96,106 +98,249 @@ public Double visitPhysicalHashJoin(PhysicalHashJoin> leftLowest - = leftChild.getLowestCostTable().get(childrenProperties.get(0)); - PhysicalProperties leftOutput = leftChild.getOutputProperties(childrenProperties.get(0)); + Optional updatedForLeft = Optional.empty(); + Optional updatedForRight = Optional.empty(); - GroupExpression rightChild = children.get(1); - Pair> rightLowest - = rightChild.getLowestCostTable().get(childrenProperties.get(1)); - PhysicalProperties rightOutput = rightChild.getOutputProperties(childrenProperties.get(1)); - - // check colocate join - if (leftHashSpec.getShuffleType() == ShuffleType.NATURAL - && rightHashSpec.getShuffleType() == ShuffleType.NATURAL) { + if ((leftHashSpec.getShuffleType() == ShuffleType.NATURAL + && rightHashSpec.getShuffleType() == ShuffleType.NATURAL)) { + // check colocate join with scan if (JoinUtils.couldColocateJoin(leftHashSpec, rightHashSpec)) { - return enforceCost; + return true; + } + updatedForRight = Optional.of(calAnotherSideRequired( + ShuffleType.STORAGE_BUCKETED, leftHashSpec, rightHashSpec, + (DistributionSpecHash) requiredProperties.get(0).getDistributionSpec(), + (DistributionSpecHash) requiredProperties.get(1).getDistributionSpec())); + } else if (leftHashSpec.getShuffleType() == ShuffleType.NATURAL + && rightHashSpec.getShuffleType() == ShuffleType.EXECUTION_BUCKETED) { + // must add enforce because shuffle algorithm is not same between NATURAL and BUCKETED + updatedForRight = Optional.of(calAnotherSideRequired( + ShuffleType.STORAGE_BUCKETED, leftHashSpec, rightHashSpec, + (DistributionSpecHash) requiredProperties.get(0).getDistributionSpec(), + (DistributionSpecHash) requiredProperties.get(1).getDistributionSpec())); + } else if (leftHashSpec.getShuffleType() == ShuffleType.NATURAL + && rightHashSpec.getShuffleType() == ShuffleType.STORAGE_BUCKETED) { + if (bothSideShuffleKeysAreSameOrder(leftHashSpec, rightHashSpec, + (DistributionSpecHash) requiredProperties.get(0).getDistributionSpec(), + (DistributionSpecHash) requiredProperties.get(1).getDistributionSpec())) { + return true; + } + updatedForRight = Optional.of(calAnotherSideRequired( + ShuffleType.STORAGE_BUCKETED, leftHashSpec, rightHashSpec, + (DistributionSpecHash) requiredProperties.get(0).getDistributionSpec(), + (DistributionSpecHash) requiredProperties.get(1).getDistributionSpec())); + } else if (leftHashSpec.getShuffleType() == ShuffleType.EXECUTION_BUCKETED + && rightHashSpec.getShuffleType() == ShuffleType.NATURAL) { + // TODO: we must do shuffle on right because coordinator could not do right be selection in this case, + // since it always to check the left most node whether olap scan node. + // after we fix coordinator problem, we could do right to left bucket shuffle + updatedForRight = Optional.of(calAnotherSideRequired( + ShuffleType.EXECUTION_BUCKETED, leftHashSpec, rightHashSpec, + (DistributionSpecHash) requiredProperties.get(0).getDistributionSpec(), + (DistributionSpecHash) requiredProperties.get(1).getDistributionSpec())); + } else if (leftHashSpec.getShuffleType() == ShuffleType.EXECUTION_BUCKETED + && rightHashSpec.getShuffleType() == ShuffleType.EXECUTION_BUCKETED) { + if (bothSideShuffleKeysAreSameOrder(rightHashSpec, leftHashSpec, + (DistributionSpecHash) requiredProperties.get(1).getDistributionSpec(), + (DistributionSpecHash) requiredProperties.get(0).getDistributionSpec())) { + return true; + } + updatedForRight = Optional.of(calAnotherSideRequired( + ShuffleType.EXECUTION_BUCKETED, leftHashSpec, rightHashSpec, + (DistributionSpecHash) requiredProperties.get(0).getDistributionSpec(), + (DistributionSpecHash) requiredProperties.get(1).getDistributionSpec())); + } else if ((leftHashSpec.getShuffleType() == ShuffleType.EXECUTION_BUCKETED + && rightHashSpec.getShuffleType() == ShuffleType.STORAGE_BUCKETED)) { + if (children.get(0).getPlan() instanceof PhysicalDistribute) { + updatedForLeft = Optional.of(calAnotherSideRequired( + ShuffleType.STORAGE_BUCKETED, rightHashSpec, leftHashSpec, + (DistributionSpecHash) requiredProperties.get(1).getDistributionSpec(), + (DistributionSpecHash) requiredProperties.get(0).getDistributionSpec())); + } else { + updatedForRight = Optional.of(calAnotherSideRequired( + ShuffleType.EXECUTION_BUCKETED, leftHashSpec, rightHashSpec, + (DistributionSpecHash) requiredProperties.get(0).getDistributionSpec(), + (DistributionSpecHash) requiredProperties.get(1).getDistributionSpec())); + } + } else if ((leftHashSpec.getShuffleType() == ShuffleType.STORAGE_BUCKETED + && rightHashSpec.getShuffleType() == ShuffleType.NATURAL)) { + // TODO: we must do shuffle on right because coordinator could not do right be selection in this case, + // since it always to check the left most node whether olap scan node. + // after we fix coordinator problem, we could do right to left bucket shuffle + updatedForRight = Optional.of(calAnotherSideRequired( + ShuffleType.STORAGE_BUCKETED, leftHashSpec, rightHashSpec, + (DistributionSpecHash) requiredProperties.get(0).getDistributionSpec(), + (DistributionSpecHash) requiredProperties.get(1).getDistributionSpec())); + } else if ((leftHashSpec.getShuffleType() == ShuffleType.STORAGE_BUCKETED + && rightHashSpec.getShuffleType() == ShuffleType.EXECUTION_BUCKETED)) { + if (children.get(0).getPlan() instanceof PhysicalDistribute) { + updatedForLeft = Optional.of(calAnotherSideRequired( + ShuffleType.EXECUTION_BUCKETED, rightHashSpec, leftHashSpec, + (DistributionSpecHash) requiredProperties.get(1).getDistributionSpec(), + (DistributionSpecHash) requiredProperties.get(0).getDistributionSpec())); + } else { + updatedForRight = Optional.of(calAnotherSideRequired( + ShuffleType.STORAGE_BUCKETED, leftHashSpec, rightHashSpec, + (DistributionSpecHash) requiredProperties.get(0).getDistributionSpec(), + (DistributionSpecHash) requiredProperties.get(1).getDistributionSpec())); } - } - // check bucket shuffle join - if (leftHashSpec.getShuffleType() != ShuffleType.ENFORCED) { - if (ConnectContext.get().getSessionVariable().isEnableBucketShuffleJoin()) { - // We need to recalculate the required property of right child, - // to make right child compatible with left child. - PhysicalProperties rightRequireProperties = calRightRequiredOfBucketShuffleJoin( - leftHashSpec, rightHashSpec); - if (!rightOutput.equals(rightRequireProperties)) { - updateChildEnforceAndCost(rightChild, rightOutput, - (DistributionSpecHash) rightRequireProperties.getDistributionSpec(), rightLowest.first); - } - childrenProperties.set(1, rightRequireProperties); - return enforceCost; + } else if ((leftHashSpec.getShuffleType() == ShuffleType.STORAGE_BUCKETED + && rightHashSpec.getShuffleType() == ShuffleType.STORAGE_BUCKETED)) { + if (bothSideShuffleKeysAreSameOrder(rightHashSpec, leftHashSpec, + (DistributionSpecHash) requiredProperties.get(1).getDistributionSpec(), + (DistributionSpecHash) requiredProperties.get(0).getDistributionSpec())) { + return true; + } + if (children.get(0).getPlan() instanceof PhysicalDistribute) { + updatedForLeft = Optional.of(calAnotherSideRequired( + ShuffleType.STORAGE_BUCKETED, rightHashSpec, leftHashSpec, + (DistributionSpecHash) requiredProperties.get(1).getDistributionSpec(), + (DistributionSpecHash) requiredProperties.get(0).getDistributionSpec())); + } else { + updatedForRight = Optional.of(calAnotherSideRequired( + ShuffleType.STORAGE_BUCKETED, leftHashSpec, rightHashSpec, + (DistributionSpecHash) requiredProperties.get(0).getDistributionSpec(), + (DistributionSpecHash) requiredProperties.get(1).getDistributionSpec())); } - updateChildEnforceAndCost(leftChild, leftOutput, - (DistributionSpecHash) requiredProperties.get(0).getDistributionSpec(), leftLowest.first); - childrenProperties.set(0, requiredProperties.get(0)); } - // check right hand must distribute. - if (rightHashSpec.getShuffleType() != ShuffleType.ENFORCED) { - updateChildEnforceAndCost(rightChild, rightOutput, - (DistributionSpecHash) requiredProperties.get(1).getDistributionSpec(), rightLowest.first); - childrenProperties.set(1, requiredProperties.get(1)); + updatedForLeft.ifPresent(physicalProperties -> updateChildEnforceAndCost(0, physicalProperties)); + updatedForRight.ifPresent(physicalProperties -> updateChildEnforceAndCost(1, physicalProperties)); + + return true; + } + + @Override + public Boolean visitPhysicalNestedLoopJoin(PhysicalNestedLoopJoin nestedLoopJoin, + Void context) { + Preconditions.checkArgument(children.size() == 2, String.format("children.size() is %d", children.size())); + Preconditions.checkArgument(childrenProperties.size() == 2); + Preconditions.checkArgument(requiredProperties.size() == 2); + DistributionSpec rightDistributionSpec = childrenProperties.get(1).getDistributionSpec(); + if (rightDistributionSpec instanceof DistributionSpecStorageGather) { + updateChildEnforceAndCost(1, PhysicalProperties.GATHER); } + return true; + } + + @Override + public Boolean visitPhysicalSetOperation(PhysicalSetOperation setOperation, Void context) { + if (children.isEmpty()) { + return true; + } + + PhysicalProperties requiredProperty = requiredProperties.get(0); + DistributionSpec requiredDistributionSpec = requiredProperty.getDistributionSpec(); + if (requiredDistributionSpec instanceof DistributionSpecGather) { + for (int i = 0; i < childrenProperties.size(); i++) { + if (childrenProperties.get(i).getDistributionSpec() instanceof DistributionSpecStorageGather) { + updateChildEnforceAndCost(i, PhysicalProperties.GATHER); + } + } + } else if (requiredDistributionSpec instanceof DistributionSpecAny) { + for (int i = 0; i < childrenProperties.size(); i++) { + if (childrenProperties.get(i).getDistributionSpec() instanceof DistributionSpecStorageAny + || childrenProperties.get(i).getDistributionSpec() instanceof DistributionSpecStorageGather + || childrenProperties.get(i).getDistributionSpec() instanceof DistributionSpecGather + || (childrenProperties.get(i).getDistributionSpec() instanceof DistributionSpecHash + && ((DistributionSpecHash) childrenProperties.get(i).getDistributionSpec()) + .getShuffleType() == ShuffleType.NATURAL)) { + updateChildEnforceAndCost(i, PhysicalProperties.EXECUTION_ANY); + } + } + } else if (requiredDistributionSpec instanceof DistributionSpecHash) { + // TODO: should use the most common hash spec as basic + DistributionSpecHash basic = (DistributionSpecHash) requiredDistributionSpec; + for (int i = 0; i < childrenProperties.size(); i++) { + DistributionSpecHash current = (DistributionSpecHash) childrenProperties.get(i).getDistributionSpec(); + if (current.getShuffleType() != ShuffleType.EXECUTION_BUCKETED + || !bothSideShuffleKeysAreSameOrder(basic, current, + (DistributionSpecHash) requiredProperties.get(0).getDistributionSpec(), + (DistributionSpecHash) requiredProperties.get(i).getDistributionSpec())) { + PhysicalProperties target = calAnotherSideRequired( + ShuffleType.EXECUTION_BUCKETED, basic, current, + (DistributionSpecHash) requiredProperties.get(0).getDistributionSpec(), + (DistributionSpecHash) requiredProperties.get(i).getDistributionSpec()); + updateChildEnforceAndCost(i, target); + } + } + } + return true; + } - return enforceCost; + private boolean bothSideShuffleKeysAreSameOrder( + DistributionSpecHash notShuffleSideOutput, DistributionSpecHash shuffleSideOutput, + DistributionSpecHash notShuffleSideRequired, DistributionSpecHash shuffleSideRequired) { + return shuffleSideOutput.getOrderedShuffledColumns().equals( + calAnotherSideRequiredShuffleIds(notShuffleSideOutput, notShuffleSideRequired, shuffleSideRequired)); } - private PhysicalProperties calRightRequiredOfBucketShuffleJoin(DistributionSpecHash leftHashSpec, - DistributionSpecHash rightHashSpec) { - Preconditions.checkArgument(leftHashSpec.getShuffleType() != ShuffleType.ENFORCED); - DistributionSpecHash leftRequireSpec = (DistributionSpecHash) requiredProperties.get(0).getDistributionSpec(); - DistributionSpecHash rightRequireSpec = (DistributionSpecHash) requiredProperties.get(1).getDistributionSpec(); + private List calAnotherSideRequiredShuffleIds(DistributionSpecHash notShuffleSideOutput, + DistributionSpecHash notShuffleSideRequired, DistributionSpecHash shuffleSideRequired) { List rightShuffleIds = new ArrayList<>(); - for (ExprId scanId : leftHashSpec.getOrderedShuffledColumns()) { - int index = leftRequireSpec.getOrderedShuffledColumns().indexOf(scanId); + for (ExprId scanId : notShuffleSideOutput.getOrderedShuffledColumns()) { + int index = notShuffleSideRequired.getOrderedShuffledColumns().indexOf(scanId); if (index == -1) { - // when there is no exprId in leftHashSpec, we need to check EquivalenceExprIds - Set equivalentExprIds = leftHashSpec.getEquivalenceExprIdsOf(scanId); + // when there is no exprId in notShuffleSideOutput, we need to check EquivalenceExprIds + Set equivalentExprIds = notShuffleSideOutput.getEquivalenceExprIdsOf(scanId); for (ExprId alternativeExpr : equivalentExprIds) { - index = leftRequireSpec.getOrderedShuffledColumns().indexOf(alternativeExpr); + index = notShuffleSideRequired.getOrderedShuffledColumns().indexOf(alternativeExpr); if (index != -1) { break; } } } Preconditions.checkArgument(index != -1); - rightShuffleIds.add(rightRequireSpec.getOrderedShuffledColumns().get(index)); + rightShuffleIds.add(shuffleSideRequired.getOrderedShuffledColumns().get(index)); } - return new PhysicalProperties(new DistributionSpecHash(rightShuffleIds, ShuffleType.ENFORCED, - rightHashSpec.getTableId(), rightHashSpec.getSelectedIndexId(), rightHashSpec.getPartitionIds())); + return rightShuffleIds; } - private double updateChildEnforceAndCost(GroupExpression child, PhysicalProperties childOutput, - DistributionSpecHash required, Cost currentCost) { + private PhysicalProperties calAnotherSideRequired(ShuffleType shuffleType, + DistributionSpecHash notShuffleSideOutput, DistributionSpecHash shuffleSideOutput, + DistributionSpecHash notShuffleSideRequired, DistributionSpecHash shuffleSideRequired) { + List shuffleSideIds = calAnotherSideRequiredShuffleIds(notShuffleSideOutput, + notShuffleSideRequired, shuffleSideRequired); + return new PhysicalProperties(new DistributionSpecHash(shuffleSideIds, shuffleType, + shuffleSideOutput.getTableId(), shuffleSideOutput.getSelectedIndexId(), + shuffleSideOutput.getPartitionIds())); + } + + private void updateChildEnforceAndCost(int index, PhysicalProperties targetProperties) { + GroupExpression child = children.get(index); + Pair> lowest = child.getLowestCostTable().get(childrenProperties.get(index)); + PhysicalProperties output = child.getOutputProperties(childrenProperties.get(index)); + DistributionSpec target = targetProperties.getDistributionSpec(); + updateChildEnforceAndCost(child, output, target, lowest.first); + childrenProperties.set(index, targetProperties); + } + + // TODO: why add enforcer according to target and target is from requiredProperties not regular + private void updateChildEnforceAndCost(GroupExpression child, PhysicalProperties childOutput, + DistributionSpec target, Cost currentCost) { if (child.getPlan() instanceof PhysicalDistribute) { //To avoid continuous distribute operator, we just enforce the child's child childOutput = child.getInputPropertiesList(childOutput).get(0); - Pair newChildAndCost - = child.getOwnerGroup().getLowestCostPlan(childOutput).get(); + Pair newChildAndCost = child.getOwnerGroup().getLowestCostPlan(childOutput).get(); child = newChildAndCost.second; currentCost = newChildAndCost.first; } - DistributionSpec outputDistributionSpec; - outputDistributionSpec = required.withShuffleType(ShuffleType.ENFORCED); - - PhysicalProperties newOutputProperty = new PhysicalProperties(outputDistributionSpec); - GroupExpression enforcer = outputDistributionSpec.addEnforcer(child.getOwnerGroup()); + PhysicalProperties newOutputProperty = new PhysicalProperties(target); + GroupExpression enforcer = target.addEnforcer(child.getOwnerGroup()); jobContext.getCascadesContext().getMemo().addEnforcerPlan(enforcer, child.getOwnerGroup()); Cost totalCost = CostCalculator.addChildCost(enforcer.getPlan(), CostCalculator.calculateCost(enforcer, Lists.newArrayList(childOutput)), @@ -207,6 +352,5 @@ private double updateChildEnforceAndCost(GroupExpression child, PhysicalProperti enforcer.putOutputPropertiesMap(newOutputProperty, newOutputProperty); } child.getOwnerGroup().setBestPlan(enforcer, totalCost, newOutputProperty); - return enforceCost; } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/DistributionSpecAny.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/DistributionSpecAny.java index 0bc75ffce8d998b..3aa7ca226b9c2f5 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/DistributionSpecAny.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/DistributionSpecAny.java @@ -18,7 +18,7 @@ package org.apache.doris.nereids.properties; /** - * Data can be anywhere on the segments (required only). + * Data can be in any instance */ public class DistributionSpecAny extends DistributionSpec { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/DistributionSpecExecutionAny.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/DistributionSpecExecutionAny.java new file mode 100644 index 000000000000000..8f4299c17946cd3 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/DistributionSpecExecutionAny.java @@ -0,0 +1,37 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.properties; + +/** + * Data can be in any instance, used in PhysicalDistribute. + * Because all candidates in group could save as DistributionSpecAny's value in LowestCostPlan map + * to distinguish DistributionSpecAny, we need a new Spec to represent must shuffle require. + */ +public class DistributionSpecExecutionAny extends DistributionSpec { + + public static final DistributionSpecExecutionAny INSTANCE = new DistributionSpecExecutionAny(); + + private DistributionSpecExecutionAny() { + super(); + } + + @Override + public boolean satisfy(DistributionSpec other) { + return other instanceof DistributionSpecAny || other instanceof DistributionSpecExecutionAny; + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/DistributionSpecGather.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/DistributionSpecGather.java index 8bbb8fff2c0794c..117dbb7222240f0 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/DistributionSpecGather.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/DistributionSpecGather.java @@ -18,7 +18,7 @@ package org.apache.doris.nereids.properties; /** - * Gather distribution which put all data into one node. + * Gather distribution which put all data into one instance. */ public class DistributionSpecGather extends DistributionSpec { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/DistributionSpecHash.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/DistributionSpecHash.java index 6a7f899c2acfd44..d61e99688a28bd4 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/DistributionSpecHash.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/DistributionSpecHash.java @@ -205,22 +205,11 @@ public boolean satisfy(DistributionSpec required) { return false; } - if (requiredHash.shuffleType == ShuffleType.NATURAL && this.shuffleType != ShuffleType.NATURAL) { - // this shuffle type is not natural but require natural - return false; - } - - if (requiredHash.shuffleType == ShuffleType.AGGREGATE) { + if (requiredHash.getShuffleType() == ShuffleType.REQUIRE) { return containsSatisfy(requiredHash.getOrderedShuffledColumns()); } - - // If the required property is from join and this property is not enforced, we only need to check to contain - // And more checking is in ChildrenPropertiesRegulator - if (requiredHash.shuffleType == shuffleType.JOIN && this.shuffleType != shuffleType.ENFORCED) { - return containsSatisfy(requiredHash.getOrderedShuffledColumns()); - } - - return equalsSatisfy(requiredHash.getOrderedShuffledColumns()); + return requiredHash.getShuffleType() == this.getShuffleType() + && equalsSatisfy(requiredHash.getOrderedShuffledColumns()); } private boolean containsSatisfy(List required) { @@ -253,13 +242,14 @@ public DistributionSpecHash withShuffleType(ShuffleType shuffleType) { /** * generate a new DistributionSpec after projection. */ - public DistributionSpec project(Map projections, Set obstructions) { + public DistributionSpec project(Map projections, + Set obstructions, DistributionSpec defaultAnySpec) { List orderedShuffledColumns = Lists.newArrayList(); List> equivalenceExprIds = Lists.newArrayList(); Map exprIdToEquivalenceSet = Maps.newHashMap(); for (ExprId shuffledColumn : this.orderedShuffledColumns) { if (obstructions.contains(shuffledColumn)) { - return DistributionSpecAny.INSTANCE; + return defaultAnySpec; } orderedShuffledColumns.add(projections.getOrDefault(shuffledColumn, shuffledColumn)); } @@ -267,7 +257,7 @@ public DistributionSpec project(Map projections, Set obs Set projectionEquivalenceSet = Sets.newHashSet(); for (ExprId equivalence : equivalenceSet) { if (obstructions.contains(equivalence)) { - return DistributionSpecAny.INSTANCE; + return defaultAnySpec; } projectionEquivalenceSet.add(projections.getOrDefault(equivalence, equivalence)); } @@ -275,7 +265,7 @@ public DistributionSpec project(Map projections, Set obs } for (Map.Entry exprIdSetKV : this.exprIdToEquivalenceSet.entrySet()) { if (obstructions.contains(exprIdSetKV.getKey())) { - return DistributionSpecAny.INSTANCE; + return defaultAnySpec; } if (projections.containsKey(exprIdSetKV.getKey())) { exprIdToEquivalenceSet.put(projections.get(exprIdSetKV.getKey()), exprIdSetKV.getValue()); @@ -317,19 +307,14 @@ public String toString() { * Enums for concrete shuffle type. */ public enum ShuffleType { - // 1. The following properties are the required properties for children - // require, need to satisfy the distribution spec by aggregation way. - AGGREGATE, - // require, need to satisfy the distribution spec by join way. - JOIN, - - // 2. The following properties are the output properties from some operators - // output, for olap scan node and colocate join + // require, need to satisfy the distribution spec by contains. + REQUIRE, + // output, execution only could be done on the node with data NATURAL, - // output, for all join except colocate join - BUCKETED, - // output, all distribute enforce - ENFORCED, + // output, for shuffle by execution hash method + EXECUTION_BUCKETED, + // output, for shuffle by storage hash method + STORAGE_BUCKETED, } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/DistributionSpecReplicated.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/DistributionSpecReplicated.java index baae83d9eacbb5b..87c38f0d6bc0ff0 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/DistributionSpecReplicated.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/DistributionSpecReplicated.java @@ -18,7 +18,7 @@ package org.apache.doris.nereids.properties; /** - * Data is replicated across all segments. + * Data is replicated across all instances. * Like: broadcast join. */ public class DistributionSpecReplicated extends DistributionSpec { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/DistributionSpecStorageAny.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/DistributionSpecStorageAny.java new file mode 100644 index 000000000000000..b8d0dd42ebb941a --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/DistributionSpecStorageAny.java @@ -0,0 +1,37 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.properties; + +/** + * Data can be in any instance, but it restricted by physical storage nodes. + * When Plan's distribution is DistributionSpecStorageAny, + * the execution on it only could be done on the node storages its data. + */ +public class DistributionSpecStorageAny extends DistributionSpec { + + public static final DistributionSpecStorageAny INSTANCE = new DistributionSpecStorageAny(); + + private DistributionSpecStorageAny() { + super(); + } + + @Override + public boolean satisfy(DistributionSpec other) { + return other instanceof DistributionSpecAny || other instanceof DistributionSpecStorageAny; + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/DistributionSpecStorageGather.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/DistributionSpecStorageGather.java new file mode 100644 index 000000000000000..5a5f6827f8c1afc --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/DistributionSpecStorageGather.java @@ -0,0 +1,38 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.properties; + +/** + * Gather distribution which put all data into one instance and + * the execution on it only could be done on the node storages its data. + */ +public class DistributionSpecStorageGather extends DistributionSpec { + + public static final DistributionSpecStorageGather INSTANCE = new DistributionSpecStorageGather(); + + public DistributionSpecStorageGather() { + super(); + } + + @Override + public boolean satisfy(DistributionSpec other) { + return other instanceof DistributionSpecGather + || other instanceof DistributionSpecStorageGather + || other instanceof DistributionSpecAny; + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/EnforceMissingPropertiesHelper.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/EnforceMissingPropertiesHelper.java index 6ea0f9f06481a75..146b165d6888031 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/EnforceMissingPropertiesHelper.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/EnforceMissingPropertiesHelper.java @@ -117,7 +117,7 @@ private PhysicalProperties enforceDistribution(PhysicalProperties oldOutputPrope DistributionSpec requiredDistributionSpec = required.getDistributionSpec(); if (requiredDistributionSpec instanceof DistributionSpecHash) { DistributionSpecHash requiredDistributionSpecHash = (DistributionSpecHash) requiredDistributionSpec; - outputDistributionSpec = requiredDistributionSpecHash.withShuffleType(ShuffleType.ENFORCED); + outputDistributionSpec = requiredDistributionSpecHash.withShuffleType(ShuffleType.EXECUTION_BUCKETED); } else { outputDistributionSpec = requiredDistributionSpec; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/PhysicalProperties.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/PhysicalProperties.java index 54f2e6cb9de8086..28bf34797788ef5 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/PhysicalProperties.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/PhysicalProperties.java @@ -34,10 +34,16 @@ public class PhysicalProperties { public static PhysicalProperties ANY = new PhysicalProperties(); + public static PhysicalProperties STORAGE_ANY = new PhysicalProperties(DistributionSpecStorageAny.INSTANCE); + + public static PhysicalProperties EXECUTION_ANY = new PhysicalProperties(DistributionSpecExecutionAny.INSTANCE); + public static PhysicalProperties REPLICATED = new PhysicalProperties(DistributionSpecReplicated.INSTANCE); public static PhysicalProperties GATHER = new PhysicalProperties(DistributionSpecGather.INSTANCE); + public static PhysicalProperties STORAGE_GATHER = new PhysicalProperties(DistributionSpecStorageGather.INSTANCE); + private final OrderSpec orderSpec; private final DistributionSpec distributionSpec; diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/RequestPropertyDeriver.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/RequestPropertyDeriver.java index 5ab11c015d8ccb7..74324101d30204a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/RequestPropertyDeriver.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/RequestPropertyDeriver.java @@ -23,29 +23,24 @@ import org.apache.doris.nereids.memo.GroupExpression; import org.apache.doris.nereids.properties.DistributionSpecHash.ShuffleType; import org.apache.doris.nereids.trees.expressions.ExprId; -import org.apache.doris.nereids.trees.expressions.Expression; -import org.apache.doris.nereids.trees.expressions.NamedExpression; -import org.apache.doris.nereids.trees.expressions.SlotReference; -import org.apache.doris.nereids.trees.expressions.functions.agg.AggregateFunction; +import org.apache.doris.nereids.trees.expressions.Slot; import org.apache.doris.nereids.trees.plans.JoinHint; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.physical.AbstractPhysicalSort; import org.apache.doris.nereids.trees.plans.physical.PhysicalAssertNumRows; -import org.apache.doris.nereids.trees.plans.physical.PhysicalGenerate; +import org.apache.doris.nereids.trees.plans.physical.PhysicalCTEAnchor; import org.apache.doris.nereids.trees.plans.physical.PhysicalHashJoin; import org.apache.doris.nereids.trees.plans.physical.PhysicalLimit; import org.apache.doris.nereids.trees.plans.physical.PhysicalNestedLoopJoin; import org.apache.doris.nereids.trees.plans.physical.PhysicalOlapTableSink; -import org.apache.doris.nereids.trees.plans.physical.PhysicalPartitionTopN; +import org.apache.doris.nereids.trees.plans.physical.PhysicalSetOperation; +import org.apache.doris.nereids.trees.plans.physical.PhysicalUnion; import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor; -import org.apache.doris.nereids.util.ExpressionUtils; import org.apache.doris.nereids.util.JoinUtils; -import com.google.common.base.Preconditions; import com.google.common.collect.Lists; import java.util.List; -import java.util.Set; import java.util.stream.Collectors; /** @@ -80,7 +75,7 @@ public List> getRequestChildrenPropertyList(GroupExpres @Override public Void visit(Plan plan, PlanContext context) { if (plan instanceof RequirePropertiesSupplier) { - RequireProperties requireProperties = ((RequirePropertiesSupplier) plan).getRequireProperties(); + RequireProperties requireProperties = ((RequirePropertiesSupplier) plan).getRequireProperties(); List requestPhysicalProperties = requireProperties.computeRequirePhysicalProperties(plan, requestPropertyFromParent); addRequestPropertyToChildren(requestPhysicalProperties); @@ -96,75 +91,68 @@ public Void visit(Plan plan, PlanContext context) { return null; } + /* ******************************************************************************************** + * sink Node, in lexicographical order + * ******************************************************************************************** */ + @Override - public Void visitAbstractPhysicalSort(AbstractPhysicalSort sort, PlanContext context) { - if (!sort.getSortPhase().isLocal()) { - addRequestPropertyToChildren(PhysicalProperties.GATHER); - } else { - addRequestPropertyToChildren(PhysicalProperties.ANY); - } + public Void visitPhysicalOlapTableSink(PhysicalOlapTableSink olapTableSink, PlanContext context) { + addRequestPropertyToChildren(olapTableSink.getRequirePhysicalProperties()); return null; } + /* ******************************************************************************************** + * Other Node, in lexicographical order + * ******************************************************************************************** */ + @Override - public Void visitPhysicalLimit(PhysicalLimit limit, PlanContext context) { - if (limit.isGlobal()) { - addRequestPropertyToChildren(PhysicalProperties.GATHER); - } else { - addRequestPropertyToChildren(PhysicalProperties.ANY); - } + public Void visitPhysicalAssertNumRows(PhysicalAssertNumRows assertNumRows, PlanContext context) { + addRequestPropertyToChildren(PhysicalProperties.GATHER); return null; } @Override - public Void visitPhysicalPartitionTopN(PhysicalPartitionTopN partitionTopN, PlanContext context) { - addRequestPropertyToChildren(PhysicalProperties.ANY); + public Void visitPhysicalCTEAnchor(PhysicalCTEAnchor cteAnchor, + PlanContext context) { + addRequestPropertyToChildren(PhysicalProperties.ANY, requestPropertyFromParent); return null; } @Override public Void visitPhysicalHashJoin(PhysicalHashJoin hashJoin, PlanContext context) { JoinHint hint = hashJoin.getHint(); - switch (hint) { - case BROADCAST_RIGHT: - addBroadcastJoinRequestProperty(); - break; - case SHUFFLE_RIGHT: - addShuffleJoinRequestProperty(hashJoin); - break; - case NONE: - default: - // for shuffle join - if (JoinUtils.couldShuffle(hashJoin)) { - addShuffleJoinRequestProperty(hashJoin); - } - // for broadcast join - if (JoinUtils.couldBroadcast(hashJoin)) { - addRequestPropertyToChildren(PhysicalProperties.ANY, PhysicalProperties.REPLICATED); - } - + if (hint == JoinHint.BROADCAST_RIGHT && JoinUtils.couldBroadcast(hashJoin)) { + addBroadcastJoinRequestProperty(); + return null; + } + if (hint == JoinHint.SHUFFLE_RIGHT && JoinUtils.couldShuffle(hashJoin)) { + addShuffleJoinRequestProperty(hashJoin); + return null; + } + // for shuffle join + if (JoinUtils.couldShuffle(hashJoin)) { + addShuffleJoinRequestProperty(hashJoin); + } + // for broadcast join + if (JoinUtils.couldBroadcast(hashJoin)) { + addBroadcastJoinRequestProperty(); } return null; } - private void addBroadcastJoinRequestProperty() { - addRequestPropertyToChildren(PhysicalProperties.ANY, PhysicalProperties.REPLICATED); - } - - private void addShuffleJoinRequestProperty(PhysicalHashJoin hashJoin) { - Pair, List> onClauseUsedSlots = hashJoin.getHashConjunctsExprIds(); - // shuffle join - addRequestPropertyToChildren( - PhysicalProperties.createHash( - new DistributionSpecHash(onClauseUsedSlots.first, ShuffleType.JOIN)), - PhysicalProperties.createHash( - new DistributionSpecHash(onClauseUsedSlots.second, ShuffleType.JOIN))); + @Override + public Void visitPhysicalLimit(PhysicalLimit limit, PlanContext context) { + if (limit.isGlobal()) { + addRequestPropertyToChildren(PhysicalProperties.GATHER); + } else { + addRequestPropertyToChildren(PhysicalProperties.ANY); + } + return null; } @Override public Void visitPhysicalNestedLoopJoin( PhysicalNestedLoopJoin nestedLoopJoin, PlanContext context) { - // TODO: currently doris only use NLJ to do cross join, update this if we use NLJ to do other joins. // see canParallelize() in NestedLoopJoinNode if (nestedLoopJoin.getJoinType().isCrossJoin() || nestedLoopJoin.getJoinType().isInnerJoin() || nestedLoopJoin.getJoinType().isLeftJoin()) { @@ -176,23 +164,102 @@ public Void visitPhysicalNestedLoopJoin( } @Override - public Void visitPhysicalAssertNumRows(PhysicalAssertNumRows assertNumRows, PlanContext context) { - addRequestPropertyToChildren(PhysicalProperties.GATHER); + public Void visitPhysicalSetOperation(PhysicalSetOperation setOperation, PlanContext context) { + // intersect and except need do distinct, so we must do distribution on it. + DistributionSpec distributionRequestFromParent = requestPropertyFromParent.getDistributionSpec(); + if (distributionRequestFromParent instanceof DistributionSpecHash) { + DistributionSpecHash distributionSpecHash = (DistributionSpecHash) distributionRequestFromParent; + addRequestPropertyToChildren(createHashRequestAccordingToParent( + setOperation, distributionSpecHash, context)); + } else { + addRequestPropertyToChildren(setOperation.children().stream() + .map(Plan::getOutputExprIds) + .map(l -> PhysicalProperties.createHash(l, ShuffleType.EXECUTION_BUCKETED)) + .collect(Collectors.toList())); + } return null; } @Override - public Void visitPhysicalGenerate(PhysicalGenerate generate, PlanContext context) { - addRequestPropertyToChildren(PhysicalProperties.ANY); + public Void visitPhysicalUnion(PhysicalUnion union, PlanContext context) { + // TODO: we do not generate gather union until we could do better cost computation on set operation + List requiredPropertyList = + Lists.newArrayListWithCapacity(context.arity()); + if (union.getConstantExprsList().isEmpty()) { + // translate requestPropertyFromParent to other children's request. + DistributionSpec distributionRequestFromParent = requestPropertyFromParent.getDistributionSpec(); + if (distributionRequestFromParent instanceof DistributionSpecHash) { + DistributionSpecHash distributionSpecHash = (DistributionSpecHash) distributionRequestFromParent; + requiredPropertyList = createHashRequestAccordingToParent(union, distributionSpecHash, context); + } else { + for (int i = context.arity(); i > 0; --i) { + requiredPropertyList.add(PhysicalProperties.ANY); + } + } + + } else { + // current be could not run const expr on appropriate node, + // so if we have constant exprs on union, the output of union always any + // then any request on children is useless. + for (int i = context.arity(); i > 0; --i) { + requiredPropertyList.add(PhysicalProperties.ANY); + } + } + addRequestPropertyToChildren(requiredPropertyList); return null; } @Override - public Void visitPhysicalOlapTableSink(PhysicalOlapTableSink olapTableSink, PlanContext context) { - addRequestPropertyToChildren(olapTableSink.getRequirePhysicalProperties()); + public Void visitAbstractPhysicalSort(AbstractPhysicalSort sort, PlanContext context) { + if (!sort.getSortPhase().isLocal()) { + addRequestPropertyToChildren(PhysicalProperties.GATHER); + } else { + addRequestPropertyToChildren(PhysicalProperties.ANY); + } return null; } + private List createHashRequestAccordingToParent( + Plan plan, DistributionSpecHash distributionRequestFromParent, PlanContext context) { + List requiredPropertyList = + Lists.newArrayListWithCapacity(context.arity()); + int[] outputOffsets = new int[distributionRequestFromParent.getOrderedShuffledColumns().size()]; + List setOperationOutputs = plan.getOutput(); + // get the offset of bucketed columns of set operation + for (int i = 0; i < setOperationOutputs.size(); i++) { + int offset = distributionRequestFromParent.getExprIdToEquivalenceSet() + .getOrDefault(setOperationOutputs.get(i).getExprId(), -1); + if (offset > 0) { + outputOffsets[offset] = i; + } + } + // use the offset to generate children's request + for (int i = 0; i < context.arity(); i++) { + List childOutput = plan.child(i).getOutput(); + List childRequest = Lists.newArrayList(); + for (int offset : outputOffsets) { + childRequest.add(childOutput.get(offset).getExprId()); + } + requiredPropertyList.add(PhysicalProperties.createHash( + childRequest, distributionRequestFromParent.getShuffleType())); + } + return requiredPropertyList; + } + + private void addBroadcastJoinRequestProperty() { + addRequestPropertyToChildren(PhysicalProperties.ANY, PhysicalProperties.REPLICATED); + } + + private void addShuffleJoinRequestProperty(PhysicalHashJoin hashJoin) { + Pair, List> onClauseUsedSlots = hashJoin.getHashConjunctsExprIds(); + // shuffle join + addRequestPropertyToChildren( + PhysicalProperties.createHash( + new DistributionSpecHash(onClauseUsedSlots.first, ShuffleType.REQUIRE)), + PhysicalProperties.createHash( + new DistributionSpecHash(onClauseUsedSlots.second, ShuffleType.REQUIRE))); + } + /** * helper function to assemble request children physical properties * @param physicalProperties one set request properties for children @@ -204,29 +271,5 @@ private void addRequestPropertyToChildren(PhysicalProperties... physicalProperti private void addRequestPropertyToChildren(List physicalProperties) { requestPropertyToChildren.add(physicalProperties); } - - private List extractExprIdFromDistinctFunction(List outputExpression) { - Set distinctAggregateFunctions = ExpressionUtils.collect(outputExpression, expr -> - expr instanceof AggregateFunction && ((AggregateFunction) expr).isDistinct() - ); - List exprIds = Lists.newArrayList(); - for (AggregateFunction aggregateFunction : distinctAggregateFunctions) { - for (Expression expr : aggregateFunction.children()) { - Preconditions.checkState(expr instanceof SlotReference, "normalize aggregate failed to" - + " normalize aggregate function " + aggregateFunction.toSql()); - exprIds.add(((SlotReference) expr).getExprId()); - } - } - return exprIds; - } - - private void addRequestHashDistribution(List hashColumns, ShuffleType shuffleType) { - List partitionedSlots = hashColumns.stream() - .map(SlotReference.class::cast) - .map(SlotReference::getExprId) - .collect(Collectors.toList()); - addRequestPropertyToChildren( - PhysicalProperties.createHash(new DistributionSpecHash(partitionedSlots, shuffleType))); - } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java index 79d89a35f82a06f..b7e75c3659ae118 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java @@ -107,6 +107,7 @@ public enum RuleType { ELIMINATE_HINT(RuleTypeClass.REWRITE), INFER_PREDICATES(RuleTypeClass.REWRITE), INFER_AGG_NOT_NULL(RuleTypeClass.REWRITE), + INFER_SET_OPERATOR_DISTINCT(RuleTypeClass.REWRITE), INFER_FILTER_NOT_NULL(RuleTypeClass.REWRITE), INFER_JOIN_NOT_NULL(RuleTypeClass.REWRITE), // subquery analyze @@ -207,6 +208,9 @@ public enum RuleType { PUSH_AGGREGATE_TO_OLAP_SCAN(RuleTypeClass.REWRITE), EXTRACT_SINGLE_TABLE_EXPRESSION_FROM_DISJUNCTION(RuleTypeClass.REWRITE), HIDE_ONE_ROW_RELATION_UNDER_UNION(RuleTypeClass.REWRITE), + PUSH_PROJECT_THROUGH_UNION(RuleTypeClass.REWRITE), + MERGE_ONE_ROW_RELATION_INTO_UNION(RuleTypeClass.REWRITE), + PUSH_PROJECT_INTO_ONE_ROW_RELATION(RuleTypeClass.REWRITE), MERGE_SET_OPERATION(RuleTypeClass.REWRITE), BUILD_AGG_FOR_UNION(RuleTypeClass.REWRITE), COUNT_DISTINCT_REWRITE(RuleTypeClass.REWRITE), diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindExpression.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindExpression.java index 286dbebfc1289a7..8ecd7b08ae8ae66 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindExpression.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindExpression.java @@ -506,11 +506,29 @@ protected boolean condition(Rule rule, Plan plan) { && (setOperation instanceof LogicalExcept || setOperation instanceof LogicalIntersect)) { throw new AnalysisException("INTERSECT and EXCEPT does not support ALL qualified"); } - + // we need to do cast before set operation, because we maybe use these slot to do shuffle + // so, we must cast it before shuffle to get correct hash code. List> castExpressions = setOperation.collectCastExpressions(); + ImmutableList.Builder newChildren = ImmutableList.builder(); + for (int i = 0; i < castExpressions.size(); i++) { + if (castExpressions.stream().allMatch(SlotReference.class::isInstance)) { + newChildren.add(setOperation.child(i)); + } else { + List projections = castExpressions.get(i).stream() + .map(e -> { + if (e instanceof SlotReference) { + return (SlotReference) e; + } else { + return new Alias(e, e.toSql()); + } + }).collect(ImmutableList.toImmutableList()); + LogicalProject logicalProject = new LogicalProject<>(projections, + setOperation.child(i)); + newChildren.add(logicalProject); + } + } List newOutputs = setOperation.buildNewOutputs(castExpressions.get(0)); - - return setOperation.withNewOutputs(newOutputs); + return setOperation.withNewOutputs(newOutputs).withChildren(newChildren.build()); }) ), RuleType.BINDING_GENERATE_SLOT.build( diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindRelation.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindRelation.java index 1b1b19d1346ca2f..43daf7e3c01efbd 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindRelation.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindRelation.java @@ -61,12 +61,24 @@ import java.util.List; import java.util.Optional; +import java.util.function.Function; +import javax.annotation.Nullable; /** * Rule to bind relations in query plan. */ public class BindRelation extends OneAnalysisRuleFactory { + private CustomTableResolver customTableResolver; + + public BindRelation() { + this(null); + } + + public BindRelation(@Nullable CustomTableResolver customTableResolver) { + this.customTableResolver = customTableResolver; + } + // TODO: cte will be copied to a sub-query with different names but the id of the unbound relation in them // are the same, so we use new relation id when binding relation, and will fix this bug later. @Override @@ -123,6 +135,9 @@ private LogicalPlan bindWithCurrentDb(CascadesContext cascadesContext, UnboundRe if (cascadesContext.getTables() != null) { table = cascadesContext.getTableByName(tableName); } + if (customTableResolver != null) { + table = customTableResolver.apply(tableQualifier); + } if (table == null) { // In some cases even if we have already called the "cascadesContext.getTableByName", // it also gets the null. So, we just check it in the catalog again for safety. @@ -136,7 +151,13 @@ private LogicalPlan bindWithCurrentDb(CascadesContext cascadesContext, UnboundRe private LogicalPlan bind(CascadesContext cascadesContext, UnboundRelation unboundRelation) { List tableQualifier = RelationUtil.getQualifierName(cascadesContext.getConnectContext(), unboundRelation.getNameParts()); - TableIf table = RelationUtil.getTable(tableQualifier, cascadesContext.getConnectContext().getEnv()); + TableIf table = null; + if (customTableResolver != null) { + table = customTableResolver.apply(tableQualifier); + } + if (table == null) { + table = RelationUtil.getTable(tableQualifier, cascadesContext.getConnectContext().getEnv()); + } return getLogicalPlan(table, unboundRelation, tableQualifier, cascadesContext); } @@ -225,4 +246,7 @@ private List getPartitionIds(TableIf t, UnboundRelation unboundRelation) { return part.getId(); }).collect(ImmutableList.toImmutableList()); } + + /** CustomTableResolver */ + public interface CustomTableResolver extends Function, TableIf> {} } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/CheckAnalysis.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/CheckAnalysis.java index bbe2a52469a0a9c..321dbe9f762fc44 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/CheckAnalysis.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/CheckAnalysis.java @@ -21,6 +21,7 @@ import org.apache.doris.nereids.rules.Rule; import org.apache.doris.nereids.rules.RuleType; import org.apache.doris.nereids.trees.expressions.Expression; +import org.apache.doris.nereids.trees.expressions.SlotReference; import org.apache.doris.nereids.trees.expressions.WindowExpression; import org.apache.doris.nereids.trees.expressions.functions.agg.AggregateFunction; import org.apache.doris.nereids.trees.expressions.functions.generator.TableGeneratingFunction; @@ -32,6 +33,7 @@ import org.apache.doris.nereids.trees.plans.logical.LogicalGenerate; import org.apache.doris.nereids.trees.plans.logical.LogicalHaving; import org.apache.doris.nereids.trees.plans.logical.LogicalJoin; +import org.apache.doris.nereids.trees.plans.logical.LogicalOneRowRelation; import org.apache.doris.nereids.trees.plans.logical.LogicalPlan; import org.apache.doris.nereids.trees.plans.logical.LogicalProject; import org.apache.doris.nereids.trees.plans.logical.LogicalSort; @@ -74,6 +76,12 @@ public class CheckAnalysis implements AnalysisRuleFactory { GroupingScalarFunction.class, TableGeneratingFunction.class, WindowExpression.class)) + .put(LogicalOneRowRelation.class, ImmutableSet.of( + AggregateFunction.class, + GroupingScalarFunction.class, + SlotReference.class, + TableGeneratingFunction.class, + WindowExpression.class)) .put(LogicalProject.class, ImmutableSet.of( TableGeneratingFunction.class)) .put(LogicalSort.class, ImmutableSet.of( diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/ProjectWithDistinctToAggregate.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/ProjectWithDistinctToAggregate.java index 1b6b270ab7c5dde..230eef9e1fee18b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/ProjectWithDistinctToAggregate.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/ProjectWithDistinctToAggregate.java @@ -22,38 +22,32 @@ import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.functions.agg.AggregateFunction; import org.apache.doris.nereids.trees.plans.logical.LogicalAggregate; - -import com.google.common.collect.Lists; +import org.apache.doris.nereids.trees.plans.logical.LogicalProject; /** * ProjectWithDistinctToAggregate. - * + *

* example sql: * 

- * select distinct value
- * from tbl
- *
- * - * origin plan: transformed plan: + * select distinct value from tbl * - * LogicalProject(projects=[distinct value]) LogicalAggregate(groupBy=[value], output=[value]) - * | => | - * LogicalOlapScan(table=tbl) LogicalOlapScan(table=tbl) + * LogicalProject(projects=[distinct value]) + * | + * LogicalOlapScan(table=tbl) + * => + * LogicalAggregate(groupBy=[value], output=[value]) + * | + * LogicalOlapScan(table=tbl) + * */ public class ProjectWithDistinctToAggregate extends OneAnalysisRuleFactory { @Override public Rule build() { return RuleType.PROJECT_WITH_DISTINCT_TO_AGGREGATE.build( - logicalProject().then(project -> { - if (project.isDistinct() && project.getProjects() - .stream() - .noneMatch(this::hasAggregateFunction)) { - return new LogicalAggregate<>(Lists.newArrayList(project.getProjects()), project.getProjects(), - project.child()); - } else { - return project; - } - }) + logicalProject() + .when(LogicalProject::isDistinct) + .whenNot(project -> project.getProjects().stream().anyMatch(this::hasAggregateFunction)) + .then(project -> new LogicalAggregate<>(project.getProjects(), project.child())) ); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/SimplifyCastRule.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/SimplifyCastRule.java index dad18d8aaf15447..d88489084dfdb61 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/SimplifyCastRule.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/SimplifyCastRule.java @@ -65,42 +65,53 @@ private Expression simplify(Cast cast, ExpressionRewriteContext context) { } if (child instanceof Literal) { - DataType castType = cast.getDataType(); - if (castType instanceof StringType) { - if (child instanceof VarcharLiteral) { - return new StringLiteral(((VarcharLiteral) child).getValue()); - } else if (child instanceof CharLiteral) { - return new StringLiteral(((CharLiteral) child).getValue()); - } - } else if (castType instanceof VarcharType) { - if (child instanceof VarcharLiteral) { - return new VarcharLiteral(((VarcharLiteral) child).getValue(), ((VarcharType) castType).getLen()); - } else if (child instanceof CharLiteral) { - return new VarcharLiteral(((CharLiteral) child).getValue(), ((VarcharType) castType).getLen()); - } - } else if (castType instanceof DecimalV2Type) { - if (child instanceof TinyIntLiteral) { - return new DecimalLiteral(new BigDecimal(((TinyIntLiteral) child).getValue())); - } else if (child instanceof SmallIntLiteral) { - return new DecimalLiteral(new BigDecimal(((SmallIntLiteral) child).getValue())); - } else if (child instanceof IntegerLiteral) { - return new DecimalLiteral(new BigDecimal(((IntegerLiteral) child).getValue())); - } else if (child instanceof BigIntLiteral) { - return new DecimalLiteral(new BigDecimal(((BigIntLiteral) child).getValue())); - } - } else if (castType instanceof DecimalV3Type) { - DecimalV3Type decimalV3Type = (DecimalV3Type) castType; - if (child instanceof TinyIntLiteral) { - return new DecimalV3Literal(decimalV3Type, new BigDecimal(((TinyIntLiteral) child).getValue())); - } else if (child instanceof SmallIntLiteral) { - return new DecimalV3Literal(decimalV3Type, new BigDecimal(((SmallIntLiteral) child).getValue())); - } else if (child instanceof IntegerLiteral) { - return new DecimalV3Literal(decimalV3Type, new BigDecimal(((IntegerLiteral) child).getValue())); - } else if (child instanceof BigIntLiteral) { - return new DecimalV3Literal(decimalV3Type, new BigDecimal(((BigIntLiteral) child).getValue())); - } else if (child instanceof DecimalV3Literal) { - return new DecimalV3Literal(decimalV3Type, ((DecimalV3Literal) child).getValue()); + try { + DataType castType = cast.getDataType(); + if (castType instanceof StringType) { + if (child instanceof VarcharLiteral) { + return new StringLiteral(((VarcharLiteral) child).getValue()); + } else if (child instanceof CharLiteral) { + return new StringLiteral(((CharLiteral) child).getValue()); + } + } else if (castType instanceof VarcharType) { + if (child instanceof VarcharLiteral) { + return new VarcharLiteral(((VarcharLiteral) child).getValue(), + ((VarcharType) castType).getLen()); + } else if (child instanceof CharLiteral) { + return new VarcharLiteral(((CharLiteral) child).getValue(), + ((VarcharType) castType).getLen()); + } + } else if (castType instanceof DecimalV2Type) { + if (child instanceof TinyIntLiteral) { + return new DecimalLiteral(new BigDecimal(((TinyIntLiteral) child).getValue())); + } else if (child instanceof SmallIntLiteral) { + return new DecimalLiteral(new BigDecimal(((SmallIntLiteral) child).getValue())); + } else if (child instanceof IntegerLiteral) { + return new DecimalLiteral(new BigDecimal(((IntegerLiteral) child).getValue())); + } else if (child instanceof BigIntLiteral) { + return new DecimalLiteral(new BigDecimal(((BigIntLiteral) child).getValue())); + } + } else if (castType instanceof DecimalV3Type) { + DecimalV3Type decimalV3Type = (DecimalV3Type) castType; + if (child instanceof TinyIntLiteral) { + return new DecimalV3Literal(decimalV3Type, + new BigDecimal(((TinyIntLiteral) child).getValue())); + } else if (child instanceof SmallIntLiteral) { + return new DecimalV3Literal(decimalV3Type, + new BigDecimal(((SmallIntLiteral) child).getValue())); + } else if (child instanceof IntegerLiteral) { + return new DecimalV3Literal(decimalV3Type, + new BigDecimal(((IntegerLiteral) child).getValue())); + } else if (child instanceof BigIntLiteral) { + return new DecimalV3Literal(decimalV3Type, + new BigDecimal(((BigIntLiteral) child).getValue())); + } else if (child instanceof DecimalV3Literal) { + return new DecimalV3Literal(decimalV3Type, + ((DecimalV3Literal) child).getValue()); + } } + } catch (Throwable t) { + return cast; } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/AggregateStrategies.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/AggregateStrategies.java index a72fa10b0d64226..551e73532cf2644 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/AggregateStrategies.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/AggregateStrategies.java @@ -369,12 +369,13 @@ private List> onePhaseAggregateWithoutDistinct( return ImmutableList.of(gatherLocalAgg); } else { RequireProperties requireHash = RequireProperties.of( - PhysicalProperties.createHash(logicalAgg.getGroupByExpressions(), ShuffleType.AGGREGATE)); + PhysicalProperties.createHash(logicalAgg.getGroupByExpressions(), ShuffleType.REQUIRE)); PhysicalHashAggregate hashLocalAgg = gatherLocalAgg .withRequire(requireHash) .withPartitionExpressions(logicalAgg.getGroupByExpressions()); return ImmutableList.>builder() - .add(gatherLocalAgg) + // TODO: usually bad, disable it until we could do better cost computation. + //.add(gatherLocalAgg) .add(hashLocalAgg) .build(); } @@ -481,7 +482,7 @@ private List> twoPhaseAggregateWithCountDistinctMult return ImmutableList.of(gatherLocalGatherDistinctAgg); } else { RequireProperties requireHash = RequireProperties.of( - PhysicalProperties.createHash(logicalAgg.getGroupByExpressions(), ShuffleType.AGGREGATE)); + PhysicalProperties.createHash(logicalAgg.getGroupByExpressions(), ShuffleType.REQUIRE)); PhysicalHashAggregate hashLocalHashGlobalAgg = gatherLocalGatherDistinctAgg .withRequireTree(requireHash.withChildren(requireHash)) .withPartitionExpressions(logicalAgg.getGroupByExpressions()); @@ -621,7 +622,7 @@ private List> threePhaseAggregateWithCount ); RequireProperties requireDistinctHash = RequireProperties.of( - PhysicalProperties.createHash(logicalAgg.getGroupByExpressions(), ShuffleType.AGGREGATE)); + PhysicalProperties.createHash(logicalAgg.getGroupByExpressions(), ShuffleType.REQUIRE)); PhysicalHashAggregate anyLocalHashGlobalGatherDistinctAgg = anyLocalGatherGlobalGatherAgg.withChildren(ImmutableList.of( anyLocalGatherGlobalAgg @@ -636,7 +637,7 @@ private List> threePhaseAggregateWithCount .build(); } else { RequireProperties requireGroupByHash = RequireProperties.of( - PhysicalProperties.createHash(logicalAgg.getGroupByExpressions(), ShuffleType.AGGREGATE)); + PhysicalProperties.createHash(logicalAgg.getGroupByExpressions(), ShuffleType.REQUIRE)); PhysicalHashAggregate> anyLocalHashGlobalHashDistinctAgg = anyLocalGatherGlobalGatherAgg.withRequirePropertiesAndChild(requireGroupByHash, anyLocalGatherGlobalAgg @@ -730,13 +731,14 @@ inputToBufferParam, maybeUsingStreamAgg(connectContext, logicalAgg), return ImmutableList.of(anyLocalGatherGlobalAgg); } else { RequireProperties requireHash = RequireProperties.of( - PhysicalProperties.createHash(logicalAgg.getGroupByExpressions(), ShuffleType.AGGREGATE)); + PhysicalProperties.createHash(logicalAgg.getGroupByExpressions(), ShuffleType.REQUIRE)); PhysicalHashAggregate anyLocalHashGlobalAgg = anyLocalGatherGlobalAgg .withRequire(requireHash) .withPartitionExpressions(logicalAgg.getGroupByExpressions()); return ImmutableList.>builder() - .add(anyLocalGatherGlobalAgg) + // TODO: usually bad, disable it until we could do better cost computation. + // .add(anyLocalGatherGlobalAgg) .add(anyLocalHashGlobalAgg) .build(); } @@ -841,7 +843,7 @@ private List> twoPhaseAggregateWithDistinc if (logicalAgg.getGroupByExpressions().isEmpty()) { RequireProperties requireDistinctHash = RequireProperties.of(PhysicalProperties.createHash( - distinctArguments, ShuffleType.AGGREGATE)); + distinctArguments, ShuffleType.REQUIRE)); PhysicalHashAggregate hashLocalGatherGlobalAgg = gatherLocalGatherGlobalAgg .withChildren(ImmutableList.of(gatherLocalAgg .withRequire(requireDistinctHash) @@ -853,7 +855,7 @@ private List> twoPhaseAggregateWithDistinc .build(); } else { RequireProperties requireGroupByHash = RequireProperties.of(PhysicalProperties.createHash( - logicalAgg.getGroupByExpressions(), ShuffleType.AGGREGATE)); + logicalAgg.getGroupByExpressions(), ShuffleType.REQUIRE)); PhysicalHashAggregate> hashLocalHashGlobalAgg = gatherLocalGatherGlobalAgg .withRequirePropertiesAndChild(requireGroupByHash, gatherLocalAgg .withRequire(requireGroupByHash) @@ -990,7 +992,7 @@ private List> threePhaseAggregateWithDisti requireGather, anyLocalGatherGlobalAgg); RequireProperties requireDistinctHash = RequireProperties.of( - PhysicalProperties.createHash(logicalAgg.getDistinctArguments(), ShuffleType.AGGREGATE)); + PhysicalProperties.createHash(logicalAgg.getDistinctArguments(), ShuffleType.REQUIRE)); PhysicalHashAggregate anyLocalHashGlobalGatherDistinctAgg = anyLocalGatherGlobalGatherDistinctAgg .withChildren(ImmutableList.of(anyLocalGatherGlobalAgg @@ -1006,7 +1008,7 @@ private List> threePhaseAggregateWithDisti .build(); } else { RequireProperties requireGroupByHash = RequireProperties.of( - PhysicalProperties.createHash(logicalAgg.getGroupByExpressions(), ShuffleType.AGGREGATE)); + PhysicalProperties.createHash(logicalAgg.getGroupByExpressions(), ShuffleType.REQUIRE)); PhysicalHashAggregate anyLocalHashGlobalHashDistinctAgg = anyLocalGatherGlobalGatherDistinctAgg .withRequirePropertiesAndChild(requireGroupByHash, anyLocalGatherGlobalAgg @@ -1069,12 +1071,13 @@ private List> onePhaseAggregateWithMultiDi return ImmutableList.of(gatherLocalAgg); } else { RequireProperties requireHash = RequireProperties.of( - PhysicalProperties.createHash(logicalAgg.getGroupByExpressions(), ShuffleType.AGGREGATE)); + PhysicalProperties.createHash(logicalAgg.getGroupByExpressions(), ShuffleType.REQUIRE)); PhysicalHashAggregate hashLocalAgg = gatherLocalAgg .withRequire(requireHash) .withPartitionExpressions(logicalAgg.getGroupByExpressions()); return ImmutableList.>builder() - .add(gatherLocalAgg) + // TODO: usually bad, disable it until we could do better cost computation. + // .add(gatherLocalAgg) .add(hashLocalAgg) .build(); } @@ -1156,7 +1159,7 @@ inputToBufferParam, maybeUsingStreamAgg(connectContext, logicalAgg), if (logicalAgg.getGroupByExpressions().isEmpty()) { Collection distinctArguments = logicalAgg.getDistinctArguments(); RequireProperties requireDistinctHash = RequireProperties.of(PhysicalProperties.createHash( - distinctArguments, ShuffleType.AGGREGATE)); + distinctArguments, ShuffleType.REQUIRE)); PhysicalHashAggregate hashLocalGatherGlobalAgg = anyLocalGatherGlobalAgg .withChildren(ImmutableList.of(anyLocalAgg .withRequire(requireDistinctHash) @@ -1168,7 +1171,7 @@ inputToBufferParam, maybeUsingStreamAgg(connectContext, logicalAgg), .build(); } else { RequireProperties requireHash = RequireProperties.of( - PhysicalProperties.createHash(logicalAgg.getGroupByExpressions(), ShuffleType.AGGREGATE)); + PhysicalProperties.createHash(logicalAgg.getGroupByExpressions(), ShuffleType.REQUIRE)); PhysicalHashAggregate anyLocalHashGlobalAgg = anyLocalGatherGlobalAgg .withRequire(requireHash) .withPartitionExpressions(logicalAgg.getGroupByExpressions()); @@ -1338,7 +1341,7 @@ private List> fourPhaseAggregateWithDistin RequireProperties requireGather = RequireProperties.of(PhysicalProperties.GATHER); RequireProperties requireDistinctHash = RequireProperties.of( - PhysicalProperties.createHash(logicalAgg.getDistinctArguments(), ShuffleType.AGGREGATE)); + PhysicalProperties.createHash(logicalAgg.getDistinctArguments(), ShuffleType.REQUIRE)); //phase 2 PhysicalHashAggregate anyLocalHashGlobalAgg = new PhysicalHashAggregate<>( diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/LogicalOlapScanToPhysicalOlapScan.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/LogicalOlapScanToPhysicalOlapScan.java index 4619e05f150c341..12399722af2e802 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/LogicalOlapScanToPhysicalOlapScan.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/LogicalOlapScanToPhysicalOlapScan.java @@ -25,9 +25,9 @@ import org.apache.doris.catalog.OlapTable; import org.apache.doris.catalog.PartitionType; import org.apache.doris.nereids.properties.DistributionSpec; -import org.apache.doris.nereids.properties.DistributionSpecAny; import org.apache.doris.nereids.properties.DistributionSpecHash; import org.apache.doris.nereids.properties.DistributionSpecHash.ShuffleType; +import org.apache.doris.nereids.properties.DistributionSpecStorageAny; import org.apache.doris.nereids.rules.Rule; import org.apache.doris.nereids.rules.RuleType; import org.apache.doris.nereids.trees.expressions.ExprId; @@ -75,54 +75,49 @@ private DistributionSpec convertDistribution(LogicalOlapScan olapScan) { && !colocateTableIndex.isGroupUnstable(colocateTableIndex.getGroup(olapTable.getId())); boolean isSelectUnpartition = olapTable.getPartitionInfo().getType() == PartitionType.UNPARTITIONED || olapScan.getSelectedPartitionIds().size() == 1; - if (isBelongStableCG || isSelectUnpartition) { - if (!(distributionInfo instanceof HashDistributionInfo) - || olapScan.getSelectedIndexId() != olapScan.getTable().getBaseIndexId()) { - // TODO if a mv is selected, we ignore base table's distributionInfo for now - // need improve this to handle the case if mv's distributionInfo is the same as base table - if (olapScan.getSelectedIndexId() != olapScan.getTable().getBaseIndexId()) { - HashDistributionInfo hashDistributionInfo = (HashDistributionInfo) distributionInfo; - List output = olapScan.getOutput(); - List baseOutput = olapScan.getOutputByIndex(olapScan.getTable().getBaseIndexId()); - List hashColumns = Lists.newArrayList(); - for (int i = 0; i < output.size(); i++) { - for (Column column : hashDistributionInfo.getDistributionColumns()) { - if (((SlotReference) output.get(i)).getColumn().get().getNameWithoutMvPrefix() - .equals(column.getName())) { - hashColumns.add(output.get(i).getExprId()); - } + // TODO: find a better way to handle both tablet num == 1 and colocate table together in future + if (distributionInfo instanceof HashDistributionInfo && (isBelongStableCG || isSelectUnpartition)) { + if (olapScan.getSelectedIndexId() != olapScan.getTable().getBaseIndexId()) { + HashDistributionInfo hashDistributionInfo = (HashDistributionInfo) distributionInfo; + List output = olapScan.getOutput(); + List baseOutput = olapScan.getOutputByIndex(olapScan.getTable().getBaseIndexId()); + List hashColumns = Lists.newArrayList(); + for (Slot slot : output) { + for (Column column : hashDistributionInfo.getDistributionColumns()) { + if (((SlotReference) slot).getColumn().get().getNameWithoutMvPrefix() + .equals(column.getName())) { + hashColumns.add(slot.getExprId()); } } - if (hashColumns.size() != hashDistributionInfo.getDistributionColumns().size()) { - for (int i = 0; i < baseOutput.size(); i++) { - for (Column column : hashDistributionInfo.getDistributionColumns()) { - if (((SlotReference) baseOutput.get(i)).getColumn().get().equals(column)) { - hashColumns.add(baseOutput.get(i).getExprId()); - } + } + if (hashColumns.size() != hashDistributionInfo.getDistributionColumns().size()) { + for (Slot slot : baseOutput) { + for (Column column : hashDistributionInfo.getDistributionColumns()) { + if (((SlotReference) slot).getColumn().get().equals(column)) { + hashColumns.add(slot.getExprId()); } } } - return new DistributionSpecHash(hashColumns, ShuffleType.NATURAL, olapScan.getTable().getId(), - olapScan.getSelectedIndexId(), Sets.newHashSet(olapScan.getSelectedPartitionIds())); } - return DistributionSpecAny.INSTANCE; - } - HashDistributionInfo hashDistributionInfo = (HashDistributionInfo) distributionInfo; - List output = olapScan.getOutput(); - List hashColumns = Lists.newArrayList(); - for (int i = 0; i < output.size(); i++) { - for (Column column : hashDistributionInfo.getDistributionColumns()) { - if (((SlotReference) output.get(i)).getColumn().get().equals(column)) { - hashColumns.add(output.get(i).getExprId()); + return new DistributionSpecHash(hashColumns, ShuffleType.NATURAL, olapScan.getTable().getId(), + olapScan.getSelectedIndexId(), Sets.newHashSet(olapScan.getSelectedPartitionIds())); + } else { + HashDistributionInfo hashDistributionInfo = (HashDistributionInfo) distributionInfo; + List output = olapScan.getOutput(); + List hashColumns = Lists.newArrayList(); + for (Slot slot : output) { + for (Column column : hashDistributionInfo.getDistributionColumns()) { + if (((SlotReference) slot).getColumn().get().equals(column)) { + hashColumns.add(slot.getExprId()); + } } } + return new DistributionSpecHash(hashColumns, ShuffleType.NATURAL, olapScan.getTable().getId(), + olapScan.getSelectedIndexId(), Sets.newHashSet(olapScan.getSelectedPartitionIds())); } - // TODO: need to consider colocate and dynamic partition and partition number - return new DistributionSpecHash(hashColumns, ShuffleType.NATURAL, olapScan.getTable().getId(), - olapScan.getSelectedIndexId(), Sets.newHashSet(olapScan.getSelectedPartitionIds())); } else { // RandomDistributionInfo - return DistributionSpecAny.INSTANCE; + return DistributionSpecStorageAny.INSTANCE; } } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/LogicalOneRowRelationToPhysicalOneRowRelation.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/LogicalOneRowRelationToPhysicalOneRowRelation.java index 95fa819bae7923b..0d007d69840f436 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/LogicalOneRowRelationToPhysicalOneRowRelation.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/LogicalOneRowRelationToPhysicalOneRowRelation.java @@ -29,7 +29,7 @@ public class LogicalOneRowRelationToPhysicalOneRowRelation extends OneImplementa public Rule build() { return logicalOneRowRelation() .then(relation -> new PhysicalOneRowRelation( - relation.getProjects(), relation.buildUnionNode(), relation.getLogicalProperties())) + relation.getProjects(), relation.getLogicalProperties())) .toRule(RuleType.LOGICAL_ONE_ROW_RELATION_TO_PHYSICAL_ONE_ROW_RELATION); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/LogicalUnionToPhysicalUnion.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/LogicalUnionToPhysicalUnion.java index 3ae88e33d524623..1b9d8e00dafc180 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/LogicalUnionToPhysicalUnion.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/LogicalUnionToPhysicalUnion.java @@ -29,6 +29,7 @@ public class LogicalUnionToPhysicalUnion extends OneImplementationRuleFactory { public Rule build() { return logicalUnion().then(union -> new PhysicalUnion(union.getQualifier(), + union.getConstantExprsList(), union.getLogicalProperties(), union.children()) ).toRule(RuleType.LOGICAL_UNION_TO_PHYSICAL_UNION); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/LogicalWindowToPhysicalWindow.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/LogicalWindowToPhysicalWindow.java index 89adcf6ef7efcfd..81acbce1ac5e5df 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/LogicalWindowToPhysicalWindow.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/LogicalWindowToPhysicalWindow.java @@ -18,7 +18,7 @@ package org.apache.doris.nereids.rules.implementation; import org.apache.doris.nereids.annotation.DependsRules; -import org.apache.doris.nereids.properties.DistributionSpecHash; +import org.apache.doris.nereids.properties.DistributionSpecHash.ShuffleType; import org.apache.doris.nereids.properties.OrderKey; import org.apache.doris.nereids.properties.OrderSpec; import org.apache.doris.nereids.properties.PhysicalProperties; @@ -145,7 +145,7 @@ private List generateKeysNeedToBeSorted(OrderKeyGroup orderKeyGroup) { if (!orderKeyGroup.orderKeys.isEmpty()) { keysNeedToBeSorted.addAll(orderKeyGroup.orderKeys.stream() - .map(orderExpression -> orderExpression.getOrderKey()) + .map(OrderExpression::getOrderKey) .collect(Collectors.toList()) ); } @@ -165,7 +165,7 @@ private PhysicalWindow createPhysicalWindow(Plan root, WindowFrameGroup wi root); if (windowFrameGroup.partitionKeys.isEmpty() && requiredOrderKeys.isEmpty()) { - return physicalWindow; + return physicalWindow.withRequirePropertiesAndChild(RequireProperties.of(PhysicalProperties.GATHER), root); } // todo: WFGs in the same OKG only need same RequiredProperties @@ -174,7 +174,7 @@ private PhysicalWindow createPhysicalWindow(Plan root, WindowFrameGroup wi properties = PhysicalProperties.GATHER.withOrderSpec(new OrderSpec(requiredOrderKeys)); } else { properties = PhysicalProperties.createHash( - windowFrameGroup.partitionKeys, DistributionSpecHash.ShuffleType.ENFORCED); + windowFrameGroup.partitionKeys, ShuffleType.REQUIRE); // requiredOrderKeys contain partitionKeys, so there is no need to check if requiredOrderKeys.isEmpty() properties = properties.withOrderSpec(new OrderSpec(requiredOrderKeys)); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/AdjustNullable.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/AdjustNullable.java index f42046d834b9288..67dc893dfdbb002 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/AdjustNullable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/AdjustNullable.java @@ -134,6 +134,9 @@ public Plan visitLogicalRepeat(LogicalRepeat repeat, Void contex @Override public Plan visitLogicalSetOperation(LogicalSetOperation setOperation, Void context) { setOperation = (LogicalSetOperation) super.visit(setOperation, context); + if (setOperation.children().isEmpty()) { + return setOperation; + } List inputNullable = setOperation.child(0).getOutput().stream() .map(ExpressionTrait::nullable).collect(Collectors.toList()); for (int i = 1; i < setOperation.arity(); i++) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/EliminateGroupByConstant.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/EliminateGroupByConstant.java index 4174e892cff0563..f5a01fe530543aa 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/EliminateGroupByConstant.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/EliminateGroupByConstant.java @@ -23,7 +23,6 @@ import org.apache.doris.nereids.rules.expression.rules.FoldConstantRule; import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.NamedExpression; -import org.apache.doris.nereids.trees.expressions.literal.Literal; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.logical.LogicalAggregate; @@ -61,7 +60,7 @@ public Rule build() { // if we do constant folding before normalize aggregate, the subtree will change and matching fail // such as: select a + 1 + 2 + 3, sum(b) from t group by a + 1 + 2 Expression foldExpression = FoldConstantRule.INSTANCE.rewrite(expression, context); - if (!(foldExpression instanceof Literal)) { + if (!foldExpression.isConstant()) { slotGroupByExprs.add(expression); } else { lit = expression; diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/HideOneRowRelationUnderUnion.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/HideOneRowRelationUnderUnion.java deleted file mode 100644 index 268f5c35b423902..000000000000000 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/HideOneRowRelationUnderUnion.java +++ /dev/null @@ -1,74 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.nereids.rules.rewrite; - -import org.apache.doris.nereids.rules.Rule; -import org.apache.doris.nereids.rules.RuleType; -import org.apache.doris.nereids.rules.analysis.AnalysisRuleFactory; -import org.apache.doris.nereids.trees.plans.Plan; -import org.apache.doris.nereids.trees.plans.logical.LogicalOneRowRelation; - -import com.google.common.collect.ImmutableList; - -import java.util.List; - -/** - * 1. Include oneRowRelation in the union, hide the oneRowRelation, and reduce the generation of union nodes. - * eg: select k1, k2 from t1 union select 1, 2 union select d1, d2 from t2; - * before: - * logicalUnion() - * / \ - * logicalUnion() logicalProject - * / \ - * logicalProject logicalOneRowRelation(BuildUnionNode:true) - * eg: select k1, k2 from t1 union select 1, 2 union select d1, d2 from t2; - * - * after: - * logicalUnion() - * / \ - * logicalUnion() logicalProject - * / \ - * logicalProject logicalOneRowRelation(BuildUnionNode:false) - */ -public class HideOneRowRelationUnderUnion implements AnalysisRuleFactory { - @Override - public List buildRules() { - return ImmutableList.of( - RuleType.HIDE_ONE_ROW_RELATION_UNDER_UNION.build( - logicalUnion(logicalOneRowRelation().when(LogicalOneRowRelation::buildUnionNode), any()) - .then(union -> { - List newChildren = new ImmutableList.Builder() - .add(((LogicalOneRowRelation) union.child(0)).withBuildUnionNode(false)) - .add(union.child(1)) - .build(); - return union.withChildren(newChildren); - }) - ), - RuleType.HIDE_ONE_ROW_RELATION_UNDER_UNION.build( - logicalUnion(any(), logicalOneRowRelation().when(LogicalOneRowRelation::buildUnionNode)) - .then(union -> { - List children = new ImmutableList.Builder() - .add(union.child(0)) - .add(((LogicalOneRowRelation) union.child(1)).withBuildUnionNode(false)) - .build(); - return union.withChildren(children); - }) - ) - ); - } -} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/InferSetOperatorDistinct.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/InferSetOperatorDistinct.java new file mode 100644 index 000000000000000..8b9f46408f57b70 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/InferSetOperatorDistinct.java @@ -0,0 +1,73 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.rules.rewrite; + +import org.apache.doris.nereids.rules.Rule; +import org.apache.doris.nereids.rules.RuleType; +import org.apache.doris.nereids.trees.plans.Plan; +import org.apache.doris.nereids.trees.plans.algebra.SetOperation.Qualifier; +import org.apache.doris.nereids.trees.plans.logical.LogicalAggregate; +import org.apache.doris.nereids.trees.plans.logical.LogicalJoin; +import org.apache.doris.nereids.trees.plans.logical.LogicalProject; + +import com.google.common.collect.ImmutableList; + +import java.util.List; + +/** + * Infer Distinct from SetOperator; + * Example: + * 
+ *                    Intersect
+ *   Intersect ->        |
+ *                  Agg for Distinct
+ *
+ */ +public class InferSetOperatorDistinct extends OneRewriteRuleFactory { + @Override + public Rule build() { + return logicalSetOperation() + .when(operation -> operation.getQualifier() == Qualifier.DISTINCT) + .when(operation -> operation.children().stream().allMatch(this::rejectNLJ)) + .then(setOperation -> { + if (setOperation.children().stream().anyMatch(child -> child instanceof LogicalAggregate)) { + return null; + } + + List newChildren = setOperation.children().stream() + .map(child -> new LogicalAggregate<>(ImmutableList.copyOf(child.getOutput()), child)) + .collect(ImmutableList.toImmutableList()); + if (newChildren.equals(setOperation.children())) { + return null; + } + return setOperation.withChildren(newChildren); + }).toRule(RuleType.INFER_SET_OPERATOR_DISTINCT); + } + + // if children exist NLJ, we can't infer distinct + private boolean rejectNLJ(Plan plan) { + if (plan instanceof LogicalProject) { + plan = plan.child(0); + } + if (plan instanceof LogicalJoin) { + LogicalJoin join = (LogicalJoin) plan; + return join.getOtherJoinConjuncts().isEmpty(); + } + return true; + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/MergeOneRowRelationIntoUnion.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/MergeOneRowRelationIntoUnion.java new file mode 100644 index 000000000000000..7b115d79d3dfd34 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/MergeOneRowRelationIntoUnion.java @@ -0,0 +1,66 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.rules.rewrite; + +import org.apache.doris.nereids.rules.Rule; +import org.apache.doris.nereids.rules.RuleType; +import org.apache.doris.nereids.trees.expressions.NamedExpression; +import org.apache.doris.nereids.trees.plans.Plan; +import org.apache.doris.nereids.trees.plans.logical.LogicalOneRowRelation; +import org.apache.doris.nereids.types.DataType; +import org.apache.doris.nereids.util.TypeCoercionUtils; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableList.Builder; +import com.google.common.collect.Lists; + +import java.util.List; + +/** + * merge one row relation into union, for easy to compute physical properties + */ +public class MergeOneRowRelationIntoUnion extends OneRewriteRuleFactory { + @Override + public Rule build() { + return logicalUnion().when(u -> u.children().stream() + .anyMatch(LogicalOneRowRelation.class::isInstance)).then(u -> { + List> constantExprsList = Lists.newArrayList(); + List newChildren = Lists.newArrayList(); + for (Plan child : u.children()) { + if (!(child instanceof LogicalOneRowRelation)) { + newChildren.add(child); + } else { + ImmutableList.Builder constantExprs = new Builder<>(); + List projects = ((LogicalOneRowRelation) child).getProjects(); + for (int i = 0; i < projects.size(); i++) { + NamedExpression project = projects.get(i); + DataType targetType = u.getOutput().get(i).getDataType(); + if (project.getDataType().equals(targetType)) { + constantExprs.add(project); + } else { + constantExprs.add((NamedExpression) project.withChildren( + TypeCoercionUtils.castIfNotSameType(project.child(0), targetType))); + } + } + constantExprsList.add(constantExprs.build()); + } + } + return u.withChildrenAndConstExprsList(newChildren, constantExprsList); + }).toRule(RuleType.MERGE_ONE_ROW_RELATION_INTO_UNION); + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushProjectIntoOneRowRelation.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushProjectIntoOneRowRelation.java new file mode 100644 index 000000000000000..04faffd9afca749 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushProjectIntoOneRowRelation.java @@ -0,0 +1,62 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.rules.rewrite; + +import org.apache.doris.nereids.rules.Rule; +import org.apache.doris.nereids.rules.RuleType; +import org.apache.doris.nereids.trees.expressions.Alias; +import org.apache.doris.nereids.trees.expressions.Expression; +import org.apache.doris.nereids.trees.expressions.NamedExpression; +import org.apache.doris.nereids.trees.expressions.SlotReference; +import org.apache.doris.nereids.util.ExpressionUtils; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.Maps; + +import java.util.Map; + +/** + * Project(OneRowRelation) -> OneRowRelation + */ +public class PushProjectIntoOneRowRelation extends OneRewriteRuleFactory { + @Override + public Rule build() { + return logicalProject(logicalOneRowRelation()).then(p -> { + Map replaceMap = Maps.newHashMap(); + Map replaceRootMap = Maps.newHashMap(); + p.child().getProjects().forEach(ne -> { + if (ne instanceof Alias) { + replaceMap.put(ne.toSlot(), ((Alias) ne).child()); + } else { + replaceMap.put(ne, ne); + } + replaceRootMap.put(ne.toSlot(), ne); + }); + ImmutableList.Builder newProjections = ImmutableList.builder(); + for (NamedExpression old : p.getProjects()) { + if (old instanceof SlotReference) { + newProjections.add(replaceRootMap.get(old)); + } else { + newProjections.add((NamedExpression) ExpressionUtils.replace(old, replaceMap)); + } + } + return p.child().withProjects(newProjections.build()); + + }).toRule(RuleType.PUSH_PROJECT_INTO_ONE_ROW_RELATION); + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushProjectThroughUnion.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushProjectThroughUnion.java new file mode 100644 index 000000000000000..882d0af36eb8707 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushProjectThroughUnion.java @@ -0,0 +1,82 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.rules.rewrite; + +import org.apache.doris.nereids.rules.Rule; +import org.apache.doris.nereids.rules.RuleType; +import org.apache.doris.nereids.trees.expressions.Alias; +import org.apache.doris.nereids.trees.expressions.Expression; +import org.apache.doris.nereids.trees.expressions.NamedExpression; +import org.apache.doris.nereids.trees.expressions.SlotReference; +import org.apache.doris.nereids.trees.plans.Plan; +import org.apache.doris.nereids.trees.plans.logical.LogicalProject; +import org.apache.doris.nereids.trees.plans.logical.LogicalSetOperation; +import org.apache.doris.nereids.util.ExpressionUtils; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.Maps; + +import java.util.List; +import java.util.Map; + +/** + * when we do analyze, we add project on the top of SetOperation's children when we need to cast children's output + * this rule push down the project under union to let MergeUnion could do better + * TODO: this rule maybe lead to unequal transformation if cast is not monomorphism, + * maybe we need to distinguish implicit cast and explicit cast + */ +public class PushProjectThroughUnion extends OneRewriteRuleFactory { + + @Override + public Rule build() { + return logicalProject(logicalSetOperation()) + .when(project -> project.getProjects().size() == project.child().getOutput().size() + && project.getProjects().stream().allMatch(e -> { + if (e instanceof SlotReference) { + return true; + } else { + Expression expr = ExpressionUtils.getExpressionCoveredByCast(e.child(0)); + return expr instanceof SlotReference; + } + } + )) + .then(project -> { + LogicalSetOperation union = project.child(); + ImmutableList.Builder newChildren = ImmutableList.builder(); + for (Plan child : union.children()) { + Map replaceMap = Maps.newHashMap(); + for (int i = 0; i < union.getOutput().size(); i++) { + replaceMap.put(union.getOutput().get(i), child.getOutput().get(i)); + } + List childProjections = project.getProjects().stream() + .map(e -> (NamedExpression) ExpressionUtils.replace(e, replaceMap)) + .map(e -> { + if (e instanceof Alias) { + return new Alias(((Alias) e).child(), e.getName()); + } + return e; + }) + .collect(ImmutableList.toImmutableList()); + newChildren.add(new LogicalProject<>(childProjections, child)); + } + List newOutput = (List) project.getOutput(); + return union.withNewOutputs(newOutput).withChildren(newChildren.build()); + }) + .toRule(RuleType.PUSH_PROJECT_THROUGH_UNION); + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/FilterEstimation.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/FilterEstimation.java index acf072fb8250f46..3ea7caaf0827400 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/FilterEstimation.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/FilterEstimation.java @@ -390,24 +390,7 @@ private Statistics estimateColumnEqualToColumn(Expression leftExpr, ColumnStatis rightBuilder.setNdv(rightIntersectLeft.getDistinctValues()); rightBuilder.setMinValue(rightIntersectLeft.getLow()); rightBuilder.setMaxValue(rightIntersectLeft.getDistinctValues()); - double sel; - double reduceRatio = 0.25; - double bothSideReducedRatio = 0.9; - if (!leftStats.rangeChanged() && !rightStats.rangeChanged() - && leftStats.ndv < leftStats.getOriginalNdv() * bothSideReducedRatio - && rightStats.ndv < rightStats.getOriginalNdv() * bothSideReducedRatio) { - double sel1; - if (leftStats.ndv > rightStats.ndv) { - sel1 = 1 / StatsMathUtil.nonZeroDivisor(leftStats.ndv); - } else { - sel1 = 1 / StatsMathUtil.nonZeroDivisor(rightStats.ndv); - } - double sel2 = Math.min(rightStats.ndv / rightStats.getOriginalNdv(), - leftStats.ndv / leftStats.getOriginalNdv()); - sel = sel1 * Math.pow(sel2, reduceRatio); - } else { - sel = 1 / StatsMathUtil.nonZeroDivisor(Math.max(leftStats.ndv, rightStats.ndv)); - } + double sel = 1 / StatsMathUtil.nonZeroDivisor(Math.max(leftStats.ndv, rightStats.ndv)); Statistics updatedStatistics = context.statistics.withSel(sel); updatedStatistics.addColumnStats(leftExpr, leftBuilder.build()); updatedStatistics.addColumnStats(rightExpr, rightBuilder.build()); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/JoinEstimation.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/JoinEstimation.java index 228b6936a72af00..afef2285e128364 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/JoinEstimation.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/JoinEstimation.java @@ -72,16 +72,7 @@ private static boolean hashJoinConditionContainsUnknownColumnStats(Statistics le return false; } - private static Statistics estimateInnerJoin(Statistics leftStats, Statistics rightStats, Join join) { - if (hashJoinConditionContainsUnknownColumnStats(leftStats, rightStats, join)) { - double rowCount = Math.max(leftStats.getRowCount(), rightStats.getRowCount()); - rowCount = Math.max(1, rowCount); - return new StatisticsBuilder() - .setRowCount(rowCount) - .putColumnStatistics(leftStats.columnStatistics()) - .putColumnStatistics(rightStats.columnStatistics()) - .build(); - } + private static Statistics estimateHashJoin(Statistics leftStats, Statistics rightStats, Join join) { /* * When we estimate filter A=B, * if any side of equation, A or B, is almost unique, the confidence level of estimation is high. @@ -95,31 +86,31 @@ private static Statistics estimateInnerJoin(Statistics leftStats, Statistics rig List trustableConditions = join.getHashJoinConjuncts().stream() .map(expression -> (EqualTo) expression) .filter( - expression -> { - // since ndv is not accurate, if ndv/rowcount < almostUniqueThreshold, - // this column is regarded as unique. - double almostUniqueThreshold = 0.9; - EqualTo equal = normalizeHashJoinCondition(expression, leftStats, rightStats); - ColumnStatistic eqLeftColStats = ExpressionEstimation.estimate(equal.left(), leftStats); - ColumnStatistic eqRightColStats = ExpressionEstimation.estimate(equal.right(), rightStats); - double rightStatsRowCount = StatsMathUtil.nonZeroDivisor(rightStats.getRowCount()); - double leftStatsRowCount = StatsMathUtil.nonZeroDivisor(leftStats.getRowCount()); - boolean trustable = eqRightColStats.ndv / rightStatsRowCount > almostUniqueThreshold - || eqLeftColStats.ndv / leftStatsRowCount > almostUniqueThreshold; - if (!trustable) { - double rNdv = StatsMathUtil.nonZeroDivisor(eqRightColStats.ndv); - double lNdv = StatsMathUtil.nonZeroDivisor(eqLeftColStats.ndv); - if (leftBigger) { - unTrustEqualRatio.add((rightStatsRowCount / rNdv) - * Math.min(eqLeftColStats.ndv, eqRightColStats.ndv) / lNdv); - } else { - unTrustEqualRatio.add((leftStatsRowCount / lNdv) - * Math.min(eqLeftColStats.ndv, eqRightColStats.ndv) / rNdv); + expression -> { + // since ndv is not accurate, if ndv/rowcount < almostUniqueThreshold, + // this column is regarded as unique. + double almostUniqueThreshold = 0.9; + EqualTo equal = normalizeHashJoinCondition(expression, leftStats, rightStats); + ColumnStatistic eqLeftColStats = ExpressionEstimation.estimate(equal.left(), leftStats); + ColumnStatistic eqRightColStats = ExpressionEstimation.estimate(equal.right(), rightStats); + double rightStatsRowCount = StatsMathUtil.nonZeroDivisor(rightStats.getRowCount()); + double leftStatsRowCount = StatsMathUtil.nonZeroDivisor(leftStats.getRowCount()); + boolean trustable = eqRightColStats.ndv / rightStatsRowCount > almostUniqueThreshold + || eqLeftColStats.ndv / leftStatsRowCount > almostUniqueThreshold; + if (!trustable) { + double rNdv = StatsMathUtil.nonZeroDivisor(eqRightColStats.ndv); + double lNdv = StatsMathUtil.nonZeroDivisor(eqLeftColStats.ndv); + if (leftBigger) { + unTrustEqualRatio.add((rightStatsRowCount / rNdv) + * Math.min(eqLeftColStats.ndv, eqRightColStats.ndv) / lNdv); + } else { + unTrustEqualRatio.add((leftStatsRowCount / lNdv) + * Math.min(eqLeftColStats.ndv, eqRightColStats.ndv) / rNdv); + } + unTrustableCondition.add(equal); } - unTrustableCondition.add(equal); + return trustable; } - return trustable; - } ).collect(Collectors.toList()); Statistics innerJoinStats; @@ -159,6 +150,34 @@ private static Statistics estimateInnerJoin(Statistics leftStats, Statistics rig } innerJoinStats = crossJoinStats.updateRowCountOnly(outputRowCount); } + return innerJoinStats; + } + + private static Statistics estimateNestLoopJoin(Statistics leftStats, Statistics rightStats, Join join) { + return new StatisticsBuilder() + .setRowCount(Math.max(1, leftStats.getRowCount() * rightStats.getRowCount())) + .putColumnStatistics(leftStats.columnStatistics()) + .putColumnStatistics(rightStats.columnStatistics()) + .build(); + } + + private static Statistics estimateInnerJoin(Statistics leftStats, Statistics rightStats, Join join) { + if (hashJoinConditionContainsUnknownColumnStats(leftStats, rightStats, join)) { + double rowCount = Math.max(leftStats.getRowCount(), rightStats.getRowCount()); + rowCount = Math.max(1, rowCount); + return new StatisticsBuilder() + .setRowCount(rowCount) + .putColumnStatistics(leftStats.columnStatistics()) + .putColumnStatistics(rightStats.columnStatistics()) + .build(); + } + + Statistics innerJoinStats; + if (join.getHashJoinConjuncts().isEmpty()) { + innerJoinStats = estimateNestLoopJoin(leftStats, rightStats, join); + } else { + innerJoinStats = estimateHashJoin(leftStats, rightStats, join); + } if (!join.getOtherJoinConjuncts().isEmpty()) { FilterEstimation filterEstimation = new FilterEstimation(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/StatsCalculator.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/StatsCalculator.java index 38856f44ba7580e..385c0227ea2687b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/StatsCalculator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/StatsCalculator.java @@ -42,13 +42,13 @@ import org.apache.doris.nereids.trees.plans.algebra.Filter; import org.apache.doris.nereids.trees.plans.algebra.Generate; import org.apache.doris.nereids.trees.plans.algebra.Limit; -import org.apache.doris.nereids.trees.plans.algebra.OneRowRelation; import org.apache.doris.nereids.trees.plans.algebra.PartitionTopN; import org.apache.doris.nereids.trees.plans.algebra.Project; import org.apache.doris.nereids.trees.plans.algebra.Repeat; import org.apache.doris.nereids.trees.plans.algebra.Scan; import org.apache.doris.nereids.trees.plans.algebra.SetOperation; import org.apache.doris.nereids.trees.plans.algebra.TopN; +import org.apache.doris.nereids.trees.plans.algebra.Union; import org.apache.doris.nereids.trees.plans.algebra.Window; import org.apache.doris.nereids.trees.plans.logical.LogicalAggregate; import org.apache.doris.nereids.trees.plans.logical.LogicalAssertNumRows; @@ -127,6 +127,7 @@ import java.util.AbstractMap.SimpleEntry; import java.util.Collections; +import java.util.Comparator; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -254,7 +255,7 @@ public Statistics visitPhysicalLimit(PhysicalLimit limit, Void c @Override public Statistics visitLogicalOneRowRelation(LogicalOneRowRelation oneRowRelation, Void context) { - return computeOneRowRelation(oneRowRelation); + return computeOneRowRelation(oneRowRelation.getProjects()); } @Override @@ -396,7 +397,7 @@ public Statistics visitPhysicalRepeat(PhysicalRepeat repeat, Voi @Override public Statistics visitPhysicalOneRowRelation(PhysicalOneRowRelation oneRowRelation, Void context) { - return computeOneRowRelation(oneRowRelation); + return computeOneRowRelation(oneRowRelation.getProjects()); } @Override @@ -689,7 +690,7 @@ private double estimateGroupByRowCount(List groupByExpressions, Stat if (groupByCount > 0) { List groupByNdvs = groupByColStats.values().stream() .map(colStats -> colStats.ndv) - .sorted().collect(Collectors.toList()); + .sorted(Comparator.reverseOrder()).collect(Collectors.toList()); rowCount = groupByNdvs.get(0); for (int groupByIndex = 1; groupByIndex < groupByCount; ++groupByIndex) { rowCount *= Math.max(1, groupByNdvs.get(groupByIndex) * Math.pow( @@ -760,9 +761,8 @@ private Statistics computeProject(Project project) { return new Statistics(childStats.getRowCount(), columnsStats, childStats.getWidth(), childStats.getPenalty()); } - private Statistics computeOneRowRelation(OneRowRelation oneRowRelation) { - Map columnStatsMap = oneRowRelation.getProjects() - .stream() + private Statistics computeOneRowRelation(List projects) { + Map columnStatsMap = projects.stream() .map(project -> { ColumnStatistic statistic = new ColumnStatisticBuilder().setNdv(1).build(); // TODO: compute the literal size @@ -789,13 +789,33 @@ private Statistics computeEmptyRelation(EmptyRelation emptyRelation) { } private Statistics computeUnion(SetOperation setOperation) { - List head = groupExpression.child(0).getLogicalProperties().getOutput(); - Statistics headStats = groupExpression.childStatistics(0); + // TODO: refactor this for one row relation + List head = null; + Statistics headStats = null; List> childOutputs = groupExpression.children() .stream().map(ge -> ge.getLogicalProperties().getOutput()).collect(Collectors.toList()); List childStats = groupExpression.children().stream().map(Group::getStatistics).collect(Collectors.toList()); + if (setOperation instanceof Union) { + childOutputs.addAll(((Union) setOperation).getConstantExprsList().stream() + .map(l -> l.stream().map(NamedExpression::toSlot).collect(Collectors.toList())) + .collect(Collectors.toList())); + childStats.addAll(((Union) setOperation).getConstantExprsList().stream() + .map(this::computeOneRowRelation) + .collect(Collectors.toList())); + if (!((Union) setOperation).getConstantExprsList().isEmpty()) { + head = ((Union) setOperation).getConstantExprsList().get(0).stream() + .map(NamedExpression::toSlot) + .collect(Collectors.toList()); + headStats = computeOneRowRelation(((Union) setOperation).getConstantExprsList().get(0)); + } + } + if (head == null) { + head = groupExpression.child(0).getLogicalProperties().getOutput(); + headStats = groupExpression.childStatistics(0); + } + StatisticsBuilder statisticsBuilder = new StatisticsBuilder(); List unionOutput = setOperation.getOutputs(); for (int i = 0; i < head.size(); i++) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/AggStateFunctionBuilder.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/AggStateFunctionBuilder.java index e07f263040f3227..054aa4767b60183 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/AggStateFunctionBuilder.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/AggStateFunctionBuilder.java @@ -18,15 +18,12 @@ package org.apache.doris.nereids.trees.expressions.functions; import org.apache.doris.nereids.trees.expressions.Expression; -import org.apache.doris.nereids.trees.expressions.SlotReference; import org.apache.doris.nereids.trees.expressions.functions.agg.AggregateFunction; import org.apache.doris.nereids.trees.expressions.functions.combinator.MergeCombinator; import org.apache.doris.nereids.trees.expressions.functions.combinator.StateCombinator; import org.apache.doris.nereids.trees.expressions.functions.combinator.UnionCombinator; import org.apache.doris.nereids.types.AggStateType; -import com.google.common.collect.ImmutableList; - import java.util.List; import java.util.Objects; import java.util.stream.Collectors; @@ -66,9 +63,7 @@ public boolean canApply(List arguments) { return false; } - return nestedBuilder.canApply(((AggStateType) argument.getDataType()).getSubTypes().stream().map(t -> { - return new SlotReference("mocked", t); - }).collect(ImmutableList.toImmutableList())); + return nestedBuilder.canApply(((AggStateType) argument.getDataType()).getMockedExpressions()); } } @@ -95,11 +90,7 @@ private AggregateFunction buildMergeOrUnion(String nestedName, List nestedArgumens = type.getSubTypes().stream().map(t -> { - return new SlotReference("mocked", t); - }).collect(Collectors.toList()); - - return (AggregateFunction) nestedBuilder.build(nestedName, nestedArgumens); + return (AggregateFunction) nestedBuilder.build(nestedName, type.getMockedExpressions()); } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/combinator/MergeCombinator.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/combinator/MergeCombinator.java index 69b9514f13c7c6c..b529ae2de3f8049 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/combinator/MergeCombinator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/combinator/MergeCombinator.java @@ -32,7 +32,6 @@ import java.util.List; import java.util.Objects; -import java.util.stream.Collectors; /** * AggState combinator merge @@ -47,13 +46,12 @@ public MergeCombinator(List arguments, AggregateFunction nested) { super(nested.getName() + AggStateFunctionBuilder.MERGE_SUFFIX, arguments); this.nested = Objects.requireNonNull(nested, "nested can not be null"); - inputType = new AggStateType(nested.getName(), nested.getArgumentsTypes(), - nested.getArguments().stream().map(Expression::nullable).collect(Collectors.toList())); + inputType = (AggStateType) arguments.get(0).getDataType(); } @Override public MergeCombinator withChildren(List children) { - return new MergeCombinator(children, nested.withChildren(children)); + return new MergeCombinator(children, nested); } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/combinator/StateCombinator.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/combinator/StateCombinator.java index d7da86c525bb93b..9b97a7afd42ada0 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/combinator/StateCombinator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/combinator/StateCombinator.java @@ -59,7 +59,7 @@ public StateCombinator(List arguments, AggregateFunction nested) { @Override public StateCombinator withChildren(List children) { - return new StateCombinator(children, nested.withChildren(children)); + return new StateCombinator(children, nested); } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/combinator/UnionCombinator.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/combinator/UnionCombinator.java index 21261998ec1aed6..67f09a50ebf9854 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/combinator/UnionCombinator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/combinator/UnionCombinator.java @@ -32,7 +32,6 @@ import java.util.List; import java.util.Objects; -import java.util.stream.Collectors; /** * AggState combinator union @@ -47,13 +46,12 @@ public UnionCombinator(List arguments, AggregateFunction nested) { super(nested.getName() + AggStateFunctionBuilder.UNION_SUFFIX, arguments); this.nested = Objects.requireNonNull(nested, "nested can not be null"); - inputType = new AggStateType(nested.getName(), nested.getArgumentsTypes(), - nested.getArguments().stream().map(Expression::nullable).collect(Collectors.toList())); + inputType = (AggStateType) arguments.get(0).getDataType(); } @Override public UnionCombinator withChildren(List children) { - return new UnionCombinator(children, nested.withChildren(children)); + return new UnionCombinator(children, nested); } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/executable/DateTimeExtractAndTransform.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/executable/DateTimeExtractAndTransform.java index f8442a810e66f65..4c7f80ab6c974e5 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/executable/DateTimeExtractAndTransform.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/executable/DateTimeExtractAndTransform.java @@ -522,9 +522,9 @@ public static Expression toDate(DateTimeLiteral date) { return new DateLiteral(date.getYear(), date.getMonth(), date.getDay()); } - @ExecFunction(name = "to_date", argTypes = {"DATETIMEV2"}, returnType = "DATE") + @ExecFunction(name = "to_date", argTypes = {"DATETIMEV2"}, returnType = "DATEV2") public static Expression toDate(DateTimeV2Literal date) { - return new DateLiteral(date.getYear(), date.getMonth(), date.getDay()); + return new DateV2Literal(date.getYear(), date.getMonth(), date.getDay()); } /** diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/table/Hdfs.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/table/Hdfs.java index 8ea2d1841421e45..2054a9ac8b11a72 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/table/Hdfs.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/table/Hdfs.java @@ -19,7 +19,6 @@ import org.apache.doris.catalog.FunctionSignature; import org.apache.doris.nereids.exceptions.AnalysisException; -import org.apache.doris.nereids.properties.PhysicalProperties; import org.apache.doris.nereids.trees.expressions.Slot; import org.apache.doris.nereids.trees.expressions.TVFProperties; import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor; @@ -63,9 +62,4 @@ public Statistics computeStats(List slots) { public R accept(ExpressionVisitor visitor, C context) { return visitor.visitHdfs(this, context); } - - @Override - public PhysicalProperties getPhysicalProperties() { - return PhysicalProperties.ANY; - } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/table/S3.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/table/S3.java index 3d995c6fcafb1f8..c2fc81489ae2069 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/table/S3.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/table/S3.java @@ -19,7 +19,6 @@ import org.apache.doris.catalog.FunctionSignature; import org.apache.doris.nereids.exceptions.AnalysisException; -import org.apache.doris.nereids.properties.PhysicalProperties; import org.apache.doris.nereids.trees.expressions.Slot; import org.apache.doris.nereids.trees.expressions.TVFProperties; import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor; @@ -63,9 +62,4 @@ public Statistics computeStats(List slots) { public R accept(ExpressionVisitor visitor, C context) { return visitor.visitS3(this, context); } - - @Override - public PhysicalProperties getPhysicalProperties() { - return PhysicalProperties.ANY; - } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/DecimalV3Literal.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/DecimalV3Literal.java index 748c44cd0905b8d..fe6040ca85ecd05 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/DecimalV3Literal.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/DecimalV3Literal.java @@ -18,9 +18,12 @@ package org.apache.doris.nereids.trees.expressions.literal; import org.apache.doris.analysis.LiteralExpr; +import org.apache.doris.nereids.exceptions.AnalysisException; import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor; import org.apache.doris.nereids.types.DecimalV3Type; +import com.google.common.base.Preconditions; + import java.math.BigDecimal; import java.math.RoundingMode; import java.util.Objects; @@ -37,9 +40,13 @@ public DecimalV3Literal(BigDecimal value) { this.value = Objects.requireNonNull(value); } + /** + * Constructor for DecimalV3Literal + */ public DecimalV3Literal(DecimalV3Type dataType, BigDecimal value) { super(DecimalV3Type.createDecimalV3Type(dataType.getPrecision(), dataType.getScale())); Objects.requireNonNull(value, "value not be null"); + checkPrecisionAndScale(dataType.getPrecision(), dataType.getScale(), value); BigDecimal adjustedValue = value.scale() < 0 ? value : value.setScale(dataType.getScale(), RoundingMode.HALF_UP); this.value = Objects.requireNonNull(adjustedValue); @@ -64,4 +71,24 @@ public LiteralExpr toLegacyLiteral() { public double getDouble() { return value.doubleValue(); } + + private void checkPrecisionAndScale(int precision, int scale, BigDecimal value) throws AnalysisException { + Preconditions.checkNotNull(value); + int realPrecision = value.precision(); + int realScale = value.scale(); + boolean valid = true; + if (precision != -1 && scale != -1) { + if (precision < realPrecision || scale < realScale) { + valid = false; + } + } else { + valid = false; + } + + if (!valid) { + throw new AnalysisException( + String.format("Invalid precision and scale - expect (%d, %d), but (%d, %d)", + precision, scale, realPrecision, realScale)); + } + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/Plan.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/Plan.java index 108474799aa1f99..a014fcc3f85b60c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/Plan.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/Plan.java @@ -95,6 +95,10 @@ default Set getOutputSet() { return ImmutableSet.copyOf(getOutput()); } + default List getOutputExprIds() { + return getOutput().stream().map(NamedExpression::getExprId).collect(Collectors.toList()); + } + default Set getOutputExprIdSet() { return getOutput().stream().map(NamedExpression::getExprId).collect(Collectors.toSet()); } diff --git a/be/src/runtime/memory/system_allocator.h b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/algebra/Union.java similarity index 64% rename from be/src/runtime/memory/system_allocator.h rename to fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/algebra/Union.java index 239e5bf3ba77c19..f6623102d400033 100644 --- a/be/src/runtime/memory/system_allocator.h +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/algebra/Union.java @@ -15,24 +15,16 @@ // specific language governing permissions and limitations // under the License. -#pragma once +package org.apache.doris.nereids.trees.plans.algebra; -#include -#include +import org.apache.doris.nereids.trees.expressions.NamedExpression; -namespace doris { +import java.util.List; -// Allocate memory from system allocator, this allocator can be configured -// to allocate memory via mmap or malloc. -class SystemAllocator { -public: - static uint8_t* allocate(size_t length); +/** + * Common interface for logical/physical Union. + */ +public interface Union extends SetOperation { - static void free(uint8_t* ptr); - -private: - static uint8_t* allocate_via_mmap(size_t length); - static uint8_t* allocate_via_malloc(size_t length); -}; - -} // namespace doris + List> getConstantExprsList(); +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalAggregate.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalAggregate.java index e9926d4d925a2b1..d6136a0812d4101 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalAggregate.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalAggregate.java @@ -76,6 +76,13 @@ public LogicalAggregate( false, Optional.empty(), child); } + /** + * Distinct Agg + */ + public LogicalAggregate(List namedExpressions, CHILD_TYPE child) { + this(ImmutableList.copyOf(namedExpressions), namedExpressions, false, Optional.empty(), child); + } + public LogicalAggregate(List groupByExpressions, List outputExpressions, boolean ordinalIsResolved, CHILD_TYPE child) { this(groupByExpressions, outputExpressions, false, ordinalIsResolved, Optional.empty(), diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalOneRowRelation.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalOneRowRelation.java index 6685640dff8c1ec..4a1f4936247e5f9 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalOneRowRelation.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalOneRowRelation.java @@ -43,14 +43,12 @@ public class LogicalOneRowRelation extends LogicalLeaf implements OneRowRelation, OutputPrunable { private final List projects; - private final boolean buildUnionNode; public LogicalOneRowRelation(List projects) { - this(projects, true, Optional.empty(), Optional.empty()); + this(projects, Optional.empty(), Optional.empty()); } private LogicalOneRowRelation(List projects, - boolean buildUnionNode, Optional groupExpression, Optional logicalProperties) { super(PlanType.LOGICAL_ONE_ROW_RELATION, groupExpression, logicalProperties); @@ -59,7 +57,6 @@ private LogicalOneRowRelation(List projects, Preconditions.checkArgument(projects.stream().noneMatch(p -> p.containsType(AggregateFunction.class)), "OneRowRelation can not contains any aggregate function"); this.projects = ImmutableList.copyOf(Objects.requireNonNull(projects, "projects can not be null")); - this.buildUnionNode = buildUnionNode; } @Override @@ -79,13 +76,12 @@ public List getExpressions() { @Override public Plan withGroupExpression(Optional groupExpression) { - return new LogicalOneRowRelation(projects, buildUnionNode, - groupExpression, Optional.of(logicalPropertiesSupplier.get())); + return new LogicalOneRowRelation(projects, groupExpression, Optional.of(logicalPropertiesSupplier.get())); } @Override public Plan withLogicalProperties(Optional logicalProperties) { - return new LogicalOneRowRelation(projects, buildUnionNode, Optional.empty(), logicalProperties); + return new LogicalOneRowRelation(projects, Optional.empty(), logicalProperties); } @Override @@ -98,8 +94,7 @@ public List computeOutput() { @Override public String toString() { return Utils.toSqlString("LogicalOneRowRelation", - "projects", projects, - "buildUnionNode", buildUnionNode + "projects", projects ); } @@ -115,25 +110,16 @@ public boolean equals(Object o) { return false; } LogicalOneRowRelation that = (LogicalOneRowRelation) o; - return Objects.equals(projects, that.projects) - && Objects.equals(buildUnionNode, that.buildUnionNode); + return Objects.equals(projects, that.projects); } @Override public int hashCode() { - return Objects.hash(projects, buildUnionNode); - } - - public boolean buildUnionNode() { - return buildUnionNode; + return Objects.hash(projects); } public LogicalOneRowRelation withProjects(List namedExpressions) { - return new LogicalOneRowRelation(namedExpressions, buildUnionNode, Optional.empty(), Optional.empty()); - } - - public Plan withBuildUnionNode(boolean buildUnionNode) { - return new LogicalOneRowRelation(projects, buildUnionNode, Optional.empty(), Optional.empty()); + return new LogicalOneRowRelation(namedExpressions, Optional.empty(), Optional.empty()); } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalUnion.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalUnion.java index 934df533f723ee7..b5feeb82c5c09ce 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalUnion.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalUnion.java @@ -19,12 +19,16 @@ import org.apache.doris.nereids.memo.GroupExpression; import org.apache.doris.nereids.properties.LogicalProperties; +import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.NamedExpression; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.PlanType; +import org.apache.doris.nereids.trees.plans.algebra.Union; import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor; import org.apache.doris.nereids.util.Utils; +import com.google.common.collect.ImmutableList; + import java.util.List; import java.util.Objects; import java.util.Optional; @@ -32,7 +36,10 @@ /** * Logical Union. */ -public class LogicalUnion extends LogicalSetOperation implements OutputPrunable { +public class LogicalUnion extends LogicalSetOperation implements Union, OutputPrunable { + + // in doris, we use union node to present one row relation + private final List> constantExprsList; // When there is an agg on the union and there is a filter on the agg, // it is necessary to keep the filter on the agg and push the filter down to each child of the union. private final boolean hasPushedFilter; @@ -40,25 +47,47 @@ public class LogicalUnion extends LogicalSetOperation implements OutputPrunable public LogicalUnion(Qualifier qualifier, List inputs) { super(PlanType.LOGICAL_UNION, qualifier, inputs); this.hasPushedFilter = false; + this.constantExprsList = ImmutableList.of(); } - public LogicalUnion(Qualifier qualifier, List outputs, boolean hasPushedFilter, - List inputs) { + public LogicalUnion(Qualifier qualifier, List outputs, + List> constantExprsList, boolean hasPushedFilter, List inputs) { super(PlanType.LOGICAL_UNION, qualifier, outputs, inputs); this.hasPushedFilter = hasPushedFilter; + this.constantExprsList = ImmutableList.copyOf( + Objects.requireNonNull(constantExprsList, "constantExprsList should not be null")); } - public LogicalUnion(Qualifier qualifier, List outputs, boolean hasPushedFilter, + public LogicalUnion(Qualifier qualifier, List outputs, + List> constantExprsList, boolean hasPushedFilter, Optional groupExpression, Optional logicalProperties, List inputs) { super(PlanType.LOGICAL_UNION, qualifier, outputs, groupExpression, logicalProperties, inputs); this.hasPushedFilter = hasPushedFilter; + this.constantExprsList = ImmutableList.copyOf( + Objects.requireNonNull(constantExprsList, "constantExprsList should not be null")); + } + + public boolean hasPushedFilter() { + return hasPushedFilter; + } + + public List> getConstantExprsList() { + return constantExprsList; + } + + @Override + public List getExpressions() { + return constantExprsList.stream().flatMap(List::stream).collect(ImmutableList.toImmutableList()); } @Override public String toString() { - return Utils.toSqlString("LogicalUnion", "qualifier", qualifier, "outputs", outputs, "hasPushedFilter", - hasPushedFilter); + return Utils.toSqlString("LogicalUnion", + "qualifier", qualifier, + "outputs", outputs, + "constantExprsList", constantExprsList, + "hasPushedFilter", hasPushedFilter); } @Override @@ -70,12 +99,13 @@ public boolean equals(Object o) { return false; } LogicalUnion that = (LogicalUnion) o; - return super.equals(that) && hasPushedFilter == that.hasPushedFilter; + return super.equals(that) && hasPushedFilter == that.hasPushedFilter + && Objects.equals(constantExprsList, that.constantExprsList); } @Override public int hashCode() { - return Objects.hash(super.hashCode(), hasPushedFilter); + return Objects.hash(super.hashCode(), hasPushedFilter, constantExprsList); } @Override @@ -85,35 +115,40 @@ public R accept(PlanVisitor visitor, C context) { @Override public LogicalUnion withChildren(List children) { - return new LogicalUnion(qualifier, outputs, hasPushedFilter, children); + return new LogicalUnion(qualifier, outputs, constantExprsList, hasPushedFilter, children); } @Override public LogicalUnion withGroupExpression(Optional groupExpression) { - return new LogicalUnion(qualifier, outputs, hasPushedFilter, groupExpression, + return new LogicalUnion(qualifier, outputs, constantExprsList, hasPushedFilter, groupExpression, Optional.of(getLogicalProperties()), children); } @Override public LogicalUnion withLogicalProperties(Optional logicalProperties) { - return new LogicalUnion(qualifier, outputs, hasPushedFilter, Optional.empty(), logicalProperties, children); + return new LogicalUnion(qualifier, outputs, constantExprsList, hasPushedFilter, + Optional.empty(), logicalProperties, children); } @Override public LogicalUnion withNewOutputs(List newOutputs) { - return new LogicalUnion(qualifier, newOutputs, hasPushedFilter, Optional.empty(), Optional.empty(), children); + return new LogicalUnion(qualifier, newOutputs, constantExprsList, + hasPushedFilter, Optional.empty(), Optional.empty(), children); } - public LogicalUnion withAllQualifier() { - return new LogicalUnion(Qualifier.ALL, outputs, hasPushedFilter, Optional.empty(), Optional.empty(), children); + public LogicalUnion withChildrenAndConstExprsList( + List children, List> constantExprsList) { + return new LogicalUnion(qualifier, outputs, constantExprsList, hasPushedFilter, children); } - public boolean hasPushedFilter() { - return hasPushedFilter; + public LogicalUnion withAllQualifier() { + return new LogicalUnion(Qualifier.ALL, outputs, constantExprsList, hasPushedFilter, + Optional.empty(), Optional.empty(), children); } public LogicalUnion withHasPushedFilter() { - return new LogicalUnion(qualifier, outputs, true, Optional.empty(), Optional.empty(), children); + return new LogicalUnion(qualifier, outputs, constantExprsList, true, + Optional.empty(), Optional.empty(), children); } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/AbstractPhysicalSort.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/AbstractPhysicalSort.java index 30850b5b463c7c1..6765400e329e3fb 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/AbstractPhysicalSort.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/AbstractPhysicalSort.java @@ -81,7 +81,7 @@ public boolean equals(Object o) { return false; } AbstractPhysicalSort that = (AbstractPhysicalSort) o; - return Objects.equals(orderKeys, that.orderKeys); + return phase == that.phase && Objects.equals(orderKeys, that.orderKeys); } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalCTEConsumer.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalCTEConsumer.java index b9215e70fbd33ac..b75b932cd871530 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalCTEConsumer.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalCTEConsumer.java @@ -17,6 +17,8 @@ package org.apache.doris.nereids.trees.plans.physical; +import org.apache.doris.catalog.OlapTable; +import org.apache.doris.nereids.exceptions.TransformException; import org.apache.doris.nereids.memo.GroupExpression; import org.apache.doris.nereids.properties.LogicalProperties; import org.apache.doris.nereids.properties.PhysicalProperties; @@ -26,6 +28,7 @@ import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.PlanType; import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor; +import org.apache.doris.nereids.util.RelationUtil; import org.apache.doris.nereids.util.Utils; import org.apache.doris.statistics.Statistics; @@ -41,7 +44,7 @@ /** * Physical CTE consumer. */ -public class PhysicalCTEConsumer extends PhysicalLeaf { +public class PhysicalCTEConsumer extends PhysicalRelation { private final CTEId cteId; private final Map producerToConsumerSlotMap; @@ -77,12 +80,23 @@ public PhysicalCTEConsumer(CTEId cteId, Map consumerToProducerSlotMa LogicalProperties logicalProperties, PhysicalProperties physicalProperties, Statistics statistics) { - super(PlanType.PHYSICAL_CTE_CONSUME, groupExpression, logicalProperties, physicalProperties, statistics); + super(RelationUtil.newRelationId(), PlanType.PHYSICAL_CTE_CONSUME, ImmutableList.of(), groupExpression, + logicalProperties, physicalProperties, statistics); this.cteId = cteId; this.consumerToProducerSlotMap = ImmutableMap.copyOf(consumerToProducerSlotMap); this.producerToConsumerSlotMap = ImmutableMap.copyOf(producerToConsumerSlotMap); } + @Override + public OlapTable getTable() { + throw new TransformException("should not reach here"); + } + + @Override + public List getQualifier() { + throw new TransformException("should not reach here"); + } + public CTEId getCteId() { return cteId; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalOneRowRelation.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalOneRowRelation.java index b780057bbdfefb0..4d2f7069c245014 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalOneRowRelation.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalOneRowRelation.java @@ -42,22 +42,18 @@ public class PhysicalOneRowRelation extends PhysicalLeaf implements OneRowRelation { private final List projects; - private final boolean buildUnionNode; - public PhysicalOneRowRelation(List projects, boolean buildUnionNode, - LogicalProperties logicalProperties) { - this(projects, buildUnionNode, Optional.empty(), logicalProperties, null, null); + public PhysicalOneRowRelation(List projects, LogicalProperties logicalProperties) { + this(projects, Optional.empty(), logicalProperties, null, null); } private PhysicalOneRowRelation(List projects, - boolean buildUnionNode, Optional groupExpression, LogicalProperties logicalProperties, PhysicalProperties physicalProperties, Statistics statistics) { super(PlanType.PHYSICAL_ONE_ROW_RELATION, groupExpression, logicalProperties, physicalProperties, statistics); this.projects = ImmutableList.copyOf(Objects.requireNonNull(projects, "projects can not be null")); - this.buildUnionNode = buildUnionNode; } @Override @@ -77,21 +73,20 @@ public List getExpressions() { @Override public Plan withGroupExpression(Optional groupExpression) { - return new PhysicalOneRowRelation(projects, buildUnionNode, groupExpression, + return new PhysicalOneRowRelation(projects, groupExpression, logicalPropertiesSupplier.get(), physicalProperties, statistics); } @Override public Plan withLogicalProperties(Optional logicalProperties) { - return new PhysicalOneRowRelation(projects, buildUnionNode, Optional.empty(), + return new PhysicalOneRowRelation(projects, Optional.empty(), logicalProperties.get(), physicalProperties, statistics); } @Override public String toString() { return Utils.toSqlString("PhysicalOneRowRelation[" + id.asInt() + "]" + getGroupIdAsString(), - "expressions", projects, - "buildUnionNode", buildUnionNode + "expressions", projects ); } @@ -104,23 +99,18 @@ public boolean equals(Object o) { return false; } PhysicalOneRowRelation that = (PhysicalOneRowRelation) o; - return Objects.equals(projects, that.projects) - && buildUnionNode == that.buildUnionNode; + return Objects.equals(projects, that.projects); } @Override public int hashCode() { - return Objects.hash(projects, buildUnionNode); + return Objects.hash(projects); } @Override public PhysicalOneRowRelation withPhysicalPropertiesAndStats(PhysicalProperties physicalProperties, Statistics statistics) { - return new PhysicalOneRowRelation(projects, buildUnionNode, groupExpression, + return new PhysicalOneRowRelation(projects, groupExpression, logicalPropertiesSupplier.get(), physicalProperties, statistics); } - - public boolean notBuildUnionNode() { - return !buildUnionNode; - } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalUnion.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalUnion.java index 025397ac920db2b..ff95c848251b5b9 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalUnion.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalUnion.java @@ -20,38 +20,59 @@ import org.apache.doris.nereids.memo.GroupExpression; import org.apache.doris.nereids.properties.LogicalProperties; import org.apache.doris.nereids.properties.PhysicalProperties; +import org.apache.doris.nereids.trees.expressions.NamedExpression; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.PlanType; +import org.apache.doris.nereids.trees.plans.algebra.Union; import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor; import org.apache.doris.nereids.util.Utils; import org.apache.doris.statistics.Statistics; +import com.google.common.collect.ImmutableList; + import java.util.List; +import java.util.Objects; import java.util.Optional; /** * Physical Union. */ -public class PhysicalUnion extends PhysicalSetOperation { +public class PhysicalUnion extends PhysicalSetOperation implements Union { + + // in doris, we use union node to present one row relation + private final List> constantExprsList; public PhysicalUnion(Qualifier qualifier, - LogicalProperties logicalProperties, - List inputs) { + List> constantExprsList, + LogicalProperties logicalProperties, + List inputs) { super(PlanType.PHYSICAL_UNION, qualifier, logicalProperties, inputs); + this.constantExprsList = ImmutableList.copyOf( + Objects.requireNonNull(constantExprsList, "constantExprsList should not be null")); } public PhysicalUnion(Qualifier qualifier, - Optional groupExpression, - LogicalProperties logicalProperties, - List inputs) { + List> constantExprsList, + Optional groupExpression, + LogicalProperties logicalProperties, + List inputs) { super(PlanType.PHYSICAL_UNION, qualifier, groupExpression, logicalProperties, inputs); + this.constantExprsList = ImmutableList.copyOf( + Objects.requireNonNull(constantExprsList, "constantExprsList should not be null")); } - public PhysicalUnion(Qualifier qualifier, + public PhysicalUnion(Qualifier qualifier, List> constantExprsList, Optional groupExpression, LogicalProperties logicalProperties, PhysicalProperties physicalProperties, Statistics statistics, List inputs) { super(PlanType.PHYSICAL_UNION, qualifier, groupExpression, logicalProperties, physicalProperties, statistics, inputs); + this.constantExprsList = ImmutableList.copyOf( + Objects.requireNonNull(constantExprsList, "constantExprsList should not be null")); + + } + + public List> getConstantExprsList() { + return constantExprsList; } @Override @@ -63,30 +84,31 @@ public R accept(PlanVisitor visitor, C context) { public String toString() { return Utils.toSqlString("PhysicalUnion" + getGroupIdAsString(), "qualifier", qualifier, + "constantExprsList", constantExprsList, "stats", statistics); } @Override public PhysicalUnion withChildren(List children) { - return new PhysicalUnion(qualifier, getLogicalProperties(), children); + return new PhysicalUnion(qualifier, constantExprsList, getLogicalProperties(), children); } @Override public PhysicalUnion withGroupExpression( Optional groupExpression) { - return new PhysicalUnion(qualifier, groupExpression, getLogicalProperties(), children); + return new PhysicalUnion(qualifier, constantExprsList, groupExpression, getLogicalProperties(), children); } @Override public PhysicalUnion withLogicalProperties( Optional logicalProperties) { - return new PhysicalUnion(qualifier, Optional.empty(), logicalProperties.get(), children); + return new PhysicalUnion(qualifier, constantExprsList, Optional.empty(), logicalProperties.get(), children); } @Override public PhysicalUnion withPhysicalPropertiesAndStats( PhysicalProperties physicalProperties, Statistics statistics) { - return new PhysicalUnion(qualifier, Optional.empty(), + return new PhysicalUnion(qualifier, constantExprsList, Optional.empty(), getLogicalProperties(), physicalProperties, statistics, children); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/types/AggStateType.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/types/AggStateType.java index cc467b0eb671edb..a999512d1b7cb2c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/types/AggStateType.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/types/AggStateType.java @@ -19,11 +19,14 @@ import org.apache.doris.analysis.Expr; import org.apache.doris.catalog.Type; +import org.apache.doris.nereids.trees.expressions.Expression; +import org.apache.doris.nereids.trees.expressions.SlotReference; import org.apache.doris.nereids.types.coercion.AbstractDataType; import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableList; +import java.util.ArrayList; import java.util.List; import java.util.Objects; import java.util.stream.Collectors; @@ -56,8 +59,12 @@ public AggStateType(String functionName, List subTypes, List this.functionName = functionName; } - public List getSubTypes() { - return subTypes; + public List getMockedExpressions() { + List result = new ArrayList(); + for (int i = 0; i < subTypes.size(); i++) { + result.add(new SlotReference("mocked", subTypes.get(i), subTypeNullables.get(i))); + } + return result; } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/util/JoinUtils.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/util/JoinUtils.java index 9e31f7467e17393..bfe1594425e8a33 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/util/JoinUtils.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/util/JoinUtils.java @@ -194,13 +194,27 @@ public static Expression swapEqualToForChildrenOrder(EqualTo equalTo, Set } } + /** + * return true if we should do bucket shuffle join when translate plan. + */ + public static boolean shouldBucketShuffleJoin(AbstractPhysicalJoin join) { + DistributionSpec rightDistributionSpec = join.right().getPhysicalProperties().getDistributionSpec(); + if (!(rightDistributionSpec instanceof DistributionSpecHash)) { + return false; + } + DistributionSpecHash rightHash = (DistributionSpecHash) rightDistributionSpec; + return rightHash.getShuffleType() == ShuffleType.STORAGE_BUCKETED; + } + /** * return true if we should do broadcast join when translate plan. */ public static boolean shouldBroadcastJoin(AbstractPhysicalJoin join) { PhysicalPlan right = join.right(); - DistributionSpec rightDistributionSpec = right.getPhysicalProperties().getDistributionSpec(); - return rightDistributionSpec instanceof DistributionSpecReplicated; + if (right instanceof PhysicalDistribute) { + return ((PhysicalDistribute) right).getDistributionSpec() instanceof DistributionSpecReplicated; + } + return false; } /** @@ -211,6 +225,7 @@ public static boolean shouldColocateJoin(AbstractPhysicalJoin join) { - if (ConnectContext.get() == null - || !ConnectContext.get().getSessionVariable().isEnableBucketShuffleJoin()) { - return false; - } - DistributionSpec joinDistributionSpec = join.getPhysicalProperties().getDistributionSpec(); - DistributionSpec leftDistributionSpec = join.left().getPhysicalProperties().getDistributionSpec(); - DistributionSpec rightDistributionSpec = join.right().getPhysicalProperties().getDistributionSpec(); - if (join.left() instanceof PhysicalDistribute) { - return false; - } - if (!(joinDistributionSpec instanceof DistributionSpecHash) - || !(leftDistributionSpec instanceof DistributionSpecHash) - || !(rightDistributionSpec instanceof DistributionSpecHash)) { - return false; - } - DistributionSpecHash leftHash = (DistributionSpecHash) leftDistributionSpec; - // when we plan a bucket shuffle join, the left should not add a distribution enforce. - // so its shuffle type should be NATURAL(olap scan node or result of colocate join / bucket shuffle join with - // left child's shuffle type is also NATURAL), or be BUCKETED(result of join / agg). - if (leftHash.getShuffleType() != ShuffleType.BUCKETED && leftHash.getShuffleType() != ShuffleType.NATURAL) { - return false; - } - // there must use left as required and join as source. - // Because after property derive upper node's properties is contains lower node - // if their properties are satisfy. - return joinDistributionSpec.satisfy(leftDistributionSpec); - } - /** * could do colocate join with left and right child distribution spec. */ diff --git a/be/test/runtime/memory/chunk_allocator_test.cpp b/fe/fe-core/src/main/java/org/apache/doris/persist/BarrierLog.java similarity index 58% rename from be/test/runtime/memory/chunk_allocator_test.cpp rename to fe/fe-core/src/main/java/org/apache/doris/persist/BarrierLog.java index 47fce96ce870dee..e44cbb2f31beca4 100644 --- a/be/test/runtime/memory/chunk_allocator_test.cpp +++ b/fe/fe-core/src/main/java/org/apache/doris/persist/BarrierLog.java @@ -15,25 +15,20 @@ // specific language governing permissions and limitations // under the License. -#include "runtime/memory/chunk_allocator.h" +package org.apache.doris.persist; -#include +import org.apache.doris.common.io.Text; +import org.apache.doris.common.io.Writable; -#include "common/config.h" -#include "common/status.h" -#include "runtime/memory/chunk.h" -#include "util/cpu_info.h" -#include "util/doris_metrics.h" +import java.io.DataOutput; +import java.io.IOException; -namespace doris { +public class BarrierLog implements Writable { + public BarrierLog() { + } -TEST(ChunkAllocatorTest, Normal) { - for (size_t size = 4096; size <= 1024 * 1024; size <<= 1) { - Chunk chunk; - EXPECT_TRUE(ChunkAllocator::instance()->allocate_align(size, &chunk).ok()); - EXPECT_NE(nullptr, chunk.data); - EXPECT_EQ(size, chunk.size); - ChunkAllocator::instance()->free(chunk); + @Override + public void write(DataOutput out) throws IOException { + Text.writeString(out, ""); } } -} // namespace doris diff --git a/fe/fe-core/src/main/java/org/apache/doris/persist/DropPartitionInfo.java b/fe/fe-core/src/main/java/org/apache/doris/persist/DropPartitionInfo.java index 5a4a07f0cd04adb..f4fac915261c619 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/persist/DropPartitionInfo.java +++ b/fe/fe-core/src/main/java/org/apache/doris/persist/DropPartitionInfo.java @@ -40,6 +40,8 @@ public class DropPartitionInfo implements Writable { private boolean forceDrop = false; @SerializedName(value = "recycleTime") private long recycleTime = 0; + @SerializedName(value = "sql") + private String sql; private DropPartitionInfo() { } @@ -52,6 +54,17 @@ public DropPartitionInfo(Long dbId, Long tableId, String partitionName, this.isTempPartition = isTempPartition; this.forceDrop = forceDrop; this.recycleTime = recycleTime; + + StringBuilder sb = new StringBuilder(); + sb.append("DROP PARTITION "); + if (isTempPartition) { + sb.append("TEMPORARY "); + } + sb.append("`").append(partitionName).append("`"); + if (forceDrop) { + sb.append(" FORCE"); + } + this.sql = sb.toString(); } public Long getDbId() { diff --git a/fe/fe-core/src/main/java/org/apache/doris/persist/EditLog.java b/fe/fe-core/src/main/java/org/apache/doris/persist/EditLog.java index 4244664d4309eef..b67ca9259e5a1df 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/persist/EditLog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/persist/EditLog.java @@ -234,8 +234,8 @@ public static void loadJournal(Env env, Long logId, JournalEntity journal) { "Begin to unprotect add partition. db = " + info.getDbId() + " table = " + info.getTableId() + " partitionName = " + info.getPartition().getName()); AddPartitionRecord addPartitionRecord = new AddPartitionRecord(logId, info); - Env.getCurrentEnv().getBinlogManager().addAddPartitionRecord(addPartitionRecord); env.replayAddPartition(info); + env.getBinlogManager().addAddPartitionRecord(addPartitionRecord); break; } case OperationType.OP_DROP_PARTITION: { @@ -243,6 +243,7 @@ public static void loadJournal(Env env, Long logId, JournalEntity journal) { LOG.info("Begin to unprotect drop partition. db = " + info.getDbId() + " table = " + info.getTableId() + " partitionName = " + info.getPartitionName()); env.replayDropPartition(info); + env.getBinlogManager().addDropPartitionRecord(info, logId); break; } case OperationType.OP_MODIFY_PARTITION: { @@ -1037,6 +1038,11 @@ public static void loadJournal(Env env, Long logId, JournalEntity journal) { env.replayGcBinlog(binlogGcInfo); break; } + case OperationType.OP_BARRIER: { + // the log only for barrier commit seq, not need to replay + LOG.info("replay barrier"); + break; + } default: { IOException e = new IOException(); LOG.error("UNKNOWN Operation Type {}", opCode, e); @@ -1199,14 +1205,17 @@ public void logRefreshExternalTableSchema(RefreshExternalTableInfo info) { logEdit(OperationType.OP_ALTER_EXTERNAL_TABLE_SCHEMA, info); } - public void logAddPartition(PartitionPersistInfo info) { + public long logAddPartition(PartitionPersistInfo info) { long logId = logEdit(OperationType.OP_ADD_PARTITION, info); AddPartitionRecord record = new AddPartitionRecord(logId, info); Env.getCurrentEnv().getBinlogManager().addAddPartitionRecord(record); + return logId; } - public void logDropPartition(DropPartitionInfo info) { - logEdit(OperationType.OP_DROP_PARTITION, info); + public long logDropPartition(DropPartitionInfo info) { + long logId = logEdit(OperationType.OP_DROP_PARTITION, info); + Env.getCurrentEnv().getBinlogManager().addDropPartitionRecord(info, logId); + return logId; } public void logErasePartition(long partitionId) { @@ -1800,11 +1809,15 @@ public void logDeleteAnalysisTask(AnalyzeDeletionLog log) { logEdit(OperationType.OP_DELETE_ANALYSIS_TASK, log); } - public void logAlterDatabaseProperty(AlterDatabasePropertyInfo log) { - logEdit(OperationType.OP_ALTER_DATABASE_PROPERTY, log); + public long logAlterDatabaseProperty(AlterDatabasePropertyInfo log) { + return logEdit(OperationType.OP_ALTER_DATABASE_PROPERTY, log); + } + + public long logGcBinlog(BinlogGcInfo log) { + return logEdit(OperationType.OP_GC_BINLOG, log); } - public void logGcBinlog(BinlogGcInfo log) { - logEdit(OperationType.OP_GC_BINLOG, log); + public long logBarrier() { + return logEdit(OperationType.OP_BARRIER, new BarrierLog()); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/persist/OperationType.java b/fe/fe-core/src/main/java/org/apache/doris/persist/OperationType.java index 674374f91795bc8..451a3eb7aae7beb 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/persist/OperationType.java +++ b/fe/fe-core/src/main/java/org/apache/doris/persist/OperationType.java @@ -308,6 +308,8 @@ public class OperationType { public static final short OP_GC_BINLOG = 435; + public static final short OP_BARRIER = 436; + /** * Get opcode name by op code. diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/EsScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/EsScanNode.java index 1e80ce17e89aa94..5da04b1f813aeff 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/EsScanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/EsScanNode.java @@ -72,20 +72,20 @@ public class EsScanNode extends ExternalScanNode { private static final Logger LOG = LogManager.getLogger(EsScanNode.class); - private EsTablePartitions esTablePartitions; - private EsTable table; + private final EsTablePartitions esTablePartitions; + private final EsTable table; private QueryBuilder queryBuilder; private boolean isFinalized = false; - public EsScanNode(PlanNodeId id, TupleDescriptor desc, String planNodeName) { - this(id, desc, planNodeName, false); + public EsScanNode(PlanNodeId id, TupleDescriptor desc) { + this(id, desc, false); } /** * For multicatalog es. **/ - public EsScanNode(PlanNodeId id, TupleDescriptor desc, String planNodeName, boolean esExternalTable) { - super(id, desc, planNodeName, StatisticalType.ES_SCAN_NODE, false); + public EsScanNode(PlanNodeId id, TupleDescriptor desc, boolean esExternalTable) { + super(id, desc, "EsScanNode", StatisticalType.ES_SCAN_NODE, false); if (esExternalTable) { EsExternalTable externalTable = (EsExternalTable) (desc.getTable()); table = externalTable.getEsTable(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/HashJoinNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/HashJoinNode.java index 1f816db3dfa70a7..02418fc01ac31c1 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/HashJoinNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/HashJoinNode.java @@ -37,6 +37,7 @@ import org.apache.doris.common.CheckedMath; import org.apache.doris.common.Pair; import org.apache.doris.common.UserException; +import org.apache.doris.nereids.trees.expressions.ExprId; import org.apache.doris.statistics.StatisticalType; import org.apache.doris.thrift.TEqJoinCondition; import org.apache.doris.thrift.TExplainLevel; @@ -47,12 +48,14 @@ import com.google.common.base.MoreObjects; import com.google.common.base.Preconditions; import com.google.common.collect.Lists; +import com.google.common.collect.Maps; import com.google.common.collect.Sets; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import java.util.ArrayList; import java.util.Collections; +import java.util.HashSet; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; @@ -76,7 +79,9 @@ public class HashJoinNode extends JoinNodeBase { private boolean isColocate = false; //the flag for colocate join private String colocateReason = ""; // if can not do colocate join, set reason here - private List hashOutputSlotIds = new ArrayList<>(); //init for nereids + private Set hashOutputSlotIds = Sets.newHashSet(); //init for nereids + + private Map hashOutputExprSlotIdMap = Maps.newHashMap(); /** * Constructor of HashJoinNode. @@ -230,7 +235,15 @@ private void initHashOutputSlotIds(List slotIdList, Analyzer analyzer) { Expr.getIds(otherJoinConjuncts, null, otherAndConjunctSlotIds); Expr.getIds(conjuncts, null, otherAndConjunctSlotIds); hashOutputSlotIdSet.addAll(otherAndConjunctSlotIds); - hashOutputSlotIds = new ArrayList<>(hashOutputSlotIdSet); + hashOutputSlotIds = new HashSet<>(hashOutputSlotIdSet); + } + + public Map getHashOutputExprSlotIdMap() { + return hashOutputExprSlotIdMap; + } + + public Set getHashOutputSlotIds() { + return hashOutputSlotIds; } @Override @@ -808,6 +821,10 @@ public void setOtherJoinConjuncts(List otherJoinConjuncts) { this.otherJoinConjuncts = otherJoinConjuncts; } + public List getOtherJoinConjuncts() { + return otherJoinConjuncts; + } + SlotRef getMappedInputSlotRef(SlotRef slotRef) { if (outputSmap != null) { Expr mappedExpr = outputSmap.mappingForRhsExpr(slotRef); diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/MultiCastPlanFragment.java b/fe/fe-core/src/main/java/org/apache/doris/planner/MultiCastPlanFragment.java index b49906fecb5d3a3..0d5b54b269bf057 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/MultiCastPlanFragment.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/MultiCastPlanFragment.java @@ -29,7 +29,8 @@ public class MultiCastPlanFragment extends PlanFragment { private final List destNodeList = Lists.newArrayList(); public MultiCastPlanFragment(PlanFragment planFragment) { - super(planFragment.getFragmentId(), planFragment.getPlanRoot(), planFragment.getDataPartition()); + super(planFragment.getFragmentId(), planFragment.getPlanRoot(), planFragment.getDataPartition(), + planFragment.getBuilderRuntimeFilterIds(), planFragment.getTargetRuntimeFilterIds()); this.outputPartition = DataPartition.RANDOM; this.children.addAll(planFragment.getChildren()); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/OlapScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/OlapScanNode.java index 879238f1ca54d95..6fa4955a319d24c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/OlapScanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/OlapScanNode.java @@ -1542,6 +1542,8 @@ public String getSelectedIndexName() { public void finalizeForNereids() { computeNumNodes(); computeStatsForNereids(); + // distributionColumnIds is used for one backend node agg optimization, nereids do not support it. + distributionColumnIds.clear(); } private void computeStatsForNereids() { @@ -1564,18 +1566,9 @@ Set getDistributionColumnNames() { public void updateRequiredSlots(PlanTranslatorContext context, Set requiredByProjectSlotIdSet) { outputColumnUniqueIds.clear(); - distributionColumnIds.clear(); - - Set distColumnName = getDistributionColumnNames(); - - int columnId = 0; for (SlotDescriptor slot : context.getTupleDesc(this.getTupleId()).getSlots()) { if (requiredByProjectSlotIdSet.contains(slot.getId()) && slot.getColumn() != null) { outputColumnUniqueIds.add(slot.getColumn().getUniqueId()); - if (distColumnName.contains(slot.getColumn().getName().toLowerCase())) { - distributionColumnIds.add(columnId); - } - columnId++; } } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/OriginalPlanner.java b/fe/fe-core/src/main/java/org/apache/doris/planner/OriginalPlanner.java index 1b388d9305e3008..218173466881eb5 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/OriginalPlanner.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/OriginalPlanner.java @@ -451,9 +451,6 @@ private SlotDescriptor injectRowIdColumnSlot(Analyzer analyzer, TupleDescriptor slotDesc.setColumn(col); slotDesc.setIsNullable(false); slotDesc.setIsMaterialized(true); - // Non-nullable slots will have 0 for the byte offset and -1 for the bit mask - slotDesc.setNullIndicatorBit(-1); - slotDesc.setNullIndicatorByte(0); return slotDesc; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/PlanFragment.java b/fe/fe-core/src/main/java/org/apache/doris/planner/PlanFragment.java index 3511ccf6fffe6c8..54903beae5209f3 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/PlanFragment.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/PlanFragment.java @@ -33,6 +33,7 @@ import org.apache.doris.thrift.TPlanFragment; import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableSet; import com.google.common.collect.Lists; import org.apache.commons.collections.CollectionUtils; import org.apache.logging.log4j.LogManager; @@ -167,6 +168,13 @@ public PlanFragment(PlanFragmentId id, PlanNode root, DataPartition partition, D this.dataPartitionForThrift = partitionForThrift; } + public PlanFragment(PlanFragmentId id, PlanNode root, DataPartition partition, + Set builderRuntimeFilterIds, Set targetRuntimeFilterIds) { + this(id, root, partition); + this.builderRuntimeFilterIds = ImmutableSet.copyOf(builderRuntimeFilterIds); + this.targetRuntimeFilterIds = ImmutableSet.copyOf(targetRuntimeFilterIds); + } + /** * Assigns 'this' as fragment of all PlanNodes in the plan tree rooted at node. * Does not traverse the children of ExchangeNodes because those must belong to a diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/SingleNodePlanner.java b/fe/fe-core/src/main/java/org/apache/doris/planner/SingleNodePlanner.java index 2a386c15c85141e..b9def52802cd187 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/SingleNodePlanner.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/SingleNodePlanner.java @@ -1976,7 +1976,7 @@ private PlanNode createScanNode(Analyzer analyzer, TableRef tblRef, SelectStmt s case BROKER: throw new RuntimeException("Broker external table is not supported, try to use table function please"); case ELASTICSEARCH: - scanNode = new EsScanNode(ctx.getNextNodeId(), tblRef.getDesc(), "EsScanNode"); + scanNode = new EsScanNode(ctx.getNextNodeId(), tblRef.getDesc()); break; case HIVE: throw new RuntimeException("Hive external table is not supported, try to use hive catalog please"); @@ -2016,7 +2016,7 @@ private PlanNode createScanNode(Analyzer analyzer, TableRef tblRef, SelectStmt s StatisticalType.MAX_COMPUTE_SCAN_NODE, true); break; case ES_EXTERNAL_TABLE: - scanNode = new EsScanNode(ctx.getNextNodeId(), tblRef.getDesc(), "EsScanNode", true); + scanNode = new EsScanNode(ctx.getNextNodeId(), tblRef.getDesc(), true); break; case JDBC_EXTERNAL_TABLE: scanNode = new JdbcScanNode(ctx.getNextNodeId(), tblRef.getDesc(), true); diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/StreamLoadPlanner.java b/fe/fe-core/src/main/java/org/apache/doris/planner/StreamLoadPlanner.java index 15089bf40c21f1f..31d7963e811f99a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/StreamLoadPlanner.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/StreamLoadPlanner.java @@ -207,9 +207,6 @@ public TExecPlanFragmentParams plan(TUniqueId loadId, int fragmentInstanceIdInde "stream load auto dynamic column"); slotDesc.setIsMaterialized(true); slotDesc.setColumn(col); - // Non-nullable slots will have 0 for the byte offset and -1 for the bit mask - slotDesc.setNullIndicatorBit(-1); - slotDesc.setNullIndicatorByte(0); slotDesc.setIsNullable(false); LOG.debug("plan tupleDesc {}", scanTupleDesc.toString()); } @@ -406,9 +403,6 @@ public TPipelineFragmentParams planForPipeline(TUniqueId loadId) throws UserExce "stream load auto dynamic column"); slotDesc.setIsMaterialized(true); slotDesc.setColumn(col); - // Non-nullable slots will have 0 for the byte offset and -1 for the bit mask - slotDesc.setNullIndicatorBit(-1); - slotDesc.setNullIndicatorByte(0); slotDesc.setIsNullable(false); LOG.debug("plan tupleDesc {}", scanTupleDesc.toString()); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/TableFunctionNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/TableFunctionNode.java index 940bbb839a32eaa..ce5aa9d1972d362 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/TableFunctionNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/TableFunctionNode.java @@ -120,7 +120,8 @@ public void projectSlots(Analyzer analyzer, SelectStmt selectStmt) throws Analys } Set outputSlotRef = Sets.newHashSet(); // case1 - List baseTblResultExprs = selectStmt.getResultExprs(); + List baseTblResultExprs = Expr.substituteList(selectStmt.getResultExprs(), + outputSmap, analyzer, false); for (Expr resultExpr : baseTblResultExprs) { // find all slotRef bound by tupleIds in resultExpr resultExpr.getSlotRefsBoundByTupleIds(tupleIds, outputSlotRef); diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/external/MetadataScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/external/MetadataScanNode.java index d16126fd8f09010..9952d9d7832eae1 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/external/MetadataScanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/external/MetadataScanNode.java @@ -21,6 +21,7 @@ import org.apache.doris.analysis.TupleDescriptor; import org.apache.doris.common.UserException; import org.apache.doris.planner.PlanNodeId; +import org.apache.doris.qe.ConnectContext; import org.apache.doris.statistics.StatisticalType; import org.apache.doris.system.Backend; import org.apache.doris.tablefunction.MetadataTableValuedFunction; @@ -31,6 +32,7 @@ import org.apache.doris.thrift.TScanRange; import org.apache.doris.thrift.TScanRangeLocation; import org.apache.doris.thrift.TScanRangeLocations; +import org.apache.doris.thrift.TUserIdentity; import com.google.common.collect.Lists; @@ -53,6 +55,8 @@ protected void toThrift(TPlanNode planNode) { TMetaScanNode metaScanNode = new TMetaScanNode(); metaScanNode.setTupleId(desc.getId().asInt()); metaScanNode.setMetadataType(this.tvf.getMetadataType()); + TUserIdentity tCurrentUser = ConnectContext.get().getCurrentUserIdentity().toThrift(); + metaScanNode.setCurrentUserIdent(tCurrentUser); planNode.setMetaScanNode(metaScanNode); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/ConnectProcessor.java b/fe/fe-core/src/main/java/org/apache/doris/qe/ConnectProcessor.java index 8c6518343139a18..36580e5c7a26b22 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/ConnectProcessor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/ConnectProcessor.java @@ -376,7 +376,7 @@ private void handleQuery() { } catch (Exception e) { // TODO: We should catch all exception here until we support all query syntax. nereidsParseException = e; - LOG.info("Nereids parse sql failed. Reason: {}. Statement: \"{}\".", + LOG.debug("Nereids parse sql failed. Reason: {}. Statement: \"{}\".", e.getMessage(), originStmt); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/Coordinator.java b/fe/fe-core/src/main/java/org/apache/doris/qe/Coordinator.java index 23bb0d4ba5e236e..d4f78b0d91bb95e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/Coordinator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/Coordinator.java @@ -2328,7 +2328,8 @@ public BucketShuffleJoinController(Map> fragmentIdT // check whether the node fragment is bucket shuffle join fragment private boolean isBucketShuffleJoin(int fragmentId, PlanNode node) { if (ConnectContext.get() != null) { - if (!ConnectContext.get().getSessionVariable().isEnableBucketShuffleJoin()) { + if (!ConnectContext.get().getSessionVariable().isEnableBucketShuffleJoin() + && !ConnectContext.get().getSessionVariable().isEnableNereidsPlanner()) { return false; } } @@ -2787,9 +2788,13 @@ public synchronized boolean cancelFragmentInstance(Types.PPlanFragmentCancelReas this.initiated, this.done, this.hasCanceled, backend.getId(), DebugUtil.printId(localParam.fragment_instance_id), cancelReason.name()); } - if (fragmentInstancesMap.get(localParam.fragment_instance_id).getIsCancel()) { + + RuntimeProfile profile = fragmentInstancesMap.get(localParam.fragment_instance_id); + if (profile.getIsDone() || profile.getIsCancel()) { continue; } + + this.hasCanceled = true; try { Span span = ConnectContext.get() != null ? ConnectContext.get().getTracer().spanBuilder("cancelPlanFragmentAsync") @@ -2811,7 +2816,10 @@ public synchronized boolean cancelFragmentInstance(Types.PPlanFragmentCancelReas return false; } } - this.hasCanceled = true; + if (!this.hasCanceled) { + return false; + } + for (int i = 0; i < this.numInstances; i++) { fragmentInstancesMap.get(rpcParams.local_params.get(i).fragment_instance_id).setIsCancel(true); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java index de14c310ff198f3..cbc78a67ce12f86 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java @@ -352,6 +352,12 @@ public class SessionVariable implements Serializable, Writable { public static final String ENABLE_STRONG_CONSISTENCY = "enable_strong_consistency_read"; + public static final String CBO_CPU_WEIGHT = "cbo_cpu_weight"; + + public static final String CBO_MEM_WEIGHT = "cbo_mem_weight"; + + public static final String CBO_NET_WEIGHT = "cbo_net_weight"; + public static final List DEBUG_VARIABLES = ImmutableList.of( SKIP_DELETE_PREDICATE, SKIP_DELETE_BITMAP, @@ -671,6 +677,39 @@ public int getBeNumberForTest() { @VariableMgr.VarAttr(name = BE_NUMBER_FOR_TEST) private int beNumberForTest = -1; + public double getCboCpuWeight() { + return cboCpuWeight; + } + + public void setCboCpuWeight(double cboCpuWeight) { + this.cboCpuWeight = cboCpuWeight; + } + + public double getCboMemWeight() { + return cboMemWeight; + } + + public void setCboMemWeight(double cboMemWeight) { + this.cboMemWeight = cboMemWeight; + } + + public double getCboNetWeight() { + return cboNetWeight; + } + + public void setCboNetWeight(double cboNetWeight) { + this.cboNetWeight = cboNetWeight; + } + + @VariableMgr.VarAttr(name = CBO_CPU_WEIGHT) + private double cboCpuWeight = 1.0; + + @VariableMgr.VarAttr(name = CBO_MEM_WEIGHT) + private double cboMemWeight = 1.0; + + @VariableMgr.VarAttr(name = CBO_NET_WEIGHT) + private double cboNetWeight = 1.5; + @VariableMgr.VarAttr(name = DISABLE_JOIN_REORDER) private boolean disableJoinReorder = false; diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java b/fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java index 1ccf2be1b3cba7e..06aa997695b4bcd 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java @@ -2363,9 +2363,16 @@ private void handleShowColumnStats() throws AnalysisException { List> columnStatistics = new ArrayList<>(); Set columnNames = showColumnStatsStmt.getColumnNames(); PartitionNames partitionNames = showColumnStatsStmt.getPartitionNames(); + boolean showCache = showColumnStatsStmt.isCached(); for (String colName : columnNames) { - if (partitionNames == null) { + // Show column statistics in columnStatisticsCache. For validation. + if (showCache) { + ColumnStatistic columnStatistic = Env.getCurrentEnv().getStatisticsCache().getColumnStatistics( + tableIf.getDatabase().getCatalog().getId(), + tableIf.getDatabase().getId(), tableIf.getId(), colName); + columnStatistics.add(Pair.of(colName, columnStatistic)); + } else if (partitionNames == null) { ColumnStatistic columnStatistic = StatisticsRepository.queryColumnStatisticsByName(tableIf.getId(), colName); columnStatistics.add(Pair.of(colName, columnStatistic)); @@ -2699,7 +2706,7 @@ private void handleMTMVTasks() throws AnalysisException { if (showStmt.isShowAllTasks()) { tasks.addAll(jobManager.getTaskManager().showAllTasks()); } else if (showStmt.isShowAllTasksFromDb()) { - tasks.addAll(jobManager.getTaskManager().showTasks(showStmt.getDbName())); + tasks.addAll(jobManager.getTaskManager().showTasksWithLock(showStmt.getDbName())); } else if (showStmt.isShowAllTasksOnMv()) { tasks.addAll(jobManager.getTaskManager().showTasks(showStmt.getDbName(), showStmt.getMVName())); } else if (showStmt.isSpecificTask()) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java b/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java index 9149d738c7c7aff..0ce889b0c3ebd58 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java @@ -320,6 +320,8 @@ private Map getSummaryInfo(boolean isFinished) { .collect(Collectors.joining(","))); builder.parallelFragmentExecInstance(String.valueOf(context.sessionVariable.getParallelExecInstanceNum())); builder.traceId(context.getSessionVariable().getTraceId()); + builder.isNereids(context.getState().isNereids ? "Yes" : "No"); + builder.isPipeline(context.getSessionVariable().enablePipelineEngine ? "Yes" : "No"); return builder.build(); } @@ -726,7 +728,7 @@ public void executeByLegacy(TUniqueId queryId) throws Exception { } else if (parsedStmt instanceof UpdateStmt) { handleUpdateStmt(); } else if (parsedStmt instanceof DdlStmt) { - if (parsedStmt instanceof DeleteStmt && ((DeleteStmt) parsedStmt).getFromClause() != null) { + if (parsedStmt instanceof DeleteStmt && ((DeleteStmt) parsedStmt).getInsertStmt() != null) { handleDeleteStmt(); } else { handleDdlStmt(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/resource/workloadgroup/WorkloadGroupMgr.java b/fe/fe-core/src/main/java/org/apache/doris/resource/workloadgroup/WorkloadGroupMgr.java index ce8717f861c70f8..c3b50d5025ff3c7 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/resource/workloadgroup/WorkloadGroupMgr.java +++ b/fe/fe-core/src/main/java/org/apache/doris/resource/workloadgroup/WorkloadGroupMgr.java @@ -20,17 +20,18 @@ import org.apache.doris.analysis.AlterWorkloadGroupStmt; import org.apache.doris.analysis.CreateWorkloadGroupStmt; import org.apache.doris.analysis.DropWorkloadGroupStmt; +import org.apache.doris.analysis.UserIdentity; import org.apache.doris.catalog.Env; import org.apache.doris.common.AnalysisException; import org.apache.doris.common.Config; import org.apache.doris.common.DdlException; import org.apache.doris.common.ErrorCode; import org.apache.doris.common.ErrorReport; +import org.apache.doris.common.Pair; import org.apache.doris.common.UserException; import org.apache.doris.common.io.Text; import org.apache.doris.common.io.Writable; import org.apache.doris.common.proc.BaseProcResult; -import org.apache.doris.common.proc.ProcNodeInterface; import org.apache.doris.common.proc.ProcResult; import org.apache.doris.mysql.privilege.PrivPredicate; import org.apache.doris.persist.DropWorkloadGroupOperationLog; @@ -38,6 +39,7 @@ import org.apache.doris.persist.gson.GsonUtils; import org.apache.doris.qe.ConnectContext; import org.apache.doris.thrift.TPipelineWorkloadGroup; +import org.apache.doris.thrift.TUserIdentity; import com.google.common.base.Strings; import com.google.common.collect.ImmutableList; @@ -240,6 +242,13 @@ public void dropWorkloadGroup(DropWorkloadGroupStmt stmt) throws DdlException { throw new DdlException("Dropping default workload group " + workloadGroupName + " is not allowed"); } + // if a workload group exists in user property, it should not be dropped + // user need to reset user property first + Pair ret = Env.getCurrentEnv().getAuth().isWorkloadGroupInUse(workloadGroupName); + if (ret.first) { + throw new DdlException("workload group " + workloadGroupName + " is set for user " + ret.second); + } + writeLock(); try { if (!nameToWorkloadGroup.containsKey(workloadGroupName)) { @@ -269,6 +278,15 @@ private void insertWorkloadGroup(WorkloadGroup workloadGroup) { } } + public boolean isWorkloadGroupExists(String workloadGroupName) { + readLock(); + try { + return nameToWorkloadGroup.containsKey(workloadGroupName); + } finally { + readUnlock(); + } + } + public void replayCreateWorkloadGroup(WorkloadGroup workloadGroup) { insertWorkloadGroup(workloadGroup); } @@ -293,7 +311,13 @@ public void replayDropWorkloadGroup(DropWorkloadGroupOperationLog operationLog) } public List> getResourcesInfo() { - return procNode.fetchResult().getRows(); + UserIdentity currentUserIdentity = ConnectContext.get().getCurrentUserIdentity(); + return procNode.fetchResult(currentUserIdentity).getRows(); + } + + public List> getResourcesInfo(TUserIdentity tcurrentUserIdentity) { + UserIdentity currentUserIdentity = UserIdentity.fromThrift(tcurrentUserIdentity); + return procNode.fetchResult(currentUserIdentity).getRows(); } // for ut @@ -323,17 +347,15 @@ public void gsonPostProcess() throws IOException { (id, workloadGroup) -> nameToWorkloadGroup.put(workloadGroup.getName(), workloadGroup)); } - public class ResourceProcNode implements ProcNodeInterface { - @Override - public ProcResult fetchResult() { + public class ResourceProcNode { + public ProcResult fetchResult(UserIdentity currentUserIdentity) { BaseProcResult result = new BaseProcResult(); result.setNames(WORKLOAD_GROUP_PROC_NODE_TITLE_NAMES); readLock(); try { for (WorkloadGroup workloadGroup : idToWorkloadGroup.values()) { - if (!Objects.isNull(ConnectContext.get()) && !Env.getCurrentEnv().getAccessManager() - .checkWorkloadGroupPriv(ConnectContext.get(), workloadGroup.getName(), - PrivPredicate.SHOW_WORKLOAD_GROUP)) { + if (!Env.getCurrentEnv().getAccessManager().checkWorkloadGroupPriv(currentUserIdentity, + workloadGroup.getName(), PrivPredicate.SHOW_WORKLOAD_GROUP)) { continue; } workloadGroup.getProcNodeData(result); diff --git a/fe/fe-core/src/main/java/org/apache/doris/scheduler/AsyncJobRegister.java b/fe/fe-core/src/main/java/org/apache/doris/scheduler/AsyncJobRegister.java new file mode 100644 index 000000000000000..59c64906e8e5639 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/scheduler/AsyncJobRegister.java @@ -0,0 +1,82 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.scheduler; + +import org.apache.doris.scheduler.executor.JobExecutor; +import org.apache.doris.scheduler.job.AsyncJobManager; +import org.apache.doris.scheduler.job.Job; +import org.apache.doris.scheduler.registry.JobRegister; + +import lombok.extern.slf4j.Slf4j; + +import java.io.IOException; + +/** + * This class registers timed scheduling events using the Netty time wheel algorithm to trigger events in a timely + * manner. + * After the event is triggered, it is produced by the Disruptor producer and consumed by the consumer, which is an + * asynchronous + * consumption model that does not guarantee strict timing accuracy. + */ +@Slf4j +public class AsyncJobRegister implements JobRegister { + + private final AsyncJobManager asyncJobManager; + + public AsyncJobRegister() { + this.asyncJobManager = new AsyncJobManager(); + } + + @Override + public Long registerJob(String name, Long intervalMs, JobExecutor executor) { + return this.registerJob(name, intervalMs, null, null, executor); + } + + @Override + public Long registerJob(String name, Long intervalMs, Long startTimeStamp, JobExecutor executor) { + return this.registerJob(name, intervalMs, startTimeStamp, null, executor); + } + + @Override + public Long registerJob(String name, Long intervalMs, Long startTimeStamp, Long endTimeStamp, + JobExecutor executor) { + + Job job = new Job(name, intervalMs, startTimeStamp, endTimeStamp, executor); + return asyncJobManager.registerJob(job); + } + + @Override + public Boolean pauseJob(Long jobId) { + return asyncJobManager.pauseJob(jobId); + } + + @Override + public Boolean stopJob(Long jobId) { + return asyncJobManager.stopJob(jobId); + } + + @Override + public Boolean resumeJob(Long jobId) { + return asyncJobManager.resumeJob(jobId); + } + + @Override + public void close() throws IOException { + asyncJobManager.close(); + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/scheduler/JobRegisterFactory.java b/fe/fe-core/src/main/java/org/apache/doris/scheduler/JobRegisterFactory.java new file mode 100644 index 000000000000000..2613a0302c265ad --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/scheduler/JobRegisterFactory.java @@ -0,0 +1,41 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.scheduler; + +import org.apache.doris.scheduler.registry.JobRegister; + +import java.util.concurrent.atomic.AtomicReference; + +/** + * This class provides a factory for creating instances of {@link JobRegister}. + * The factory ensures that only one instance of the client is created in a lazy manner. + */ +public class JobRegisterFactory { + private static final AtomicReference INSTANCE = new AtomicReference<>(); + + public static JobRegister getInstance() { + JobRegister instance = INSTANCE.get(); + if (instance == null) { + instance = new AsyncJobRegister(); + if (!INSTANCE.compareAndSet(null, instance)) { + instance = INSTANCE.get(); + } + } + return instance; + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/scheduler/constants/JobStatus.java b/fe/fe-core/src/main/java/org/apache/doris/scheduler/constants/JobStatus.java new file mode 100644 index 000000000000000..5c4af0b64972a61 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/scheduler/constants/JobStatus.java @@ -0,0 +1,38 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.scheduler.constants; + +public enum JobStatus { + + /** + * When the task is not started, the initial state will be triggered. + * The initial state can be started + */ + RUNNING, + /** + * When the task execution encounters an exception or manually suspends the task, + * the pause state will be triggered. + * Pause state can be resumed + */ + PAUSED, + /** + * When the task is manually stopped, the stop state will be triggered. + * The stop state cannot be resumed + */ + STOPPED, +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/scheduler/constants/SystemJob.java b/fe/fe-core/src/main/java/org/apache/doris/scheduler/constants/SystemJob.java new file mode 100644 index 000000000000000..f24f6e4e19b0656 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/scheduler/constants/SystemJob.java @@ -0,0 +1,42 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.scheduler.constants; + +import lombok.Getter; + +/** + * System scheduler event job + * They will start when scheduler starts + */ +public enum SystemJob { + + /** + * System cycle scheduler event job, it will start cycle scheduler + */ + SYSTEM_SCHEDULER_JOB("system_scheduler_event_job", 1L); + + @Getter + private final String description; + @Getter + private final Long id; + + SystemJob(String description, Long id) { + this.description = description; + this.id = id; + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/scheduler/disruptor/TimerTaskDisruptor.java b/fe/fe-core/src/main/java/org/apache/doris/scheduler/disruptor/TimerTaskDisruptor.java new file mode 100644 index 000000000000000..98a27365421239f --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/scheduler/disruptor/TimerTaskDisruptor.java @@ -0,0 +1,133 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.scheduler.disruptor; + +import org.apache.doris.scheduler.job.AsyncJobManager; + +import com.lmax.disruptor.BlockingWaitStrategy; +import com.lmax.disruptor.EventTranslatorTwoArg; +import com.lmax.disruptor.TimeoutException; +import com.lmax.disruptor.WorkHandler; +import com.lmax.disruptor.dsl.Disruptor; +import com.lmax.disruptor.dsl.ProducerType; +import com.lmax.disruptor.util.DaemonThreadFactory; +import lombok.extern.slf4j.Slf4j; + +import java.io.Closeable; +import java.util.concurrent.ThreadFactory; +import java.util.concurrent.TimeUnit; + +/** + * This class represents a disruptor for processing event tasks consumed by a Disruptor. + * + *

The work handler retrieves the associated event job and executes it if it is running. + * If the event job is not running, the work handler logs an error message. If the event job execution fails, + * the work handler logs an error message and pauses the event job. + * + *

The work handler also handles system events by scheduling batch scheduler tasks. + */ +@Slf4j +public class TimerTaskDisruptor implements Closeable { + + private final Disruptor disruptor; + private static final int DEFAULT_RING_BUFFER_SIZE = 1024; + + /** + * The default timeout for {@link #close()} in seconds. + */ + private static final int DEFAULT_CLOSE_WAIT_TIME_SECONDS = 5; + + /** + * The default number of consumers to create for each {@link Disruptor} instance. + */ + private static final int DEFAULT_CONSUMER_COUNT = System.getProperty("event.task.disruptor.consumer.count") + == null ? Runtime.getRuntime().availableProcessors() + : Integer.parseInt(System.getProperty("event.task.disruptor.consumer.count")); + + /** + * Whether this disruptor has been closed. + * if true, then we can't publish any more events. + */ + private boolean isClosed = false; + + /** + * The default {@link EventTranslatorTwoArg} to use for {@link #tryPublish(Long, Long)}. + * This is used to avoid creating a new object for each publish. + */ + private static final EventTranslatorTwoArg TRANSLATOR + = (event, sequence, jobId, taskId) -> { + event.setJobId(jobId); + event.setTaskId(taskId); + }; + + public TimerTaskDisruptor(AsyncJobManager asyncJobManager) { + ThreadFactory producerThreadFactory = DaemonThreadFactory.INSTANCE; + disruptor = new Disruptor<>(TimerTaskEvent.FACTORY, DEFAULT_RING_BUFFER_SIZE, producerThreadFactory, + ProducerType.SINGLE, new BlockingWaitStrategy()); + WorkHandler[] workers = new TimerTaskExpirationHandler[DEFAULT_CONSUMER_COUNT]; + for (int i = 0; i < DEFAULT_CONSUMER_COUNT; i++) { + workers[i] = new TimerTaskExpirationHandler(asyncJobManager); + } + disruptor.handleEventsWithWorkerPool(workers); + disruptor.start(); + } + + /** + * Publishes an event to the disruptor. + * + * @param eventId event job id + * @param taskId event task id + */ + public void tryPublish(Long eventId, Long taskId) { + if (isClosed) { + log.info("tryPublish failed, disruptor is closed, eventId: {}", eventId); + return; + } + try { + disruptor.publishEvent(TRANSLATOR, eventId, taskId); + } catch (Exception e) { + log.error("tryPublish failed, eventId: {}", eventId, e); + } + } + + public boolean tryPublish(TimerTaskEvent timerTaskEvent) { + if (isClosed) { + log.info("tryPublish failed, disruptor is closed, eventJobId: {}", timerTaskEvent.getJobId()); + return false; + } + try { + disruptor.publishEvent(TRANSLATOR, timerTaskEvent.getJobId(), timerTaskEvent.getTaskId()); + return true; + } catch (Exception e) { + log.error("tryPublish failed, eventJobId: {}", timerTaskEvent.getJobId(), e); + return false; + } + } + + + @Override + public void close() { + try { + isClosed = true; + // we can wait for 5 seconds, so that backlog can be committed + disruptor.shutdown(DEFAULT_CLOSE_WAIT_TIME_SECONDS, TimeUnit.SECONDS); + } catch (TimeoutException e) { + log.warn("close disruptor failed", e); + } + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/scheduler/disruptor/TimerTaskEvent.java b/fe/fe-core/src/main/java/org/apache/doris/scheduler/disruptor/TimerTaskEvent.java new file mode 100644 index 000000000000000..3c1cfe440d5014f --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/scheduler/disruptor/TimerTaskEvent.java @@ -0,0 +1,38 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.scheduler.disruptor; + +import com.lmax.disruptor.EventFactory; +import lombok.Data; + +/** + * This class represents an event task that can be produced and consumed by the Disruptor. + * The event task contains the ID of the event job and the ID of the event task itself. + * The class also provides an event factory to create instances of {@link TimerTaskEvent}. + *

+ * it's used by {@link TimerTaskDisruptor} and {@link TimerTaskExpirationHandler} + */ +@Data +public class TimerTaskEvent { + + private Long jobId; + + private Long taskId; + + public static final EventFactory FACTORY = TimerTaskEvent::new; +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/scheduler/disruptor/TimerTaskExpirationHandler.java b/fe/fe-core/src/main/java/org/apache/doris/scheduler/disruptor/TimerTaskExpirationHandler.java new file mode 100644 index 000000000000000..8c4a5db6816410c --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/scheduler/disruptor/TimerTaskExpirationHandler.java @@ -0,0 +1,125 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.scheduler.disruptor; + +import org.apache.doris.scheduler.constants.SystemJob; +import org.apache.doris.scheduler.job.AsyncJobManager; +import org.apache.doris.scheduler.job.Job; + +import com.lmax.disruptor.WorkHandler; +import lombok.extern.slf4j.Slf4j; + +import java.util.Objects; + +/** + * This class represents a work handler for processing event tasks consumed by a Disruptor. + * The work handler retrieves the associated event job and executes it if it is running. + * If the event job is not running, the work handler logs an error message. + * If the event job execution fails, the work handler logs an error message and pauses the event job. + * The work handler also handles system events by scheduling batch scheduler tasks. + */ +@Slf4j +public class TimerTaskExpirationHandler implements WorkHandler { + + /** + * The event job manager used to retrieve and execute event jobs. + */ + private AsyncJobManager asyncJobManager; + + /** + * Constructs a new {@link TimerTaskExpirationHandler} instance with the specified event job manager. + * + * @param asyncJobManager The event job manager used to retrieve and execute event jobs. + */ + public TimerTaskExpirationHandler(AsyncJobManager asyncJobManager) { + this.asyncJobManager = asyncJobManager; + } + + /** + * Processes an event task by retrieving the associated event job and executing it if it is running. + * If the event job is not running, it logs an error message. + * If the event job execution fails, it logs an error message and pauses the event job. + * + * @param event The event task to be processed. + */ + @Override + public void onEvent(TimerTaskEvent event) { + if (checkIsSystemEvent(event)) { + onSystemEvent(); + return; + } + onEventTask(event); + } + + /** + * Processes an event task by retrieving the associated event job and executing it if it is running. + * + * @param timerTaskEvent The event task to be processed. + */ + @SuppressWarnings("checkstyle:UnusedLocalVariable") + public void onEventTask(TimerTaskEvent timerTaskEvent) { + long jobId = timerTaskEvent.getJobId(); + Job job = asyncJobManager.getJob(jobId); + if (job == null) { + log.info("Event job is null, eventJobId: {}", jobId); + return; + } + if (!job.isRunning()) { + log.info("Event job is not running, eventJobId: {}", jobId); + return; + } + log.debug("Event job is running, eventJobId: {}", jobId); + checkJobIsExpired(job); + try { + // TODO: We should record the result of the event task. + //Object result = job.getExecutor().execute(); + job.getExecutor().execute(); + job.setLatestCompleteExecuteTimestamp(System.currentTimeMillis()); + } catch (Exception e) { + log.error("Event job execute failed, jobId: {}", jobId, e); + job.pause(e.getMessage()); + } + } + + /** + * Handles a system event by scheduling batch scheduler tasks. + */ + private void onSystemEvent() { + try { + asyncJobManager.batchSchedulerTasks(); + } catch (Exception e) { + log.error("System batch scheduler execute failed", e); + } + } + + /** + * Checks whether the specified event task is a system event. + * + * @param event The event task to be checked. + * @return true if the event task is a system event, false otherwise. + */ + private boolean checkIsSystemEvent(TimerTaskEvent event) { + return Objects.equals(event.getJobId(), SystemJob.SYSTEM_SCHEDULER_JOB.getId()); + } + + private void checkJobIsExpired(Job job) { + if (job.isExpired()) { + job.pause(); + } + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/scheduler/executor/JobExecutor.java b/fe/fe-core/src/main/java/org/apache/doris/scheduler/executor/JobExecutor.java new file mode 100644 index 000000000000000..cd96b6a6e4ad4cf --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/scheduler/executor/JobExecutor.java @@ -0,0 +1,37 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.scheduler.executor; + +/** + * This interface represents a callback for an event registration. All event registrations + * must implement this interface to provide an execution method. + * + * @param The result type of the event job execution. + */ +@FunctionalInterface +public interface JobExecutor { + + /** + * Executes the event job and returns the result. + * Exceptions will be caught internally, so there is no need to define or throw them separately. + * + * @return The result of the event job execution. + */ + T execute(); +} + diff --git a/fe/fe-core/src/main/java/org/apache/doris/scheduler/job/AsyncJobManager.java b/fe/fe-core/src/main/java/org/apache/doris/scheduler/job/AsyncJobManager.java new file mode 100644 index 000000000000000..e0944bf24ac80dc --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/scheduler/job/AsyncJobManager.java @@ -0,0 +1,262 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.scheduler.job; + +import org.apache.doris.scheduler.disruptor.TimerTaskDisruptor; + +import io.netty.util.HashedWheelTimer; +import io.netty.util.Timeout; +import lombok.extern.slf4j.Slf4j; + +import java.io.Closeable; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.TimeUnit; + +@Slf4j +public class AsyncJobManager implements Closeable { + + private final ConcurrentHashMap jobMap = new ConcurrentHashMap<>(128); + + private long lastBatchSchedulerTimestamp; + + /** + * batch scheduler interval time + */ + private static final long BATCH_SCHEDULER_INTERVAL_MILLI_SECONDS = 10 * 60 * 1000L; + + private static final TimeUnit TIME_UNIT = TimeUnit.MILLISECONDS; + + + private boolean isClosed = false; + + /** + * key: jobid + * value: timeout list for one job + * it's used to cancel task, if task has started, it can't be canceled + */ + private final ConcurrentHashMap> jobTimeoutMap = + new ConcurrentHashMap<>(128); + + /** + * scheduler tasks, it's used to scheduler job + */ + private final HashedWheelTimer dorisTimer = new HashedWheelTimer(1, TimeUnit.SECONDS, + 660); + + /** + * Producer and Consumer model + * disruptor is used to handle task + * disruptor will start a thread pool to handle task + */ + private final TimerTaskDisruptor disruptor; + + public AsyncJobManager() { + dorisTimer.start(); + this.disruptor = new TimerTaskDisruptor(this); + this.lastBatchSchedulerTimestamp = System.currentTimeMillis(); + batchSchedulerTasks(); + cycleSystemSchedulerTasks(); + } + + public Long registerJob(Job job) { + if (!job.checkJobParam()) { + log.warn("registerJob failed, job: {} param is invalid", job); + return null; + } + if (job.getStartTimestamp() != 0L) { + job.setNextExecuteTimestamp(job.getStartTimestamp() + job.getIntervalMilliSeconds()); + } else { + job.setNextExecuteTimestamp(System.currentTimeMillis() + job.getIntervalMilliSeconds()); + } + + if (job.getNextExecuteTimestamp() < BATCH_SCHEDULER_INTERVAL_MILLI_SECONDS + lastBatchSchedulerTimestamp) { + List executeTimestamp = findTasksBetweenTime(job, System.currentTimeMillis(), + BATCH_SCHEDULER_INTERVAL_MILLI_SECONDS + lastBatchSchedulerTimestamp, + job.getNextExecuteTimestamp()); + if (!executeTimestamp.isEmpty()) { + for (Long timestamp : executeTimestamp) { + putOneTask(job.getJobId(), timestamp); + } + } + } + + jobMap.putIfAbsent(job.getJobId(), job); + return job.getJobId(); + } + + public void unregisterJob(Long jobId) { + jobMap.remove(jobId); + } + + public boolean pauseJob(Long jobId) { + if (jobMap.get(jobId) == null) { + log.warn("pauseJob failed, jobId: {} not exist", jobId); + return false; + } + cancelJobAllTask(jobId); + jobMap.get(jobId).pause(); + return true; + } + + public boolean resumeJob(Long jobId) { + if (jobMap.get(jobId) == null) { + log.warn("resumeJob failed, jobId: {} not exist", jobId); + return false; + } + jobMap.get(jobId).resume(); + return true; + } + + public boolean stopJob(Long jobId) { + if (jobMap.get(jobId) == null) { + log.warn("stopJob failed, jobId: {} not exist", jobId); + return false; + } + cancelJobAllTask(jobId); + jobMap.get(jobId).stop(); + return true; + } + + public Job getJob(Long jobId) { + return jobMap.get(jobId); + } + + public Map getAllJob() { + return jobMap; + } + + public boolean batchSchedulerTasks() { + executeJobIdsWithinLastTenMinutesWindow(); + return true; + } + + public List findTasksBetweenTime(Job job, Long startTime, Long endTime, Long nextExecuteTime) { + List jobExecuteTimes = new ArrayList<>(); + if (System.currentTimeMillis() < startTime) { + return jobExecuteTimes; + } + while (endTime >= nextExecuteTime) { + if (job.isTaskTimeExceeded()) { + break; + } + jobExecuteTimes.add(nextExecuteTime); + nextExecuteTime = job.getExecuteTimestampAndGeneratorNext(); + } + return jobExecuteTimes; + } + + /** + * We will get the task in the next time window, and then hand it over to the time wheel for timing trigger + */ + private void executeJobIdsWithinLastTenMinutesWindow() { + if (jobMap.isEmpty()) { + return; + } + jobMap.forEach((k, v) -> { + if (v.isRunning() && (v.getNextExecuteTimestamp() + v.getIntervalMilliSeconds() + < lastBatchSchedulerTimestamp + BATCH_SCHEDULER_INTERVAL_MILLI_SECONDS)) { + List executeTimes = findTasksBetweenTime(v, lastBatchSchedulerTimestamp, + lastBatchSchedulerTimestamp + BATCH_SCHEDULER_INTERVAL_MILLI_SECONDS, + v.getNextExecuteTimestamp()); + if (!executeTimes.isEmpty()) { + for (Long executeTime : executeTimes) { + putOneTask(v.getJobId(), executeTime); + } + } + } + }); + this.lastBatchSchedulerTimestamp += BATCH_SCHEDULER_INTERVAL_MILLI_SECONDS; + } + + /** + * We will cycle system scheduler tasks every 10 minutes. + * Jobs will be re-registered after the task is completed + */ + private void cycleSystemSchedulerTasks() { + dorisTimer.newTimeout(timeout -> { + batchSchedulerTasks(); + cycleSystemSchedulerTasks(); + }, BATCH_SCHEDULER_INTERVAL_MILLI_SECONDS, TimeUnit.MILLISECONDS); + } + + public void putOneTask(Long jobId, Long startExecuteTime) { + DorisTimerTask task = new DorisTimerTask(jobId, startExecuteTime, disruptor); + if (isClosed) { + log.info("putOneTask failed, scheduler is closed, jobId: {}", task.getJobId()); + return; + } + long delay = getDelaySecond(task.getStartTimestamp()); + Timeout timeout = dorisTimer.newTimeout(task, delay, TimeUnit.SECONDS); + if (timeout == null) { + log.error("putOneTask failed, jobId: {}", task.getJobId()); + return; + } + if (jobTimeoutMap.containsKey(task.getJobId())) { + jobTimeoutMap.get(task.getJobId()).put(task.getTaskId(), timeout); + return; + } + Map timeoutMap = new ConcurrentHashMap<>(); + timeoutMap.put(task.getTaskId(), timeout); + jobTimeoutMap.put(task.getJobId(), timeoutMap); + } + + // cancel all task for one job + // if task has started, it can't be canceled + public void cancelJobAllTask(Long jobId) { + if (!jobTimeoutMap.containsKey(jobId)) { + return; + } + + jobTimeoutMap.get(jobId).values().forEach(timeout -> { + if (!timeout.isExpired() || timeout.isCancelled()) { + timeout.cancel(); + } + }); + } + + public void stopTask(Long jobId, Long taskId) { + if (!jobTimeoutMap.containsKey(jobId)) { + return; + } + cancelJobAllTask(jobId); + jobTimeoutMap.get(jobId).remove(taskId); + } + + // get delay time, if startTimestamp is less than now, return 0 + private long getDelaySecond(long startTimestamp) { + long delay = 0; + long now = System.currentTimeMillis(); + if (startTimestamp > now) { + delay = startTimestamp - now; + } else { + log.warn("startTimestamp is less than now, startTimestamp: {}, now: {}", startTimestamp, now); + } + return delay / 1000; + } + + @Override + public void close() throws IOException { + isClosed = true; + dorisTimer.stop(); + disruptor.close(); + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/scheduler/job/DorisTimerTask.java b/fe/fe-core/src/main/java/org/apache/doris/scheduler/job/DorisTimerTask.java new file mode 100644 index 000000000000000..7522548ad6f3e06 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/scheduler/job/DorisTimerTask.java @@ -0,0 +1,58 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.scheduler.job; + +import org.apache.doris.scheduler.disruptor.TimerTaskDisruptor; + +import io.netty.util.Timeout; +import io.netty.util.TimerTask; +import lombok.Getter; + +import java.util.UUID; + +/** + * This class represents a timer task that can be scheduled by a Netty timer. + * When the timer task is triggered, it produces an event task using the Disruptor. + * The event task contains the ID of the event and the ID of the task itself. + */ +@Getter +public class DorisTimerTask implements TimerTask { + + private final Long jobId; + + // more fields should be added here and record in feature + private final Long taskId = UUID.randomUUID().getMostSignificantBits(); + + private final Long startTimestamp; + + private final TimerTaskDisruptor timerTaskDisruptor; + + public DorisTimerTask(Long jobId, Long startTimestamp, TimerTaskDisruptor timerTaskDisruptor) { + this.jobId = jobId; + this.startTimestamp = startTimestamp; + this.timerTaskDisruptor = timerTaskDisruptor; + } + + @Override + public void run(Timeout timeout) { + if (timeout.isCancelled()) { + return; + } + timerTaskDisruptor.tryPublish(jobId, taskId); + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/scheduler/job/Job.java b/fe/fe-core/src/main/java/org/apache/doris/scheduler/job/Job.java new file mode 100644 index 000000000000000..6923e2277f192a4 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/scheduler/job/Job.java @@ -0,0 +1,148 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.scheduler.job; + +import org.apache.doris.scheduler.constants.JobStatus; +import org.apache.doris.scheduler.executor.JobExecutor; + +import lombok.Data; + +import java.util.UUID; + +/** + * Job is the core of the scheduler module, which is used to store the Job information of the job module. + * We can use the job to uniquely identify a Job. + * The jobName is used to identify the job, which is not unique. + * The jobStatus is used to identify the status of the Job, which is used to control the execution of the + * job. + */ +@Data +public class Job { + + public Job(String jobName, Long intervalMilliSeconds, Long startTimestamp, Long endTimestamp, + JobExecutor executor) { + this.jobName = jobName; + this.executor = executor; + this.intervalMilliSeconds = intervalMilliSeconds; + this.startTimestamp = null == startTimestamp ? 0L : startTimestamp; + this.endTimestamp = null == endTimestamp ? 0L : endTimestamp; + } + + private Long jobId = UUID.randomUUID().getMostSignificantBits(); + + private String jobName; + + /** + * The status of the job, which is used to control the execution of the job. + * + * @see JobStatus + */ + private JobStatus jobStatus = JobStatus.RUNNING; + + /** + * The executor of the job. + * + * @see JobExecutor + */ + private JobExecutor executor; + + private String user; + + private String errMsg; + + private Long intervalMilliSeconds; + + private Long updateTime; + + private Long nextExecuteTimestamp; + private Long startTimestamp = 0L; + + private Long endTimestamp = 0L; + + private Long firstExecuteTimestamp = 0L; + + private Long latestStartExecuteTimestamp = 0L; + private Long latestCompleteExecuteTimestamp = 0L; + + public boolean isRunning() { + return jobStatus == JobStatus.RUNNING; + } + + public boolean isStopped() { + return jobStatus == JobStatus.STOPPED; + } + + public boolean isExpired(long nextExecuteTimestamp) { + if (endTimestamp == 0L) { + return false; + } + return nextExecuteTimestamp > endTimestamp; + } + + public boolean isTaskTimeExceeded() { + if (endTimestamp == 0L) { + return false; + } + return System.currentTimeMillis() >= endTimestamp || nextExecuteTimestamp > endTimestamp; + } + + public boolean isExpired() { + if (endTimestamp == 0L) { + return false; + } + return System.currentTimeMillis() >= endTimestamp; + } + + public Long getExecuteTimestampAndGeneratorNext() { + this.latestStartExecuteTimestamp = nextExecuteTimestamp; + // todo The problem of delay should be considered. If it is greater than the ten-minute time window, + // should the task be lost or executed on a new time window? + this.nextExecuteTimestamp = latestStartExecuteTimestamp + intervalMilliSeconds; + return nextExecuteTimestamp; + } + + public void pause() { + this.jobStatus = JobStatus.PAUSED; + } + + public void pause(String errMsg) { + this.jobStatus = JobStatus.PAUSED; + this.errMsg = errMsg; + } + + public void resume() { + this.jobStatus = JobStatus.RUNNING; + } + + public void stop() { + this.jobStatus = JobStatus.STOPPED; + } + + public boolean checkJobParam() { + if (startTimestamp != 0L && startTimestamp < System.currentTimeMillis()) { + return false; + } + if (endTimestamp != 0L && endTimestamp < System.currentTimeMillis()) { + return false; + } + if (intervalMilliSeconds == null || intervalMilliSeconds <= 0L) { + return false; + } + return null != executor; + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/scheduler/registry/JobRegister.java b/fe/fe-core/src/main/java/org/apache/doris/scheduler/registry/JobRegister.java new file mode 100644 index 000000000000000..ebb6b0d590a3612 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/scheduler/registry/JobRegister.java @@ -0,0 +1,111 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.scheduler.registry; + +import org.apache.doris.scheduler.executor.JobExecutor; + +import java.io.IOException; + +/** + * This interface provides a contract for registering timed scheduling events. + * The implementation should trigger events in a timely manner using a specific algorithm. + * The execution of the events may be asynchronous and not guarantee strict timing accuracy. + */ +public interface JobRegister { + + /** + * Register a job + * + * @param name job name,it's not unique + * @param intervalMs job interval, unit: ms + * @param executor job executor @See {@link JobExecutor} + * @return event job id + */ + Long registerJob(String name, Long intervalMs, JobExecutor executor); + + /** + * Register a job + * + * @param name job name,it's not unique + * @param intervalMs job interval, unit: ms + * @param startTimeStamp job start time stamp, unit: ms + * if startTimeStamp is null, event job will start immediately in the next cycle + * startTimeStamp should be greater than current time + * @param executor event job executor @See {@link JobExecutor} + * @return job id + */ + Long registerJob(String name, Long intervalMs, Long startTimeStamp, JobExecutor executor); + + + /** + * Register a event job + * + * @param name job name,it's not unique + * @param intervalMs job interval, unit: ms + * @param startTimeStamp job start time stamp, unit: ms + * if startTimeStamp is null, job will start immediately in the next cycle + * startTimeStamp should be greater than current time + * @param endTimeStamp job end time stamp, unit: ms + * if endTimeStamp is null, job will never stop + * endTimeStamp must be greater than startTimeStamp and endTimeStamp should be greater + * than current time + * @param executor event job executor @See {@link JobExecutor} + * @return event job id + */ + Long registerJob(String name, Long intervalMs, Long startTimeStamp, Long endTimeStamp, + JobExecutor executor); + + /** + * if job is running, pause it + * pause means event job will not be executed in the next cycle,but current cycle will not be interrupted + * we can resume it by {@link #resumeJob(Long)} + * + * @param eventId event job id + * if eventId not exist, return false + * @return true if pause success, false if pause failed + */ + Boolean pauseJob(Long jodId); + + /** + * if job is running, stop it + * stop means event job will not be executed in the next cycle and current cycle will be interrupted + * stop not can be resumed, if you want to resume it, you should register it again + * we will delete stopped event job + * + * @param jobId event job id + * @return true if stop success, false if stop failed + */ + Boolean stopJob(Long jobId); + + /** + * if job is paused, resume it + * + * @param jobId job id + * @return true if resume success, false if resume failed + */ + Boolean resumeJob(Long jobId); + + /** + * close job scheduler register + * close means job scheduler register will not accept new job + * Jobs that have not reached the trigger time will not be executed. Jobs that have reached the trigger time will + * have an execution time of 5 seconds, and will not be executed if the time exceeds + */ + void close() throws IOException; + +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/ColumnStatisticsCacheLoader.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/ColumnStatisticsCacheLoader.java index 160dc3a56a14a28..ecd082aa110dc49 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/statistics/ColumnStatisticsCacheLoader.java +++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/ColumnStatisticsCacheLoader.java @@ -53,7 +53,7 @@ protected Optional doLoad(StatisticsCacheKey key) { try { TableIf table = Env.getCurrentEnv().getCatalogMgr().getCatalog(key.catalogId) .getDbOrMetaException(key.dbId).getTableOrMetaException(key.tableId); - columnStatistic = table.getColumnStatistic(); + columnStatistic = table.getColumnStatistic(key.colName); } catch (Exception e) { LOG.warn(String.format("Exception to get column statistics by metadata. [Catalog:%d, DB:%d, Table:%d]", key.catalogId, key.dbId, key.tableId), e); diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/HiveAnalysisTask.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/HiveAnalysisTask.java index cc580b74df6a492..8ae74206deb3855 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/statistics/HiveAnalysisTask.java +++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/HiveAnalysisTask.java @@ -25,18 +25,14 @@ import org.apache.doris.statistics.util.InternalQueryResult; import org.apache.doris.statistics.util.StatisticsUtil; -import com.google.common.base.Preconditions; -import com.google.common.collect.Lists; import org.apache.commons.lang3.StringUtils; import org.apache.commons.text.StringSubstitutor; import org.apache.hadoop.hive.metastore.api.ColumnStatisticsData; -import org.apache.hadoop.hive.metastore.api.ColumnStatisticsObj; import org.apache.hadoop.hive.metastore.api.DateColumnStatsData; import org.apache.hadoop.hive.metastore.api.Decimal; import org.apache.hadoop.hive.metastore.api.DecimalColumnStatsData; import org.apache.hadoop.hive.metastore.api.DoubleColumnStatsData; import org.apache.hadoop.hive.metastore.api.LongColumnStatsData; -import org.apache.hadoop.hive.metastore.api.Partition; import org.apache.hadoop.hive.metastore.api.StringColumnStatsData; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -60,7 +56,6 @@ public class HiveAnalysisTask extends HMSAnalysisTask { public static final String NUM_ROWS = "numRows"; public static final String NUM_FILES = "numFiles"; public static final String TIMESTAMP = "transient_lastDdlTime"; - public static final String DELIMITER = "-"; private static final String ANALYZE_SQL_TABLE_TEMPLATE = "INSERT INTO " + "${internalDB}.${columnStatTbl}" @@ -279,107 +274,7 @@ private Map buildTableStatsParams(String partId) { @Override protected void getStatsByMeta() throws Exception { - if (isTableLevelTask) { - getTableStatsByMeta(); - } else { - getColumnStatsByMeta(); - } - } - - protected void getTableStatsByMeta() throws Exception { - // Get table level information. - Map parameters = table.getRemoteTable().getParameters(); - if (isPartitionOnly) { - for (String partId : partitionNames) { - Map params = buildTableStatsParams(partId); - // Collect table level row count, null number and timestamp. - setParameterData(parameters, params); - StatisticsRepository.persistTableStats(params); - } - } else { - Map params = buildTableStatsParams("NULL"); - // Collect table level row count, null number and timestamp. - setParameterData(parameters, params); - StatisticsRepository.persistTableStats(params); - } - } - - protected void getColumnStatsByMeta() throws Exception { - List columns = new ArrayList<>(); - columns.add(col.getName()); - Map params = new HashMap<>(); - params.put("internalDB", FeConstants.INTERNAL_DB_NAME); - params.put("columnStatTbl", StatisticConstants.STATISTIC_TBL_NAME); - params.put("catalogId", String.valueOf(catalog.getId())); - params.put("dbId", String.valueOf(db.getId())); - params.put("tblId", String.valueOf(tbl.getId())); - params.put("colId", String.valueOf(col.getName())); - - // Get table level information. - Map parameters = table.getRemoteTable().getParameters(); - long rowCount; - StringSubstitutor stringSubstitutor; - if (isPartitionOnly) { - // Collect table level row count, null number and timestamp. - setParameterData(parameters, params); - params.put("id", genColumnStatId(tbl.getId(), -1, col.getName(), null)); - List tableStats = table.getHiveTableColumnStats(columns); - rowCount = parameters.containsKey(NUM_ROWS) ? Long.parseLong(parameters.get(NUM_ROWS)) : 0; - // Collect table level ndv, nulls, min and max. tableStats contains at most 1 item; - for (ColumnStatisticsObj tableStat : tableStats) { - if (!tableStat.isSetStatsData()) { - continue; - } - ColumnStatisticsData data = tableStat.getStatsData(); - getStatData(data, params, rowCount); - } - stringSubstitutor = new StringSubstitutor(params); - String sql = stringSubstitutor.replace(ANALYZE_META_TABLE_COLUMN_TEMPLATE); - try (AutoCloseConnectContext r = StatisticsUtil.buildConnectContext()) { - r.connectContext.getSessionVariable().disableNereidsPlannerOnce(); - this.stmtExecutor = new StmtExecutor(r.connectContext, sql); - this.stmtExecutor.execute(); - } - } - - // Get partition level information. - Map> columnStats - = table.getHivePartitionColumnStats(Lists.newArrayList(partitionNames), columns); - List partitionAnalysisSQLs = new ArrayList<>(); - for (Map.Entry> entry : columnStats.entrySet()) { - String partName = entry.getKey(); - List partitionValues = new ArrayList<>(); - for (String p : partName.split("/")) { - partitionValues.add(p.split("=")[1]); - } - Partition partition = table.getPartition(partitionValues); - parameters = partition.getParameters(); - // Collect row count, null number and timestamp. - setParameterData(parameters, params); - params.put("id", genColumnStatId(tbl.getId(), -1, col.getName(), partName)); - params.put("partId", partName); - List value = entry.getValue(); - Preconditions.checkState(value.size() == 1); - ColumnStatisticsObj stat = value.get(0); - if (!stat.isSetStatsData()) { - continue; - } - rowCount = parameters.containsKey(NUM_ROWS) ? Long.parseLong(parameters.get(NUM_ROWS)) : 0; - // Collect ndv, nulls, min and max for different data type. - ColumnStatisticsData data = stat.getStatsData(); - getStatData(data, params, rowCount); - stringSubstitutor = new StringSubstitutor(params); - partitionAnalysisSQLs.add(stringSubstitutor.replace(ANALYZE_META_PARTITION_COLUMN_TEMPLATE)); - } - // Update partition level stats for this column. - for (String partitionSql : partitionAnalysisSQLs) { - try (AutoCloseConnectContext r = StatisticsUtil.buildConnectContext()) { - r.connectContext.getSessionVariable().disableNereidsPlannerOnce(); - this.stmtExecutor = new StmtExecutor(r.connectContext, partitionSql); - this.stmtExecutor.execute(); - } - } - Env.getCurrentEnv().getStatisticsCache().refreshColStatsSync(tbl.getId(), -1, col.getName()); + // To be removed. } private void getStatData(ColumnStatisticsData data, Map params, long rowCount) { @@ -472,18 +367,4 @@ private void setParameterData(Map parameters, Map validParams) throws AnalysisE case "json": this.fileFormatType = TFileFormatType.FORMAT_JSON; break; + case "avro": + this.fileFormatType = TFileFormatType.FORMAT_AVRO; + break; default: throw new AnalysisException("format:" + formatString + " is not supported."); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/FrontendsTableValuedFunction.java b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/FrontendsTableValuedFunction.java index 122ecc9e48e7b15..92109d05a9a71c6 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/FrontendsTableValuedFunction.java +++ b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/FrontendsTableValuedFunction.java @@ -32,14 +32,14 @@ /** * The Implement of table valued function - * backends(). + * frontends(). */ public class FrontendsTableValuedFunction extends MetadataTableValuedFunction { public static final String NAME = "frontends"; private static final ImmutableList SCHEMA = ImmutableList.of( new Column("Name", ScalarType.createStringType()), - new Column("HOST", ScalarType.createStringType()), + new Column("Host", ScalarType.createStringType()), new Column("EditLogPort", ScalarType.createStringType()), new Column("HttpPort", ScalarType.createStringType()), new Column("QueryPort", ScalarType.createStringType()), diff --git a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/MetadataGenerator.java b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/MetadataGenerator.java index d2a0a2a5eea9882..ff2d9ce55c9d3e4 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/MetadataGenerator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/MetadataGenerator.java @@ -37,6 +37,7 @@ import org.apache.doris.thrift.TRow; import org.apache.doris.thrift.TStatus; import org.apache.doris.thrift.TStatusCode; +import org.apache.doris.thrift.TUserIdentity; import com.google.common.base.Stopwatch; import com.google.common.base.Strings; @@ -257,8 +258,13 @@ private static TFetchSchemaTableDataResult frontendsMetadataResult(TMetadataTabl } private static TFetchSchemaTableDataResult workloadGroupsMetadataResult(TMetadataTableRequestParams params) { + if (!params.isSetCurrentUserIdent()) { + return errorResult("current user ident is not set."); + } + + TUserIdentity tcurrentUserIdentity = params.getCurrentUserIdent(); List> workloadGroupsInfo = Env.getCurrentEnv().getWorkloadGroupMgr() - .getResourcesInfo(); + .getResourcesInfo(tcurrentUserIdentity); TFetchSchemaTableDataResult result = new TFetchSchemaTableDataResult(); List dataBatch = Lists.newArrayList(); for (List rGroupsInfo : workloadGroupsInfo) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/S3TableValuedFunction.java b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/S3TableValuedFunction.java index 9b820fa18503447..aa3777652d5797c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/S3TableValuedFunction.java +++ b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/S3TableValuedFunction.java @@ -67,6 +67,7 @@ public class S3TableValuedFunction extends ExternalFileTableValuedFunction { private final S3URI s3uri; private final boolean forceVirtualHosted; private String virtualBucket; + private String virtualKey; public S3TableValuedFunction(Map params) throws AnalysisException { Map tvfParams = getValidParams(params); @@ -83,12 +84,15 @@ public S3TableValuedFunction(Map params) throws AnalysisExceptio credential.setSessionToken(tvfParams.get(S3Properties.SESSION_TOKEN)); } - parseProperties(tvfParams); // set S3 location properties // these five properties is necessary, no one can be lost. locationProperties = S3Properties.credentialToMap(credential); String usePathStyle = tvfParams.getOrDefault(PropertyConverter.USE_PATH_STYLE, "false"); locationProperties.put(PropertyConverter.USE_PATH_STYLE, usePathStyle); + locationProperties.put(S3Properties.VIRTUAL_BUCKET, virtualBucket); + locationProperties.put(S3Properties.VIRTUAL_KEY, getVirtualKey()); + + parseProperties(tvfParams); if (FeConstants.runningUnitTest) { // Just check FileSystemFactory.getS3FileSystem(locationProperties); @@ -116,6 +120,11 @@ private static Map getValidParams(Map params) th return S3Properties.requiredS3TVFProperties(validParams); } + private String getVirtualKey() { + virtualKey = s3uri.getBucket() + S3URI.PATH_DELIM + s3uri.getKey(); + return virtualKey; + } + private String getEndpointAndSetVirtualBucket(Map params) throws AnalysisException { Preconditions.checkState(forceVirtualHosted, "only invoked when force virtual hosted."); String[] fileds = s3uri.getVirtualBucket().split("\\.", 2); @@ -162,8 +171,7 @@ public TFileType getTFileType() { public String getFilePath() { // must be "s3://..." if (forceVirtualHosted) { - return NAME + S3URI.SCHEME_DELIM + virtualBucket + S3URI.PATH_DELIM - + s3uri.getBucket() + S3URI.PATH_DELIM + s3uri.getKey(); + return NAME + S3URI.SCHEME_DELIM + virtualBucket + S3URI.PATH_DELIM + virtualKey; } return NAME + S3URI.SCHEME_DELIM + s3uri.getKey(); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/task/AlterInvertedIndexTask.java b/fe/fe-core/src/main/java/org/apache/doris/task/AlterInvertedIndexTask.java index c199d1d4829d1ab..caf7733165e99ad 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/task/AlterInvertedIndexTask.java +++ b/fe/fe-core/src/main/java/org/apache/doris/task/AlterInvertedIndexTask.java @@ -44,11 +44,13 @@ public class AlterInvertedIndexTask extends AgentTask { private List schemaColumns; private List existIndexes; private boolean isDropOp = false; + private long jobId; public AlterInvertedIndexTask(long backendId, long dbId, long tableId, long partitionId, long indexId, long tabletId, int schemaHash, List existIndexes, List alterInvertedIndexes, - List schemaColumns, boolean isDropOp, long taskSignature) { + List schemaColumns, boolean isDropOp, long taskSignature, + long jobId) { super(null, backendId, TTaskType.ALTER_INVERTED_INDEX, dbId, tableId, partitionId, indexId, tabletId, taskSignature); this.tabletId = tabletId; @@ -57,6 +59,7 @@ public AlterInvertedIndexTask(long backendId, long dbId, long tableId, this.alterInvertedIndexes = alterInvertedIndexes; this.schemaColumns = schemaColumns; this.isDropOp = isDropOp; + this.jobId = jobId; } public long getTabletId() { @@ -94,6 +97,8 @@ public TAlterInvertedIndexReq toThrift() { req.setTabletId(tabletId); req.setSchemaHash(schemaHash); req.setIsDropOp(isDropOp); + // set jonId for debugging in BE + req.setJobId(jobId); if (!alterInvertedIndexes.isEmpty()) { List tIndexes = new ArrayList<>(); diff --git a/fe/fe-core/src/main/jflex/sql_scanner.flex b/fe/fe-core/src/main/jflex/sql_scanner.flex index 5c228844b15bbdb..28a8ece6157f57b 100644 --- a/fe/fe-core/src/main/jflex/sql_scanner.flex +++ b/fe/fe-core/src/main/jflex/sql_scanner.flex @@ -130,6 +130,7 @@ import org.apache.doris.qe.SqlModeHelper; keywordMap.put("build", new Integer(SqlParserSymbols.KW_BUILD)); keywordMap.put("builtin", new Integer(SqlParserSymbols.KW_BUILTIN)); keywordMap.put("by", new Integer(SqlParserSymbols.KW_BY)); + keywordMap.put("cached", new Integer(SqlParserSymbols.KW_CACHED)); keywordMap.put("cancel", new Integer(SqlParserSymbols.KW_CANCEL)); keywordMap.put("case", new Integer(SqlParserSymbols.KW_CASE)); keywordMap.put("cast", new Integer(SqlParserSymbols.KW_CAST)); diff --git a/fe/fe-core/src/test/java/org/apache/doris/analysis/CreateTableStmtTest.java b/fe/fe-core/src/test/java/org/apache/doris/analysis/CreateTableStmtTest.java index 4086b207e632cf9..9138248c01d4ccf 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/analysis/CreateTableStmtTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/analysis/CreateTableStmtTest.java @@ -147,6 +147,27 @@ public void testCreateTableUniqueKeyNormal() throws UserException { cols.remove(col4); } + @Test + public void testCreateTableUniqueKeyNoProperties() throws UserException { + // setup + ColumnDef col3 = new ColumnDef("col3", new TypeDef(ScalarType.createType(PrimitiveType.BIGINT))); + col3.setIsKey(false); + cols.add(col3); + ColumnDef col4 = new ColumnDef("col4", new TypeDef(ScalarType.createType(PrimitiveType.STRING))); + col4.setIsKey(false); + cols.add(col4); + // test normal case + CreateTableStmt stmt = new CreateTableStmt(false, false, tblName, cols, "olap", + new KeysDesc(KeysType.UNIQUE_KEYS, colsName), null, + new HashDistributionDesc(10, Lists.newArrayList("col1")), null, null, ""); + stmt.analyze(analyzer); + Assert.assertEquals(col3.getAggregateType(), AggregateType.REPLACE); + Assert.assertEquals(col4.getAggregateType(), AggregateType.REPLACE); + // clear + cols.remove(col3); + cols.remove(col4); + } + @Test public void testCreateTableUniqueKeyMoW() throws UserException { // setup diff --git a/fe/fe-core/src/test/java/org/apache/doris/analysis/ExprTest.java b/fe/fe-core/src/test/java/org/apache/doris/analysis/ExprTest.java index 5262a29034c3f29..a91199189ac1099 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/analysis/ExprTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/analysis/ExprTest.java @@ -264,7 +264,7 @@ public void testPersist() throws IOException { DataInputStream dis = new DataInputStream(new FileInputStream(file)); Expr readExpr = Expr.readIn(dis); Assert.assertTrue(readExpr instanceof ArithmeticExpr); - Assert.assertEquals("cos(1) + 100 / 200", readExpr.toSql()); + Assert.assertEquals("(cos(1) + (100 / 200))", readExpr.toSql()); // 3. delete files dis.close(); diff --git a/fe/fe-core/src/test/java/org/apache/doris/analysis/LoadStmtTest.java b/fe/fe-core/src/test/java/org/apache/doris/analysis/LoadStmtTest.java index 59070dc80cd98b2..e8f83098671503a 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/analysis/LoadStmtTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/analysis/LoadStmtTest.java @@ -150,13 +150,13 @@ public void testRewrite() throws Exception { columnDescs.descs = columns1; columnDescs.isColumnDescsRewrited = false; Load.rewriteColumns(columnDescs); - String orig = "`c1` + 1 + 1"; + String orig = "((`c1` + 1) + 1)"; Assert.assertEquals(orig, columns1.get(4).getExpr().toString()); List columns2 = getColumns("c1,c2,c3,tmp_c5 = tmp_c4+1, tmp_c4=c1 + 1"); columnDescs.descs = columns2; columnDescs.isColumnDescsRewrited = false; - String orig2 = "`tmp_c4` + 1"; + String orig2 = "(`tmp_c4` + 1)"; Load.rewriteColumns(columnDescs); Assert.assertEquals(orig2, columns2.get(3).getExpr().toString()); diff --git a/fe/fe-core/src/test/java/org/apache/doris/analysis/S3TvfLoadStmtTest.java b/fe/fe-core/src/test/java/org/apache/doris/analysis/S3TvfLoadStmtTest.java index dd40ecb63705832..cc41e31e3c338ca 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/analysis/S3TvfLoadStmtTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/analysis/S3TvfLoadStmtTest.java @@ -180,7 +180,7 @@ public void testColumnMappings() throws Exception { Deencapsulation.invoke(s3TvfLoadStmt, "rewriteExpr", columnsDescList); Assert.assertEquals(columnsDescList.size(), 5); - final String orig4 = "upper(`c1`) + 1 + 1"; + final String orig4 = "((upper(`c1`) + 1) + 1)"; Assert.assertEquals(orig4, columnsDescList.get(4).getExpr().toString()); final List filterColumns = Deencapsulation.invoke(s3TvfLoadStmt, diff --git a/fe/fe-core/src/test/java/org/apache/doris/backup/BackupHandlerTest.java b/fe/fe-core/src/test/java/org/apache/doris/backup/BackupHandlerTest.java index 50d39e0ab6d287a..97e689b697256c4 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/backup/BackupHandlerTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/backup/BackupHandlerTest.java @@ -228,7 +228,7 @@ public Status getSnapshotInfoFile(String label, String backupTimestamp, List 0); + Assertions.assertTrue(jobManager.getTaskManager().getHistoryTasksByJobName(jobName).size() > 0); } @Test public void testOnceJob() throws DdlException, InterruptedException { - MTMVJobManager jobManager = new MTMVJobManager(); - jobManager.start(); - MTMVJob job = MTMVUtilsTest.createOnceJob(); + String jobName = "testOnceJob"; + String mvName = "testOnceJobMv"; + MTMVJobManager jobManager = Env.getCurrentEnv().getMTMVJobManager(); + MTMVJob job = MTMVUtilsTest.createOnceJob(mvName, jobName); jobManager.createJob(job, false); - Assertions.assertEquals(1, jobManager.showAllJobs().size()); - Assertions.assertEquals(1, jobManager.showJobs(MTMVUtilsTest.dbName).size()); - Assertions.assertEquals(1, jobManager.showJobs(MTMVUtilsTest.dbName, MTMVUtilsTest.MV_NAME).size()); - while (!jobManager.getJob(MTMVUtilsTest.O_JOB).getState().equals(JobState.COMPLETE)) { + Assertions.assertNotNull(jobManager.getJob(jobName)); + while (!jobManager.getJob(jobName).getState().equals(JobState.COMPLETE)) { Thread.sleep(1000L); System.out.println("Loop once"); } - Assertions.assertEquals(1, jobManager.getTaskManager().getHistoryTasks().size()); - Assertions.assertEquals(1, jobManager.getTaskManager().showAllTasks().size()); - Assertions.assertEquals(1, jobManager.getTaskManager().showTasks(MTMVUtilsTest.dbName).size()); + Assertions.assertEquals(1, jobManager.getTaskManager().getHistoryTasksByJobName(jobName).size()); Assertions.assertEquals(1, - jobManager.getTaskManager().showTasks(MTMVUtilsTest.dbName, MTMVUtilsTest.MV_NAME).size()); + jobManager.getTaskManager().showTasks(MTMVUtilsTest.dbName, mvName).size()); // verify job meta - MTMVJob metaJob = jobManager.showAllJobs().get(0); + MTMVJob metaJob = jobManager.getJob(jobName); List jobRow = metaJob.toStringRow(); Assertions.assertEquals(13, jobRow.size()); // index 1: Name - Assertions.assertEquals(MTMVUtilsTest.O_JOB, jobRow.get(1)); + Assertions.assertEquals(jobName, jobRow.get(1)); // index 2: TriggerMode Assertions.assertEquals("ONCE", jobRow.get(2)); // index 3: Schedule @@ -102,7 +100,7 @@ public void testOnceJob() throws DdlException, InterruptedException { // index 4: DBName Assertions.assertEquals(MTMVUtilsTest.dbName, jobRow.get(4)); // index 5: MVName - Assertions.assertEquals(MTMVUtilsTest.MV_NAME, jobRow.get(5)); + Assertions.assertEquals(mvName, jobRow.get(5)); // index 6: Query Assertions.assertEquals("", jobRow.get(6)); // index 7: User @@ -113,15 +111,15 @@ public void testOnceJob() throws DdlException, InterruptedException { Assertions.assertEquals("COMPLETE", jobRow.get(9)); // verify task meta - MTMVTask metaTask = jobManager.getTaskManager().showAllTasks().get(0); + MTMVTask metaTask = jobManager.getTaskManager().getHistoryTasksByJobName(jobName).get(0); List taskRow = metaTask.toStringRow(); Assertions.assertEquals(14, taskRow.size()); // index 1: JobName - Assertions.assertEquals(MTMVUtilsTest.O_JOB, taskRow.get(1)); + Assertions.assertEquals(jobName, taskRow.get(1)); // index 2: DBName Assertions.assertEquals(MTMVUtilsTest.dbName, taskRow.get(2)); // index 3: MVName - Assertions.assertEquals(MTMVUtilsTest.MV_NAME, taskRow.get(3)); + Assertions.assertEquals(mvName, taskRow.get(3)); // index 4: Query Assertions.assertEquals("", taskRow.get(4)); // index 5: User @@ -140,9 +138,6 @@ public void testOnceJob() throws DdlException, InterruptedException { @Test public void testMetrics() { - MTMVJobManager jobManager = new MTMVJobManager(); - jobManager.start(); - int jobMetricCount = 0; int taskMetricCount = 0; List metrics = MetricRepo.DORIS_METRIC_REGISTER.getMetrics(); diff --git a/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVTaskExecutorTest.java b/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVTaskExecutorTest.java index bee3c2e8bd01790..7bd58b145a0fa29 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVTaskExecutorTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVTaskExecutorTest.java @@ -23,6 +23,7 @@ import org.apache.doris.utframe.TestWithFeService; import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; import java.util.UUID; @@ -31,10 +32,12 @@ public class MTMVTaskExecutorTest extends TestWithFeService { @Test public void testSubmitTask() throws InterruptedException, ExecutionException { + String mvName = "testSubmitTaskMv"; + String jobName = "testSubmitTaskJob"; MTMVTaskExecutorPool pool = new MTMVTaskExecutorPool(); MTMVTaskExecutor executor = new MTMVTaskExecutor(); executor.setProcessor(new MTMVTaskProcessor()); - executor.setJob(MTMVUtilsTest.createDummyJob()); + executor.setJob(MTMVUtilsTest.createDummyJob(mvName, jobName)); executor.initTask(UUID.randomUUID().toString(), System.currentTimeMillis()); pool.executeTask(executor); executor.getFuture().get(); @@ -44,10 +47,12 @@ public void testSubmitTask() throws InterruptedException, ExecutionException { @Test public void testFailTask() throws InterruptedException, ExecutionException { + String mvName = "testFailTaskMv"; + String jobName = "testFailTaskJob"; MTMVTaskExecutorPool pool = new MTMVTaskExecutorPool(); MTMVTaskExecutor executor = new MTMVTaskExecutor(); executor.setProcessor(new MTMVTaskProcessorTest(1)); - executor.setJob(MTMVUtilsTest.createDummyJob()); + executor.setJob(MTMVUtilsTest.createDummyJob(mvName, jobName)); executor.initTask(UUID.randomUUID().toString(), System.currentTimeMillis()); pool.executeTask(executor); executor.getFuture().get(); @@ -56,12 +61,15 @@ public void testFailTask() throws InterruptedException, ExecutionException { } @Test + @Disabled public void testRetryTask() throws InterruptedException, ExecutionException { + String mvName = "testRetryTaskMv"; + String jobName = "testRetryTaskJob"; MTMVTaskExecutorPool pool = new MTMVTaskExecutorPool(); MTMVTaskExecutor executor = new MTMVTaskExecutor(); executor.setProcessor(new MTMVTaskProcessorTest(3)); - MTMVJob job = MTMVUtilsTest.createDummyJob(); + MTMVJob job = MTMVUtilsTest.createDummyJob(mvName, jobName); job.setRetryPolicy(TaskRetryPolicy.TIMES); executor.setJob(job); executor.initTask(UUID.randomUUID().toString(), System.currentTimeMillis()); @@ -71,12 +79,15 @@ public void testRetryTask() throws InterruptedException, ExecutionException { } @Test + @Disabled public void testRetryFailTask() throws InterruptedException, ExecutionException { + String mvName = "testRetryTaskMv"; + String jobName = "testRetryTaskJob"; MTMVTaskExecutorPool pool = new MTMVTaskExecutorPool(); MTMVTaskExecutor executor = new MTMVTaskExecutor(); executor.setProcessor(new MTMVTaskProcessorTest(4)); - MTMVJob job = MTMVUtilsTest.createDummyJob(); + MTMVJob job = MTMVUtilsTest.createDummyJob(mvName, jobName); job.setRetryPolicy(TaskRetryPolicy.TIMES); executor.setJob(job); executor.initTask(UUID.randomUUID().toString(), System.currentTimeMillis()); diff --git a/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVUtilsTest.java b/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVUtilsTest.java index 5a81173715bd2b2..e76e26ec0deb6cf 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVUtilsTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVUtilsTest.java @@ -33,41 +33,35 @@ public class MTMVUtilsTest { public static final String dbName = "test"; public static final String MV_NAME = "mvName"; - public static final String S_JOB = "SchedulerJob"; - public static final String O_JOB = "OnceJob"; - - - public static MTMVJob createDummyJob() { - MTMVJob job = new MTMVJob("dummy"); + public static MTMVJob createDummyJob(String mvName, String jobName) { + MTMVJob job = new MTMVJob(jobName); job.setDBName(dbName); - job.setMVName(MV_NAME); + job.setMVName(mvName); return job; } - public static MTMVJob createOnceJob() { - MTMVJob job = new MTMVJob(""); + public static MTMVJob createOnceJob(String mvName, String jobName) { + MTMVJob job = new MTMVJob(jobName); job.setTriggerMode(TriggerMode.ONCE); job.setDBName(dbName); - job.setName(O_JOB); - job.setMVName(MV_NAME); + job.setMVName(mvName); return job; } - public static MTMVJob createSchedulerJob() { - MTMVJob job = new MTMVJob(""); + public static MTMVJob createSchedulerJob(String mvName, String jobName) { + MTMVJob job = new MTMVJob(jobName); JobSchedule jobSchedule = new JobSchedule(System.currentTimeMillis() / 1000, 1, TimeUnit.SECONDS); job.setSchedule(jobSchedule); job.setTriggerMode(TriggerMode.PERIODICAL); job.setDBName(dbName); - job.setName(S_JOB); - job.setMVName(MV_NAME); + job.setMVName(mvName); return job; } @Test public void testGetDelaySeconds() { - MTMVJob job = MTMVUtilsTest.createDummyJob(); + MTMVJob job = MTMVUtilsTest.createDummyJob("testGetDelaySecondsMv", "testGetDelaySecondsJob"); // 2022-10-03 15:00:00 JobSchedule jobSchedule = new JobSchedule(1664780400L, 1, TimeUnit.HOURS); diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/JoinHintTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/JoinHintTest.java index 55c2921bb4ef475..631a89b7762402f 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/JoinHintTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/JoinHintTest.java @@ -121,7 +121,7 @@ public void testHintWithReorderCrossJoin() throws Exception { DistributionSpec spec = dis.getDistributionSpec(); Assertions.assertTrue(spec instanceof DistributionSpecHash); DistributionSpecHash hashSpec = (DistributionSpecHash) spec; - Assertions.assertEquals(ShuffleType.ENFORCED, + Assertions.assertEquals(ShuffleType.EXECUTION_BUCKETED, hashSpec.getShuffleType()); return true; }), physicalDistribute()), diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/postprocess/RuntimeFilterTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/postprocess/RuntimeFilterTest.java index 699df19ba40a39b..fc08b0c35d71d6e 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/postprocess/RuntimeFilterTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/postprocess/RuntimeFilterTest.java @@ -101,9 +101,9 @@ public void testPushDownEncounterUnsupportedJoinType() { + " inner join (select c_custkey from customer inner join supplier on c_custkey = s_suppkey) b" + " on b.c_custkey = a.lo_custkey"; List filters = getRuntimeFilters(sql).get(); - Assertions.assertEquals(1, filters.size()); + Assertions.assertEquals(2, filters.size()); checkRuntimeFilterExprs(filters, ImmutableList.of( - Pair.of("s_suppkey", "c_custkey"))); + Pair.of("s_suppkey", "c_custkey"), Pair.of("c_custkey", "lo_custkey"))); } @Test @@ -207,10 +207,10 @@ public void testPushDownThroughUnsupportedJoinType() { + " on b.c_custkey = a.lo_custkey) c inner join (select lo_custkey from customer inner join lineorder" + " on c_custkey = lo_custkey) d on c.c_custkey = d.lo_custkey"; List filters = getRuntimeFilters(sql).get(); - Assertions.assertEquals(3, filters.size()); + Assertions.assertEquals(4, filters.size()); checkRuntimeFilterExprs(filters, ImmutableList.of( Pair.of("c_custkey", "lo_custkey"), Pair.of("d_datekey", "lo_orderdate"), - Pair.of("lo_custkey", "c_custkey"))); + Pair.of("lo_custkey", "c_custkey"), Pair.of("lo_custkey", "c_custkey"))); } @Test diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/properties/ChildOutputPropertyDeriverTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/properties/ChildOutputPropertyDeriverTest.java index 5ac1e8cc2f1fc00..2f6a973503a2bcf 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/properties/ChildOutputPropertyDeriverTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/properties/ChildOutputPropertyDeriverTest.java @@ -219,7 +219,7 @@ Pair, List> getOnClauseUsedSlots( leftMap.put(new ExprId(1), 0); PhysicalProperties left = new PhysicalProperties(new DistributionSpecHash( Lists.newArrayList(new ExprId(0)), - ShuffleType.ENFORCED, + ShuffleType.EXECUTION_BUCKETED, 0, Sets.newHashSet(0L), ImmutableList.of(Sets.newHashSet(new ExprId(0), new ExprId(1))), @@ -228,7 +228,7 @@ Pair, List> getOnClauseUsedSlots( PhysicalProperties right = new PhysicalProperties(new DistributionSpecHash( Lists.newArrayList(new ExprId(2)), - ShuffleType.ENFORCED, + ShuffleType.EXECUTION_BUCKETED, 1, Sets.newHashSet(1L) )); @@ -240,7 +240,7 @@ Pair, List> getOnClauseUsedSlots( Assertions.assertTrue(result.getOrderSpec().getOrderKeys().isEmpty()); Assertions.assertTrue(result.getDistributionSpec() instanceof DistributionSpecHash); DistributionSpecHash actual = (DistributionSpecHash) result.getDistributionSpec(); - Assertions.assertEquals(ShuffleType.BUCKETED, actual.getShuffleType()); + Assertions.assertEquals(ShuffleType.EXECUTION_BUCKETED, actual.getShuffleType()); // check merged Assertions.assertEquals(3, actual.getExprIdToEquivalenceSet().size()); } @@ -308,12 +308,12 @@ public void testGlobalPhaseAggregate() { new AggregateParam(AggPhase.GLOBAL, AggMode.BUFFER_TO_RESULT), true, logicalProperties, - RequireProperties.of(PhysicalProperties.createHash(ImmutableList.of(partition), ShuffleType.AGGREGATE)), + RequireProperties.of(PhysicalProperties.createHash(ImmutableList.of(partition), ShuffleType.REQUIRE)), groupPlan ); GroupExpression groupExpression = new GroupExpression(aggregate); DistributionSpecHash childHash = new DistributionSpecHash(Lists.newArrayList(partition.getExprId()), - ShuffleType.BUCKETED); + ShuffleType.EXECUTION_BUCKETED); PhysicalProperties child = new PhysicalProperties(childHash, new OrderSpec(Lists.newArrayList( new OrderKey(new SlotReference("ignored", IntegerType.INSTANCE), true, true)))); @@ -323,7 +323,7 @@ public void testGlobalPhaseAggregate() { Assertions.assertTrue(result.getOrderSpec().getOrderKeys().isEmpty()); Assertions.assertTrue(result.getDistributionSpec() instanceof DistributionSpecHash); DistributionSpecHash actual = (DistributionSpecHash) result.getDistributionSpec(); - Assertions.assertEquals(ShuffleType.BUCKETED, actual.getShuffleType()); + Assertions.assertEquals(ShuffleType.EXECUTION_BUCKETED, actual.getShuffleType()); Assertions.assertEquals(Lists.newArrayList(partition).stream() .map(SlotReference::getExprId).collect(Collectors.toList()), actual.getOrderedShuffledColumns()); @@ -355,7 +355,7 @@ public void testAggregateWithoutGroupBy() { public void testLocalQuickSort() { SlotReference key = new SlotReference("col1", IntegerType.INSTANCE); List orderKeys = Lists.newArrayList(new OrderKey(key, true, true)); - PhysicalQuickSort sort = new PhysicalQuickSort(orderKeys, SortPhase.LOCAL_SORT, logicalProperties, groupPlan); + PhysicalQuickSort sort = new PhysicalQuickSort<>(orderKeys, SortPhase.LOCAL_SORT, logicalProperties, groupPlan); GroupExpression groupExpression = new GroupExpression(sort); PhysicalProperties child = new PhysicalProperties(DistributionSpecReplicated.INSTANCE, new OrderSpec(Lists.newArrayList( @@ -417,7 +417,7 @@ public void testLimit() { List orderKeys = Lists.newArrayList(new OrderKey(key, true, true)); PhysicalLimit limit = new PhysicalLimit<>(10, 10, LimitPhase.ORIGIN, logicalProperties, groupPlan); GroupExpression groupExpression = new GroupExpression(limit); - PhysicalProperties child = new PhysicalProperties(DistributionSpecReplicated.INSTANCE, + PhysicalProperties child = new PhysicalProperties(DistributionSpecGather.INSTANCE, new OrderSpec(orderKeys)); ChildOutputPropertyDeriver deriver = new ChildOutputPropertyDeriver(Lists.newArrayList(child)); @@ -434,7 +434,7 @@ public void testAssertNumRows() { groupPlan ); GroupExpression groupExpression = new GroupExpression(assertNumRows); - PhysicalProperties child = new PhysicalProperties(DistributionSpecReplicated.INSTANCE, new OrderSpec()); + PhysicalProperties child = new PhysicalProperties(DistributionSpecGather.INSTANCE, new OrderSpec()); ChildOutputPropertyDeriver deriver = new ChildOutputPropertyDeriver(Lists.newArrayList(child)); PhysicalProperties result = deriver.getOutputProperties(groupExpression); Assertions.assertEquals(PhysicalProperties.GATHER, result); diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/properties/DistributionSpecHashTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/properties/DistributionSpecHashTest.java index 49b3837dad3fc12..bb38f668015e959 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/properties/DistributionSpecHashTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/properties/DistributionSpecHashTest.java @@ -47,18 +47,18 @@ public void testMerge() { naturalMap ); - Map joinMap = Maps.newHashMap(); - joinMap.put(new ExprId(1), 0); - joinMap.put(new ExprId(4), 0); - joinMap.put(new ExprId(3), 1); - joinMap.put(new ExprId(5), 1); + Map requireMap = Maps.newHashMap(); + requireMap.put(new ExprId(1), 0); + requireMap.put(new ExprId(4), 0); + requireMap.put(new ExprId(3), 1); + requireMap.put(new ExprId(5), 1); DistributionSpecHash join = new DistributionSpecHash( Lists.newArrayList(new ExprId(1), new ExprId(5)), - ShuffleType.JOIN, + ShuffleType.REQUIRE, 1, Sets.newHashSet(1L), Lists.newArrayList(Sets.newHashSet(new ExprId(1), new ExprId(4)), Sets.newHashSet(new ExprId(3), new ExprId(5))), - joinMap + requireMap ); Map expectedMap = Maps.newHashMap(); @@ -101,7 +101,7 @@ public void testProject() { projects.put(new ExprId(2), new ExprId(5)); Set obstructions = Sets.newHashSet(); - DistributionSpec after = original.project(projects, obstructions); + DistributionSpec after = original.project(projects, obstructions, DistributionSpecAny.INSTANCE); Assertions.assertTrue(after instanceof DistributionSpecHash); DistributionSpecHash afterHash = (DistributionSpecHash) after; Assertions.assertEquals(Lists.newArrayList(new ExprId(0), new ExprId(5)), afterHash.getOrderedShuffledColumns()); @@ -119,41 +119,41 @@ public void testProject() { // have obstructions obstructions.add(new ExprId(3)); - after = original.project(projects, obstructions); + after = original.project(projects, obstructions, DistributionSpecAny.INSTANCE); Assertions.assertTrue(after instanceof DistributionSpecAny); } @Test public void testSatisfyAny() { DistributionSpec required = DistributionSpecAny.INSTANCE; - DistributionSpecHash join = new DistributionSpecHash(Lists.newArrayList(), ShuffleType.JOIN); - DistributionSpecHash aggregate = new DistributionSpecHash(Lists.newArrayList(), ShuffleType.AGGREGATE); - DistributionSpecHash enforce = new DistributionSpecHash(Lists.newArrayList(), ShuffleType.ENFORCED); + DistributionSpecHash require = new DistributionSpecHash(Lists.newArrayList(), ShuffleType.REQUIRE); + DistributionSpecHash storageBucketed = new DistributionSpecHash(Lists.newArrayList(), ShuffleType.STORAGE_BUCKETED); + DistributionSpecHash executionBucketed = new DistributionSpecHash(Lists.newArrayList(), ShuffleType.EXECUTION_BUCKETED); DistributionSpecHash natural = new DistributionSpecHash(Lists.newArrayList(), ShuffleType.NATURAL); - Assertions.assertTrue(join.satisfy(required)); - Assertions.assertTrue(aggregate.satisfy(required)); - Assertions.assertTrue(enforce.satisfy(required)); + Assertions.assertTrue(require.satisfy(required)); + Assertions.assertTrue(storageBucketed.satisfy(required)); + Assertions.assertTrue(executionBucketed.satisfy(required)); Assertions.assertTrue(natural.satisfy(required)); } @Test public void testNotSatisfyOther() { - DistributionSpecHash join = new DistributionSpecHash(Lists.newArrayList(), ShuffleType.JOIN); - DistributionSpecHash aggregate = new DistributionSpecHash(Lists.newArrayList(), ShuffleType.AGGREGATE); - DistributionSpecHash enforce = new DistributionSpecHash(Lists.newArrayList(), ShuffleType.ENFORCED); + DistributionSpecHash require = new DistributionSpecHash(Lists.newArrayList(), ShuffleType.REQUIRE); + DistributionSpecHash storageBucketed = new DistributionSpecHash(Lists.newArrayList(), ShuffleType.STORAGE_BUCKETED); + DistributionSpecHash executionBucketed = new DistributionSpecHash(Lists.newArrayList(), ShuffleType.EXECUTION_BUCKETED); DistributionSpecHash natural = new DistributionSpecHash(Lists.newArrayList(), ShuffleType.NATURAL); DistributionSpec gather = DistributionSpecGather.INSTANCE; - Assertions.assertFalse(join.satisfy(gather)); - Assertions.assertFalse(aggregate.satisfy(gather)); - Assertions.assertFalse(enforce.satisfy(gather)); + Assertions.assertFalse(require.satisfy(gather)); + Assertions.assertFalse(storageBucketed.satisfy(gather)); + Assertions.assertFalse(executionBucketed.satisfy(gather)); Assertions.assertFalse(natural.satisfy(gather)); DistributionSpec replicated = DistributionSpecReplicated.INSTANCE; - Assertions.assertFalse(join.satisfy(replicated)); - Assertions.assertFalse(aggregate.satisfy(replicated)); - Assertions.assertFalse(enforce.satisfy(replicated)); + Assertions.assertFalse(require.satisfy(replicated)); + Assertions.assertFalse(storageBucketed.satisfy(replicated)); + Assertions.assertFalse(executionBucketed.satisfy(replicated)); Assertions.assertFalse(natural.satisfy(replicated)); } @@ -197,27 +197,27 @@ public void testSatisfyNaturalHash() { natural3Map ); - DistributionSpecHash join = new DistributionSpecHash( + DistributionSpecHash require = new DistributionSpecHash( Lists.newArrayList(new ExprId(1), new ExprId(2)), - ShuffleType.JOIN, + ShuffleType.REQUIRE, 1, Sets.newHashSet(1L), Lists.newArrayList(Sets.newHashSet(new ExprId(1)), Sets.newHashSet(new ExprId(2))), natural2Map ); - DistributionSpecHash enforce = new DistributionSpecHash( + DistributionSpecHash storageBucketed = new DistributionSpecHash( Lists.newArrayList(new ExprId(1), new ExprId(2)), - ShuffleType.ENFORCED, + ShuffleType.STORAGE_BUCKETED, 1, Sets.newHashSet(1L), Lists.newArrayList(Sets.newHashSet(new ExprId(1)), Sets.newHashSet(new ExprId(2))), natural2Map ); - DistributionSpecHash aggregate = new DistributionSpecHash( + DistributionSpecHash executionBucketed = new DistributionSpecHash( Lists.newArrayList(new ExprId(1), new ExprId(2)), - ShuffleType.AGGREGATE, + ShuffleType.EXECUTION_BUCKETED, 1, Sets.newHashSet(1L), Lists.newArrayList(Sets.newHashSet(new ExprId(1)), Sets.newHashSet(new ExprId(2))), @@ -231,144 +231,49 @@ public void testSatisfyNaturalHash() { // require slots is not contained by target Assertions.assertFalse(natural2.satisfy(natural1)); // other shuffle type with same order - Assertions.assertFalse(join.satisfy(natural2)); - Assertions.assertFalse(aggregate.satisfy(natural2)); - Assertions.assertFalse(enforce.satisfy(natural2)); + Assertions.assertFalse(require.satisfy(natural2)); + Assertions.assertFalse(executionBucketed.satisfy(natural2)); + Assertions.assertFalse(storageBucketed.satisfy(natural2)); } @Test - public void testSatisfyJoinHash() { - Map join1Map = Maps.newHashMap(); - join1Map.put(new ExprId(0), 0); - join1Map.put(new ExprId(1), 0); - join1Map.put(new ExprId(2), 1); - join1Map.put(new ExprId(3), 1); - DistributionSpecHash join1 = new DistributionSpecHash( + public void testSatisfyRequiredHash() { + Map require1Map = Maps.newHashMap(); + require1Map.put(new ExprId(0), 0); + require1Map.put(new ExprId(1), 0); + require1Map.put(new ExprId(2), 1); + require1Map.put(new ExprId(3), 1); + DistributionSpecHash require1 = new DistributionSpecHash( Lists.newArrayList(new ExprId(0), new ExprId(2)), - ShuffleType.JOIN, + ShuffleType.REQUIRE, 0, Sets.newHashSet(0L), Lists.newArrayList(Sets.newHashSet(new ExprId(0), new ExprId(1)), Sets.newHashSet(new ExprId(2), new ExprId(3))), - join1Map + require1Map ); - Map join2Map = Maps.newHashMap(); - join2Map.put(new ExprId(1), 0); - join2Map.put(new ExprId(2), 1); - DistributionSpecHash join2 = new DistributionSpecHash( + Map require2Map = Maps.newHashMap(); + require2Map.put(new ExprId(1), 0); + require2Map.put(new ExprId(2), 1); + DistributionSpecHash require2 = new DistributionSpecHash( Lists.newArrayList(new ExprId(1), new ExprId(2)), - ShuffleType.JOIN, + ShuffleType.REQUIRE, 1, Sets.newHashSet(1L), Lists.newArrayList(Sets.newHashSet(new ExprId(1)), Sets.newHashSet(new ExprId(2))), - join2Map + require2Map ); - Map join3Map = Maps.newHashMap(); - join3Map.put(new ExprId(2), 0); - join3Map.put(new ExprId(1), 1); - DistributionSpecHash join3 = new DistributionSpecHash( + Map require3Map = Maps.newHashMap(); + require3Map.put(new ExprId(2), 0); + require3Map.put(new ExprId(1), 1); + DistributionSpecHash require3 = new DistributionSpecHash( Lists.newArrayList(new ExprId(2), new ExprId(1)), - ShuffleType.JOIN, + ShuffleType.REQUIRE, 1, Sets.newHashSet(1L), Lists.newArrayList(Sets.newHashSet(new ExprId(2)), Sets.newHashSet(new ExprId(1))), - join3Map - ); - - DistributionSpecHash natural = new DistributionSpecHash( - Lists.newArrayList(new ExprId(1), new ExprId(2)), - ShuffleType.NATURAL, - 1, - Sets.newHashSet(1L), - Lists.newArrayList(Sets.newHashSet(new ExprId(1)), Sets.newHashSet(new ExprId(2))), - join2Map - ); - - DistributionSpecHash enforce = new DistributionSpecHash( - Lists.newArrayList(new ExprId(1), new ExprId(2)), - ShuffleType.ENFORCED, - 1, - Sets.newHashSet(1L), - Lists.newArrayList(Sets.newHashSet(new ExprId(1)), Sets.newHashSet(new ExprId(2))), - join2Map - ); - - DistributionSpecHash aggregate = new DistributionSpecHash( - Lists.newArrayList(new ExprId(1), new ExprId(2)), - ShuffleType.AGGREGATE, - 1, - Sets.newHashSet(1L), - Lists.newArrayList(Sets.newHashSet(new ExprId(1)), Sets.newHashSet(new ExprId(2))), - join2Map - ); - - // require is same order - Assertions.assertTrue(join1.satisfy(join2)); - // require contains all sets but order is not same - Assertions.assertTrue(join1.satisfy(join3)); - // require slots is not contained by target - Assertions.assertFalse(join3.satisfy(join1)); - // other shuffle type with same order - Assertions.assertTrue(natural.satisfy(join2)); - Assertions.assertTrue(aggregate.satisfy(join2)); - Assertions.assertTrue(enforce.satisfy(join2)); - // other shuffle type contain all set but order is not same - Assertions.assertTrue(natural.satisfy(join3)); - Assertions.assertTrue(aggregate.satisfy(join3)); - Assertions.assertFalse(enforce.satisfy(join3)); - } - - @Test - public void testSatisfyAggregateHash() { - Map aggregate1Map = Maps.newHashMap(); - aggregate1Map.put(new ExprId(0), 0); - aggregate1Map.put(new ExprId(1), 0); - aggregate1Map.put(new ExprId(2), 1); - aggregate1Map.put(new ExprId(3), 1); - DistributionSpecHash aggregate1 = new DistributionSpecHash( - Lists.newArrayList(new ExprId(0), new ExprId(2)), - ShuffleType.AGGREGATE, - 0, - Sets.newHashSet(0L), - Lists.newArrayList(Sets.newHashSet(new ExprId(0), new ExprId(1)), Sets.newHashSet(new ExprId(2), new ExprId(3))), - aggregate1Map - ); - - Map aggregate2Map = Maps.newHashMap(); - aggregate2Map.put(new ExprId(1), 0); - aggregate2Map.put(new ExprId(2), 1); - DistributionSpecHash aggregate2 = new DistributionSpecHash( - Lists.newArrayList(new ExprId(1), new ExprId(2)), - ShuffleType.AGGREGATE, - 1, - Sets.newHashSet(1L), - Lists.newArrayList(Sets.newHashSet(new ExprId(1)), Sets.newHashSet(new ExprId(2))), - aggregate2Map - ); - - Map aggregate3Map = Maps.newHashMap(); - aggregate3Map.put(new ExprId(2), 0); - aggregate3Map.put(new ExprId(1), 1); - DistributionSpecHash aggregate3 = new DistributionSpecHash( - Lists.newArrayList(new ExprId(2), new ExprId(1)), - ShuffleType.AGGREGATE, - 1, - Sets.newHashSet(1L), - Lists.newArrayList(Sets.newHashSet(new ExprId(2)), Sets.newHashSet(new ExprId(1))), - aggregate3Map - ); - - Map aggregate4Map = Maps.newHashMap(); - aggregate4Map.put(new ExprId(2), 0); - aggregate4Map.put(new ExprId(3), 1); - DistributionSpecHash aggregate4 = new DistributionSpecHash( - Lists.newArrayList(new ExprId(2), new ExprId(3)), - ShuffleType.AGGREGATE, - 1, - Sets.newHashSet(1L), - Lists.newArrayList(Sets.newHashSet(new ExprId(2)), Sets.newHashSet(new ExprId(3))), - aggregate4Map + require3Map ); DistributionSpecHash natural = new DistributionSpecHash( @@ -377,152 +282,71 @@ public void testSatisfyAggregateHash() { 1, Sets.newHashSet(1L), Lists.newArrayList(Sets.newHashSet(new ExprId(1)), Sets.newHashSet(new ExprId(2))), - aggregate2Map + require2Map ); - DistributionSpecHash enforce = new DistributionSpecHash( + DistributionSpecHash storageBucketed = new DistributionSpecHash( Lists.newArrayList(new ExprId(1), new ExprId(2)), - ShuffleType.ENFORCED, + ShuffleType.STORAGE_BUCKETED, 1, Sets.newHashSet(1L), Lists.newArrayList(Sets.newHashSet(new ExprId(1)), Sets.newHashSet(new ExprId(2))), - aggregate2Map + require2Map ); - DistributionSpecHash join = new DistributionSpecHash( + DistributionSpecHash executionBucketed = new DistributionSpecHash( Lists.newArrayList(new ExprId(1), new ExprId(2)), - ShuffleType.JOIN, + ShuffleType.EXECUTION_BUCKETED, 1, Sets.newHashSet(1L), Lists.newArrayList(Sets.newHashSet(new ExprId(1)), Sets.newHashSet(new ExprId(2))), - aggregate2Map + require2Map ); // require is same order - Assertions.assertTrue(aggregate1.satisfy(aggregate2)); + Assertions.assertTrue(require1.satisfy(require2)); // require contains all sets but order is not same - Assertions.assertTrue(aggregate1.satisfy(aggregate3)); - // require do not contain all set but has the same number slot - Assertions.assertFalse(aggregate1.satisfy(aggregate4)); + Assertions.assertTrue(require1.satisfy(require3)); // require slots is not contained by target - Assertions.assertFalse(aggregate3.satisfy(aggregate1)); + Assertions.assertFalse(require3.satisfy(require1)); // other shuffle type with same order - Assertions.assertTrue(natural.satisfy(aggregate2)); - Assertions.assertTrue(join.satisfy(aggregate2)); - Assertions.assertTrue(enforce.satisfy(aggregate2)); + Assertions.assertTrue(natural.satisfy(require2)); + Assertions.assertTrue(executionBucketed.satisfy(require2)); + Assertions.assertTrue(storageBucketed.satisfy(require2)); // other shuffle type contain all set but order is not same - Assertions.assertTrue(natural.satisfy(aggregate3)); - Assertions.assertTrue(join.satisfy(aggregate3)); - Assertions.assertTrue(enforce.satisfy(aggregate3)); - } - - @Test - public void testSatisfyEnforceHash() { - Map enforce1Map = Maps.newHashMap(); - enforce1Map.put(new ExprId(0), 0); - enforce1Map.put(new ExprId(1), 0); - enforce1Map.put(new ExprId(2), 1); - enforce1Map.put(new ExprId(3), 1); - DistributionSpecHash enforce1 = new DistributionSpecHash( - Lists.newArrayList(new ExprId(0), new ExprId(2)), - ShuffleType.ENFORCED, - 0, - Sets.newHashSet(0L), - Lists.newArrayList(Sets.newHashSet(new ExprId(0), new ExprId(1)), Sets.newHashSet(new ExprId(2), new ExprId(3))), - enforce1Map - ); - - Map enforce2Map = Maps.newHashMap(); - enforce2Map.put(new ExprId(1), 0); - enforce2Map.put(new ExprId(2), 1); - DistributionSpecHash enforce2 = new DistributionSpecHash( - Lists.newArrayList(new ExprId(1), new ExprId(2)), - ShuffleType.ENFORCED, - 1, - Sets.newHashSet(1L), - Lists.newArrayList(Sets.newHashSet(new ExprId(1)), Sets.newHashSet(new ExprId(2))), - enforce2Map - ); - - Map enforce3Map = Maps.newHashMap(); - enforce3Map.put(new ExprId(2), 0); - enforce3Map.put(new ExprId(1), 1); - DistributionSpecHash enforce3 = new DistributionSpecHash( - Lists.newArrayList(new ExprId(2), new ExprId(1)), - ShuffleType.ENFORCED, - 1, - Sets.newHashSet(1L), - Lists.newArrayList(Sets.newHashSet(new ExprId(2)), Sets.newHashSet(new ExprId(1))), - enforce3Map - ); - - DistributionSpecHash join = new DistributionSpecHash( - Lists.newArrayList(new ExprId(1), new ExprId(2)), - ShuffleType.JOIN, - 1, - Sets.newHashSet(1L), - Lists.newArrayList(Sets.newHashSet(new ExprId(1)), Sets.newHashSet(new ExprId(2))), - enforce2Map - ); - - DistributionSpecHash natural = new DistributionSpecHash( - Lists.newArrayList(new ExprId(1), new ExprId(2)), - ShuffleType.NATURAL, - 1, - Sets.newHashSet(1L), - Lists.newArrayList(Sets.newHashSet(new ExprId(1)), Sets.newHashSet(new ExprId(2))), - enforce2Map - ); - - DistributionSpecHash aggregate = new DistributionSpecHash( - Lists.newArrayList(new ExprId(1), new ExprId(2)), - ShuffleType.AGGREGATE, - 1, - Sets.newHashSet(1L), - Lists.newArrayList(Sets.newHashSet(new ExprId(1)), Sets.newHashSet(new ExprId(2))), - enforce2Map - ); - - // require is same order - Assertions.assertTrue(enforce1.satisfy(enforce2)); - // require contains all sets but order is not same - Assertions.assertFalse(enforce1.satisfy(enforce3)); - // require slots is not contained by target - Assertions.assertFalse(enforce2.satisfy(enforce1)); - // other shuffle type with same order - Assertions.assertTrue(join.satisfy(enforce2)); - Assertions.assertTrue(aggregate.satisfy(enforce2)); - Assertions.assertTrue(natural.satisfy(enforce2)); + Assertions.assertTrue(natural.satisfy(require3)); + Assertions.assertTrue(executionBucketed.satisfy(require3)); + Assertions.assertTrue(storageBucketed.satisfy(require3)); } @Test public void testHashEqualSatisfyWithDifferentLength() { - Map enforce1Map = Maps.newHashMap(); - enforce1Map.put(new ExprId(0), 0); - enforce1Map.put(new ExprId(1), 1); - enforce1Map.put(new ExprId(2), 2); - DistributionSpecHash enforce1 = new DistributionSpecHash( + Map bucketed1Map = Maps.newHashMap(); + bucketed1Map.put(new ExprId(0), 0); + bucketed1Map.put(new ExprId(1), 1); + bucketed1Map.put(new ExprId(2), 2); + DistributionSpecHash bucketed1 = new DistributionSpecHash( Lists.newArrayList(new ExprId(0), new ExprId(1), new ExprId(2)), - ShuffleType.ENFORCED, + ShuffleType.EXECUTION_BUCKETED, 0, Sets.newHashSet(0L), Lists.newArrayList(Sets.newHashSet(new ExprId(0)), Sets.newHashSet(new ExprId(1)), Sets.newHashSet(new ExprId(2))), - enforce1Map + bucketed1Map ); - Map enforce2Map = Maps.newHashMap(); - enforce2Map.put(new ExprId(0), 0); - enforce2Map.put(new ExprId(1), 1); - DistributionSpecHash enforce2 = new DistributionSpecHash( + Map bucketed2Map = Maps.newHashMap(); + bucketed2Map.put(new ExprId(0), 0); + bucketed2Map.put(new ExprId(1), 1); + DistributionSpecHash bucketed2 = new DistributionSpecHash( Lists.newArrayList(new ExprId(0), new ExprId(1)), - ShuffleType.ENFORCED, + ShuffleType.EXECUTION_BUCKETED, 1, Sets.newHashSet(1L), Lists.newArrayList(Sets.newHashSet(new ExprId(0)), Sets.newHashSet(new ExprId(1))), - enforce2Map + bucketed2Map ); - Assertions.assertFalse(enforce1.satisfy(enforce2)); - Assertions.assertFalse(enforce2.satisfy(enforce1)); + Assertions.assertFalse(bucketed1.satisfy(bucketed2)); + Assertions.assertFalse(bucketed2.satisfy(bucketed1)); } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/properties/DistributionSpecTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/properties/DistributionSpecTest.java index 0a09c8a63af32fd..688e99e4fa981f0 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/properties/DistributionSpecTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/properties/DistributionSpecTest.java @@ -30,7 +30,7 @@ public void testSatisfy() { DistributionSpec replicated = DistributionSpecReplicated.INSTANCE; DistributionSpec any = DistributionSpecAny.INSTANCE; DistributionSpec gather = DistributionSpecGather.INSTANCE; - DistributionSpec hash = new DistributionSpecHash(Lists.newArrayList(), ShuffleType.JOIN); + DistributionSpec hash = new DistributionSpecHash(Lists.newArrayList(), ShuffleType.REQUIRE); Assertions.assertTrue(replicated.satisfy(any)); Assertions.assertTrue(gather.satisfy(any)); diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/properties/RequestPropertyDeriverTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/properties/RequestPropertyDeriverTest.java index 0c9044eb3df190e..1df7067c2b088d9 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/properties/RequestPropertyDeriverTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/properties/RequestPropertyDeriverTest.java @@ -110,8 +110,8 @@ Pair, List> getHashConjunctsExprIds() { List> expected = Lists.newArrayList(); expected.add(Lists.newArrayList( - new PhysicalProperties(new DistributionSpecHash(Lists.newArrayList(new ExprId(0)), ShuffleType.JOIN)), - new PhysicalProperties(new DistributionSpecHash(Lists.newArrayList(new ExprId(1)), ShuffleType.JOIN)) + new PhysicalProperties(new DistributionSpecHash(Lists.newArrayList(new ExprId(0)), ShuffleType.REQUIRE)), + new PhysicalProperties(new DistributionSpecHash(Lists.newArrayList(new ExprId(1)), ShuffleType.REQUIRE)) )); Assertions.assertEquals(expected, actual); } @@ -136,8 +136,8 @@ Pair, List> getHashConjunctsExprIds() { List> expected = Lists.newArrayList(); expected.add(Lists.newArrayList( - new PhysicalProperties(new DistributionSpecHash(Lists.newArrayList(new ExprId(0)), ShuffleType.JOIN)), - new PhysicalProperties(new DistributionSpecHash(Lists.newArrayList(new ExprId(1)), ShuffleType.JOIN)) + new PhysicalProperties(new DistributionSpecHash(Lists.newArrayList(new ExprId(0)), ShuffleType.REQUIRE)), + new PhysicalProperties(new DistributionSpecHash(Lists.newArrayList(new ExprId(1)), ShuffleType.REQUIRE)) )); expected.add(Lists.newArrayList(PhysicalProperties.ANY, PhysicalProperties.REPLICATED)); Assertions.assertEquals(expected, actual); @@ -174,7 +174,7 @@ public void testGlobalAggregate() { new AggregateParam(AggPhase.GLOBAL, AggMode.BUFFER_TO_RESULT), true, logicalProperties, - RequireProperties.of(PhysicalProperties.createHash(ImmutableList.of(partition), ShuffleType.AGGREGATE)), + RequireProperties.of(PhysicalProperties.createHash(ImmutableList.of(partition), ShuffleType.REQUIRE)), groupPlan ); GroupExpression groupExpression = new GroupExpression(aggregate); @@ -184,7 +184,7 @@ public void testGlobalAggregate() { List> expected = Lists.newArrayList(); expected.add(Lists.newArrayList(PhysicalProperties.createHash(new DistributionSpecHash( Lists.newArrayList(partition.getExprId()), - ShuffleType.AGGREGATE + ShuffleType.REQUIRE )))); Assertions.assertEquals(expected, actual); } diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/analysis/BindRelationTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/analysis/BindRelationTest.java index 4b3316e9cc2736a..f9550dce0b9f306 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/analysis/BindRelationTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/analysis/BindRelationTest.java @@ -17,9 +17,19 @@ package org.apache.doris.nereids.rules.analysis; +import org.apache.doris.catalog.Column; +import org.apache.doris.catalog.KeysType; +import org.apache.doris.catalog.OlapTable; +import org.apache.doris.catalog.PartitionInfo; +import org.apache.doris.catalog.RandomDistributionInfo; +import org.apache.doris.catalog.Type; import org.apache.doris.nereids.analyzer.UnboundRelation; +import org.apache.doris.nereids.pattern.GeneratedPlanPatterns; +import org.apache.doris.nereids.rules.RulePromise; +import org.apache.doris.nereids.rules.analysis.BindRelation.CustomTableResolver; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.logical.LogicalOlapScan; +import org.apache.doris.nereids.util.PlanChecker; import org.apache.doris.nereids.util.PlanRewriter; import org.apache.doris.nereids.util.RelationUtil; import org.apache.doris.utframe.TestWithFeService; @@ -28,7 +38,10 @@ import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; -class BindRelationTest extends TestWithFeService { +import java.util.List; +import java.util.Optional; + +class BindRelationTest extends TestWithFeService implements GeneratedPlanPatterns { private static final String DB1 = "db1"; private static final String DB2 = "db2"; @@ -66,4 +79,54 @@ void bindByDbQualifier() { ImmutableList.of(DEFAULT_CLUSTER_PREFIX + DB1, "t"), ((LogicalOlapScan) plan).qualified()); } + + @Test + public void bindExternalRelation() { + connectContext.setDatabase(DEFAULT_CLUSTER_PREFIX + DB1); + String tableName = "external_table"; + + List externalTableColumns = ImmutableList.of( + new Column("id", Type.INT), + new Column("name", Type.VARCHAR) + ); + + OlapTable externalOlapTable = new OlapTable(1, tableName, externalTableColumns, KeysType.DUP_KEYS, + new PartitionInfo(), new RandomDistributionInfo(10)) { + @Override + public List getBaseSchema() { + return externalTableColumns; + } + + @Override + public boolean hasDeleteSign() { + return false; + } + }; + + CustomTableResolver customTableResolver = qualifiedTable -> { + if (qualifiedTable.get(2).equals(tableName)) { + return externalOlapTable; + } else { + return null; + } + }; + + PlanChecker.from(connectContext) + .parse("select * from " + tableName + " as et join db1.t on et.id = t.a") + .customAnalyzer(Optional.of(customTableResolver)) // analyze internal relation + .rewrite() + .matchesFromRoot( + logicalProject( + logicalJoin( + logicalOlapScan().when(r -> r.getTable() == externalOlapTable), + logicalOlapScan().when(r -> r.getTable().getName().equals("t")) + ) + ) + ); + } + + @Override + public RulePromise defaultPromise() { + return RulePromise.REWRITE; + } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/util/PlanChecker.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/util/PlanChecker.java index 7380c90528df36c..ab57a650f6795bf 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/util/PlanChecker.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/util/PlanChecker.java @@ -47,6 +47,7 @@ import org.apache.doris.nereids.rules.RuleFactory; import org.apache.doris.nereids.rules.RuleSet; import org.apache.doris.nereids.rules.RuleType; +import org.apache.doris.nereids.rules.analysis.BindRelation.CustomTableResolver; import org.apache.doris.nereids.rules.rewrite.OneRewriteRuleFactory; import org.apache.doris.nereids.trees.expressions.Slot; import org.apache.doris.nereids.trees.plans.GroupPlan; @@ -68,6 +69,7 @@ import java.util.ArrayList; import java.util.List; +import java.util.Optional; import java.util.function.Consumer; /** @@ -110,8 +112,13 @@ public PlanChecker checkParse(String sql, Consumer consumer) { return this; } - public PlanChecker analyze(String sql) { + public PlanChecker parse(String sql) { this.cascadesContext = MemoTestUtils.createCascadesContext(connectContext, sql); + this.cascadesContext.toMemo(); + return this; + } + + public PlanChecker analyze() { this.cascadesContext.newAnalyzer().analyze(); this.cascadesContext.toMemo(); return this; @@ -125,6 +132,24 @@ public PlanChecker analyze(Plan plan) { return this; } + public PlanChecker analyze(String sql) { + this.cascadesContext = MemoTestUtils.createCascadesContext(connectContext, sql); + this.cascadesContext.newAnalyzer().analyze(); + this.cascadesContext.toMemo(); + return this; + } + + public PlanChecker customAnalyzer(Optional customTableResolver) { + this.cascadesContext.newAnalyzer(customTableResolver).analyze(); + this.cascadesContext.toMemo(); + return this; + } + + public PlanChecker setRewritePlanFromMemo() { + this.cascadesContext.setRewritePlan(this.cascadesContext.getMemo().copyOut()); + return this; + } + public PlanChecker customRewrite(CustomRewriter customRewriter) { new CustomRewriteJob(() -> customRewriter, RuleType.TEST_REWRITE).execute(cascadesContext.getCurrentJobContext()); cascadesContext.toMemo(); diff --git a/fe/fe-core/src/test/java/org/apache/doris/planner/DistributedPlannerTest.java b/fe/fe-core/src/test/java/org/apache/doris/planner/DistributedPlannerTest.java index 2551a9d06a7438f..45aab8366fde1c8 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/planner/DistributedPlannerTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/planner/DistributedPlannerTest.java @@ -52,6 +52,7 @@ public class DistributedPlannerTest { public static void setUp() throws Exception { UtFrameUtils.createDorisCluster(runningDir); ctx = UtFrameUtils.createDefaultCtx(); + ctx.getSessionVariable().setEnableNereidsPlanner(false); String createDbStmtStr = "create database db1;"; CreateDbStmt createDbStmt = (CreateDbStmt) UtFrameUtils.parseAndAnalyzeStmt(createDbStmtStr, ctx); Env.getCurrentEnv().createDb(createDbStmt); diff --git a/fe/fe-core/src/test/java/org/apache/doris/planner/PlannerTest.java b/fe/fe-core/src/test/java/org/apache/doris/planner/PlannerTest.java index 7cf92fc5058c61f..8e99eadd1aa4d1d 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/planner/PlannerTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/planner/PlannerTest.java @@ -560,71 +560,60 @@ public void testPushSortToOlapScan() throws Exception { @Test public void testEliminatingSortNode() throws Exception { - // success case 1 + // fail case 1 { - String sql1 = "explain select /*+ SET_VAR(enable_nereids_planner=false) */ k1 from db1.tbl1 where k1 = 1 order by k1, k2 limit " - + connectContext.getSessionVariable().topnOptLimitThreshold; + String sql1 = "explain select /*+ SET_VAR(enable_nereids_planner=false) */ k1 from db1.tbl1 where k1 = 1 order by k1, k2"; StmtExecutor stmtExecutor1 = new StmtExecutor(connectContext, sql1); stmtExecutor1.execute(); Planner planner1 = stmtExecutor1.planner(); String plan1 = planner1.getExplainString(new ExplainOptions(false, false)); - Assertions.assertTrue(plan1.contains("SORT INFO:\n `k1`\n `k2`")); - Assertions.assertTrue(plan1.contains("SORT LIMIT:")); + Assertions.assertTrue(plan1.contains("order by:")); } // fail case 2 { - String sql1 = "explain select /*+ SET_VAR(enable_nereids_planner=false) */ k1 from db1.tbl1 where k1 = 1 and k3 = 2 order by k1, k2 limit " - + connectContext.getSessionVariable().topnOptLimitThreshold; + String sql1 = "explain select /*+ SET_VAR(enable_nereids_planner=false) */ k1 from db1.tbl1 where k1 = 1 and k3 = 2 order by k1, k2"; StmtExecutor stmtExecutor1 = new StmtExecutor(connectContext, sql1); stmtExecutor1.execute(); Planner planner1 = stmtExecutor1.planner(); String plan1 = planner1.getExplainString(new ExplainOptions(false, false)); - Assertions.assertTrue(plan1.contains("SORT INFO:\n `k1`\n `k2`")); - Assertions.assertTrue(plan1.contains("SORT LIMIT:")); + Assertions.assertTrue(plan1.contains("order by:")); } // fail case 3 { - String sql1 = "explain select /*+ SET_VAR(enable_nereids_planner=false) */ k1 from db1.tbl1 where k1 = 1 and k2 != 2 order by k1, k2 limit " - + connectContext.getSessionVariable().topnOptLimitThreshold; + String sql1 = "explain select /*+ SET_VAR(enable_nereids_planner=false) */ k1 from db1.tbl1 where k1 = 1 and k2 != 2 order by k1, k2"; StmtExecutor stmtExecutor1 = new StmtExecutor(connectContext, sql1); stmtExecutor1.execute(); Planner planner1 = stmtExecutor1.planner(); String plan1 = planner1.getExplainString(new ExplainOptions(false, false)); - Assertions.assertTrue(plan1.contains("SORT INFO:\n `k1`\n `k2`")); - Assertions.assertTrue(plan1.contains("SORT LIMIT:")); + Assertions.assertTrue(plan1.contains("order by:")); } // fail case 4 { - String sql1 = "explain select /*+ SET_VAR(enable_nereids_planner=false) */ k1 from db1.tbl1 where k1 = 1 or k2 = 2 order by k1, k2 limit " - + connectContext.getSessionVariable().topnOptLimitThreshold; + String sql1 = "explain select /*+ SET_VAR(enable_nereids_planner=false) */ k1 from db1.tbl1 where k1 = 1 or k2 = 2 order by k1, k2"; StmtExecutor stmtExecutor1 = new StmtExecutor(connectContext, sql1); stmtExecutor1.execute(); Planner planner1 = stmtExecutor1.planner(); String plan1 = planner1.getExplainString(new ExplainOptions(false, false)); - Assertions.assertTrue(plan1.contains("SORT INFO:\n `k1`\n `k2`")); - Assertions.assertTrue(plan1.contains("SORT LIMIT:")); + Assertions.assertTrue(plan1.contains("order by:")); } // fail case 5 { - String sql1 = "explain select /*+ SET_VAR(enable_nereids_planner=false) */ k1 from db1.tbl1 where k1 = 1 and k2 = 2 or k3 = 3 order by k1, k2 limit " - + connectContext.getSessionVariable().topnOptLimitThreshold; + String sql1 = "explain select /*+ SET_VAR(enable_nereids_planner=false) */ k1 from db1.tbl1 where k1 = 1 and k2 = 2 or k3 = 3 order by k1, k2"; StmtExecutor stmtExecutor1 = new StmtExecutor(connectContext, sql1); stmtExecutor1.execute(); Planner planner1 = stmtExecutor1.planner(); String plan1 = planner1.getExplainString(new ExplainOptions(false, false)); - Assertions.assertTrue(plan1.contains("SORT INFO:\n `k1`\n `k2`")); - Assertions.assertTrue(plan1.contains("SORT LIMIT:")); + Assertions.assertTrue(plan1.contains("order by:")); } // fail case 6 // TODO, support: in (select 1) { - String sql1 = "explain select k1 from db1.tbl1 where k1 in (select 1) and k2 = 2 order by k1, k2 limit " - + connectContext.getSessionVariable().topnOptLimitThreshold; + String sql1 = "explain select k1 from db1.tbl1 where k1 in (select 1) and k2 = 2 order by k1, k2"; StmtExecutor stmtExecutor1 = new StmtExecutor(connectContext, sql1); stmtExecutor1.execute(); Planner planner1 = stmtExecutor1.planner(); @@ -634,8 +623,7 @@ public void testEliminatingSortNode() throws Exception { // fail case 7 { - String sql1 = "explain select k1 from db1.tbl1 where k1 not in (1) and k2 = 2 order by k1, k2 limit " - + connectContext.getSessionVariable().topnOptLimitThreshold; + String sql1 = "explain select k1 from db1.tbl1 where k1 not in (1) and k2 = 2 order by k1, k2"; StmtExecutor stmtExecutor1 = new StmtExecutor(connectContext, sql1); stmtExecutor1.execute(); Planner planner1 = stmtExecutor1.planner(); @@ -645,63 +633,53 @@ public void testEliminatingSortNode() throws Exception { // success case 1 { - String sql1 = "explain select /*+ SET_VAR(enable_nereids_planner=false) */ k1 from db1.tbl1 where k1 = 1 and k2 = 2 order by k1, k2 limit " - + connectContext.getSessionVariable().topnOptLimitThreshold; + String sql1 = "explain select /*+ SET_VAR(enable_nereids_planner=false) */ k1 from db1.tbl1 where k1 = 1 and k2 = 2 order by k1, k2"; StmtExecutor stmtExecutor1 = new StmtExecutor(connectContext, sql1); stmtExecutor1.execute(); Planner planner1 = stmtExecutor1.planner(); String plan1 = planner1.getExplainString(new ExplainOptions(false, false)); - Assertions.assertFalse(plan1.contains("SORT INFO:\n `k1`\n `k2`")); - Assertions.assertFalse(plan1.contains("SORT LIMIT:")); + Assertions.assertFalse(plan1.contains("order by:")); } // success case 2 { - String sql1 = "explain select /*+ SET_VAR(enable_nereids_planner=false) */ k1 from db1.tbl1 where k3 = 3 and k2 = 2 and k1 = 1 order by k1, k2 limit " - + connectContext.getSessionVariable().topnOptLimitThreshold; + String sql1 = "explain select /*+ SET_VAR(enable_nereids_planner=false) */ k1 from db1.tbl1 where k3 = 3 and k2 = 2 and k1 = 1 order by k1, k2"; StmtExecutor stmtExecutor1 = new StmtExecutor(connectContext, sql1); stmtExecutor1.execute(); Planner planner1 = stmtExecutor1.planner(); String plan1 = planner1.getExplainString(new ExplainOptions(false, false)); - Assertions.assertFalse(plan1.contains("SORT INFO:\n `k1`\n `k2`")); - Assertions.assertFalse(plan1.contains("SORT LIMIT:")); + Assertions.assertFalse(plan1.contains("order by:")); } // success case 3 { - String sql1 = "explain select /*+ SET_VAR(enable_nereids_planner=false) */ k1 from db1.tbl1 where k1 in (1) and k2 in (2) and k2 !=2 order by k1, k2 limit " - + connectContext.getSessionVariable().topnOptLimitThreshold; + String sql1 = "explain select /*+ SET_VAR(enable_nereids_planner=false) */ k1 from db1.tbl1 where k1 in (1) and k2 in (2) and k2 !=2 order by k1, k2"; StmtExecutor stmtExecutor1 = new StmtExecutor(connectContext, sql1); stmtExecutor1.execute(); Planner planner1 = stmtExecutor1.planner(); String plan1 = planner1.getExplainString(new ExplainOptions(false, false)); - Assertions.assertFalse(plan1.contains("SORT INFO:\n `k1`\n `k2`")); - Assertions.assertFalse(plan1.contains("SORT LIMIT:")); + Assertions.assertFalse(plan1.contains("order by:")); } // success case 4 { - String sql1 = "explain select /*+ SET_VAR(enable_nereids_planner=false) */ k1 from db1.tbl1 where k1 in (concat('1','2')) and k2 = 2 order by k1, k2 limit " - + connectContext.getSessionVariable().topnOptLimitThreshold; + String sql1 = "explain select /*+ SET_VAR(enable_nereids_planner=false) */ k1 from db1.tbl1 where k1 in (concat('1','2')) and k2 = 2 order by k1, k2"; StmtExecutor stmtExecutor1 = new StmtExecutor(connectContext, sql1); stmtExecutor1.execute(); Planner planner1 = stmtExecutor1.planner(); String plan1 = planner1.getExplainString(new ExplainOptions(false, false)); - Assertions.assertFalse(plan1.contains("SORT INFO:\n `k1`\n `k2`")); - Assertions.assertFalse(plan1.contains("SORT LIMIT:")); + Assertions.assertFalse(plan1.contains("order by:")); } // success case 5 { String sql1 = "explain select tbl1.k1 from db1.tbl1 join db1.tbl2 on tbl1.k1 = tbl2.k1" - + " where tbl1.k1 = 1 and tbl2.k1 = 2 and tbl1.k2 = 3 order by tbl1.k1, tbl2.k1 limit " - + connectContext.getSessionVariable().topnOptLimitThreshold; + + " where tbl1.k1 = 1 and tbl2.k1 = 2 and tbl1.k2 = 3 order by tbl1.k1, tbl2.k1"; StmtExecutor stmtExecutor1 = new StmtExecutor(connectContext, sql1); stmtExecutor1.execute(); Planner planner1 = stmtExecutor1.planner(); String plan1 = planner1.getExplainString(new ExplainOptions(false, false)); - Assertions.assertFalse(plan1.contains("SORT INFO:")); - Assertions.assertFalse(plan1.contains("SORT LIMIT:")); + Assertions.assertFalse(plan1.contains("order by:")); } } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/planner/RepeatNodeTest.java b/fe/fe-core/src/test/java/org/apache/doris/planner/RepeatNodeTest.java index 1687d0c1a691dae..a4a09d0870d6120 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/planner/RepeatNodeTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/planner/RepeatNodeTest.java @@ -67,9 +67,9 @@ public void testExpr() throws Exception { String sql2 = "select /*+ SET_VAR(enable_nereids_planner=false) */ (id + 1) id_, name, sum(cost) from mycost group by grouping sets((id_, name),());"; String explainString2 = getSQLPlanOrErrorMsg("explain " + sql2); System.out.println(explainString2); - Assertions.assertTrue(explainString2.contains("exprs: (`id` + 1), `name`, `cost`")); + Assertions.assertTrue(explainString2.contains("exprs: ((`id` + 1)), `name`, `cost`")); Assertions.assertTrue( - explainString2.contains(" output slots: `(`id` + 1)`, ``name``, ``cost``, ``GROUPING_ID``")); + explainString2.contains(" output slots: `((`id` + 1))`, ``name``, ``cost``, ``GROUPING_ID``")); String sql3 = "select /*+ SET_VAR(enable_nereids_planner=false) */ 1 as id_, name, sum(cost) from mycost group by grouping sets((id_, name),());"; String explainString3 = getSQLPlanOrErrorMsg("explain " + sql3); diff --git a/fe/fe-core/src/test/java/org/apache/doris/planner/TableFunctionPlanTest.java b/fe/fe-core/src/test/java/org/apache/doris/planner/TableFunctionPlanTest.java index 9de505d693a2a05..0b1308c9b1ac237 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/planner/TableFunctionPlanTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/planner/TableFunctionPlanTest.java @@ -78,7 +78,7 @@ public void normalTableFunction() throws Exception { Assert.assertTrue( explainString.contains("table function: explode_split(`default_cluster:db1`.`tbl1`.`k2`, ',')")); Assert.assertTrue(explainString.contains("tuple ids: 0 1")); - Assert.assertTrue(explainString.contains("TupleDescriptor{id=1, tbl=tmp, byteSize=32}")); + Assert.assertTrue(explainString.contains("TupleDescriptor{id=1, tbl=tmp, byteSize=16}")); Assert.assertTrue(explainString.contains("SlotDescriptor{id=1, col=e1, colUniqueId=-1, type=varchar(*)")); } @@ -94,7 +94,7 @@ public void withoutOutputExplodeColumn() throws Exception { Assert.assertTrue( explainString.contains("table function: explode_split(`default_cluster:db1`.`tbl1`.`k2`, ',')")); Assert.assertTrue(explainString.contains("tuple ids: 0 1")); - Assert.assertTrue(explainString.contains("TupleDescriptor{id=1, tbl=tmp, byteSize=32}")); + Assert.assertTrue(explainString.contains("TupleDescriptor{id=1, tbl=tmp, byteSize=16}")); Assert.assertTrue(explainString.contains("SlotDescriptor{id=1, col=e1, colUniqueId=-1, type=varchar(*)")); } @@ -115,7 +115,7 @@ public void groupByExplodeColumn() throws Exception { Assert.assertTrue( explainString.contains("table function: explode_split(`default_cluster:db1`.`tbl1`.`k2`, ',')")); Assert.assertTrue(explainString.contains("tuple ids: 0 1")); - Assert.assertTrue(explainString.contains("TupleDescriptor{id=1, tbl=tmp, byteSize=32}")); + Assert.assertTrue(explainString.contains("TupleDescriptor{id=1, tbl=tmp, byteSize=16}")); Assert.assertTrue(explainString.contains("SlotDescriptor{id=1, col=e1, colUniqueId=-1, type=varchar(*)")); // group by tuple Assert.assertTrue(explainString.contains("TupleDescriptor{id=2, tbl=null, byteSize=32}")); @@ -134,7 +134,7 @@ public void whereExplodeColumn() throws Exception { explainString.contains("table function: explode_split(`default_cluster:db1`.`tbl1`.`k2`, ',')")); Assert.assertTrue(explainString.contains("PREDICATES: `e1` = '1'")); Assert.assertTrue(explainString.contains("tuple ids: 0 1")); - Assert.assertTrue(explainString.contains("TupleDescriptor{id=1, tbl=tmp, byteSize=32}")); + Assert.assertTrue(explainString.contains("TupleDescriptor{id=1, tbl=tmp, byteSize=16}")); Assert.assertTrue(explainString.contains("SlotDescriptor{id=1, col=e1, colUniqueId=-1, type=varchar(*)")); } @@ -150,7 +150,7 @@ public void whereNormalColumn() throws Exception { Assert.assertTrue( explainString.contains("table function: explode_split(`default_cluster:db1`.`tbl1`.`k2`, ',')")); Assert.assertTrue(explainString.contains("tuple ids: 0 1")); - Assert.assertTrue(explainString.contains("TupleDescriptor{id=1, tbl=tmp, byteSize=32}")); + Assert.assertTrue(explainString.contains("TupleDescriptor{id=1, tbl=tmp, byteSize=16}")); Assert.assertTrue(explainString.contains("SlotDescriptor{id=1, col=e1, colUniqueId=-1, type=varchar(*)")); Assert.assertTrue(UtFrameUtils.checkPlanResultContainsNode(explainString, 0, "OlapScanNode")); Assert.assertTrue(explainString.contains("PREDICATES: `k1` = 1")); @@ -170,10 +170,10 @@ public void testMultiLateralView() throws Exception { "table function: explode_split(`default_cluster:db1`.`tbl1`.`k2`, ',') explode_split(`default_cluster:db1`.`tbl1`.`k2`, ',')")); Assert.assertTrue(explainString.contains("lateral view tuple id: 1 2")); // lateral view 2 tuple - Assert.assertTrue(explainString.contains("TupleDescriptor{id=1, tbl=tmp2, byteSize=32}")); + Assert.assertTrue(explainString.contains("TupleDescriptor{id=1, tbl=tmp2, byteSize=16}")); Assert.assertTrue(explainString.contains("SlotDescriptor{id=1, col=e2, colUniqueId=-1, type=varchar(*)")); // lateral view 1 tuple - Assert.assertTrue(explainString.contains("TupleDescriptor{id=2, tbl=tmp1, byteSize=32}")); + Assert.assertTrue(explainString.contains("TupleDescriptor{id=2, tbl=tmp1, byteSize=16}")); Assert.assertTrue(explainString.contains("SlotDescriptor{id=2, col=e1, colUniqueId=-1, type=varchar(*)")); } diff --git a/fe/fe-core/src/test/java/org/apache/doris/resource/workloadgroup/WorkloadGroupMgrTest.java b/fe/fe-core/src/test/java/org/apache/doris/resource/workloadgroup/WorkloadGroupMgrTest.java index 630c1be21fa0d61..89062d87cda4e09 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/resource/workloadgroup/WorkloadGroupMgrTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/resource/workloadgroup/WorkloadGroupMgrTest.java @@ -23,8 +23,10 @@ import org.apache.doris.catalog.Env; import org.apache.doris.common.Config; import org.apache.doris.common.DdlException; +import org.apache.doris.common.Pair; import org.apache.doris.common.UserException; import org.apache.doris.mysql.privilege.AccessControllerManager; +import org.apache.doris.mysql.privilege.Auth; import org.apache.doris.mysql.privilege.PrivPredicate; import org.apache.doris.persist.EditLog; import org.apache.doris.qe.ConnectContext; @@ -54,6 +56,10 @@ public class WorkloadGroupMgrTest { @Mocked AccessControllerManager accessControllerManager; + @Mocked + private Auth auth; + + private AtomicLong id = new AtomicLong(10); @Before @@ -86,6 +92,18 @@ long delegate() { accessControllerManager.checkWorkloadGroupPriv((ConnectContext) any, anyString, (PrivPredicate) any); minTimes = 0; result = true; + + env.getAuth(); + minTimes = 0; + result = auth; + + auth.isWorkloadGroupInUse(anyString); + minTimes = 0; + result = new Delegate() { + Pair list() { + return Pair.of(false, ""); + } + }; } }; } diff --git a/fe/fe-core/src/test/java/org/apache/doris/scheduler/disruptor/AsyncJobManagerTest.java b/fe/fe-core/src/test/java/org/apache/doris/scheduler/disruptor/AsyncJobManagerTest.java new file mode 100644 index 000000000000000..dceb8049cdc7c03 --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/scheduler/disruptor/AsyncJobManagerTest.java @@ -0,0 +1,117 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.scheduler.disruptor; + +import org.apache.doris.scheduler.executor.JobExecutor; +import org.apache.doris.scheduler.job.AsyncJobManager; +import org.apache.doris.scheduler.job.Job; + +import lombok.extern.slf4j.Slf4j; +import org.awaitility.Awaitility; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import java.io.IOException; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; + +@Slf4j +public class AsyncJobManagerTest { + + AsyncJobManager asyncJobManager; + + private static AtomicInteger testExecuteCount = new AtomicInteger(0); + Job job = new Job("test", 6000L, null, + null, new TestExecutor()); + + @BeforeEach + public void init() { + testExecuteCount.set(0); + asyncJobManager = new AsyncJobManager(); + } + + @Test + public void testCycleScheduler() { + asyncJobManager.registerJob(job); + //consider the time of the first execution and give some buffer time + Awaitility.await().atMost(30, TimeUnit.SECONDS).until(() -> testExecuteCount.get() >= 3); + } + + @Test + public void testCycleSchedulerAndStop() { + asyncJobManager.registerJob(job); + long startTime = System.currentTimeMillis(); + Awaitility.await().atMost(8, TimeUnit.SECONDS).until(() -> testExecuteCount.get() >= 1); + asyncJobManager.unregisterJob(job.getJobId()); + //consider the time of the first execution and give some buffer time + Awaitility.await().atMost(30, TimeUnit.SECONDS).until(() -> System.currentTimeMillis() >= startTime + 13000L); + Assertions.assertEquals(1, testExecuteCount.get()); + + } + + @Test + public void testCycleSchedulerWithIncludeStartTimeAndEndTime() { + job.setStartTimestamp(System.currentTimeMillis() + 6000L); + long endTimestamp = System.currentTimeMillis() + 19000L; + job.setEndTimestamp(endTimestamp); + asyncJobManager.registerJob(job); + //consider the time of the first execution and give some buffer time + + Awaitility.await().atMost(60, TimeUnit.SECONDS).until(() -> System.currentTimeMillis() + >= endTimestamp + 12000L); + Assertions.assertEquals(2, testExecuteCount.get()); + } + + @Test + public void testCycleSchedulerWithIncludeEndTime() { + long endTimestamp = System.currentTimeMillis() + 13000; + job.setEndTimestamp(endTimestamp); + asyncJobManager.registerJob(job); + //consider the time of the first execution and give some buffer time + Awaitility.await().atMost(36, TimeUnit.SECONDS).until(() -> System.currentTimeMillis() + >= endTimestamp + 12000L); + Assertions.assertEquals(2, testExecuteCount.get()); + } + + @Test + public void testCycleSchedulerWithIncludeStartTime() { + + long startTimestamp = System.currentTimeMillis() + 6000L; + job.setStartTimestamp(startTimestamp); + asyncJobManager.registerJob(job); + //consider the time of the first execution and give some buffer time + Awaitility.await().atMost(14, TimeUnit.SECONDS).until(() -> System.currentTimeMillis() + >= startTimestamp + 7000L); + Assertions.assertEquals(1, testExecuteCount.get()); + } + + @AfterEach + public void after() throws IOException { + asyncJobManager.close(); + } + + class TestExecutor implements JobExecutor { + @Override + public Boolean execute() { + log.info("test execute count:{}", testExecuteCount.incrementAndGet()); + return true; + } + } +} diff --git a/fe/fe-core/src/test/java/org/apache/doris/scheduler/disruptor/TimerTaskDisruptorTest.java b/fe/fe-core/src/test/java/org/apache/doris/scheduler/disruptor/TimerTaskDisruptorTest.java new file mode 100644 index 000000000000000..1630b1f864d0eca --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/scheduler/disruptor/TimerTaskDisruptorTest.java @@ -0,0 +1,77 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.scheduler.disruptor; + +import org.apache.doris.scheduler.executor.JobExecutor; +import org.apache.doris.scheduler.job.AsyncJobManager; +import org.apache.doris.scheduler.job.Job; + +import mockit.Expectations; +import mockit.Injectable; +import mockit.Tested; +import org.awaitility.Awaitility; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import java.util.UUID; +import java.util.concurrent.TimeUnit; + +public class TimerTaskDisruptorTest { + + @Tested + private TimerTaskDisruptor timerTaskDisruptor; + + @Injectable + private AsyncJobManager asyncJobManager; + + private static boolean testEventExecuteFlag = false; + + @BeforeEach + public void init() { + timerTaskDisruptor = new TimerTaskDisruptor(asyncJobManager); + } + + @Test + void testPublishEventAndConsumer() { + Job job = new Job("test", 6000L, null, + null, new TestExecutor()); + new Expectations() {{ + asyncJobManager.getJob(anyLong); + result = job; + }}; + timerTaskDisruptor.tryPublish(job.getJobId(), UUID.randomUUID().getMostSignificantBits()); + Awaitility.await().atMost(1, TimeUnit.SECONDS).until(() -> testEventExecuteFlag); + Assertions.assertTrue(testEventExecuteFlag); + } + + + class TestExecutor implements JobExecutor { + @Override + public Boolean execute() { + testEventExecuteFlag = true; + return true; + } + } + + @AfterEach + public void after() { + timerTaskDisruptor.close(); + } +} diff --git a/fe/fe-core/src/test/java/org/apache/doris/statistics/CacheTest.java b/fe/fe-core/src/test/java/org/apache/doris/statistics/CacheTest.java index f97e1cb171d8fbb..eacab62034bb057 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/statistics/CacheTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/statistics/CacheTest.java @@ -281,7 +281,7 @@ public Env getCurrentEnv() { db.getTableOrMetaException(1); result = table; - table.getColumnStatistic(); + table.getColumnStatistic("col"); result = new ColumnStatistic(1, 2, null, 3, 4, 5, 6, 7, 8, null, null, false, null); } }; diff --git a/fe/pom.xml b/fe/pom.xml index 3c12f0c30bbcaa4..048257fdb53e3d5 100644 --- a/fe/pom.xml +++ b/fe/pom.xml @@ -215,7 +215,7 @@ under the License. 1.7 2.8.9 30.0-jre - 2.12.3 + 2.14.3 0.11-a-czt02-cdh 3.18.2-GA 3.1.0 @@ -233,9 +233,10 @@ under the License. 2.18.0 2.0.6 4.0.2 - 4.1.89.Final + 4.1.94.Final 2.1 1.30.0 + 3.32.0 3.21.12 @@ -243,7 +244,7 @@ under the License. com.google.protobuf:protoc:${protoc.artifact.version} io.grpc:protoc-gen-grpc-java:${grpc.version} 3.1.5 - 1.1.7.2 + 1.1.10.1 1.11-8 1.0.1 5.12.2 @@ -275,7 +276,7 @@ under the License. 9.0.0 1.11.1 - 0.13.0 + 0.13.1 2.7.4-11 3.0.0-8 @@ -295,7 +296,8 @@ under the License. 3.3.5 2.8.1 github - 2.7.8 + 2.7.13 + 1.8.4 3.4.14 2.3 11.2.0.4 @@ -308,7 +310,7 @@ under the License. 3.0.0 0.4.0-incubating - 3.3.4 + 3.4.4 @@ -373,6 +375,11 @@ under the License. opentelemetry-exporter-zipkin ${opentelemetry.version} + + org.apache.orc + orc-core + ${orc.version} + org.springframework.boot spring-boot-starter @@ -482,6 +489,18 @@ under the License. hadoop-auth ${hadoop.version} + + org.apache.hadoop + hadoop-aliyun + ${hadoop.version} + + + + org.ini4j + ini4j + + + org.apache.doris hive-catalog-shade @@ -846,6 +865,11 @@ under the License. protobuf-java ${protobuf.version} + + org.checkerframework + checker + ${check.freamework.version} + com.squareup @@ -1025,7 +1049,7 @@ under the License. org.apache.spark spark-launcher_2.12 ${spark.version} - + provided @@ -1040,6 +1064,12 @@ under the License. provided + + org.apache.spark + spark-tags_2.12 + ${spark.version} + provided + org.apache.hadoop @@ -1300,22 +1330,42 @@ under the License. org.apache.hive hive-common ${hive.common.version} + + + log4j + log4j + + + io.netty + netty + + org.apache.hadoop hadoop-mapreduce-client-core - ${mapreduce.client.version} + ${hadoop.version} javax.servlet servlet-api + + io.netty + netty + org.apache.hadoop hadoop-mapreduce-client-common - ${mapreduce.client.version} + ${hadoop.version} + + + io.netty + netty + + com.alibaba @@ -1382,6 +1432,12 @@ under the License. org.jmockit jmockit + + + org.awaitility + awaitility + + diff --git a/gensrc/thrift/FrontendService.thrift b/gensrc/thrift/FrontendService.thrift index c58bebf80cb0e94..a771d40ed98c226 100644 --- a/gensrc/thrift/FrontendService.thrift +++ b/gensrc/thrift/FrontendService.thrift @@ -807,6 +807,7 @@ struct TMetadataTableRequestParams { 3: optional PlanNodes.TBackendsMetadataParams backends_metadata_params 4: optional list columns_name 5: optional PlanNodes.TFrontendsMetadataParams frontends_metadata_params + 6: optional Types.TUserIdentity current_user_ident } struct TFetchSchemaTableDataRequest { @@ -956,6 +957,8 @@ enum TBinlogType { UPSERT = 0, ADD_PARTITION = 1, CREATE_TABLE = 2, + DROP_PARTITION = 3, + DROP_TABLE = 4, } struct TBinlog { diff --git a/gensrc/thrift/PlanNodes.thrift b/gensrc/thrift/PlanNodes.thrift index 6f5963f4fe71a87..978efee422151e1 100644 --- a/gensrc/thrift/PlanNodes.thrift +++ b/gensrc/thrift/PlanNodes.thrift @@ -115,6 +115,7 @@ enum TFileFormatType { FORMAT_JSON, FORMAT_PROTO, FORMAT_JNI, + FORMAT_AVRO, } // In previous versions, the data compression format and file format were stored together, as TFileFormatType, @@ -583,6 +584,7 @@ struct TSchemaScanNode { struct TMetaScanNode { 1: required Types.TTupleId tuple_id 2: optional Types.TMetadataType metadata_type + 3: optional Types.TUserIdentity current_user_ident } struct TTestExternalScanNode { diff --git a/regression-test/data/bloom_filter_p0/test_create_table_with_bloom_filter.out b/regression-test/data/bloom_filter_p0/test_create_table_with_bloom_filter.out index 19d1ea9180a6af9..eb7280263ba2023 100644 --- a/regression-test/data/bloom_filter_p0/test_create_table_with_bloom_filter.out +++ b/regression-test/data/bloom_filter_p0/test_create_table_with_bloom_filter.out @@ -20,8 +20,8 @@ date_key DATE No true \N BLOOM_FILTER datetime_key DATETIME No true \N BLOOM_FILTER datev2_key DATE No true \N BLOOM_FILTER datetimev2_key_1 DATETIME No true \N BLOOM_FILTER -datetimev2_key_2 DATETIME No true \N BLOOM_FILTER -datetimev2_key_3 DATETIME No true \N BLOOM_FILTER +datetimev2_key_2 DATETIME(3) No true \N BLOOM_FILTER +datetimev2_key_3 DATETIME(6) No true \N BLOOM_FILTER tinyint_value TINYINT No false \N SUM smallint_value SMALLINT No false \N SUM int_value INT No false \N SUM @@ -47,12 +47,12 @@ datev2_value_min DATE No false \N MIN datetimev2_value_1_max DATETIME No false \N MAX datetimev2_value_1_replace DATETIME No false \N REPLACE datetimev2_value_1_min DATETIME No false \N MIN -datetimev2_value_2_max DATETIME No false \N MAX -datetimev2_value_2_replace DATETIME No false \N REPLACE -datetimev2_value_2_min DATETIME No false \N MIN -datetimev2_value_3_max DATETIME No false \N MAX -datetimev2_value_3_replace DATETIME No false \N REPLACE -datetimev2_value_3_min DATETIME No false \N MIN +datetimev2_value_2_max DATETIME(3) No false \N MAX +datetimev2_value_2_replace DATETIME(3) No false \N REPLACE +datetimev2_value_2_min DATETIME(3) No false \N MIN +datetimev2_value_3_max DATETIME(6) No false \N MAX +datetimev2_value_3_replace DATETIME(6) No false \N REPLACE +datetimev2_value_3_min DATETIME(6) No false \N MIN float_value FLOAT No false \N SUM double_value DOUBLE No false \N SUM diff --git a/regression-test/data/correctness_p0/test_first_value_window.out b/regression-test/data/correctness_p0/test_first_value_window.out index 5a2ad3022b8ac38..d0cecdaccf8cdcf 100644 --- a/regression-test/data/correctness_p0/test_first_value_window.out +++ b/regression-test/data/correctness_p0/test_first_value_window.out @@ -6,3 +6,10 @@ 23 04-23-10 1 1 24 02-24-10-21 1 1 +-- !select_default -- +21 04-21-11 ["amory", "clever"] ["amory", "clever"] +22 04-22-10-21 ["doris", "aws", "greate"] ["doris", "aws", "greate"] +22 04-22-10-21 ["is ", "cute", "tea"] ["is ", "cute", "tea"] +23 04-23-10 ["p7", "year4"] ["p7", "year4"] +24 02-24-10-21 [""] [""] + diff --git a/regression-test/data/correctness_p0/test_last_value_window.out b/regression-test/data/correctness_p0/test_last_value_window.out index 2b56500191257fa..2ca759725eec3f2 100644 --- a/regression-test/data/correctness_p0/test_last_value_window.out +++ b/regression-test/data/correctness_p0/test_last_value_window.out @@ -6,3 +6,10 @@ 23 04-23-10 1 1 24 02-24-10-21 1 1 +-- !select_default -- +21 04-21-11 ["amory", "clever"] ["amory", "clever"] +22 04-22-10-21 ["doris", "aws", "greate"] ["doris", "aws", "greate"] +22 04-22-10-21 ["is ", "cute", "tea"] ["doris", "aws", "greate"] +23 04-23-10 ["p7", "year4"] ["p7", "year4"] +24 02-24-10-21 [""] [""] + diff --git a/regression-test/data/correctness_p0/test_table_function.out b/regression-test/data/correctness_p0/test_table_function.out new file mode 100644 index 000000000000000..1969d634862791e --- /dev/null +++ b/regression-test/data/correctness_p0/test_table_function.out @@ -0,0 +1,4 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !select -- +3 + diff --git a/regression-test/data/data_model_p0/aggregate/test_aggregate_table.out b/regression-test/data/data_model_p0/aggregate/test_aggregate_table.out index eac6262f1da1742..d147f656433df58 100644 --- a/regression-test/data/data_model_p0/aggregate/test_aggregate_table.out +++ b/regression-test/data/data_model_p0/aggregate/test_aggregate_table.out @@ -37,12 +37,12 @@ datetimev2_value_max DATETIME Yes false \N MAX datetimev2_value_min DATETIME Yes false \N MIN datetimev2_value_replace DATETIME Yes false \N REPLACE datetimev2_value_replace_if_not_null DATETIME Yes false \N REPLACE_IF_NOT_NULL -datetimev2_value_max_1 DATETIME Yes false \N MAX -datetimev2_value_min_1 DATETIME Yes false \N MIN -datetimev2_value_replace_1 DATETIME Yes false \N REPLACE -datetimev2_value_replace_if_not_null_1 DATETIME Yes false \N REPLACE_IF_NOT_NULL -datetimev2_value_max_2 DATETIME Yes false \N MAX -datetimev2_value_min_2 DATETIME Yes false \N MIN -datetimev2_value_replace_2 DATETIME Yes false \N REPLACE -datetimev2_value_replace_if_not_null_2 DATETIME Yes false \N REPLACE_IF_NOT_NULL +datetimev2_value_max_1 DATETIME(3) Yes false \N MAX +datetimev2_value_min_1 DATETIME(3) Yes false \N MIN +datetimev2_value_replace_1 DATETIME(3) Yes false \N REPLACE +datetimev2_value_replace_if_not_null_1 DATETIME(3) Yes false \N REPLACE_IF_NOT_NULL +datetimev2_value_max_2 DATETIME(6) Yes false \N MAX +datetimev2_value_min_2 DATETIME(6) Yes false \N MIN +datetimev2_value_replace_2 DATETIME(6) Yes false \N REPLACE +datetimev2_value_replace_if_not_null_2 DATETIME(6) Yes false \N REPLACE_IF_NOT_NULL diff --git a/regression-test/data/data_model_p0/duplicate/test_duplicate_table.out b/regression-test/data/data_model_p0/duplicate/test_duplicate_table.out index 59973fcdd750265..ca53c5f958dfa66 100644 --- a/regression-test/data/data_model_p0/duplicate/test_duplicate_table.out +++ b/regression-test/data/data_model_p0/duplicate/test_duplicate_table.out @@ -11,8 +11,8 @@ char_value CHAR(10) Yes false \N NONE date_value DATE Yes false \N NONE date_value2 DATE Yes false \N NONE date_value3 DATETIME Yes false \N NONE -date_value4 DATETIME Yes false \N NONE -date_value5 DATETIME Yes false \N NONE +date_value4 DATETIME(3) Yes false \N NONE +date_value5 DATETIME(6) Yes false \N NONE -- !select_dup_table -- 0 1 2 3 diff --git a/regression-test/data/datatype_p0/agg_state/group_concat/test_agg_state_group_concat.out b/regression-test/data/datatype_p0/agg_state/group_concat/test_agg_state_group_concat.out new file mode 100644 index 000000000000000..34376e46bb833bf --- /dev/null +++ b/regression-test/data/datatype_p0/agg_state/group_concat/test_agg_state_group_concat.out @@ -0,0 +1,22 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !length1 -- +1 15 + +-- !group1 -- +1 ccc,bb,a + +-- !merge1 -- +ccc,bb,a + +-- !length2 -- +1 15 + +-- !group2 -- +1 ccc,bb,a + +-- !merge2 -- +ccc,bb,a + +-- !union -- +ccc,bb,a + diff --git a/regression-test/data/datatype_p0/agg_state/max/test_agg_state_max.out b/regression-test/data/datatype_p0/agg_state/max/test_agg_state_max.out new file mode 100644 index 000000000000000..ea6bed327d6b335 --- /dev/null +++ b/regression-test/data/datatype_p0/agg_state/max/test_agg_state_max.out @@ -0,0 +1,55 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !select -- +0 999 +1 1999 +2 2999 +3 3999 +4 4999 +5 5999 +6 6999 +7 7999 + +-- !select -- +7999 + +-- !select -- +0 999 +1 1999 +2 2999 +3 3999 +4 4999 +5 5999 +6 6999 +7 7999 + +-- !select -- +7999 + +-- !select -- +0 999 +1 1999 +2 2999 +3 3999 +4 4999 +5 5999 +6 6999 +7 7999 +100 \N + +-- !select -- +7999 + +-- !select -- +0 999 +1 1999 +2 2999 +3 3999 +4 4999 +5 5999 +6 6999 +7 7999 +100 \N + +-- !select -- +7999 + diff --git a/regression-test/data/delete_p0/test_delete_where_in.out b/regression-test/data/delete_p0/test_delete_where_in.out index b840a2898420046..37855fcdc3b4bea 100644 --- a/regression-test/data/delete_p0/test_delete_where_in.out +++ b/regression-test/data/delete_p0/test_delete_where_in.out @@ -18,4 +18,9 @@ 0 -- !sql -- -0 \ No newline at end of file +0 + +-- !delete_in -- +2 20 2 2 2.0 2000-01-02 +3 30 3 3 3.0 2000-01-03 + diff --git a/regression-test/data/external_table_emr_p2/hive/test_hive_statistic_cache.out b/regression-test/data/external_table_emr_p2/hive/test_hive_statistic_cache.out new file mode 100644 index 000000000000000..dc5ded38375caeb --- /dev/null +++ b/regression-test/data/external_table_emr_p2/hive/test_hive_statistic_cache.out @@ -0,0 +1,58 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !1 -- +100 + +-- !2 -- +lo_orderkey 100.0 26.0 0.0 404.0 4.0 1 98 + +-- !3 -- +lo_linenumber 100.0 6.0 0.0 404.0 4.0 1 7 + +-- !4 -- +lo_custkey 100.0 30.0 0.0 404.0 4.0 67423 2735521 + +-- !5 -- +lo_partkey 100.0 107.0 0.0 404.0 4.0 2250 989601 + +-- !6 -- +lo_suppkey 100.0 90.0 0.0 404.0 4.0 4167 195845 + +-- !7 -- +lo_orderdate 100.0 22.0 0.0 404.0 4.0 19920221 19980721 + +-- !8 -- +lo_orderpriority 100.0 5.0 0.0 888.8000000000001 8.8 N/A N/A + +-- !9 -- +lo_shippriority 100.0 2.0 0.0 404.0 4.0 0 0 + +-- !10 -- +lo_extendedprice 100.0 112.0 0.0 404.0 4.0 104300 9066094 + +-- !11 -- +lo_ordtotalprice 100.0 31.0 0.0 404.0 4.0 3428256 36771805 + +-- !12 -- +lo_discount 100.0 11.0 0.0 404.0 4.0 0 10 + +-- !13 -- +lo_revenue 100.0 127.0 0.0 404.0 4.0 101171 8703450 + +-- !14 -- +lo_supplycost 100.0 112.0 0.0 404.0 4.0 58023 121374 + +-- !15 -- +lo_tax 100.0 9.0 0.0 404.0 4.0 0 8 + +-- !16 -- +lo_commitdate 100.0 122.0 0.0 404.0 4.0 19920515 19981016 + +-- !17 -- +lo_shipmode 100.0 7.0 0.0 425.21 4.21 N/A N/A + +-- !18 -- +lo_orderkey 100.0 26.0 0.0 404.0 4.0 1 98 + +-- !19 -- +lo_quantity 100.0 34.0 0.0 404.0 4.0 1 50 + diff --git a/regression-test/data/external_table_emr_p2/hive/test_hive_to_date.out b/regression-test/data/external_table_emr_p2/hive/test_hive_to_date.out new file mode 100644 index 000000000000000..97df02275602b8d --- /dev/null +++ b/regression-test/data/external_table_emr_p2/hive/test_hive_to_date.out @@ -0,0 +1,25 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !1 -- +1 2000-01-01 + +-- !2 -- +1 2000-01-01 + +-- !3 -- +1 2000-01-01 + +-- !4 -- +1 2000-01-01 + +-- !5 -- +1 2000-01-01 + +-- !6 -- +1 2000-01-01 + +-- !7 -- + +-- !8 -- + +-- !9 -- + diff --git a/regression-test/data/index_p0/test_bitmap_index.out b/regression-test/data/index_p0/test_bitmap_index.out index 37a16688053f65a..1994ffc43f3c74e 100644 --- a/regression-test/data/index_p0/test_bitmap_index.out +++ b/regression-test/data/index_p0/test_bitmap_index.out @@ -13,8 +13,8 @@ k10 DECIMALV3(9, 0) Yes false \N NONE k11 BOOLEAN Yes false \N NONE k12 DATE Yes false \N NONE k13 DATETIME Yes false \N NONE -k14 DATETIME Yes false \N NONE -k15 DATETIME Yes false \N NONE +k14 DATETIME(3) Yes false \N NONE +k15 DATETIME(6) Yes false \N NONE -- !sql -- default_cluster:regression_test_index_p0.test_bitmap_index_dup index1 k1 BITMAP @@ -50,8 +50,8 @@ k10 DECIMALV3(9, 0) Yes true \N k11 BOOLEAN Yes true \N k12 DATE Yes true \N k13 DATETIME Yes true \N -k14 DATETIME Yes true \N -k15 DATETIME Yes true \N +k14 DATETIME(3) Yes true \N +k15 DATETIME(6) Yes true \N v1 INT Yes false \N SUM -- !sql -- @@ -88,8 +88,8 @@ k10 DECIMALV3(9, 0) Yes true \N k11 BOOLEAN Yes true \N k12 DATE Yes false \N REPLACE k13 DATETIME Yes false \N REPLACE -k14 DATETIME Yes false \N REPLACE -k15 DATETIME Yes false \N REPLACE +k14 DATETIME(3) Yes false \N REPLACE +k15 DATETIME(6) Yes false \N REPLACE v1 INT Yes false \N REPLACE -- !sql -- diff --git a/regression-test/data/inverted_index_p0/test_inverted_index.out b/regression-test/data/inverted_index_p0/test_inverted_index.out index ca62006d0852340..82b525127f0b81d 100644 --- a/regression-test/data/inverted_index_p0/test_inverted_index.out +++ b/regression-test/data/inverted_index_p0/test_inverted_index.out @@ -13,8 +13,8 @@ k10 DECIMALV3(9, 0) Yes false \N NONE k11 BOOLEAN Yes false \N NONE k12 DATE Yes false \N NONE k13 DATETIME Yes false \N NONE -k14 DATETIME Yes false \N NONE -k15 DATETIME Yes false \N NONE +k14 DATETIME(3) Yes false \N NONE +k15 DATETIME(6) Yes false \N NONE -- !sql -- default_cluster:regression_test_inverted_index_p0.test_inverted_index_dup index1 k1 INVERTED @@ -50,8 +50,8 @@ k10 DECIMALV3(9, 0) Yes true \N k11 BOOLEAN Yes true \N k12 DATE Yes true \N k13 DATETIME Yes true \N -k14 DATETIME Yes true \N -k15 DATETIME Yes true \N +k14 DATETIME(3) Yes true \N +k15 DATETIME(6) Yes true \N v1 INT Yes false \N SUM -- !sql -- @@ -88,8 +88,8 @@ k10 DECIMALV3(9, 0) Yes true \N k11 BOOLEAN Yes true \N k12 DATE Yes false \N NONE k13 DATETIME Yes false \N NONE -k14 DATETIME Yes false \N NONE -k15 DATETIME Yes false \N NONE +k14 DATETIME(3) Yes false \N NONE +k15 DATETIME(6) Yes false \N NONE v1 INT Yes false \N NONE -- !sql -- diff --git a/regression-test/data/jdbc_catalog_p0/test_mysql_jdbc_catalog.out b/regression-test/data/jdbc_catalog_p0/test_mysql_jdbc_catalog.out index e1a5431b00ae486..458b5fa14a7c18c 100644 --- a/regression-test/data/jdbc_catalog_p0/test_mysql_jdbc_catalog.out +++ b/regression-test/data/jdbc_catalog_p0/test_mysql_jdbc_catalog.out @@ -252,6 +252,10 @@ VIEW_TABLE_USAGE -- !dt -- 2023-06-17T10:00 2023-06-17T10:00:01.100 2023-06-17T10:00:02.220 2023-06-17T10:00:03.333 2023-06-17T10:00:04.444400 2023-06-17T10:00:05.555550 2023-06-17T10:00:06.666666 +-- !dt_null -- +\N +2023-06-17T10:00 + -- !test_insert1 -- doris1 18 diff --git a/regression-test/data/jdbc_catalog_p0/test_oracle_jdbc_catalog.out b/regression-test/data/jdbc_catalog_p0/test_oracle_jdbc_catalog.out index 32b7070f1714989..df7d46628a56083 100644 --- a/regression-test/data/jdbc_catalog_p0/test_oracle_jdbc_catalog.out +++ b/regression-test/data/jdbc_catalog_p0/test_oracle_jdbc_catalog.out @@ -32,14 +32,15 @@ 5 \N \N 12 10:23:1.123457 -- !test6 -- -1 2013-01-21T05:23:01 \N \N \N \N \N -2 2013-11-12T20:32:56 \N \N \N \N \N -3 \N 2019-11-12T20:33:57.999998 \N \N \N \N -4 \N \N 2019-11-12T20:33:57.999996 \N \N \N -5 \N \N \N 2019-11-12T20:33:57.999997 \N \N -6 \N \N \N \N 11-0 \N -7 \N \N \N \N 223-9 \N -8 \N \N \N \N \N 12 10:23:1.123457 +1 2013-01-21T05:23:01 \N \N \N \N \N \N +2 2013-11-12T20:32:56 \N \N \N \N \N \N +3 \N 2019-11-12T20:33:57.999 \N \N \N \N \N +4 \N \N 2019-11-12T20:33:57.999998 \N \N \N \N +5 \N \N \N 2019-11-12T20:33:57.999996 \N \N \N +6 \N \N \N \N 2019-11-12T20:33:57.999997 \N \N +7 \N \N \N \N \N 11-0 \N +8 \N \N \N \N \N 223-9 \N +9 \N \N \N \N \N \N 12 10:23:1.123457 -- !test7 -- 1 123.45 12300 0.0012345 diff --git a/regression-test/data/json_p0/test_json_load_and_function.out b/regression-test/data/json_p0/test_json_load_and_function.out index 51d4485d6a8b657..21c25c4ff94e7cf 100644 --- a/regression-test/data/json_p0/test_json_load_and_function.out +++ b/regression-test/data/json_p0/test_json_load_and_function.out @@ -191,17 +191,17 @@ 8 1152921504606846976 \N 9 6.18 \N 10 "abcd" \N -11 {} \N -12 {"k1":"v31","k2":300} \N +11 {} {} +12 {"k1":"v31","k2":300} {"k1":"v31","k2":300} 13 [] \N 14 [123,456] 123 15 ["abc","def"] "abc" 16 [null,true,false,100,6.18,"abc"] null 17 [{"k1":"v41","k2":400},1,"a",3.14] {"k1":"v41","k2":400} -18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} \N +18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} 26 \N \N -27 {"k1":"v1","k2":200} \N -28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} \N +27 {"k1":"v1","k2":200} {"k1":"v1","k2":200} +28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} -- !select -- 1 \N \N @@ -697,17 +697,17 @@ 8 1152921504606846976 \N 9 6.18 \N 10 "abcd" \N -11 {} \N -12 {"k1":"v31","k2":300} \N +11 {} {} +12 {"k1":"v31","k2":300} {"k1":"v31","k2":300} 13 [] \N 14 [123,456] 123 15 ["abc","def"] abc 16 [null,true,false,100,6.18,"abc"] null 17 [{"k1":"v41","k2":400},1,"a",3.14] {"k1":"v41","k2":400} -18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} \N +18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} 26 \N \N -27 {"k1":"v1","k2":200} \N -28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} \N +27 {"k1":"v1","k2":200} {"k1":"v1","k2":200} +28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} -- !select -- 1 \N \N @@ -2767,17 +2767,17 @@ 8 1152921504606846976 \N 9 6.18 \N 10 "abcd" \N -11 {} \N -12 {"k1":"v31","k2":300} \N +11 {} false +12 {"k1":"v31","k2":300} false 13 [] \N 14 [123,456] false 15 ["abc","def"] false 16 [null,true,false,100,6.18,"abc"] true 17 [{"k1":"v41","k2":400},1,"a",3.14] false -18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} \N +18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} false 26 \N \N -27 {"k1":"v1","k2":200} \N -28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} \N +27 {"k1":"v1","k2":200} false +28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} false -- !select -- 1 \N \N @@ -3181,17 +3181,17 @@ 8 1152921504606846976 false 9 6.18 false 10 "abcd" false -11 {} false -12 {"k1":"v31","k2":300} false +11 {} true +12 {"k1":"v31","k2":300} true 13 [] false 14 [123,456] true 15 ["abc","def"] true 16 [null,true,false,100,6.18,"abc"] true 17 [{"k1":"v41","k2":400},1,"a",3.14] true -18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} false +18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} true 26 \N \N -27 {"k1":"v1","k2":200} false -28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} false +27 {"k1":"v1","k2":200} true +28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} true -- !select -- 1 \N \N @@ -3595,17 +3595,17 @@ 8 1152921504606846976 \N 9 6.18 \N 10 "abcd" \N -11 {} \N -12 {"k1":"v31","k2":300} \N +11 {} object +12 {"k1":"v31","k2":300} object 13 [] \N 14 [123,456] int 15 ["abc","def"] string 16 [null,true,false,100,6.18,"abc"] null 17 [{"k1":"v41","k2":400},1,"a",3.14] object -18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} \N +18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} object 26 \N \N -27 {"k1":"v1","k2":200} \N -28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} \N +27 {"k1":"v1","k2":200} object +28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} object -- !select -- 1 \N \N diff --git a/regression-test/data/json_p0/test_json_load_unique_key_and_function.out b/regression-test/data/json_p0/test_json_load_unique_key_and_function.out index 01bc0b3de78e95d..f63959c2e02aa38 100644 --- a/regression-test/data/json_p0/test_json_load_unique_key_and_function.out +++ b/regression-test/data/json_p0/test_json_load_unique_key_and_function.out @@ -191,17 +191,17 @@ 8 1152921504606846976 \N 9 6.18 \N 10 "abcd" \N -11 {} \N -12 {"k1":"v31","k2":300} \N +11 {} {} +12 {"k1":"v31","k2":300} {"k1":"v31","k2":300} 13 [] \N 14 [123,456] 123 15 ["abc","def"] "abc" 16 [null,true,false,100,6.18,"abc"] null 17 [{"k1":"v41","k2":400},1,"a",3.14] {"k1":"v41","k2":400} -18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} \N +18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} 26 \N \N -27 {"k1":"v1","k2":200} \N -28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} \N +27 {"k1":"v1","k2":200} {"k1":"v1","k2":200} +28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} -- !select -- 1 \N \N @@ -697,17 +697,17 @@ 8 1152921504606846976 \N 9 6.18 \N 10 "abcd" \N -11 {} \N -12 {"k1":"v31","k2":300} \N +11 {} {} +12 {"k1":"v31","k2":300} {"k1":"v31","k2":300} 13 [] \N 14 [123,456] 123 15 ["abc","def"] abc 16 [null,true,false,100,6.18,"abc"] null 17 [{"k1":"v41","k2":400},1,"a",3.14] {"k1":"v41","k2":400} -18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} \N +18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} 26 \N \N -27 {"k1":"v1","k2":200} \N -28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} \N +27 {"k1":"v1","k2":200} {"k1":"v1","k2":200} +28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} -- !select -- 1 \N \N @@ -2767,17 +2767,17 @@ 8 1152921504606846976 \N 9 6.18 \N 10 "abcd" \N -11 {} \N -12 {"k1":"v31","k2":300} \N +11 {} false +12 {"k1":"v31","k2":300} false 13 [] \N 14 [123,456] false 15 ["abc","def"] false 16 [null,true,false,100,6.18,"abc"] true 17 [{"k1":"v41","k2":400},1,"a",3.14] false -18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} \N +18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} false 26 \N \N -27 {"k1":"v1","k2":200} \N -28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} \N +27 {"k1":"v1","k2":200} false +28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} false -- !select -- 1 \N \N @@ -3181,17 +3181,17 @@ 8 1152921504606846976 false 9 6.18 false 10 "abcd" false -11 {} false -12 {"k1":"v31","k2":300} false +11 {} true +12 {"k1":"v31","k2":300} true 13 [] false 14 [123,456] true 15 ["abc","def"] true 16 [null,true,false,100,6.18,"abc"] true 17 [{"k1":"v41","k2":400},1,"a",3.14] true -18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} false +18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} true 26 \N \N -27 {"k1":"v1","k2":200} false -28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} false +27 {"k1":"v1","k2":200} true +28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} true -- !select -- 1 \N \N @@ -3595,17 +3595,17 @@ 8 1152921504606846976 \N 9 6.18 \N 10 "abcd" \N -11 {} \N -12 {"k1":"v31","k2":300} \N +11 {} object +12 {"k1":"v31","k2":300} object 13 [] \N 14 [123,456] int 15 ["abc","def"] string 16 [null,true,false,100,6.18,"abc"] null 17 [{"k1":"v41","k2":400},1,"a",3.14] object -18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} \N +18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} object 26 \N \N -27 {"k1":"v1","k2":200} \N -28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} \N +27 {"k1":"v1","k2":200} object +28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} object -- !select -- 1 \N \N diff --git a/regression-test/data/jsonb_p0/test_jsonb_load_and_function.out b/regression-test/data/jsonb_p0/test_jsonb_load_and_function.out index cd10f5cb1da0c7d..00944828bdc83f3 100644 --- a/regression-test/data/jsonb_p0/test_jsonb_load_and_function.out +++ b/regression-test/data/jsonb_p0/test_jsonb_load_and_function.out @@ -42,7 +42,7 @@ 27 {"k1":"v1","k2":200} 28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} --- !select -- +-- !jsonb_extract_select -- 1 \N \N 2 null null 3 true true @@ -191,17 +191,17 @@ 8 1152921504606846976 \N 9 6.18 \N 10 "abcd" \N -11 {} \N -12 {"k1":"v31","k2":300} \N +11 {} {} +12 {"k1":"v31","k2":300} {"k1":"v31","k2":300} 13 [] \N 14 [123,456] 123 15 ["abc","def"] "abc" 16 [null,true,false,100,6.18,"abc"] null 17 [{"k1":"v41","k2":400},1,"a",3.14] {"k1":"v41","k2":400} -18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} \N +18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} 26 \N \N -27 {"k1":"v1","k2":200} \N -28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} \N +27 {"k1":"v1","k2":200} {"k1":"v1","k2":200} +28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} -- !select -- 1 \N \N @@ -548,6 +548,29 @@ 27 {"k1":"v1","k2":200} \N 28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} \N +-- !select -- +1 \N \N +2 null \N +3 true \N +4 false \N +5 100 \N +6 10000 \N +7 1000000000 \N +8 1152921504606846976 \N +9 6.18 \N +10 "abcd" \N +11 {} \N +12 {"k1":"v31","k2":300} \N +13 [] \N +14 [123,456] \N +15 ["abc","def"] \N +16 [null,true,false,100,6.18,"abc"] \N +17 [{"k1":"v41","k2":400},1,"a",3.14] \N +18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} 3.14 +26 \N \N +27 {"k1":"v1","k2":200} \N +28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} \N + -- !select -- 1 \N \N 2 null \N @@ -619,6 +642,75 @@ -- !select -- 1 \N \N +2 null \N +3 true \N +4 false \N +5 100 \N +6 10000 \N +7 1000000000 \N +8 1152921504606846976 \N +9 6.18 \N +10 "abcd" \N +11 {} \N +12 {"k1":"v31","k2":300} \N +13 [] \N +14 [123,456] \N +15 ["abc","def"] \N +16 [null,true,false,100,6.18,"abc"] \N +17 [{"k1":"v41","k2":400},1,"a",3.14] \N +18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} {"k1":"v41","k2":400} +26 \N \N +27 {"k1":"v1","k2":200} \N +28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} \N + +-- !select -- +1 \N \N +2 null \N +3 true \N +4 false \N +5 100 \N +6 10000 \N +7 1000000000 \N +8 1152921504606846976 \N +9 6.18 \N +10 "abcd" \N +11 {} \N +12 {"k1":"v31","k2":300} \N +13 [] \N +14 [123,456] \N +15 ["abc","def"] \N +16 [null,true,false,100,6.18,"abc"] \N +17 [{"k1":"v41","k2":400},1,"a",3.14] \N +18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} 3.14 +26 \N \N +27 {"k1":"v1","k2":200} \N +28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} \N + +-- !select -- +1 \N \N +2 null \N +3 true \N +4 false \N +5 100 \N +6 10000 \N +7 1000000000 \N +8 1152921504606846976 \N +9 6.18 \N +10 "abcd" \N +11 {} \N +12 {"k1":"v31","k2":300} \N +13 [] \N +14 [123,456] \N +15 ["abc","def"] \N +16 [null,true,false,100,6.18,"abc"] \N +17 [{"k1":"v41","k2":400},1,"a",3.14] \N +18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} \N +26 \N \N +27 {"k1":"v1","k2":200} \N +28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} \N + +-- !jsonb_extract_string_select -- +1 \N \N 2 null null 3 true true 4 false false @@ -697,17 +789,17 @@ 8 1152921504606846976 \N 9 6.18 \N 10 "abcd" \N -11 {} \N -12 {"k1":"v31","k2":300} \N +11 {} {} +12 {"k1":"v31","k2":300} {"k1":"v31","k2":300} 13 [] \N 14 [123,456] 123 15 ["abc","def"] abc 16 [null,true,false,100,6.18,"abc"] null 17 [{"k1":"v41","k2":400},1,"a",3.14] {"k1":"v41","k2":400} -18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} \N +18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} 26 \N \N -27 {"k1":"v1","k2":200} \N -28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} \N +27 {"k1":"v1","k2":200} {"k1":"v1","k2":200} +28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} -- !select -- 1 \N \N @@ -1036,9 +1128,9 @@ 2 null \N 3 true \N 4 false \N -5 100 100 -6 10000 10000 -7 1000000000 1000000000 +5 100 \N +6 10000 \N +7 1000000000 \N 8 1152921504606846976 \N 9 6.18 \N 10 "abcd" \N @@ -1049,7 +1141,7 @@ 15 ["abc","def"] \N 16 [null,true,false,100,6.18,"abc"] \N 17 [{"k1":"v41","k2":400},1,"a",3.14] \N -18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} \N +18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} 3.14 26 \N \N 27 {"k1":"v1","k2":200} \N 28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} \N @@ -1072,7 +1164,7 @@ 15 ["abc","def"] \N 16 [null,true,false,100,6.18,"abc"] \N 17 [{"k1":"v41","k2":400},1,"a",3.14] \N -18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} \N +18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} 3.14 26 \N \N 27 {"k1":"v1","k2":200} \N 28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} \N @@ -1089,15 +1181,15 @@ 9 6.18 \N 10 "abcd" \N 11 {} \N -12 {"k1":"v31","k2":300} 300 +12 {"k1":"v31","k2":300} \N 13 [] \N 14 [123,456] \N 15 ["abc","def"] \N 16 [null,true,false,100,6.18,"abc"] \N 17 [{"k1":"v41","k2":400},1,"a",3.14] \N -18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} 300 +18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} a 26 \N \N -27 {"k1":"v1","k2":200} 200 +27 {"k1":"v1","k2":200} \N 28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} \N -- !select -- @@ -1114,11 +1206,11 @@ 11 {} \N 12 {"k1":"v31","k2":300} \N 13 [] \N -14 [123,456] 123 +14 [123,456] \N 15 ["abc","def"] \N 16 [null,true,false,100,6.18,"abc"] \N 17 [{"k1":"v41","k2":400},1,"a",3.14] \N -18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} \N +18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} 1 26 \N \N 27 {"k1":"v1","k2":200} \N 28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} \N @@ -1137,10 +1229,10 @@ 11 {} \N 12 {"k1":"v31","k2":300} \N 13 [] \N -14 [123,456] 456 +14 [123,456] \N 15 ["abc","def"] \N 16 [null,true,false,100,6.18,"abc"] \N -17 [{"k1":"v41","k2":400},1,"a",3.14] 1 +17 [{"k1":"v41","k2":400},1,"a",3.14] \N 18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} \N 26 \N \N 27 {"k1":"v1","k2":200} \N @@ -1164,7 +1256,7 @@ 15 ["abc","def"] \N 16 [null,true,false,100,6.18,"abc"] \N 17 [{"k1":"v41","k2":400},1,"a",3.14] \N -18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} \N +18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} {"k1":"v41","k2":400} 26 \N \N 27 {"k1":"v1","k2":200} \N 28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} \N @@ -1185,9 +1277,9 @@ 13 [] \N 14 [123,456] \N 15 ["abc","def"] \N -16 [null,true,false,100,6.18,"abc"] 100 +16 [null,true,false,100,6.18,"abc"] \N 17 [{"k1":"v41","k2":400},1,"a",3.14] \N -18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} \N +18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} 3.14 26 \N \N 27 {"k1":"v1","k2":200} \N 28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} \N @@ -1215,14 +1307,14 @@ 27 {"k1":"v1","k2":200} \N 28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} \N --- !select -- +-- !jsonb_extract_int_select -- 1 \N \N 2 null \N 3 true \N 4 false \N -5 100 \N -6 10000 \N -7 1000000000 \N +5 100 100 +6 10000 10000 +7 1000000000 1000000000 8 1152921504606846976 \N 9 6.18 \N 10 "abcd" \N @@ -1273,15 +1365,15 @@ 9 6.18 \N 10 "abcd" \N 11 {} \N -12 {"k1":"v31","k2":300} \N +12 {"k1":"v31","k2":300} 300 13 [] \N 14 [123,456] \N 15 ["abc","def"] \N 16 [null,true,false,100,6.18,"abc"] \N 17 [{"k1":"v41","k2":400},1,"a",3.14] \N -18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} \N +18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} 300 26 \N \N -27 {"k1":"v1","k2":200} \N +27 {"k1":"v1","k2":200} 200 28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} \N -- !select -- @@ -1298,7 +1390,7 @@ 11 {} \N 12 {"k1":"v31","k2":300} \N 13 [] \N -14 [123,456] \N +14 [123,456] 123 15 ["abc","def"] \N 16 [null,true,false,100,6.18,"abc"] \N 17 [{"k1":"v41","k2":400},1,"a",3.14] \N @@ -1321,10 +1413,10 @@ 11 {} \N 12 {"k1":"v31","k2":300} \N 13 [] \N -14 [123,456] \N +14 [123,456] 456 15 ["abc","def"] \N 16 [null,true,false,100,6.18,"abc"] \N -17 [{"k1":"v41","k2":400},1,"a",3.14] \N +17 [{"k1":"v41","k2":400},1,"a",3.14] 1 18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} \N 26 \N \N 27 {"k1":"v1","k2":200} \N @@ -1348,7 +1440,7 @@ 15 ["abc","def"] \N 16 [null,true,false,100,6.18,"abc"] \N 17 [{"k1":"v41","k2":400},1,"a",3.14] \N -18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} 1 +18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} \N 26 \N \N 27 {"k1":"v1","k2":200} \N 28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} \N @@ -1369,7 +1461,7 @@ 13 [] \N 14 [123,456] \N 15 ["abc","def"] \N -16 [null,true,false,100,6.18,"abc"] \N +16 [null,true,false,100,6.18,"abc"] 100 17 [{"k1":"v41","k2":400},1,"a",3.14] \N 18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} \N 26 \N \N @@ -1450,10 +1542,10 @@ 2 null \N 3 true \N 4 false \N -5 100 100 -6 10000 10000 -7 1000000000 1000000000 -8 1152921504606846976 1152921504606846976 +5 100 \N +6 10000 \N +7 1000000000 \N +8 1152921504606846976 \N 9 6.18 \N 10 "abcd" \N 11 {} \N @@ -1503,15 +1595,15 @@ 9 6.18 \N 10 "abcd" \N 11 {} \N -12 {"k1":"v31","k2":300} 300 +12 {"k1":"v31","k2":300} \N 13 [] \N 14 [123,456] \N 15 ["abc","def"] \N 16 [null,true,false,100,6.18,"abc"] \N 17 [{"k1":"v41","k2":400},1,"a",3.14] \N -18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} 300 +18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} \N 26 \N \N -27 {"k1":"v1","k2":200} 200 +27 {"k1":"v1","k2":200} \N 28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} \N -- !select -- @@ -1528,11 +1620,11 @@ 11 {} \N 12 {"k1":"v31","k2":300} \N 13 [] \N -14 [123,456] 123 +14 [123,456] \N 15 ["abc","def"] \N 16 [null,true,false,100,6.18,"abc"] \N 17 [{"k1":"v41","k2":400},1,"a",3.14] \N -18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} \N +18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} 1 26 \N \N 27 {"k1":"v1","k2":200} \N 28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} \N @@ -1551,10 +1643,10 @@ 11 {} \N 12 {"k1":"v31","k2":300} \N 13 [] \N -14 [123,456] 456 +14 [123,456] \N 15 ["abc","def"] \N 16 [null,true,false,100,6.18,"abc"] \N -17 [{"k1":"v41","k2":400},1,"a",3.14] 1 +17 [{"k1":"v41","k2":400},1,"a",3.14] \N 18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} \N 26 \N \N 27 {"k1":"v1","k2":200} \N @@ -1599,7 +1691,7 @@ 13 [] \N 14 [123,456] \N 15 ["abc","def"] \N -16 [null,true,false,100,6.18,"abc"] 100 +16 [null,true,false,100,6.18,"abc"] \N 17 [{"k1":"v41","k2":400},1,"a",3.14] \N 18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} \N 26 \N \N @@ -1716,7 +1808,7 @@ 15 ["abc","def"] \N 16 [null,true,false,100,6.18,"abc"] \N 17 [{"k1":"v41","k2":400},1,"a",3.14] \N -18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} \N +18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} 1 26 \N \N 27 {"k1":"v1","k2":200} \N 28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} \N @@ -1762,7 +1854,7 @@ 15 ["abc","def"] \N 16 [null,true,false,100,6.18,"abc"] \N 17 [{"k1":"v41","k2":400},1,"a",3.14] \N -18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} 1 +18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} \N 26 \N \N 27 {"k1":"v1","k2":200} \N 28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} \N @@ -1813,15 +1905,15 @@ 27 {"k1":"v1","k2":200} \N 28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} \N --- !select -- +-- !jsonb_extract_bigint_select -- 1 \N \N 2 null \N 3 true \N 4 false \N -5 100 \N -6 10000 \N -7 1000000000 \N -8 1152921504606846976 \N +5 100 100 +6 10000 10000 +7 1000000000 1000000000 +8 1152921504606846976 1152921504606846976 9 6.18 \N 10 "abcd" \N 11 {} \N @@ -1864,22 +1956,22 @@ 2 null \N 3 true \N 4 false \N -5 100 100.0 -6 10000 10000.0 -7 1000000000 1.0E9 -8 1152921504606846976 1.15292150460684698E18 -9 6.18 6.18 +5 100 \N +6 10000 \N +7 1000000000 \N +8 1152921504606846976 \N +9 6.18 \N 10 "abcd" \N 11 {} \N -12 {"k1":"v31","k2":300} \N +12 {"k1":"v31","k2":300} 300 13 [] \N 14 [123,456] \N 15 ["abc","def"] \N 16 [null,true,false,100,6.18,"abc"] \N 17 [{"k1":"v41","k2":400},1,"a",3.14] \N -18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} \N +18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} 300 26 \N \N -27 {"k1":"v1","k2":200} \N +27 {"k1":"v1","k2":200} 200 28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} \N -- !select -- @@ -1896,7 +1988,7 @@ 11 {} \N 12 {"k1":"v31","k2":300} \N 13 [] \N -14 [123,456] \N +14 [123,456] 123 15 ["abc","def"] \N 16 [null,true,false,100,6.18,"abc"] \N 17 [{"k1":"v41","k2":400},1,"a",3.14] \N @@ -1917,15 +2009,15 @@ 9 6.18 \N 10 "abcd" \N 11 {} \N -12 {"k1":"v31","k2":300} 300.0 +12 {"k1":"v31","k2":300} \N 13 [] \N -14 [123,456] \N +14 [123,456] 456 15 ["abc","def"] \N 16 [null,true,false,100,6.18,"abc"] \N -17 [{"k1":"v41","k2":400},1,"a",3.14] \N -18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} 300.0 +17 [{"k1":"v41","k2":400},1,"a",3.14] 1 +18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} \N 26 \N \N -27 {"k1":"v1","k2":200} 200.0 +27 {"k1":"v1","k2":200} \N 28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} \N -- !select -- @@ -1942,7 +2034,7 @@ 11 {} \N 12 {"k1":"v31","k2":300} \N 13 [] \N -14 [123,456] 123.0 +14 [123,456] \N 15 ["abc","def"] \N 16 [null,true,false,100,6.18,"abc"] \N 17 [{"k1":"v41","k2":400},1,"a",3.14] \N @@ -1965,10 +2057,10 @@ 11 {} \N 12 {"k1":"v31","k2":300} \N 13 [] \N -14 [123,456] 456.0 +14 [123,456] \N 15 ["abc","def"] \N -16 [null,true,false,100,6.18,"abc"] \N -17 [{"k1":"v41","k2":400},1,"a",3.14] 1.0 +16 [null,true,false,100,6.18,"abc"] 100 +17 [{"k1":"v41","k2":400},1,"a",3.14] \N 18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} \N 26 \N \N 27 {"k1":"v1","k2":200} \N @@ -2013,8 +2105,8 @@ 13 [] \N 14 [123,456] \N 15 ["abc","def"] \N -16 [null,true,false,100,6.18,"abc"] 100.0 -17 [{"k1":"v41","k2":400},1,"a",3.14] 3.14 +16 [null,true,false,100,6.18,"abc"] \N +17 [{"k1":"v41","k2":400},1,"a",3.14] \N 18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} \N 26 \N \N 27 {"k1":"v1","k2":200} \N @@ -2036,7 +2128,7 @@ 13 [] \N 14 [123,456] \N 15 ["abc","def"] \N -16 [null,true,false,100,6.18,"abc"] 6.18 +16 [null,true,false,100,6.18,"abc"] \N 17 [{"k1":"v41","k2":400},1,"a",3.14] \N 18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} \N 26 \N \N @@ -2130,7 +2222,7 @@ 15 ["abc","def"] \N 16 [null,true,false,100,6.18,"abc"] \N 17 [{"k1":"v41","k2":400},1,"a",3.14] \N -18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} \N +18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} 1 26 \N \N 27 {"k1":"v1","k2":200} \N 28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} \N @@ -2176,7 +2268,7 @@ 15 ["abc","def"] \N 16 [null,true,false,100,6.18,"abc"] \N 17 [{"k1":"v41","k2":400},1,"a",3.14] \N -18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} 1.0 +18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} \N 26 \N \N 27 {"k1":"v1","k2":200} \N 28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} \N @@ -2222,7 +2314,7 @@ 15 ["abc","def"] \N 16 [null,true,false,100,6.18,"abc"] \N 17 [{"k1":"v41","k2":400},1,"a",3.14] \N -18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} 3.14 +18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} \N 26 \N \N 27 {"k1":"v1","k2":200} \N 28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} \N @@ -2276,8 +2368,8 @@ -- !select -- 1 \N \N 2 null \N -3 true true -4 false false +3 true \N +4 false \N 5 100 \N 6 10000 \N 7 1000000000 \N @@ -2314,7 +2406,7 @@ 15 ["abc","def"] \N 16 [null,true,false,100,6.18,"abc"] \N 17 [{"k1":"v41","k2":400},1,"a",3.14] \N -18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} \N +18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} 1 26 \N \N 27 {"k1":"v1","k2":200} \N 28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} \N @@ -2381,7 +2473,7 @@ 13 [] \N 14 [123,456] \N 15 ["abc","def"] \N -16 [null,true,false,100,6.18,"abc"] true +16 [null,true,false,100,6.18,"abc"] \N 17 [{"k1":"v41","k2":400},1,"a",3.14] \N 18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} \N 26 \N \N @@ -2404,7 +2496,30 @@ 13 [] \N 14 [123,456] \N 15 ["abc","def"] \N -16 [null,true,false,100,6.18,"abc"] false +16 [null,true,false,100,6.18,"abc"] \N +17 [{"k1":"v41","k2":400},1,"a",3.14] \N +18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} \N +26 \N \N +27 {"k1":"v1","k2":200} \N +28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} \N + +-- !jsonb_extract_double_select -- +1 \N \N +2 null \N +3 true \N +4 false \N +5 100 100.0 +6 10000 10000.0 +7 1000000000 1.0E9 +8 1152921504606846976 1.15292150460684698E18 +9 6.18 6.18 +10 "abcd" \N +11 {} \N +12 {"k1":"v31","k2":300} \N +13 [] \N +14 [123,456] \N +15 ["abc","def"] \N +16 [null,true,false,100,6.18,"abc"] \N 17 [{"k1":"v41","k2":400},1,"a",3.14] \N 18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} \N 26 \N \N @@ -2446,12 +2561,58 @@ 9 6.18 \N 10 "abcd" \N 11 {} \N -12 {"k1":"v31","k2":300} \N +12 {"k1":"v31","k2":300} 300.0 13 [] \N 14 [123,456] \N 15 ["abc","def"] \N 16 [null,true,false,100,6.18,"abc"] \N 17 [{"k1":"v41","k2":400},1,"a",3.14] \N +18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} 300.0 +26 \N \N +27 {"k1":"v1","k2":200} 200.0 +28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} \N + +-- !select -- +1 \N \N +2 null \N +3 true \N +4 false \N +5 100 \N +6 10000 \N +7 1000000000 \N +8 1152921504606846976 \N +9 6.18 \N +10 "abcd" \N +11 {} \N +12 {"k1":"v31","k2":300} \N +13 [] \N +14 [123,456] 123.0 +15 ["abc","def"] \N +16 [null,true,false,100,6.18,"abc"] \N +17 [{"k1":"v41","k2":400},1,"a",3.14] \N +18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} \N +26 \N \N +27 {"k1":"v1","k2":200} \N +28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} \N + +-- !select -- +1 \N \N +2 null \N +3 true \N +4 false \N +5 100 \N +6 10000 \N +7 1000000000 \N +8 1152921504606846976 \N +9 6.18 \N +10 "abcd" \N +11 {} \N +12 {"k1":"v31","k2":300} \N +13 [] \N +14 [123,456] 456.0 +15 ["abc","def"] \N +16 [null,true,false,100,6.18,"abc"] \N +17 [{"k1":"v41","k2":400},1,"a",3.14] 1.0 18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} \N 26 \N \N 27 {"k1":"v1","k2":200} \N @@ -2496,7 +2657,30 @@ 13 [] \N 14 [123,456] \N 15 ["abc","def"] \N -16 [null,true,false,100,6.18,"abc"] \N +16 [null,true,false,100,6.18,"abc"] 100.0 +17 [{"k1":"v41","k2":400},1,"a",3.14] 3.14 +18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} \N +26 \N \N +27 {"k1":"v1","k2":200} \N +28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} \N + +-- !select -- +1 \N \N +2 null \N +3 true \N +4 false \N +5 100 \N +6 10000 \N +7 1000000000 \N +8 1152921504606846976 \N +9 6.18 \N +10 "abcd" \N +11 {} \N +12 {"k1":"v31","k2":300} \N +13 [] \N +14 [123,456] \N +15 ["abc","def"] \N +16 [null,true,false,100,6.18,"abc"] 6.18 17 [{"k1":"v41","k2":400},1,"a",3.14] \N 18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} \N 26 \N \N @@ -2636,7 +2820,7 @@ 15 ["abc","def"] \N 16 [null,true,false,100,6.18,"abc"] \N 17 [{"k1":"v41","k2":400},1,"a",3.14] \N -18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} \N +18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} 1.0 26 \N \N 27 {"k1":"v1","k2":200} \N 28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} \N @@ -2664,6 +2848,29 @@ 27 {"k1":"v1","k2":200} \N 28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} \N +-- !select -- +1 \N \N +2 null \N +3 true \N +4 false \N +5 100 \N +6 10000 \N +7 1000000000 \N +8 1152921504606846976 \N +9 6.18 \N +10 "abcd" \N +11 {} \N +12 {"k1":"v31","k2":300} \N +13 [] \N +14 [123,456] \N +15 ["abc","def"] \N +16 [null,true,false,100,6.18,"abc"] \N +17 [{"k1":"v41","k2":400},1,"a",3.14] \N +18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} 3.14 +26 \N \N +27 {"k1":"v1","k2":200} \N +28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} \N + -- !select -- 1 \N \N 2 null \N @@ -2689,26 +2896,26 @@ -- !select -- 1 \N \N -2 null true -3 true false -4 false false -5 100 false -6 10000 false -7 1000000000 false -8 1152921504606846976 false -9 6.18 false -10 "abcd" false -11 {} false -12 {"k1":"v31","k2":300} false -13 [] false -14 [123,456] false -15 ["abc","def"] false -16 [null,true,false,100,6.18,"abc"] false -17 [{"k1":"v41","k2":400},1,"a",3.14] false -18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} false +2 null \N +3 true \N +4 false \N +5 100 \N +6 10000 \N +7 1000000000 \N +8 1152921504606846976 \N +9 6.18 \N +10 "abcd" \N +11 {} \N +12 {"k1":"v31","k2":300} \N +13 [] \N +14 [123,456] \N +15 ["abc","def"] \N +16 [null,true,false,100,6.18,"abc"] \N +17 [{"k1":"v41","k2":400},1,"a",3.14] \N +18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} \N 26 \N \N -27 {"k1":"v1","k2":200} false -28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} false +27 {"k1":"v1","k2":200} \N +28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} \N -- !select -- 1 \N \N @@ -2722,15 +2929,15 @@ 9 6.18 \N 10 "abcd" \N 11 {} \N -12 {"k1":"v31","k2":300} false +12 {"k1":"v31","k2":300} \N 13 [] \N 14 [123,456] \N 15 ["abc","def"] \N 16 [null,true,false,100,6.18,"abc"] \N 17 [{"k1":"v41","k2":400},1,"a",3.14] \N -18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} false +18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} 3.14 26 \N \N -27 {"k1":"v1","k2":200} false +27 {"k1":"v1","k2":200} \N 28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} \N -- !select -- @@ -2745,15 +2952,15 @@ 9 6.18 \N 10 "abcd" \N 11 {} \N -12 {"k1":"v31","k2":300} false +12 {"k1":"v31","k2":300} \N 13 [] \N 14 [123,456] \N 15 ["abc","def"] \N 16 [null,true,false,100,6.18,"abc"] \N 17 [{"k1":"v41","k2":400},1,"a",3.14] \N -18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} false +18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} 3.14 26 \N \N -27 {"k1":"v1","k2":200} false +27 {"k1":"v1","k2":200} \N 28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} \N -- !select -- @@ -2770,10 +2977,10 @@ 11 {} \N 12 {"k1":"v31","k2":300} \N 13 [] \N -14 [123,456] false -15 ["abc","def"] false -16 [null,true,false,100,6.18,"abc"] true -17 [{"k1":"v41","k2":400},1,"a",3.14] false +14 [123,456] \N +15 ["abc","def"] \N +16 [null,true,false,100,6.18,"abc"] \N +17 [{"k1":"v41","k2":400},1,"a",3.14] \N 18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} \N 26 \N \N 27 {"k1":"v1","k2":200} \N @@ -2793,11 +3000,11 @@ 11 {} \N 12 {"k1":"v31","k2":300} \N 13 [] \N -14 [123,456] false -15 ["abc","def"] false -16 [null,true,false,100,6.18,"abc"] false -17 [{"k1":"v41","k2":400},1,"a",3.14] false -18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} \N +14 [123,456] \N +15 ["abc","def"] \N +16 [null,true,false,100,6.18,"abc"] \N +17 [{"k1":"v41","k2":400},1,"a",3.14] \N +18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} 1.0 26 \N \N 27 {"k1":"v1","k2":200} \N 28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} \N @@ -2818,8 +3025,8 @@ 13 [] \N 14 [123,456] \N 15 ["abc","def"] \N -16 [null,true,false,100,6.18,"abc"] false -17 [{"k1":"v41","k2":400},1,"a",3.14] false +16 [null,true,false,100,6.18,"abc"] \N +17 [{"k1":"v41","k2":400},1,"a",3.14] \N 18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} \N 26 \N \N 27 {"k1":"v1","k2":200} \N @@ -2841,8 +3048,8 @@ 13 [] \N 14 [123,456] \N 15 ["abc","def"] \N -16 [null,true,false,100,6.18,"abc"] false -17 [{"k1":"v41","k2":400},1,"a",3.14] false +16 [null,true,false,100,6.18,"abc"] \N +17 [{"k1":"v41","k2":400},1,"a",3.14] \N 18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} \N 26 \N \N 27 {"k1":"v1","k2":200} \N @@ -2864,9 +3071,9 @@ 13 [] \N 14 [123,456] \N 15 ["abc","def"] \N -16 [null,true,false,100,6.18,"abc"] false +16 [null,true,false,100,6.18,"abc"] \N 17 [{"k1":"v41","k2":400},1,"a",3.14] \N -18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} \N +18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} 3.14 26 \N \N 27 {"k1":"v1","k2":200} \N 28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} \N @@ -2887,7 +3094,30 @@ 13 [] \N 14 [123,456] \N 15 ["abc","def"] \N -16 [null,true,false,100,6.18,"abc"] false +16 [null,true,false,100,6.18,"abc"] \N +17 [{"k1":"v41","k2":400},1,"a",3.14] \N +18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} \N +26 \N \N +27 {"k1":"v1","k2":200} \N +28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} \N + +-- !jsonb_extract_bool_select -- +1 \N \N +2 null \N +3 true true +4 false false +5 100 \N +6 10000 \N +7 1000000000 \N +8 1152921504606846976 \N +9 6.18 \N +10 "abcd" \N +11 {} \N +12 {"k1":"v31","k2":300} \N +13 [] \N +14 [123,456] \N +15 ["abc","def"] \N +16 [null,true,false,100,6.18,"abc"] \N 17 [{"k1":"v41","k2":400},1,"a",3.14] \N 18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} \N 26 \N \N @@ -2958,7 +3188,7 @@ 15 ["abc","def"] \N 16 [null,true,false,100,6.18,"abc"] \N 17 [{"k1":"v41","k2":400},1,"a",3.14] \N -18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} false +18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} \N 26 \N \N 27 {"k1":"v1","k2":200} \N 28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} \N @@ -2979,9 +3209,9 @@ 13 [] \N 14 [123,456] \N 15 ["abc","def"] \N -16 [null,true,false,100,6.18,"abc"] \N +16 [null,true,false,100,6.18,"abc"] true 17 [{"k1":"v41","k2":400},1,"a",3.14] \N -18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} false +18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} \N 26 \N \N 27 {"k1":"v1","k2":200} \N 28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} \N @@ -3002,9 +3232,9 @@ 13 [] \N 14 [123,456] \N 15 ["abc","def"] \N -16 [null,true,false,100,6.18,"abc"] \N +16 [null,true,false,100,6.18,"abc"] false 17 [{"k1":"v41","k2":400},1,"a",3.14] \N -18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} false +18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} \N 26 \N \N 27 {"k1":"v1","k2":200} \N 28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} \N @@ -3027,7 +3257,7 @@ 15 ["abc","def"] \N 16 [null,true,false,100,6.18,"abc"] \N 17 [{"k1":"v41","k2":400},1,"a",3.14] \N -18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} false +18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} \N 26 \N \N 27 {"k1":"v1","k2":200} \N 28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} \N @@ -3050,7 +3280,7 @@ 15 ["abc","def"] \N 16 [null,true,false,100,6.18,"abc"] \N 17 [{"k1":"v41","k2":400},1,"a",3.14] \N -18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} false +18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} \N 26 \N \N 27 {"k1":"v1","k2":200} \N 28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} \N @@ -3103,26 +3333,1176 @@ -- !select -- 1 \N \N -2 null true -3 true true -4 false true -5 100 true -6 10000 true -7 1000000000 true -8 1152921504606846976 true -9 6.18 true -10 "abcd" true -11 {} true -12 {"k1":"v31","k2":300} true -13 [] true -14 [123,456] true -15 ["abc","def"] true +2 null \N +3 true \N +4 false \N +5 100 \N +6 10000 \N +7 1000000000 \N +8 1152921504606846976 \N +9 6.18 \N +10 "abcd" \N +11 {} \N +12 {"k1":"v31","k2":300} \N +13 [] \N +14 [123,456] \N +15 ["abc","def"] \N +16 [null,true,false,100,6.18,"abc"] \N +17 [{"k1":"v41","k2":400},1,"a",3.14] \N +18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} \N +26 \N \N +27 {"k1":"v1","k2":200} \N +28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} \N + +-- !select -- +1 \N \N +2 null \N +3 true \N +4 false \N +5 100 \N +6 10000 \N +7 1000000000 \N +8 1152921504606846976 \N +9 6.18 \N +10 "abcd" \N +11 {} \N +12 {"k1":"v31","k2":300} \N +13 [] \N +14 [123,456] \N +15 ["abc","def"] \N +16 [null,true,false,100,6.18,"abc"] \N +17 [{"k1":"v41","k2":400},1,"a",3.14] \N +18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} \N +26 \N \N +27 {"k1":"v1","k2":200} \N +28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} \N + +-- !select -- +1 \N \N +2 null \N +3 true \N +4 false \N +5 100 \N +6 10000 \N +7 1000000000 \N +8 1152921504606846976 \N +9 6.18 \N +10 "abcd" \N +11 {} \N +12 {"k1":"v31","k2":300} \N +13 [] \N +14 [123,456] \N +15 ["abc","def"] \N +16 [null,true,false,100,6.18,"abc"] \N +17 [{"k1":"v41","k2":400},1,"a",3.14] \N +18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} \N +26 \N \N +27 {"k1":"v1","k2":200} \N +28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} \N + +-- !select -- +1 \N \N +2 null \N +3 true \N +4 false \N +5 100 \N +6 10000 \N +7 1000000000 \N +8 1152921504606846976 \N +9 6.18 \N +10 "abcd" \N +11 {} \N +12 {"k1":"v31","k2":300} \N +13 [] \N +14 [123,456] \N +15 ["abc","def"] \N +16 [null,true,false,100,6.18,"abc"] \N +17 [{"k1":"v41","k2":400},1,"a",3.14] \N +18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} \N +26 \N \N +27 {"k1":"v1","k2":200} \N +28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} \N + +-- !select -- +1 \N \N +2 null \N +3 true \N +4 false \N +5 100 \N +6 10000 \N +7 1000000000 \N +8 1152921504606846976 \N +9 6.18 \N +10 "abcd" \N +11 {} \N +12 {"k1":"v31","k2":300} \N +13 [] \N +14 [123,456] \N +15 ["abc","def"] \N +16 [null,true,false,100,6.18,"abc"] \N +17 [{"k1":"v41","k2":400},1,"a",3.14] \N +18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} \N +26 \N \N +27 {"k1":"v1","k2":200} \N +28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} \N + +-- !select -- +1 \N \N +2 null \N +3 true \N +4 false \N +5 100 \N +6 10000 \N +7 1000000000 \N +8 1152921504606846976 \N +9 6.18 \N +10 "abcd" \N +11 {} \N +12 {"k1":"v31","k2":300} \N +13 [] \N +14 [123,456] \N +15 ["abc","def"] \N +16 [null,true,false,100,6.18,"abc"] \N +17 [{"k1":"v41","k2":400},1,"a",3.14] \N +18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} \N +26 \N \N +27 {"k1":"v1","k2":200} \N +28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} \N + +-- !select -- +1 \N \N +2 null \N +3 true \N +4 false \N +5 100 \N +6 10000 \N +7 1000000000 \N +8 1152921504606846976 \N +9 6.18 \N +10 "abcd" \N +11 {} \N +12 {"k1":"v31","k2":300} \N +13 [] \N +14 [123,456] \N +15 ["abc","def"] \N +16 [null,true,false,100,6.18,"abc"] \N +17 [{"k1":"v41","k2":400},1,"a",3.14] \N +18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} \N +26 \N \N +27 {"k1":"v1","k2":200} \N +28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} \N + +-- !select -- +1 \N \N +2 null \N +3 true \N +4 false \N +5 100 \N +6 10000 \N +7 1000000000 \N +8 1152921504606846976 \N +9 6.18 \N +10 "abcd" \N +11 {} \N +12 {"k1":"v31","k2":300} \N +13 [] \N +14 [123,456] \N +15 ["abc","def"] \N +16 [null,true,false,100,6.18,"abc"] \N +17 [{"k1":"v41","k2":400},1,"a",3.14] \N +18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} \N +26 \N \N +27 {"k1":"v1","k2":200} \N +28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} \N + +-- !select -- +1 \N \N +2 null \N +3 true \N +4 false \N +5 100 \N +6 10000 \N +7 1000000000 \N +8 1152921504606846976 \N +9 6.18 \N +10 "abcd" \N +11 {} \N +12 {"k1":"v31","k2":300} \N +13 [] \N +14 [123,456] \N +15 ["abc","def"] \N +16 [null,true,false,100,6.18,"abc"] \N +17 [{"k1":"v41","k2":400},1,"a",3.14] \N +18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} \N +26 \N \N +27 {"k1":"v1","k2":200} \N +28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} \N + +-- !select -- +1 \N \N +2 null \N +3 true \N +4 false \N +5 100 \N +6 10000 \N +7 1000000000 \N +8 1152921504606846976 \N +9 6.18 \N +10 "abcd" \N +11 {} \N +12 {"k1":"v31","k2":300} \N +13 [] \N +14 [123,456] \N +15 ["abc","def"] \N +16 [null,true,false,100,6.18,"abc"] \N +17 [{"k1":"v41","k2":400},1,"a",3.14] \N +18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} \N +26 \N \N +27 {"k1":"v1","k2":200} \N +28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} \N + +-- !select -- +1 \N \N +2 null \N +3 true \N +4 false \N +5 100 \N +6 10000 \N +7 1000000000 \N +8 1152921504606846976 \N +9 6.18 \N +10 "abcd" \N +11 {} \N +12 {"k1":"v31","k2":300} \N +13 [] \N +14 [123,456] \N +15 ["abc","def"] \N +16 [null,true,false,100,6.18,"abc"] \N +17 [{"k1":"v41","k2":400},1,"a",3.14] \N +18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} \N +26 \N \N +27 {"k1":"v1","k2":200} \N +28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} \N + +-- !select -- +1 \N \N +2 null \N +3 true \N +4 false \N +5 100 \N +6 10000 \N +7 1000000000 \N +8 1152921504606846976 \N +9 6.18 \N +10 "abcd" \N +11 {} \N +12 {"k1":"v31","k2":300} \N +13 [] \N +14 [123,456] \N +15 ["abc","def"] \N +16 [null,true,false,100,6.18,"abc"] \N +17 [{"k1":"v41","k2":400},1,"a",3.14] \N +18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} \N +26 \N \N +27 {"k1":"v1","k2":200} \N +28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} \N + +-- !select -- +1 \N \N +2 null \N +3 true \N +4 false \N +5 100 \N +6 10000 \N +7 1000000000 \N +8 1152921504606846976 \N +9 6.18 \N +10 "abcd" \N +11 {} \N +12 {"k1":"v31","k2":300} \N +13 [] \N +14 [123,456] \N +15 ["abc","def"] \N +16 [null,true,false,100,6.18,"abc"] \N +17 [{"k1":"v41","k2":400},1,"a",3.14] \N +18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} \N +26 \N \N +27 {"k1":"v1","k2":200} \N +28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} \N + +-- !select -- +1 \N \N +2 null \N +3 true \N +4 false \N +5 100 \N +6 10000 \N +7 1000000000 \N +8 1152921504606846976 \N +9 6.18 \N +10 "abcd" \N +11 {} \N +12 {"k1":"v31","k2":300} \N +13 [] \N +14 [123,456] \N +15 ["abc","def"] \N +16 [null,true,false,100,6.18,"abc"] \N +17 [{"k1":"v41","k2":400},1,"a",3.14] \N +18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} \N +26 \N \N +27 {"k1":"v1","k2":200} \N +28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} \N + +-- !select -- +1 \N \N +2 null \N +3 true \N +4 false \N +5 100 \N +6 10000 \N +7 1000000000 \N +8 1152921504606846976 \N +9 6.18 \N +10 "abcd" \N +11 {} \N +12 {"k1":"v31","k2":300} \N +13 [] \N +14 [123,456] \N +15 ["abc","def"] \N +16 [null,true,false,100,6.18,"abc"] \N +17 [{"k1":"v41","k2":400},1,"a",3.14] \N +18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} \N +26 \N \N +27 {"k1":"v1","k2":200} \N +28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} \N + +-- !select -- +1 \N \N +2 null \N +3 true \N +4 false \N +5 100 \N +6 10000 \N +7 1000000000 \N +8 1152921504606846976 \N +9 6.18 \N +10 "abcd" \N +11 {} \N +12 {"k1":"v31","k2":300} \N +13 [] \N +14 [123,456] \N +15 ["abc","def"] \N +16 [null,true,false,100,6.18,"abc"] \N +17 [{"k1":"v41","k2":400},1,"a",3.14] \N +18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} \N +26 \N \N +27 {"k1":"v1","k2":200} \N +28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} \N + +-- !jsonb_extract_isnull_select -- +1 \N \N +2 null true +3 true false +4 false false +5 100 false +6 10000 false +7 1000000000 false +8 1152921504606846976 false +9 6.18 false +10 "abcd" false +11 {} false +12 {"k1":"v31","k2":300} false +13 [] false +14 [123,456] false +15 ["abc","def"] false +16 [null,true,false,100,6.18,"abc"] false +17 [{"k1":"v41","k2":400},1,"a",3.14] false +18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} false +26 \N \N +27 {"k1":"v1","k2":200} false +28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} false + +-- !select -- +1 \N \N +2 null \N +3 true \N +4 false \N +5 100 \N +6 10000 \N +7 1000000000 \N +8 1152921504606846976 \N +9 6.18 \N +10 "abcd" \N +11 {} \N +12 {"k1":"v31","k2":300} false +13 [] \N +14 [123,456] \N +15 ["abc","def"] \N +16 [null,true,false,100,6.18,"abc"] \N +17 [{"k1":"v41","k2":400},1,"a",3.14] \N +18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} false +26 \N \N +27 {"k1":"v1","k2":200} false +28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} \N + +-- !select -- +1 \N \N +2 null \N +3 true \N +4 false \N +5 100 \N +6 10000 \N +7 1000000000 \N +8 1152921504606846976 \N +9 6.18 \N +10 "abcd" \N +11 {} \N +12 {"k1":"v31","k2":300} false +13 [] \N +14 [123,456] \N +15 ["abc","def"] \N +16 [null,true,false,100,6.18,"abc"] \N +17 [{"k1":"v41","k2":400},1,"a",3.14] \N +18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} false +26 \N \N +27 {"k1":"v1","k2":200} false +28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} \N + +-- !select -- +1 \N \N +2 null \N +3 true \N +4 false \N +5 100 \N +6 10000 \N +7 1000000000 \N +8 1152921504606846976 \N +9 6.18 \N +10 "abcd" \N +11 {} false +12 {"k1":"v31","k2":300} false +13 [] \N +14 [123,456] false +15 ["abc","def"] false +16 [null,true,false,100,6.18,"abc"] true +17 [{"k1":"v41","k2":400},1,"a",3.14] false +18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} false +26 \N \N +27 {"k1":"v1","k2":200} false +28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} false + +-- !select -- +1 \N \N +2 null \N +3 true \N +4 false \N +5 100 \N +6 10000 \N +7 1000000000 \N +8 1152921504606846976 \N +9 6.18 \N +10 "abcd" \N +11 {} \N +12 {"k1":"v31","k2":300} \N +13 [] \N +14 [123,456] false +15 ["abc","def"] false +16 [null,true,false,100,6.18,"abc"] false +17 [{"k1":"v41","k2":400},1,"a",3.14] false +18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} \N +26 \N \N +27 {"k1":"v1","k2":200} \N +28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} \N + +-- !select -- +1 \N \N +2 null \N +3 true \N +4 false \N +5 100 \N +6 10000 \N +7 1000000000 \N +8 1152921504606846976 \N +9 6.18 \N +10 "abcd" \N +11 {} \N +12 {"k1":"v31","k2":300} \N +13 [] \N +14 [123,456] \N +15 ["abc","def"] \N +16 [null,true,false,100,6.18,"abc"] false +17 [{"k1":"v41","k2":400},1,"a",3.14] false +18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} \N +26 \N \N +27 {"k1":"v1","k2":200} \N +28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} \N + +-- !select -- +1 \N \N +2 null \N +3 true \N +4 false \N +5 100 \N +6 10000 \N +7 1000000000 \N +8 1152921504606846976 \N +9 6.18 \N +10 "abcd" \N +11 {} \N +12 {"k1":"v31","k2":300} \N +13 [] \N +14 [123,456] \N +15 ["abc","def"] \N +16 [null,true,false,100,6.18,"abc"] false +17 [{"k1":"v41","k2":400},1,"a",3.14] false +18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} \N +26 \N \N +27 {"k1":"v1","k2":200} \N +28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} \N + +-- !select -- +1 \N \N +2 null \N +3 true \N +4 false \N +5 100 \N +6 10000 \N +7 1000000000 \N +8 1152921504606846976 \N +9 6.18 \N +10 "abcd" \N +11 {} \N +12 {"k1":"v31","k2":300} \N +13 [] \N +14 [123,456] \N +15 ["abc","def"] \N +16 [null,true,false,100,6.18,"abc"] false +17 [{"k1":"v41","k2":400},1,"a",3.14] \N +18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} \N +26 \N \N +27 {"k1":"v1","k2":200} \N +28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} \N + +-- !select -- +1 \N \N +2 null \N +3 true \N +4 false \N +5 100 \N +6 10000 \N +7 1000000000 \N +8 1152921504606846976 \N +9 6.18 \N +10 "abcd" \N +11 {} \N +12 {"k1":"v31","k2":300} \N +13 [] \N +14 [123,456] \N +15 ["abc","def"] \N +16 [null,true,false,100,6.18,"abc"] false +17 [{"k1":"v41","k2":400},1,"a",3.14] \N +18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} \N +26 \N \N +27 {"k1":"v1","k2":200} \N +28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} \N + +-- !select -- +1 \N \N +2 null \N +3 true \N +4 false \N +5 100 \N +6 10000 \N +7 1000000000 \N +8 1152921504606846976 \N +9 6.18 \N +10 "abcd" \N +11 {} \N +12 {"k1":"v31","k2":300} \N +13 [] \N +14 [123,456] \N +15 ["abc","def"] \N +16 [null,true,false,100,6.18,"abc"] \N +17 [{"k1":"v41","k2":400},1,"a",3.14] \N +18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} \N +26 \N \N +27 {"k1":"v1","k2":200} \N +28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} \N + +-- !select -- +1 \N \N +2 null \N +3 true \N +4 false \N +5 100 \N +6 10000 \N +7 1000000000 \N +8 1152921504606846976 \N +9 6.18 \N +10 "abcd" \N +11 {} \N +12 {"k1":"v31","k2":300} \N +13 [] \N +14 [123,456] \N +15 ["abc","def"] \N +16 [null,true,false,100,6.18,"abc"] \N +17 [{"k1":"v41","k2":400},1,"a",3.14] \N +18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} \N +26 \N \N +27 {"k1":"v1","k2":200} \N +28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} \N + +-- !select -- +1 \N \N +2 null \N +3 true \N +4 false \N +5 100 \N +6 10000 \N +7 1000000000 \N +8 1152921504606846976 \N +9 6.18 \N +10 "abcd" \N +11 {} \N +12 {"k1":"v31","k2":300} \N +13 [] \N +14 [123,456] \N +15 ["abc","def"] \N +16 [null,true,false,100,6.18,"abc"] \N +17 [{"k1":"v41","k2":400},1,"a",3.14] \N +18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} false +26 \N \N +27 {"k1":"v1","k2":200} \N +28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} \N + +-- !select -- +1 \N \N +2 null \N +3 true \N +4 false \N +5 100 \N +6 10000 \N +7 1000000000 \N +8 1152921504606846976 \N +9 6.18 \N +10 "abcd" \N +11 {} \N +12 {"k1":"v31","k2":300} \N +13 [] \N +14 [123,456] \N +15 ["abc","def"] \N +16 [null,true,false,100,6.18,"abc"] \N +17 [{"k1":"v41","k2":400},1,"a",3.14] \N +18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} false +26 \N \N +27 {"k1":"v1","k2":200} \N +28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} \N + +-- !select -- +1 \N \N +2 null \N +3 true \N +4 false \N +5 100 \N +6 10000 \N +7 1000000000 \N +8 1152921504606846976 \N +9 6.18 \N +10 "abcd" \N +11 {} \N +12 {"k1":"v31","k2":300} \N +13 [] \N +14 [123,456] \N +15 ["abc","def"] \N +16 [null,true,false,100,6.18,"abc"] \N +17 [{"k1":"v41","k2":400},1,"a",3.14] \N +18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} false +26 \N \N +27 {"k1":"v1","k2":200} \N +28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} \N + +-- !select -- +1 \N \N +2 null \N +3 true \N +4 false \N +5 100 \N +6 10000 \N +7 1000000000 \N +8 1152921504606846976 \N +9 6.18 \N +10 "abcd" \N +11 {} \N +12 {"k1":"v31","k2":300} \N +13 [] \N +14 [123,456] \N +15 ["abc","def"] \N +16 [null,true,false,100,6.18,"abc"] \N +17 [{"k1":"v41","k2":400},1,"a",3.14] \N +18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} false +26 \N \N +27 {"k1":"v1","k2":200} \N +28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} \N + +-- !select -- +1 \N \N +2 null \N +3 true \N +4 false \N +5 100 \N +6 10000 \N +7 1000000000 \N +8 1152921504606846976 \N +9 6.18 \N +10 "abcd" \N +11 {} \N +12 {"k1":"v31","k2":300} \N +13 [] \N +14 [123,456] \N +15 ["abc","def"] \N +16 [null,true,false,100,6.18,"abc"] \N +17 [{"k1":"v41","k2":400},1,"a",3.14] \N +18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} false +26 \N \N +27 {"k1":"v1","k2":200} \N +28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} \N + +-- !select -- +1 \N \N +2 null \N +3 true \N +4 false \N +5 100 \N +6 10000 \N +7 1000000000 \N +8 1152921504606846976 \N +9 6.18 \N +10 "abcd" \N +11 {} \N +12 {"k1":"v31","k2":300} \N +13 [] \N +14 [123,456] \N +15 ["abc","def"] \N +16 [null,true,false,100,6.18,"abc"] \N +17 [{"k1":"v41","k2":400},1,"a",3.14] \N +18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} \N +26 \N \N +27 {"k1":"v1","k2":200} \N +28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} \N + +-- !select -- +1 \N \N +2 null \N +3 true \N +4 false \N +5 100 \N +6 10000 \N +7 1000000000 \N +8 1152921504606846976 \N +9 6.18 \N +10 "abcd" \N +11 {} \N +12 {"k1":"v31","k2":300} \N +13 [] \N +14 [123,456] \N +15 ["abc","def"] \N +16 [null,true,false,100,6.18,"abc"] \N +17 [{"k1":"v41","k2":400},1,"a",3.14] \N +18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} \N +26 \N \N +27 {"k1":"v1","k2":200} \N +28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} \N + +-- !select -- +1 \N \N +2 null \N +3 true \N +4 false \N +5 100 \N +6 10000 \N +7 1000000000 \N +8 1152921504606846976 \N +9 6.18 \N +10 "abcd" \N +11 {} \N +12 {"k1":"v31","k2":300} \N +13 [] \N +14 [123,456] \N +15 ["abc","def"] \N +16 [null,true,false,100,6.18,"abc"] \N +17 [{"k1":"v41","k2":400},1,"a",3.14] \N +18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} false +26 \N \N +27 {"k1":"v1","k2":200} \N +28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} \N + +-- !select -- +1 \N \N +2 null \N +3 true \N +4 false \N +5 100 \N +6 10000 \N +7 1000000000 \N +8 1152921504606846976 \N +9 6.18 \N +10 "abcd" \N +11 {} \N +12 {"k1":"v31","k2":300} \N +13 [] \N +14 [123,456] \N +15 ["abc","def"] \N +16 [null,true,false,100,6.18,"abc"] \N +17 [{"k1":"v41","k2":400},1,"a",3.14] \N +18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} false +26 \N \N +27 {"k1":"v1","k2":200} \N +28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} \N + +-- !select -- +1 \N \N +2 null \N +3 true \N +4 false \N +5 100 \N +6 10000 \N +7 1000000000 \N +8 1152921504606846976 \N +9 6.18 \N +10 "abcd" \N +11 {} \N +12 {"k1":"v31","k2":300} \N +13 [] \N +14 [123,456] \N +15 ["abc","def"] \N +16 [null,true,false,100,6.18,"abc"] \N +17 [{"k1":"v41","k2":400},1,"a",3.14] \N +18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} false +26 \N \N +27 {"k1":"v1","k2":200} \N +28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} \N + +-- !select -- +1 \N \N +2 null \N +3 true \N +4 false \N +5 100 \N +6 10000 \N +7 1000000000 \N +8 1152921504606846976 \N +9 6.18 \N +10 "abcd" \N +11 {} \N +12 {"k1":"v31","k2":300} \N +13 [] \N +14 [123,456] \N +15 ["abc","def"] \N +16 [null,true,false,100,6.18,"abc"] \N +17 [{"k1":"v41","k2":400},1,"a",3.14] \N +18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} false +26 \N \N +27 {"k1":"v1","k2":200} \N +28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} \N + +-- !select -- +1 \N \N +2 null \N +3 true \N +4 false \N +5 100 \N +6 10000 \N +7 1000000000 \N +8 1152921504606846976 \N +9 6.18 \N +10 "abcd" \N +11 {} \N +12 {"k1":"v31","k2":300} \N +13 [] \N +14 [123,456] \N +15 ["abc","def"] \N +16 [null,true,false,100,6.18,"abc"] \N +17 [{"k1":"v41","k2":400},1,"a",3.14] \N +18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} \N +26 \N \N +27 {"k1":"v1","k2":200} \N +28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} \N + +-- !select -- +1 \N \N +2 null \N +3 true \N +4 false \N +5 100 \N +6 10000 \N +7 1000000000 \N +8 1152921504606846976 \N +9 6.18 \N +10 "abcd" \N +11 {} \N +12 {"k1":"v31","k2":300} \N +13 [] \N +14 [123,456] \N +15 ["abc","def"] \N +16 [null,true,false,100,6.18,"abc"] \N +17 [{"k1":"v41","k2":400},1,"a",3.14] \N +18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} false +26 \N \N +27 {"k1":"v1","k2":200} \N +28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} \N + +-- !select -- +1 \N \N +2 null \N +3 true \N +4 false \N +5 100 \N +6 10000 \N +7 1000000000 \N +8 1152921504606846976 \N +9 6.18 \N +10 "abcd" \N +11 {} \N +12 {"k1":"v31","k2":300} \N +13 [] \N +14 [123,456] \N +15 ["abc","def"] \N +16 [null,true,false,100,6.18,"abc"] \N +17 [{"k1":"v41","k2":400},1,"a",3.14] \N +18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} false +26 \N \N +27 {"k1":"v1","k2":200} \N +28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} \N + +-- !select -- +1 \N \N +2 null \N +3 true \N +4 false \N +5 100 \N +6 10000 \N +7 1000000000 \N +8 1152921504606846976 \N +9 6.18 \N +10 "abcd" \N +11 {} \N +12 {"k1":"v31","k2":300} \N +13 [] \N +14 [123,456] \N +15 ["abc","def"] \N +16 [null,true,false,100,6.18,"abc"] \N +17 [{"k1":"v41","k2":400},1,"a",3.14] \N +18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} \N +26 \N \N +27 {"k1":"v1","k2":200} \N +28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} \N + +-- !jsonb_exists_path_select -- +1 \N \N +2 null true +3 true true +4 false true +5 100 true +6 10000 true +7 1000000000 true +8 1152921504606846976 true +9 6.18 true +10 "abcd" true +11 {} true +12 {"k1":"v31","k2":300} true +13 [] true +14 [123,456] true +15 ["abc","def"] true +16 [null,true,false,100,6.18,"abc"] true +17 [{"k1":"v41","k2":400},1,"a",3.14] true +18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} true +26 \N \N +27 {"k1":"v1","k2":200} true +28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} true + +-- !select -- +1 \N \N +2 null false +3 true false +4 false false +5 100 false +6 10000 false +7 1000000000 false +8 1152921504606846976 false +9 6.18 false +10 "abcd" false +11 {} false +12 {"k1":"v31","k2":300} true +13 [] false +14 [123,456] false +15 ["abc","def"] false +16 [null,true,false,100,6.18,"abc"] false +17 [{"k1":"v41","k2":400},1,"a",3.14] false +18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} true +26 \N \N +27 {"k1":"v1","k2":200} true +28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} false + +-- !select -- +1 \N \N +2 null false +3 true false +4 false false +5 100 false +6 10000 false +7 1000000000 false +8 1152921504606846976 false +9 6.18 false +10 "abcd" false +11 {} false +12 {"k1":"v31","k2":300} true +13 [] false +14 [123,456] false +15 ["abc","def"] false +16 [null,true,false,100,6.18,"abc"] false +17 [{"k1":"v41","k2":400},1,"a",3.14] false +18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} true +26 \N \N +27 {"k1":"v1","k2":200} true +28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} false + +-- !select -- +1 \N \N +2 null false +3 true false +4 false false +5 100 false +6 10000 false +7 1000000000 false +8 1152921504606846976 false +9 6.18 false +10 "abcd" false +11 {} true +12 {"k1":"v31","k2":300} true +13 [] false +14 [123,456] true +15 ["abc","def"] true 16 [null,true,false,100,6.18,"abc"] true 17 [{"k1":"v41","k2":400},1,"a",3.14] true 18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} true 26 \N \N -27 {"k1":"v1","k2":200} true -28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} true +27 {"k1":"v1","k2":200} true +28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} true + +-- !select -- +1 \N \N +2 null false +3 true false +4 false false +5 100 false +6 10000 false +7 1000000000 false +8 1152921504606846976 false +9 6.18 false +10 "abcd" false +11 {} false +12 {"k1":"v31","k2":300} false +13 [] false +14 [123,456] true +15 ["abc","def"] true +16 [null,true,false,100,6.18,"abc"] true +17 [{"k1":"v41","k2":400},1,"a",3.14] true +18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} false +26 \N \N +27 {"k1":"v1","k2":200} false +28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} false + +-- !select -- +1 \N \N +2 null false +3 true false +4 false false +5 100 false +6 10000 false +7 1000000000 false +8 1152921504606846976 false +9 6.18 false +10 "abcd" false +11 {} false +12 {"k1":"v31","k2":300} false +13 [] false +14 [123,456] false +15 ["abc","def"] false +16 [null,true,false,100,6.18,"abc"] true +17 [{"k1":"v41","k2":400},1,"a",3.14] true +18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} false +26 \N \N +27 {"k1":"v1","k2":200} false +28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} false + +-- !select -- +1 \N \N +2 null false +3 true false +4 false false +5 100 false +6 10000 false +7 1000000000 false +8 1152921504606846976 false +9 6.18 false +10 "abcd" false +11 {} false +12 {"k1":"v31","k2":300} false +13 [] false +14 [123,456] false +15 ["abc","def"] false +16 [null,true,false,100,6.18,"abc"] true +17 [{"k1":"v41","k2":400},1,"a",3.14] true +18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} false +26 \N \N +27 {"k1":"v1","k2":200} false +28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} false + +-- !select -- +1 \N \N +2 null false +3 true false +4 false false +5 100 false +6 10000 false +7 1000000000 false +8 1152921504606846976 false +9 6.18 false +10 "abcd" false +11 {} false +12 {"k1":"v31","k2":300} false +13 [] false +14 [123,456] false +15 ["abc","def"] false +16 [null,true,false,100,6.18,"abc"] true +17 [{"k1":"v41","k2":400},1,"a",3.14] false +18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} false +26 \N \N +27 {"k1":"v1","k2":200} false +28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} false + +-- !select -- +1 \N \N +2 null false +3 true false +4 false false +5 100 false +6 10000 false +7 1000000000 false +8 1152921504606846976 false +9 6.18 false +10 "abcd" false +11 {} false +12 {"k1":"v31","k2":300} false +13 [] false +14 [123,456] false +15 ["abc","def"] false +16 [null,true,false,100,6.18,"abc"] true +17 [{"k1":"v41","k2":400},1,"a",3.14] false +18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} false +26 \N \N +27 {"k1":"v1","k2":200} false +28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} false -- !select -- 1 \N \N @@ -3136,15 +4516,15 @@ 9 6.18 false 10 "abcd" false 11 {} false -12 {"k1":"v31","k2":300} true +12 {"k1":"v31","k2":300} false 13 [] false 14 [123,456] false 15 ["abc","def"] false 16 [null,true,false,100,6.18,"abc"] false 17 [{"k1":"v41","k2":400},1,"a",3.14] false -18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} true +18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} false 26 \N \N -27 {"k1":"v1","k2":200} true +27 {"k1":"v1","k2":200} false 28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} false -- !select -- @@ -3159,15 +4539,15 @@ 9 6.18 false 10 "abcd" false 11 {} false -12 {"k1":"v31","k2":300} true +12 {"k1":"v31","k2":300} false 13 [] false 14 [123,456] false 15 ["abc","def"] false 16 [null,true,false,100,6.18,"abc"] false 17 [{"k1":"v41","k2":400},1,"a",3.14] false -18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} true +18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} false 26 \N \N -27 {"k1":"v1","k2":200} true +27 {"k1":"v1","k2":200} false 28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} false -- !select -- @@ -3184,11 +4564,11 @@ 11 {} false 12 {"k1":"v31","k2":300} false 13 [] false -14 [123,456] true -15 ["abc","def"] true -16 [null,true,false,100,6.18,"abc"] true -17 [{"k1":"v41","k2":400},1,"a",3.14] true -18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} false +14 [123,456] false +15 ["abc","def"] false +16 [null,true,false,100,6.18,"abc"] false +17 [{"k1":"v41","k2":400},1,"a",3.14] false +18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} true 26 \N \N 27 {"k1":"v1","k2":200} false 28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} false @@ -3207,11 +4587,11 @@ 11 {} false 12 {"k1":"v31","k2":300} false 13 [] false -14 [123,456] true -15 ["abc","def"] true -16 [null,true,false,100,6.18,"abc"] true -17 [{"k1":"v41","k2":400},1,"a",3.14] true -18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} false +14 [123,456] false +15 ["abc","def"] false +16 [null,true,false,100,6.18,"abc"] false +17 [{"k1":"v41","k2":400},1,"a",3.14] false +18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} true 26 \N \N 27 {"k1":"v1","k2":200} false 28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} false @@ -3232,9 +4612,9 @@ 13 [] false 14 [123,456] false 15 ["abc","def"] false -16 [null,true,false,100,6.18,"abc"] true -17 [{"k1":"v41","k2":400},1,"a",3.14] true -18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} false +16 [null,true,false,100,6.18,"abc"] false +17 [{"k1":"v41","k2":400},1,"a",3.14] false +18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} true 26 \N \N 27 {"k1":"v1","k2":200} false 28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} false @@ -3255,9 +4635,9 @@ 13 [] false 14 [123,456] false 15 ["abc","def"] false -16 [null,true,false,100,6.18,"abc"] true -17 [{"k1":"v41","k2":400},1,"a",3.14] true -18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} false +16 [null,true,false,100,6.18,"abc"] false +17 [{"k1":"v41","k2":400},1,"a",3.14] false +18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} true 26 \N \N 27 {"k1":"v1","k2":200} false 28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} false @@ -3278,9 +4658,9 @@ 13 [] false 14 [123,456] false 15 ["abc","def"] false -16 [null,true,false,100,6.18,"abc"] true +16 [null,true,false,100,6.18,"abc"] false 17 [{"k1":"v41","k2":400},1,"a",3.14] false -18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} false +18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} true 26 \N \N 27 {"k1":"v1","k2":200} false 28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} false @@ -3301,7 +4681,7 @@ 13 [] false 14 [123,456] false 15 ["abc","def"] false -16 [null,true,false,100,6.18,"abc"] true +16 [null,true,false,100,6.18,"abc"] false 17 [{"k1":"v41","k2":400},1,"a",3.14] false 18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} false 26 \N \N @@ -3349,7 +4729,7 @@ 15 ["abc","def"] false 16 [null,true,false,100,6.18,"abc"] false 17 [{"k1":"v41","k2":400},1,"a",3.14] false -18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} false +18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} true 26 \N \N 27 {"k1":"v1","k2":200} false 28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} false @@ -3441,7 +4821,7 @@ 15 ["abc","def"] false 16 [null,true,false,100,6.18,"abc"] false 17 [{"k1":"v41","k2":400},1,"a",3.14] false -18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} true +18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} false 26 \N \N 27 {"k1":"v1","k2":200} false 28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} false @@ -3487,7 +4867,7 @@ 15 ["abc","def"] false 16 [null,true,false,100,6.18,"abc"] false 17 [{"k1":"v41","k2":400},1,"a",3.14] false -18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} false +18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} true 26 \N \N 27 {"k1":"v1","k2":200} false 28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} false @@ -3515,7 +4895,7 @@ 27 {"k1":"v1","k2":200} false 28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} false --- !select -- +-- !jsonb_type_select -- 1 \N \N 2 null null 3 true bool @@ -3595,17 +4975,17 @@ 8 1152921504606846976 \N 9 6.18 \N 10 "abcd" \N -11 {} \N -12 {"k1":"v31","k2":300} \N +11 {} object +12 {"k1":"v31","k2":300} object 13 [] \N 14 [123,456] int 15 ["abc","def"] string 16 [null,true,false,100,6.18,"abc"] null 17 [{"k1":"v41","k2":400},1,"a",3.14] object -18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} \N +18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} object 26 \N \N -27 {"k1":"v1","k2":200} \N -28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} \N +27 {"k1":"v1","k2":200} object +28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} object -- !select -- 1 \N \N @@ -3932,6 +5312,190 @@ -- !select -- 1 \N \N 2 null \N +3 true \N +4 false \N +5 100 \N +6 10000 \N +7 1000000000 \N +8 1152921504606846976 \N +9 6.18 \N +10 "abcd" \N +11 {} \N +12 {"k1":"v31","k2":300} \N +13 [] \N +14 [123,456] \N +15 ["abc","def"] \N +16 [null,true,false,100,6.18,"abc"] \N +17 [{"k1":"v41","k2":400},1,"a",3.14] \N +18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} double +26 \N \N +27 {"k1":"v1","k2":200} \N +28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} \N + +-- !select -- +1 \N \N +2 null \N +3 true \N +4 false \N +5 100 \N +6 10000 \N +7 1000000000 \N +8 1152921504606846976 \N +9 6.18 \N +10 "abcd" \N +11 {} \N +12 {"k1":"v31","k2":300} \N +13 [] \N +14 [123,456] \N +15 ["abc","def"] \N +16 [null,true,false,100,6.18,"abc"] \N +17 [{"k1":"v41","k2":400},1,"a",3.14] \N +18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} double +26 \N \N +27 {"k1":"v1","k2":200} \N +28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} \N + +-- !select -- +1 \N \N +2 null \N +3 true \N +4 false \N +5 100 \N +6 10000 \N +7 1000000000 \N +8 1152921504606846976 \N +9 6.18 \N +10 "abcd" \N +11 {} \N +12 {"k1":"v31","k2":300} \N +13 [] \N +14 [123,456] \N +15 ["abc","def"] \N +16 [null,true,false,100,6.18,"abc"] \N +17 [{"k1":"v41","k2":400},1,"a",3.14] \N +18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} string +26 \N \N +27 {"k1":"v1","k2":200} \N +28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} \N + +-- !select -- +1 \N \N +2 null \N +3 true \N +4 false \N +5 100 \N +6 10000 \N +7 1000000000 \N +8 1152921504606846976 \N +9 6.18 \N +10 "abcd" \N +11 {} \N +12 {"k1":"v31","k2":300} \N +13 [] \N +14 [123,456] \N +15 ["abc","def"] \N +16 [null,true,false,100,6.18,"abc"] \N +17 [{"k1":"v41","k2":400},1,"a",3.14] \N +18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} int +26 \N \N +27 {"k1":"v1","k2":200} \N +28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} \N + +-- !select -- +1 \N \N +2 null \N +3 true \N +4 false \N +5 100 \N +6 10000 \N +7 1000000000 \N +8 1152921504606846976 \N +9 6.18 \N +10 "abcd" \N +11 {} \N +12 {"k1":"v31","k2":300} \N +13 [] \N +14 [123,456] \N +15 ["abc","def"] \N +16 [null,true,false,100,6.18,"abc"] \N +17 [{"k1":"v41","k2":400},1,"a",3.14] \N +18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} \N +26 \N \N +27 {"k1":"v1","k2":200} \N +28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} \N + +-- !select -- +1 \N \N +2 null \N +3 true \N +4 false \N +5 100 \N +6 10000 \N +7 1000000000 \N +8 1152921504606846976 \N +9 6.18 \N +10 "abcd" \N +11 {} \N +12 {"k1":"v31","k2":300} \N +13 [] \N +14 [123,456] \N +15 ["abc","def"] \N +16 [null,true,false,100,6.18,"abc"] \N +17 [{"k1":"v41","k2":400},1,"a",3.14] \N +18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} object +26 \N \N +27 {"k1":"v1","k2":200} \N +28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} \N + +-- !select -- +1 \N \N +2 null \N +3 true \N +4 false \N +5 100 \N +6 10000 \N +7 1000000000 \N +8 1152921504606846976 \N +9 6.18 \N +10 "abcd" \N +11 {} \N +12 {"k1":"v31","k2":300} \N +13 [] \N +14 [123,456] \N +15 ["abc","def"] \N +16 [null,true,false,100,6.18,"abc"] \N +17 [{"k1":"v41","k2":400},1,"a",3.14] \N +18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} double +26 \N \N +27 {"k1":"v1","k2":200} \N +28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} \N + +-- !select -- +1 \N \N +2 null \N +3 true \N +4 false \N +5 100 \N +6 10000 \N +7 1000000000 \N +8 1152921504606846976 \N +9 6.18 \N +10 "abcd" \N +11 {} \N +12 {"k1":"v31","k2":300} \N +13 [] \N +14 [123,456] \N +15 ["abc","def"] \N +16 [null,true,false,100,6.18,"abc"] \N +17 [{"k1":"v41","k2":400},1,"a",3.14] \N +18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} \N +26 \N \N +27 {"k1":"v1","k2":200} \N +28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} \N + +-- !cast_from_select -- +1 \N \N +2 null \N 3 true true 4 false false 5 100 \N @@ -4067,7 +5631,7 @@ 27 {"k1":"v1","k2":200} {"k1":"v1","k2":200} 28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} --- !select -- +-- !cast_to_select -- 1 \N \N 2 null \N 3 true true diff --git a/regression-test/data/jsonb_p0/test_jsonb_load_unique_key_and_function.out b/regression-test/data/jsonb_p0/test_jsonb_load_unique_key_and_function.out index 01bc0b3de78e95d..04946beaa120164 100644 --- a/regression-test/data/jsonb_p0/test_jsonb_load_unique_key_and_function.out +++ b/regression-test/data/jsonb_p0/test_jsonb_load_unique_key_and_function.out @@ -42,7 +42,7 @@ 27 {"k1":"v1","k2":200} 28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} --- !select -- +-- !jsonb_extract_select -- 1 \N \N 2 null null 3 true true @@ -191,17 +191,17 @@ 8 1152921504606846976 \N 9 6.18 \N 10 "abcd" \N -11 {} \N -12 {"k1":"v31","k2":300} \N +11 {} {} +12 {"k1":"v31","k2":300} {"k1":"v31","k2":300} 13 [] \N 14 [123,456] 123 15 ["abc","def"] "abc" 16 [null,true,false,100,6.18,"abc"] null 17 [{"k1":"v41","k2":400},1,"a",3.14] {"k1":"v41","k2":400} -18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} \N +18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} 26 \N \N -27 {"k1":"v1","k2":200} \N -28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} \N +27 {"k1":"v1","k2":200} {"k1":"v1","k2":200} +28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} -- !select -- 1 \N \N @@ -548,6 +548,29 @@ 27 {"k1":"v1","k2":200} \N 28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} \N +-- !select -- +1 \N \N +2 null \N +3 true \N +4 false \N +5 100 \N +6 10000 \N +7 1000000000 \N +8 1152921504606846976 \N +9 6.18 \N +10 "abcd" \N +11 {} \N +12 {"k1":"v31","k2":300} \N +13 [] \N +14 [123,456] \N +15 ["abc","def"] \N +16 [null,true,false,100,6.18,"abc"] \N +17 [{"k1":"v41","k2":400},1,"a",3.14] \N +18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} 3.14 +26 \N \N +27 {"k1":"v1","k2":200} \N +28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} \N + -- !select -- 1 \N \N 2 null \N @@ -619,6 +642,75 @@ -- !select -- 1 \N \N +2 null \N +3 true \N +4 false \N +5 100 \N +6 10000 \N +7 1000000000 \N +8 1152921504606846976 \N +9 6.18 \N +10 "abcd" \N +11 {} \N +12 {"k1":"v31","k2":300} \N +13 [] \N +14 [123,456] \N +15 ["abc","def"] \N +16 [null,true,false,100,6.18,"abc"] \N +17 [{"k1":"v41","k2":400},1,"a",3.14] \N +18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} {"k1":"v41","k2":400} +26 \N \N +27 {"k1":"v1","k2":200} \N +28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} \N + +-- !select -- +1 \N \N +2 null \N +3 true \N +4 false \N +5 100 \N +6 10000 \N +7 1000000000 \N +8 1152921504606846976 \N +9 6.18 \N +10 "abcd" \N +11 {} \N +12 {"k1":"v31","k2":300} \N +13 [] \N +14 [123,456] \N +15 ["abc","def"] \N +16 [null,true,false,100,6.18,"abc"] \N +17 [{"k1":"v41","k2":400},1,"a",3.14] \N +18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} 3.14 +26 \N \N +27 {"k1":"v1","k2":200} \N +28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} \N + +-- !select -- +1 \N \N +2 null \N +3 true \N +4 false \N +5 100 \N +6 10000 \N +7 1000000000 \N +8 1152921504606846976 \N +9 6.18 \N +10 "abcd" \N +11 {} \N +12 {"k1":"v31","k2":300} \N +13 [] \N +14 [123,456] \N +15 ["abc","def"] \N +16 [null,true,false,100,6.18,"abc"] \N +17 [{"k1":"v41","k2":400},1,"a",3.14] \N +18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} \N +26 \N \N +27 {"k1":"v1","k2":200} \N +28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} \N + +-- !jsonb_extract_string_select -- +1 \N \N 2 null null 3 true true 4 false false @@ -697,17 +789,17 @@ 8 1152921504606846976 \N 9 6.18 \N 10 "abcd" \N -11 {} \N -12 {"k1":"v31","k2":300} \N +11 {} {} +12 {"k1":"v31","k2":300} {"k1":"v31","k2":300} 13 [] \N 14 [123,456] 123 15 ["abc","def"] abc 16 [null,true,false,100,6.18,"abc"] null 17 [{"k1":"v41","k2":400},1,"a",3.14] {"k1":"v41","k2":400} -18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} \N +18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} 26 \N \N -27 {"k1":"v1","k2":200} \N -28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} \N +27 {"k1":"v1","k2":200} {"k1":"v1","k2":200} +28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} -- !select -- 1 \N \N @@ -1036,9 +1128,9 @@ 2 null \N 3 true \N 4 false \N -5 100 100 -6 10000 10000 -7 1000000000 1000000000 +5 100 \N +6 10000 \N +7 1000000000 \N 8 1152921504606846976 \N 9 6.18 \N 10 "abcd" \N @@ -1049,7 +1141,7 @@ 15 ["abc","def"] \N 16 [null,true,false,100,6.18,"abc"] \N 17 [{"k1":"v41","k2":400},1,"a",3.14] \N -18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} \N +18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} 3.14 26 \N \N 27 {"k1":"v1","k2":200} \N 28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} \N @@ -1072,7 +1164,7 @@ 15 ["abc","def"] \N 16 [null,true,false,100,6.18,"abc"] \N 17 [{"k1":"v41","k2":400},1,"a",3.14] \N -18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} \N +18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} 3.14 26 \N \N 27 {"k1":"v1","k2":200} \N 28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} \N @@ -1089,15 +1181,15 @@ 9 6.18 \N 10 "abcd" \N 11 {} \N -12 {"k1":"v31","k2":300} 300 +12 {"k1":"v31","k2":300} \N 13 [] \N 14 [123,456] \N 15 ["abc","def"] \N 16 [null,true,false,100,6.18,"abc"] \N 17 [{"k1":"v41","k2":400},1,"a",3.14] \N -18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} 300 +18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} a 26 \N \N -27 {"k1":"v1","k2":200} 200 +27 {"k1":"v1","k2":200} \N 28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} \N -- !select -- @@ -1114,11 +1206,11 @@ 11 {} \N 12 {"k1":"v31","k2":300} \N 13 [] \N -14 [123,456] 123 +14 [123,456] \N 15 ["abc","def"] \N 16 [null,true,false,100,6.18,"abc"] \N 17 [{"k1":"v41","k2":400},1,"a",3.14] \N -18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} \N +18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} 1 26 \N \N 27 {"k1":"v1","k2":200} \N 28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} \N @@ -1137,10 +1229,10 @@ 11 {} \N 12 {"k1":"v31","k2":300} \N 13 [] \N -14 [123,456] 456 +14 [123,456] \N 15 ["abc","def"] \N 16 [null,true,false,100,6.18,"abc"] \N -17 [{"k1":"v41","k2":400},1,"a",3.14] 1 +17 [{"k1":"v41","k2":400},1,"a",3.14] \N 18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} \N 26 \N \N 27 {"k1":"v1","k2":200} \N @@ -1164,7 +1256,7 @@ 15 ["abc","def"] \N 16 [null,true,false,100,6.18,"abc"] \N 17 [{"k1":"v41","k2":400},1,"a",3.14] \N -18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} \N +18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} {"k1":"v41","k2":400} 26 \N \N 27 {"k1":"v1","k2":200} \N 28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} \N @@ -1185,9 +1277,9 @@ 13 [] \N 14 [123,456] \N 15 ["abc","def"] \N -16 [null,true,false,100,6.18,"abc"] 100 +16 [null,true,false,100,6.18,"abc"] \N 17 [{"k1":"v41","k2":400},1,"a",3.14] \N -18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} \N +18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} 3.14 26 \N \N 27 {"k1":"v1","k2":200} \N 28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} \N @@ -1215,14 +1307,14 @@ 27 {"k1":"v1","k2":200} \N 28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} \N --- !select -- +-- !jsonb_extract_int_select -- 1 \N \N 2 null \N 3 true \N 4 false \N -5 100 \N -6 10000 \N -7 1000000000 \N +5 100 100 +6 10000 10000 +7 1000000000 1000000000 8 1152921504606846976 \N 9 6.18 \N 10 "abcd" \N @@ -1273,15 +1365,15 @@ 9 6.18 \N 10 "abcd" \N 11 {} \N -12 {"k1":"v31","k2":300} \N +12 {"k1":"v31","k2":300} 300 13 [] \N 14 [123,456] \N 15 ["abc","def"] \N 16 [null,true,false,100,6.18,"abc"] \N 17 [{"k1":"v41","k2":400},1,"a",3.14] \N -18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} \N +18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} 300 26 \N \N -27 {"k1":"v1","k2":200} \N +27 {"k1":"v1","k2":200} 200 28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} \N -- !select -- @@ -1298,7 +1390,7 @@ 11 {} \N 12 {"k1":"v31","k2":300} \N 13 [] \N -14 [123,456] \N +14 [123,456] 123 15 ["abc","def"] \N 16 [null,true,false,100,6.18,"abc"] \N 17 [{"k1":"v41","k2":400},1,"a",3.14] \N @@ -1321,10 +1413,10 @@ 11 {} \N 12 {"k1":"v31","k2":300} \N 13 [] \N -14 [123,456] \N +14 [123,456] 456 15 ["abc","def"] \N 16 [null,true,false,100,6.18,"abc"] \N -17 [{"k1":"v41","k2":400},1,"a",3.14] \N +17 [{"k1":"v41","k2":400},1,"a",3.14] 1 18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} \N 26 \N \N 27 {"k1":"v1","k2":200} \N @@ -1348,7 +1440,7 @@ 15 ["abc","def"] \N 16 [null,true,false,100,6.18,"abc"] \N 17 [{"k1":"v41","k2":400},1,"a",3.14] \N -18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} 1 +18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} \N 26 \N \N 27 {"k1":"v1","k2":200} \N 28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} \N @@ -1369,7 +1461,7 @@ 13 [] \N 14 [123,456] \N 15 ["abc","def"] \N -16 [null,true,false,100,6.18,"abc"] \N +16 [null,true,false,100,6.18,"abc"] 100 17 [{"k1":"v41","k2":400},1,"a",3.14] \N 18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} \N 26 \N \N @@ -1450,10 +1542,10 @@ 2 null \N 3 true \N 4 false \N -5 100 100 -6 10000 10000 -7 1000000000 1000000000 -8 1152921504606846976 1152921504606846976 +5 100 \N +6 10000 \N +7 1000000000 \N +8 1152921504606846976 \N 9 6.18 \N 10 "abcd" \N 11 {} \N @@ -1503,15 +1595,15 @@ 9 6.18 \N 10 "abcd" \N 11 {} \N -12 {"k1":"v31","k2":300} 300 +12 {"k1":"v31","k2":300} \N 13 [] \N 14 [123,456] \N 15 ["abc","def"] \N 16 [null,true,false,100,6.18,"abc"] \N 17 [{"k1":"v41","k2":400},1,"a",3.14] \N -18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} 300 +18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} \N 26 \N \N -27 {"k1":"v1","k2":200} 200 +27 {"k1":"v1","k2":200} \N 28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} \N -- !select -- @@ -1528,11 +1620,11 @@ 11 {} \N 12 {"k1":"v31","k2":300} \N 13 [] \N -14 [123,456] 123 +14 [123,456] \N 15 ["abc","def"] \N 16 [null,true,false,100,6.18,"abc"] \N 17 [{"k1":"v41","k2":400},1,"a",3.14] \N -18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} \N +18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} 1 26 \N \N 27 {"k1":"v1","k2":200} \N 28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} \N @@ -1551,10 +1643,10 @@ 11 {} \N 12 {"k1":"v31","k2":300} \N 13 [] \N -14 [123,456] 456 +14 [123,456] \N 15 ["abc","def"] \N 16 [null,true,false,100,6.18,"abc"] \N -17 [{"k1":"v41","k2":400},1,"a",3.14] 1 +17 [{"k1":"v41","k2":400},1,"a",3.14] \N 18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} \N 26 \N \N 27 {"k1":"v1","k2":200} \N @@ -1599,7 +1691,7 @@ 13 [] \N 14 [123,456] \N 15 ["abc","def"] \N -16 [null,true,false,100,6.18,"abc"] 100 +16 [null,true,false,100,6.18,"abc"] \N 17 [{"k1":"v41","k2":400},1,"a",3.14] \N 18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} \N 26 \N \N @@ -1716,7 +1808,7 @@ 15 ["abc","def"] \N 16 [null,true,false,100,6.18,"abc"] \N 17 [{"k1":"v41","k2":400},1,"a",3.14] \N -18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} \N +18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} 1 26 \N \N 27 {"k1":"v1","k2":200} \N 28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} \N @@ -1762,7 +1854,7 @@ 15 ["abc","def"] \N 16 [null,true,false,100,6.18,"abc"] \N 17 [{"k1":"v41","k2":400},1,"a",3.14] \N -18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} 1 +18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} \N 26 \N \N 27 {"k1":"v1","k2":200} \N 28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} \N @@ -1813,15 +1905,15 @@ 27 {"k1":"v1","k2":200} \N 28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} \N --- !select -- +-- !jsonb_extract_bigint_select -- 1 \N \N 2 null \N 3 true \N 4 false \N -5 100 \N -6 10000 \N -7 1000000000 \N -8 1152921504606846976 \N +5 100 100 +6 10000 10000 +7 1000000000 1000000000 +8 1152921504606846976 1152921504606846976 9 6.18 \N 10 "abcd" \N 11 {} \N @@ -1864,22 +1956,22 @@ 2 null \N 3 true \N 4 false \N -5 100 100.0 -6 10000 10000.0 -7 1000000000 1.0E9 -8 1152921504606846976 1.15292150460684698E18 -9 6.18 6.18 +5 100 \N +6 10000 \N +7 1000000000 \N +8 1152921504606846976 \N +9 6.18 \N 10 "abcd" \N 11 {} \N -12 {"k1":"v31","k2":300} \N +12 {"k1":"v31","k2":300} 300 13 [] \N 14 [123,456] \N 15 ["abc","def"] \N 16 [null,true,false,100,6.18,"abc"] \N 17 [{"k1":"v41","k2":400},1,"a",3.14] \N -18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} \N +18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} 300 26 \N \N -27 {"k1":"v1","k2":200} \N +27 {"k1":"v1","k2":200} 200 28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} \N -- !select -- @@ -1896,7 +1988,7 @@ 11 {} \N 12 {"k1":"v31","k2":300} \N 13 [] \N -14 [123,456] \N +14 [123,456] 123 15 ["abc","def"] \N 16 [null,true,false,100,6.18,"abc"] \N 17 [{"k1":"v41","k2":400},1,"a",3.14] \N @@ -1917,15 +2009,15 @@ 9 6.18 \N 10 "abcd" \N 11 {} \N -12 {"k1":"v31","k2":300} 300.0 +12 {"k1":"v31","k2":300} \N 13 [] \N -14 [123,456] \N +14 [123,456] 456 15 ["abc","def"] \N 16 [null,true,false,100,6.18,"abc"] \N -17 [{"k1":"v41","k2":400},1,"a",3.14] \N -18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} 300.0 +17 [{"k1":"v41","k2":400},1,"a",3.14] 1 +18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} \N 26 \N \N -27 {"k1":"v1","k2":200} 200.0 +27 {"k1":"v1","k2":200} \N 28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} \N -- !select -- @@ -1942,7 +2034,7 @@ 11 {} \N 12 {"k1":"v31","k2":300} \N 13 [] \N -14 [123,456] 123.0 +14 [123,456] \N 15 ["abc","def"] \N 16 [null,true,false,100,6.18,"abc"] \N 17 [{"k1":"v41","k2":400},1,"a",3.14] \N @@ -1965,10 +2057,10 @@ 11 {} \N 12 {"k1":"v31","k2":300} \N 13 [] \N -14 [123,456] 456.0 +14 [123,456] \N 15 ["abc","def"] \N -16 [null,true,false,100,6.18,"abc"] \N -17 [{"k1":"v41","k2":400},1,"a",3.14] 1.0 +16 [null,true,false,100,6.18,"abc"] 100 +17 [{"k1":"v41","k2":400},1,"a",3.14] \N 18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} \N 26 \N \N 27 {"k1":"v1","k2":200} \N @@ -2013,8 +2105,8 @@ 13 [] \N 14 [123,456] \N 15 ["abc","def"] \N -16 [null,true,false,100,6.18,"abc"] 100.0 -17 [{"k1":"v41","k2":400},1,"a",3.14] 3.14 +16 [null,true,false,100,6.18,"abc"] \N +17 [{"k1":"v41","k2":400},1,"a",3.14] \N 18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} \N 26 \N \N 27 {"k1":"v1","k2":200} \N @@ -2036,7 +2128,7 @@ 13 [] \N 14 [123,456] \N 15 ["abc","def"] \N -16 [null,true,false,100,6.18,"abc"] 6.18 +16 [null,true,false,100,6.18,"abc"] \N 17 [{"k1":"v41","k2":400},1,"a",3.14] \N 18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} \N 26 \N \N @@ -2130,7 +2222,7 @@ 15 ["abc","def"] \N 16 [null,true,false,100,6.18,"abc"] \N 17 [{"k1":"v41","k2":400},1,"a",3.14] \N -18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} \N +18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} 1 26 \N \N 27 {"k1":"v1","k2":200} \N 28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} \N @@ -2176,7 +2268,7 @@ 15 ["abc","def"] \N 16 [null,true,false,100,6.18,"abc"] \N 17 [{"k1":"v41","k2":400},1,"a",3.14] \N -18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} 1.0 +18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} \N 26 \N \N 27 {"k1":"v1","k2":200} \N 28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} \N @@ -2222,7 +2314,7 @@ 15 ["abc","def"] \N 16 [null,true,false,100,6.18,"abc"] \N 17 [{"k1":"v41","k2":400},1,"a",3.14] \N -18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} 3.14 +18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} \N 26 \N \N 27 {"k1":"v1","k2":200} \N 28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} \N @@ -2276,8 +2368,8 @@ -- !select -- 1 \N \N 2 null \N -3 true true -4 false false +3 true \N +4 false \N 5 100 \N 6 10000 \N 7 1000000000 \N @@ -2314,7 +2406,7 @@ 15 ["abc","def"] \N 16 [null,true,false,100,6.18,"abc"] \N 17 [{"k1":"v41","k2":400},1,"a",3.14] \N -18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} \N +18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} 1 26 \N \N 27 {"k1":"v1","k2":200} \N 28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} \N @@ -2381,7 +2473,7 @@ 13 [] \N 14 [123,456] \N 15 ["abc","def"] \N -16 [null,true,false,100,6.18,"abc"] true +16 [null,true,false,100,6.18,"abc"] \N 17 [{"k1":"v41","k2":400},1,"a",3.14] \N 18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} \N 26 \N \N @@ -2404,7 +2496,30 @@ 13 [] \N 14 [123,456] \N 15 ["abc","def"] \N -16 [null,true,false,100,6.18,"abc"] false +16 [null,true,false,100,6.18,"abc"] \N +17 [{"k1":"v41","k2":400},1,"a",3.14] \N +18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} \N +26 \N \N +27 {"k1":"v1","k2":200} \N +28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} \N + +-- !jsonb_extract_double_select -- +1 \N \N +2 null \N +3 true \N +4 false \N +5 100 100.0 +6 10000 10000.0 +7 1000000000 1.0E9 +8 1152921504606846976 1.15292150460684698E18 +9 6.18 6.18 +10 "abcd" \N +11 {} \N +12 {"k1":"v31","k2":300} \N +13 [] \N +14 [123,456] \N +15 ["abc","def"] \N +16 [null,true,false,100,6.18,"abc"] \N 17 [{"k1":"v41","k2":400},1,"a",3.14] \N 18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} \N 26 \N \N @@ -2446,12 +2561,58 @@ 9 6.18 \N 10 "abcd" \N 11 {} \N -12 {"k1":"v31","k2":300} \N +12 {"k1":"v31","k2":300} 300.0 13 [] \N 14 [123,456] \N 15 ["abc","def"] \N 16 [null,true,false,100,6.18,"abc"] \N 17 [{"k1":"v41","k2":400},1,"a",3.14] \N +18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} 300.0 +26 \N \N +27 {"k1":"v1","k2":200} 200.0 +28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} \N + +-- !select -- +1 \N \N +2 null \N +3 true \N +4 false \N +5 100 \N +6 10000 \N +7 1000000000 \N +8 1152921504606846976 \N +9 6.18 \N +10 "abcd" \N +11 {} \N +12 {"k1":"v31","k2":300} \N +13 [] \N +14 [123,456] 123.0 +15 ["abc","def"] \N +16 [null,true,false,100,6.18,"abc"] \N +17 [{"k1":"v41","k2":400},1,"a",3.14] \N +18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} \N +26 \N \N +27 {"k1":"v1","k2":200} \N +28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} \N + +-- !select -- +1 \N \N +2 null \N +3 true \N +4 false \N +5 100 \N +6 10000 \N +7 1000000000 \N +8 1152921504606846976 \N +9 6.18 \N +10 "abcd" \N +11 {} \N +12 {"k1":"v31","k2":300} \N +13 [] \N +14 [123,456] 456.0 +15 ["abc","def"] \N +16 [null,true,false,100,6.18,"abc"] \N +17 [{"k1":"v41","k2":400},1,"a",3.14] 1.0 18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} \N 26 \N \N 27 {"k1":"v1","k2":200} \N @@ -2496,7 +2657,30 @@ 13 [] \N 14 [123,456] \N 15 ["abc","def"] \N -16 [null,true,false,100,6.18,"abc"] \N +16 [null,true,false,100,6.18,"abc"] 100.0 +17 [{"k1":"v41","k2":400},1,"a",3.14] 3.14 +18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} \N +26 \N \N +27 {"k1":"v1","k2":200} \N +28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} \N + +-- !select -- +1 \N \N +2 null \N +3 true \N +4 false \N +5 100 \N +6 10000 \N +7 1000000000 \N +8 1152921504606846976 \N +9 6.18 \N +10 "abcd" \N +11 {} \N +12 {"k1":"v31","k2":300} \N +13 [] \N +14 [123,456] \N +15 ["abc","def"] \N +16 [null,true,false,100,6.18,"abc"] 6.18 17 [{"k1":"v41","k2":400},1,"a",3.14] \N 18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} \N 26 \N \N @@ -2636,7 +2820,7 @@ 15 ["abc","def"] \N 16 [null,true,false,100,6.18,"abc"] \N 17 [{"k1":"v41","k2":400},1,"a",3.14] \N -18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} \N +18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} 1.0 26 \N \N 27 {"k1":"v1","k2":200} \N 28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} \N @@ -2664,6 +2848,29 @@ 27 {"k1":"v1","k2":200} \N 28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} \N +-- !select -- +1 \N \N +2 null \N +3 true \N +4 false \N +5 100 \N +6 10000 \N +7 1000000000 \N +8 1152921504606846976 \N +9 6.18 \N +10 "abcd" \N +11 {} \N +12 {"k1":"v31","k2":300} \N +13 [] \N +14 [123,456] \N +15 ["abc","def"] \N +16 [null,true,false,100,6.18,"abc"] \N +17 [{"k1":"v41","k2":400},1,"a",3.14] \N +18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} 3.14 +26 \N \N +27 {"k1":"v1","k2":200} \N +28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} \N + -- !select -- 1 \N \N 2 null \N @@ -2689,26 +2896,26 @@ -- !select -- 1 \N \N -2 null true -3 true false -4 false false -5 100 false -6 10000 false -7 1000000000 false -8 1152921504606846976 false -9 6.18 false -10 "abcd" false -11 {} false -12 {"k1":"v31","k2":300} false -13 [] false -14 [123,456] false -15 ["abc","def"] false -16 [null,true,false,100,6.18,"abc"] false -17 [{"k1":"v41","k2":400},1,"a",3.14] false -18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} false +2 null \N +3 true \N +4 false \N +5 100 \N +6 10000 \N +7 1000000000 \N +8 1152921504606846976 \N +9 6.18 \N +10 "abcd" \N +11 {} \N +12 {"k1":"v31","k2":300} \N +13 [] \N +14 [123,456] \N +15 ["abc","def"] \N +16 [null,true,false,100,6.18,"abc"] \N +17 [{"k1":"v41","k2":400},1,"a",3.14] \N +18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} \N 26 \N \N -27 {"k1":"v1","k2":200} false -28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} false +27 {"k1":"v1","k2":200} \N +28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} \N -- !select -- 1 \N \N @@ -2722,15 +2929,15 @@ 9 6.18 \N 10 "abcd" \N 11 {} \N -12 {"k1":"v31","k2":300} false +12 {"k1":"v31","k2":300} \N 13 [] \N 14 [123,456] \N 15 ["abc","def"] \N 16 [null,true,false,100,6.18,"abc"] \N 17 [{"k1":"v41","k2":400},1,"a",3.14] \N -18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} false +18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} 3.14 26 \N \N -27 {"k1":"v1","k2":200} false +27 {"k1":"v1","k2":200} \N 28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} \N -- !select -- @@ -2745,15 +2952,15 @@ 9 6.18 \N 10 "abcd" \N 11 {} \N -12 {"k1":"v31","k2":300} false +12 {"k1":"v31","k2":300} \N 13 [] \N 14 [123,456] \N 15 ["abc","def"] \N 16 [null,true,false,100,6.18,"abc"] \N 17 [{"k1":"v41","k2":400},1,"a",3.14] \N -18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} false +18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} 3.14 26 \N \N -27 {"k1":"v1","k2":200} false +27 {"k1":"v1","k2":200} \N 28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} \N -- !select -- @@ -2770,10 +2977,10 @@ 11 {} \N 12 {"k1":"v31","k2":300} \N 13 [] \N -14 [123,456] false -15 ["abc","def"] false -16 [null,true,false,100,6.18,"abc"] true -17 [{"k1":"v41","k2":400},1,"a",3.14] false +14 [123,456] \N +15 ["abc","def"] \N +16 [null,true,false,100,6.18,"abc"] \N +17 [{"k1":"v41","k2":400},1,"a",3.14] \N 18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} \N 26 \N \N 27 {"k1":"v1","k2":200} \N @@ -2793,11 +3000,11 @@ 11 {} \N 12 {"k1":"v31","k2":300} \N 13 [] \N -14 [123,456] false -15 ["abc","def"] false -16 [null,true,false,100,6.18,"abc"] false -17 [{"k1":"v41","k2":400},1,"a",3.14] false -18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} \N +14 [123,456] \N +15 ["abc","def"] \N +16 [null,true,false,100,6.18,"abc"] \N +17 [{"k1":"v41","k2":400},1,"a",3.14] \N +18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} 1.0 26 \N \N 27 {"k1":"v1","k2":200} \N 28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} \N @@ -2818,8 +3025,8 @@ 13 [] \N 14 [123,456] \N 15 ["abc","def"] \N -16 [null,true,false,100,6.18,"abc"] false -17 [{"k1":"v41","k2":400},1,"a",3.14] false +16 [null,true,false,100,6.18,"abc"] \N +17 [{"k1":"v41","k2":400},1,"a",3.14] \N 18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} \N 26 \N \N 27 {"k1":"v1","k2":200} \N @@ -2841,8 +3048,8 @@ 13 [] \N 14 [123,456] \N 15 ["abc","def"] \N -16 [null,true,false,100,6.18,"abc"] false -17 [{"k1":"v41","k2":400},1,"a",3.14] false +16 [null,true,false,100,6.18,"abc"] \N +17 [{"k1":"v41","k2":400},1,"a",3.14] \N 18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} \N 26 \N \N 27 {"k1":"v1","k2":200} \N @@ -2864,9 +3071,9 @@ 13 [] \N 14 [123,456] \N 15 ["abc","def"] \N -16 [null,true,false,100,6.18,"abc"] false +16 [null,true,false,100,6.18,"abc"] \N 17 [{"k1":"v41","k2":400},1,"a",3.14] \N -18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} \N +18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} 3.14 26 \N \N 27 {"k1":"v1","k2":200} \N 28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} \N @@ -2887,7 +3094,30 @@ 13 [] \N 14 [123,456] \N 15 ["abc","def"] \N -16 [null,true,false,100,6.18,"abc"] false +16 [null,true,false,100,6.18,"abc"] \N +17 [{"k1":"v41","k2":400},1,"a",3.14] \N +18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} \N +26 \N \N +27 {"k1":"v1","k2":200} \N +28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} \N + +-- !jsonb_extract_bool_select -- +1 \N \N +2 null \N +3 true true +4 false false +5 100 \N +6 10000 \N +7 1000000000 \N +8 1152921504606846976 \N +9 6.18 \N +10 "abcd" \N +11 {} \N +12 {"k1":"v31","k2":300} \N +13 [] \N +14 [123,456] \N +15 ["abc","def"] \N +16 [null,true,false,100,6.18,"abc"] \N 17 [{"k1":"v41","k2":400},1,"a",3.14] \N 18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} \N 26 \N \N @@ -2958,7 +3188,7 @@ 15 ["abc","def"] \N 16 [null,true,false,100,6.18,"abc"] \N 17 [{"k1":"v41","k2":400},1,"a",3.14] \N -18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} false +18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} \N 26 \N \N 27 {"k1":"v1","k2":200} \N 28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} \N @@ -2979,9 +3209,9 @@ 13 [] \N 14 [123,456] \N 15 ["abc","def"] \N -16 [null,true,false,100,6.18,"abc"] \N +16 [null,true,false,100,6.18,"abc"] true 17 [{"k1":"v41","k2":400},1,"a",3.14] \N -18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} false +18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} \N 26 \N \N 27 {"k1":"v1","k2":200} \N 28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} \N @@ -3002,9 +3232,9 @@ 13 [] \N 14 [123,456] \N 15 ["abc","def"] \N -16 [null,true,false,100,6.18,"abc"] \N +16 [null,true,false,100,6.18,"abc"] false 17 [{"k1":"v41","k2":400},1,"a",3.14] \N -18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} false +18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} \N 26 \N \N 27 {"k1":"v1","k2":200} \N 28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} \N @@ -3027,7 +3257,7 @@ 15 ["abc","def"] \N 16 [null,true,false,100,6.18,"abc"] \N 17 [{"k1":"v41","k2":400},1,"a",3.14] \N -18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} false +18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} \N 26 \N \N 27 {"k1":"v1","k2":200} \N 28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} \N @@ -3050,7 +3280,7 @@ 15 ["abc","def"] \N 16 [null,true,false,100,6.18,"abc"] \N 17 [{"k1":"v41","k2":400},1,"a",3.14] \N -18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} false +18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} \N 26 \N \N 27 {"k1":"v1","k2":200} \N 28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} \N @@ -3103,26 +3333,1176 @@ -- !select -- 1 \N \N -2 null true -3 true true -4 false true -5 100 true -6 10000 true -7 1000000000 true -8 1152921504606846976 true -9 6.18 true -10 "abcd" true -11 {} true -12 {"k1":"v31","k2":300} true -13 [] true -14 [123,456] true -15 ["abc","def"] true +2 null \N +3 true \N +4 false \N +5 100 \N +6 10000 \N +7 1000000000 \N +8 1152921504606846976 \N +9 6.18 \N +10 "abcd" \N +11 {} \N +12 {"k1":"v31","k2":300} \N +13 [] \N +14 [123,456] \N +15 ["abc","def"] \N +16 [null,true,false,100,6.18,"abc"] \N +17 [{"k1":"v41","k2":400},1,"a",3.14] \N +18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} \N +26 \N \N +27 {"k1":"v1","k2":200} \N +28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} \N + +-- !select -- +1 \N \N +2 null \N +3 true \N +4 false \N +5 100 \N +6 10000 \N +7 1000000000 \N +8 1152921504606846976 \N +9 6.18 \N +10 "abcd" \N +11 {} \N +12 {"k1":"v31","k2":300} \N +13 [] \N +14 [123,456] \N +15 ["abc","def"] \N +16 [null,true,false,100,6.18,"abc"] \N +17 [{"k1":"v41","k2":400},1,"a",3.14] \N +18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} \N +26 \N \N +27 {"k1":"v1","k2":200} \N +28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} \N + +-- !select -- +1 \N \N +2 null \N +3 true \N +4 false \N +5 100 \N +6 10000 \N +7 1000000000 \N +8 1152921504606846976 \N +9 6.18 \N +10 "abcd" \N +11 {} \N +12 {"k1":"v31","k2":300} \N +13 [] \N +14 [123,456] \N +15 ["abc","def"] \N +16 [null,true,false,100,6.18,"abc"] \N +17 [{"k1":"v41","k2":400},1,"a",3.14] \N +18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} \N +26 \N \N +27 {"k1":"v1","k2":200} \N +28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} \N + +-- !select -- +1 \N \N +2 null \N +3 true \N +4 false \N +5 100 \N +6 10000 \N +7 1000000000 \N +8 1152921504606846976 \N +9 6.18 \N +10 "abcd" \N +11 {} \N +12 {"k1":"v31","k2":300} \N +13 [] \N +14 [123,456] \N +15 ["abc","def"] \N +16 [null,true,false,100,6.18,"abc"] \N +17 [{"k1":"v41","k2":400},1,"a",3.14] \N +18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} \N +26 \N \N +27 {"k1":"v1","k2":200} \N +28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} \N + +-- !select -- +1 \N \N +2 null \N +3 true \N +4 false \N +5 100 \N +6 10000 \N +7 1000000000 \N +8 1152921504606846976 \N +9 6.18 \N +10 "abcd" \N +11 {} \N +12 {"k1":"v31","k2":300} \N +13 [] \N +14 [123,456] \N +15 ["abc","def"] \N +16 [null,true,false,100,6.18,"abc"] \N +17 [{"k1":"v41","k2":400},1,"a",3.14] \N +18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} \N +26 \N \N +27 {"k1":"v1","k2":200} \N +28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} \N + +-- !select -- +1 \N \N +2 null \N +3 true \N +4 false \N +5 100 \N +6 10000 \N +7 1000000000 \N +8 1152921504606846976 \N +9 6.18 \N +10 "abcd" \N +11 {} \N +12 {"k1":"v31","k2":300} \N +13 [] \N +14 [123,456] \N +15 ["abc","def"] \N +16 [null,true,false,100,6.18,"abc"] \N +17 [{"k1":"v41","k2":400},1,"a",3.14] \N +18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} \N +26 \N \N +27 {"k1":"v1","k2":200} \N +28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} \N + +-- !select -- +1 \N \N +2 null \N +3 true \N +4 false \N +5 100 \N +6 10000 \N +7 1000000000 \N +8 1152921504606846976 \N +9 6.18 \N +10 "abcd" \N +11 {} \N +12 {"k1":"v31","k2":300} \N +13 [] \N +14 [123,456] \N +15 ["abc","def"] \N +16 [null,true,false,100,6.18,"abc"] \N +17 [{"k1":"v41","k2":400},1,"a",3.14] \N +18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} \N +26 \N \N +27 {"k1":"v1","k2":200} \N +28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} \N + +-- !select -- +1 \N \N +2 null \N +3 true \N +4 false \N +5 100 \N +6 10000 \N +7 1000000000 \N +8 1152921504606846976 \N +9 6.18 \N +10 "abcd" \N +11 {} \N +12 {"k1":"v31","k2":300} \N +13 [] \N +14 [123,456] \N +15 ["abc","def"] \N +16 [null,true,false,100,6.18,"abc"] \N +17 [{"k1":"v41","k2":400},1,"a",3.14] \N +18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} \N +26 \N \N +27 {"k1":"v1","k2":200} \N +28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} \N + +-- !select -- +1 \N \N +2 null \N +3 true \N +4 false \N +5 100 \N +6 10000 \N +7 1000000000 \N +8 1152921504606846976 \N +9 6.18 \N +10 "abcd" \N +11 {} \N +12 {"k1":"v31","k2":300} \N +13 [] \N +14 [123,456] \N +15 ["abc","def"] \N +16 [null,true,false,100,6.18,"abc"] \N +17 [{"k1":"v41","k2":400},1,"a",3.14] \N +18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} \N +26 \N \N +27 {"k1":"v1","k2":200} \N +28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} \N + +-- !select -- +1 \N \N +2 null \N +3 true \N +4 false \N +5 100 \N +6 10000 \N +7 1000000000 \N +8 1152921504606846976 \N +9 6.18 \N +10 "abcd" \N +11 {} \N +12 {"k1":"v31","k2":300} \N +13 [] \N +14 [123,456] \N +15 ["abc","def"] \N +16 [null,true,false,100,6.18,"abc"] \N +17 [{"k1":"v41","k2":400},1,"a",3.14] \N +18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} \N +26 \N \N +27 {"k1":"v1","k2":200} \N +28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} \N + +-- !select -- +1 \N \N +2 null \N +3 true \N +4 false \N +5 100 \N +6 10000 \N +7 1000000000 \N +8 1152921504606846976 \N +9 6.18 \N +10 "abcd" \N +11 {} \N +12 {"k1":"v31","k2":300} \N +13 [] \N +14 [123,456] \N +15 ["abc","def"] \N +16 [null,true,false,100,6.18,"abc"] \N +17 [{"k1":"v41","k2":400},1,"a",3.14] \N +18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} \N +26 \N \N +27 {"k1":"v1","k2":200} \N +28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} \N + +-- !select -- +1 \N \N +2 null \N +3 true \N +4 false \N +5 100 \N +6 10000 \N +7 1000000000 \N +8 1152921504606846976 \N +9 6.18 \N +10 "abcd" \N +11 {} \N +12 {"k1":"v31","k2":300} \N +13 [] \N +14 [123,456] \N +15 ["abc","def"] \N +16 [null,true,false,100,6.18,"abc"] \N +17 [{"k1":"v41","k2":400},1,"a",3.14] \N +18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} \N +26 \N \N +27 {"k1":"v1","k2":200} \N +28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} \N + +-- !select -- +1 \N \N +2 null \N +3 true \N +4 false \N +5 100 \N +6 10000 \N +7 1000000000 \N +8 1152921504606846976 \N +9 6.18 \N +10 "abcd" \N +11 {} \N +12 {"k1":"v31","k2":300} \N +13 [] \N +14 [123,456] \N +15 ["abc","def"] \N +16 [null,true,false,100,6.18,"abc"] \N +17 [{"k1":"v41","k2":400},1,"a",3.14] \N +18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} \N +26 \N \N +27 {"k1":"v1","k2":200} \N +28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} \N + +-- !select -- +1 \N \N +2 null \N +3 true \N +4 false \N +5 100 \N +6 10000 \N +7 1000000000 \N +8 1152921504606846976 \N +9 6.18 \N +10 "abcd" \N +11 {} \N +12 {"k1":"v31","k2":300} \N +13 [] \N +14 [123,456] \N +15 ["abc","def"] \N +16 [null,true,false,100,6.18,"abc"] \N +17 [{"k1":"v41","k2":400},1,"a",3.14] \N +18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} \N +26 \N \N +27 {"k1":"v1","k2":200} \N +28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} \N + +-- !select -- +1 \N \N +2 null \N +3 true \N +4 false \N +5 100 \N +6 10000 \N +7 1000000000 \N +8 1152921504606846976 \N +9 6.18 \N +10 "abcd" \N +11 {} \N +12 {"k1":"v31","k2":300} \N +13 [] \N +14 [123,456] \N +15 ["abc","def"] \N +16 [null,true,false,100,6.18,"abc"] \N +17 [{"k1":"v41","k2":400},1,"a",3.14] \N +18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} \N +26 \N \N +27 {"k1":"v1","k2":200} \N +28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} \N + +-- !select -- +1 \N \N +2 null \N +3 true \N +4 false \N +5 100 \N +6 10000 \N +7 1000000000 \N +8 1152921504606846976 \N +9 6.18 \N +10 "abcd" \N +11 {} \N +12 {"k1":"v31","k2":300} \N +13 [] \N +14 [123,456] \N +15 ["abc","def"] \N +16 [null,true,false,100,6.18,"abc"] \N +17 [{"k1":"v41","k2":400},1,"a",3.14] \N +18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} \N +26 \N \N +27 {"k1":"v1","k2":200} \N +28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} \N + +-- !jsonb_extract_isnull_select -- +1 \N \N +2 null true +3 true false +4 false false +5 100 false +6 10000 false +7 1000000000 false +8 1152921504606846976 false +9 6.18 false +10 "abcd" false +11 {} false +12 {"k1":"v31","k2":300} false +13 [] false +14 [123,456] false +15 ["abc","def"] false +16 [null,true,false,100,6.18,"abc"] false +17 [{"k1":"v41","k2":400},1,"a",3.14] false +18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} false +26 \N \N +27 {"k1":"v1","k2":200} false +28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} false + +-- !select -- +1 \N \N +2 null \N +3 true \N +4 false \N +5 100 \N +6 10000 \N +7 1000000000 \N +8 1152921504606846976 \N +9 6.18 \N +10 "abcd" \N +11 {} \N +12 {"k1":"v31","k2":300} false +13 [] \N +14 [123,456] \N +15 ["abc","def"] \N +16 [null,true,false,100,6.18,"abc"] \N +17 [{"k1":"v41","k2":400},1,"a",3.14] \N +18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} false +26 \N \N +27 {"k1":"v1","k2":200} false +28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} \N + +-- !select -- +1 \N \N +2 null \N +3 true \N +4 false \N +5 100 \N +6 10000 \N +7 1000000000 \N +8 1152921504606846976 \N +9 6.18 \N +10 "abcd" \N +11 {} \N +12 {"k1":"v31","k2":300} false +13 [] \N +14 [123,456] \N +15 ["abc","def"] \N +16 [null,true,false,100,6.18,"abc"] \N +17 [{"k1":"v41","k2":400},1,"a",3.14] \N +18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} false +26 \N \N +27 {"k1":"v1","k2":200} false +28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} \N + +-- !select -- +1 \N \N +2 null \N +3 true \N +4 false \N +5 100 \N +6 10000 \N +7 1000000000 \N +8 1152921504606846976 \N +9 6.18 \N +10 "abcd" \N +11 {} false +12 {"k1":"v31","k2":300} false +13 [] \N +14 [123,456] false +15 ["abc","def"] false +16 [null,true,false,100,6.18,"abc"] true +17 [{"k1":"v41","k2":400},1,"a",3.14] false +18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} false +26 \N \N +27 {"k1":"v1","k2":200} false +28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} false + +-- !select -- +1 \N \N +2 null \N +3 true \N +4 false \N +5 100 \N +6 10000 \N +7 1000000000 \N +8 1152921504606846976 \N +9 6.18 \N +10 "abcd" \N +11 {} \N +12 {"k1":"v31","k2":300} \N +13 [] \N +14 [123,456] false +15 ["abc","def"] false +16 [null,true,false,100,6.18,"abc"] false +17 [{"k1":"v41","k2":400},1,"a",3.14] false +18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} \N +26 \N \N +27 {"k1":"v1","k2":200} \N +28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} \N + +-- !select -- +1 \N \N +2 null \N +3 true \N +4 false \N +5 100 \N +6 10000 \N +7 1000000000 \N +8 1152921504606846976 \N +9 6.18 \N +10 "abcd" \N +11 {} \N +12 {"k1":"v31","k2":300} \N +13 [] \N +14 [123,456] \N +15 ["abc","def"] \N +16 [null,true,false,100,6.18,"abc"] false +17 [{"k1":"v41","k2":400},1,"a",3.14] false +18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} \N +26 \N \N +27 {"k1":"v1","k2":200} \N +28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} \N + +-- !select -- +1 \N \N +2 null \N +3 true \N +4 false \N +5 100 \N +6 10000 \N +7 1000000000 \N +8 1152921504606846976 \N +9 6.18 \N +10 "abcd" \N +11 {} \N +12 {"k1":"v31","k2":300} \N +13 [] \N +14 [123,456] \N +15 ["abc","def"] \N +16 [null,true,false,100,6.18,"abc"] false +17 [{"k1":"v41","k2":400},1,"a",3.14] false +18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} \N +26 \N \N +27 {"k1":"v1","k2":200} \N +28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} \N + +-- !select -- +1 \N \N +2 null \N +3 true \N +4 false \N +5 100 \N +6 10000 \N +7 1000000000 \N +8 1152921504606846976 \N +9 6.18 \N +10 "abcd" \N +11 {} \N +12 {"k1":"v31","k2":300} \N +13 [] \N +14 [123,456] \N +15 ["abc","def"] \N +16 [null,true,false,100,6.18,"abc"] false +17 [{"k1":"v41","k2":400},1,"a",3.14] \N +18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} \N +26 \N \N +27 {"k1":"v1","k2":200} \N +28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} \N + +-- !select -- +1 \N \N +2 null \N +3 true \N +4 false \N +5 100 \N +6 10000 \N +7 1000000000 \N +8 1152921504606846976 \N +9 6.18 \N +10 "abcd" \N +11 {} \N +12 {"k1":"v31","k2":300} \N +13 [] \N +14 [123,456] \N +15 ["abc","def"] \N +16 [null,true,false,100,6.18,"abc"] false +17 [{"k1":"v41","k2":400},1,"a",3.14] \N +18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} \N +26 \N \N +27 {"k1":"v1","k2":200} \N +28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} \N + +-- !select -- +1 \N \N +2 null \N +3 true \N +4 false \N +5 100 \N +6 10000 \N +7 1000000000 \N +8 1152921504606846976 \N +9 6.18 \N +10 "abcd" \N +11 {} \N +12 {"k1":"v31","k2":300} \N +13 [] \N +14 [123,456] \N +15 ["abc","def"] \N +16 [null,true,false,100,6.18,"abc"] \N +17 [{"k1":"v41","k2":400},1,"a",3.14] \N +18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} \N +26 \N \N +27 {"k1":"v1","k2":200} \N +28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} \N + +-- !select -- +1 \N \N +2 null \N +3 true \N +4 false \N +5 100 \N +6 10000 \N +7 1000000000 \N +8 1152921504606846976 \N +9 6.18 \N +10 "abcd" \N +11 {} \N +12 {"k1":"v31","k2":300} \N +13 [] \N +14 [123,456] \N +15 ["abc","def"] \N +16 [null,true,false,100,6.18,"abc"] \N +17 [{"k1":"v41","k2":400},1,"a",3.14] \N +18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} \N +26 \N \N +27 {"k1":"v1","k2":200} \N +28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} \N + +-- !select -- +1 \N \N +2 null \N +3 true \N +4 false \N +5 100 \N +6 10000 \N +7 1000000000 \N +8 1152921504606846976 \N +9 6.18 \N +10 "abcd" \N +11 {} \N +12 {"k1":"v31","k2":300} \N +13 [] \N +14 [123,456] \N +15 ["abc","def"] \N +16 [null,true,false,100,6.18,"abc"] \N +17 [{"k1":"v41","k2":400},1,"a",3.14] \N +18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} false +26 \N \N +27 {"k1":"v1","k2":200} \N +28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} \N + +-- !select -- +1 \N \N +2 null \N +3 true \N +4 false \N +5 100 \N +6 10000 \N +7 1000000000 \N +8 1152921504606846976 \N +9 6.18 \N +10 "abcd" \N +11 {} \N +12 {"k1":"v31","k2":300} \N +13 [] \N +14 [123,456] \N +15 ["abc","def"] \N +16 [null,true,false,100,6.18,"abc"] \N +17 [{"k1":"v41","k2":400},1,"a",3.14] \N +18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} false +26 \N \N +27 {"k1":"v1","k2":200} \N +28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} \N + +-- !select -- +1 \N \N +2 null \N +3 true \N +4 false \N +5 100 \N +6 10000 \N +7 1000000000 \N +8 1152921504606846976 \N +9 6.18 \N +10 "abcd" \N +11 {} \N +12 {"k1":"v31","k2":300} \N +13 [] \N +14 [123,456] \N +15 ["abc","def"] \N +16 [null,true,false,100,6.18,"abc"] \N +17 [{"k1":"v41","k2":400},1,"a",3.14] \N +18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} false +26 \N \N +27 {"k1":"v1","k2":200} \N +28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} \N + +-- !select -- +1 \N \N +2 null \N +3 true \N +4 false \N +5 100 \N +6 10000 \N +7 1000000000 \N +8 1152921504606846976 \N +9 6.18 \N +10 "abcd" \N +11 {} \N +12 {"k1":"v31","k2":300} \N +13 [] \N +14 [123,456] \N +15 ["abc","def"] \N +16 [null,true,false,100,6.18,"abc"] \N +17 [{"k1":"v41","k2":400},1,"a",3.14] \N +18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} false +26 \N \N +27 {"k1":"v1","k2":200} \N +28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} \N + +-- !select -- +1 \N \N +2 null \N +3 true \N +4 false \N +5 100 \N +6 10000 \N +7 1000000000 \N +8 1152921504606846976 \N +9 6.18 \N +10 "abcd" \N +11 {} \N +12 {"k1":"v31","k2":300} \N +13 [] \N +14 [123,456] \N +15 ["abc","def"] \N +16 [null,true,false,100,6.18,"abc"] \N +17 [{"k1":"v41","k2":400},1,"a",3.14] \N +18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} false +26 \N \N +27 {"k1":"v1","k2":200} \N +28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} \N + +-- !select -- +1 \N \N +2 null \N +3 true \N +4 false \N +5 100 \N +6 10000 \N +7 1000000000 \N +8 1152921504606846976 \N +9 6.18 \N +10 "abcd" \N +11 {} \N +12 {"k1":"v31","k2":300} \N +13 [] \N +14 [123,456] \N +15 ["abc","def"] \N +16 [null,true,false,100,6.18,"abc"] \N +17 [{"k1":"v41","k2":400},1,"a",3.14] \N +18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} \N +26 \N \N +27 {"k1":"v1","k2":200} \N +28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} \N + +-- !select -- +1 \N \N +2 null \N +3 true \N +4 false \N +5 100 \N +6 10000 \N +7 1000000000 \N +8 1152921504606846976 \N +9 6.18 \N +10 "abcd" \N +11 {} \N +12 {"k1":"v31","k2":300} \N +13 [] \N +14 [123,456] \N +15 ["abc","def"] \N +16 [null,true,false,100,6.18,"abc"] \N +17 [{"k1":"v41","k2":400},1,"a",3.14] \N +18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} \N +26 \N \N +27 {"k1":"v1","k2":200} \N +28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} \N + +-- !select -- +1 \N \N +2 null \N +3 true \N +4 false \N +5 100 \N +6 10000 \N +7 1000000000 \N +8 1152921504606846976 \N +9 6.18 \N +10 "abcd" \N +11 {} \N +12 {"k1":"v31","k2":300} \N +13 [] \N +14 [123,456] \N +15 ["abc","def"] \N +16 [null,true,false,100,6.18,"abc"] \N +17 [{"k1":"v41","k2":400},1,"a",3.14] \N +18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} false +26 \N \N +27 {"k1":"v1","k2":200} \N +28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} \N + +-- !select -- +1 \N \N +2 null \N +3 true \N +4 false \N +5 100 \N +6 10000 \N +7 1000000000 \N +8 1152921504606846976 \N +9 6.18 \N +10 "abcd" \N +11 {} \N +12 {"k1":"v31","k2":300} \N +13 [] \N +14 [123,456] \N +15 ["abc","def"] \N +16 [null,true,false,100,6.18,"abc"] \N +17 [{"k1":"v41","k2":400},1,"a",3.14] \N +18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} false +26 \N \N +27 {"k1":"v1","k2":200} \N +28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} \N + +-- !select -- +1 \N \N +2 null \N +3 true \N +4 false \N +5 100 \N +6 10000 \N +7 1000000000 \N +8 1152921504606846976 \N +9 6.18 \N +10 "abcd" \N +11 {} \N +12 {"k1":"v31","k2":300} \N +13 [] \N +14 [123,456] \N +15 ["abc","def"] \N +16 [null,true,false,100,6.18,"abc"] \N +17 [{"k1":"v41","k2":400},1,"a",3.14] \N +18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} false +26 \N \N +27 {"k1":"v1","k2":200} \N +28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} \N + +-- !select -- +1 \N \N +2 null \N +3 true \N +4 false \N +5 100 \N +6 10000 \N +7 1000000000 \N +8 1152921504606846976 \N +9 6.18 \N +10 "abcd" \N +11 {} \N +12 {"k1":"v31","k2":300} \N +13 [] \N +14 [123,456] \N +15 ["abc","def"] \N +16 [null,true,false,100,6.18,"abc"] \N +17 [{"k1":"v41","k2":400},1,"a",3.14] \N +18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} false +26 \N \N +27 {"k1":"v1","k2":200} \N +28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} \N + +-- !select -- +1 \N \N +2 null \N +3 true \N +4 false \N +5 100 \N +6 10000 \N +7 1000000000 \N +8 1152921504606846976 \N +9 6.18 \N +10 "abcd" \N +11 {} \N +12 {"k1":"v31","k2":300} \N +13 [] \N +14 [123,456] \N +15 ["abc","def"] \N +16 [null,true,false,100,6.18,"abc"] \N +17 [{"k1":"v41","k2":400},1,"a",3.14] \N +18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} \N +26 \N \N +27 {"k1":"v1","k2":200} \N +28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} \N + +-- !select -- +1 \N \N +2 null \N +3 true \N +4 false \N +5 100 \N +6 10000 \N +7 1000000000 \N +8 1152921504606846976 \N +9 6.18 \N +10 "abcd" \N +11 {} \N +12 {"k1":"v31","k2":300} \N +13 [] \N +14 [123,456] \N +15 ["abc","def"] \N +16 [null,true,false,100,6.18,"abc"] \N +17 [{"k1":"v41","k2":400},1,"a",3.14] \N +18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} false +26 \N \N +27 {"k1":"v1","k2":200} \N +28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} \N + +-- !select -- +1 \N \N +2 null \N +3 true \N +4 false \N +5 100 \N +6 10000 \N +7 1000000000 \N +8 1152921504606846976 \N +9 6.18 \N +10 "abcd" \N +11 {} \N +12 {"k1":"v31","k2":300} \N +13 [] \N +14 [123,456] \N +15 ["abc","def"] \N +16 [null,true,false,100,6.18,"abc"] \N +17 [{"k1":"v41","k2":400},1,"a",3.14] \N +18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} false +26 \N \N +27 {"k1":"v1","k2":200} \N +28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} \N + +-- !select -- +1 \N \N +2 null \N +3 true \N +4 false \N +5 100 \N +6 10000 \N +7 1000000000 \N +8 1152921504606846976 \N +9 6.18 \N +10 "abcd" \N +11 {} \N +12 {"k1":"v31","k2":300} \N +13 [] \N +14 [123,456] \N +15 ["abc","def"] \N +16 [null,true,false,100,6.18,"abc"] \N +17 [{"k1":"v41","k2":400},1,"a",3.14] \N +18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} \N +26 \N \N +27 {"k1":"v1","k2":200} \N +28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} \N + +-- !jsonb_exists_path_select -- +1 \N \N +2 null true +3 true true +4 false true +5 100 true +6 10000 true +7 1000000000 true +8 1152921504606846976 true +9 6.18 true +10 "abcd" true +11 {} true +12 {"k1":"v31","k2":300} true +13 [] true +14 [123,456] true +15 ["abc","def"] true +16 [null,true,false,100,6.18,"abc"] true +17 [{"k1":"v41","k2":400},1,"a",3.14] true +18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} true +26 \N \N +27 {"k1":"v1","k2":200} true +28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} true + +-- !select -- +1 \N \N +2 null false +3 true false +4 false false +5 100 false +6 10000 false +7 1000000000 false +8 1152921504606846976 false +9 6.18 false +10 "abcd" false +11 {} false +12 {"k1":"v31","k2":300} true +13 [] false +14 [123,456] false +15 ["abc","def"] false +16 [null,true,false,100,6.18,"abc"] false +17 [{"k1":"v41","k2":400},1,"a",3.14] false +18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} true +26 \N \N +27 {"k1":"v1","k2":200} true +28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} false + +-- !select -- +1 \N \N +2 null false +3 true false +4 false false +5 100 false +6 10000 false +7 1000000000 false +8 1152921504606846976 false +9 6.18 false +10 "abcd" false +11 {} false +12 {"k1":"v31","k2":300} true +13 [] false +14 [123,456] false +15 ["abc","def"] false +16 [null,true,false,100,6.18,"abc"] false +17 [{"k1":"v41","k2":400},1,"a",3.14] false +18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} true +26 \N \N +27 {"k1":"v1","k2":200} true +28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} false + +-- !select -- +1 \N \N +2 null false +3 true false +4 false false +5 100 false +6 10000 false +7 1000000000 false +8 1152921504606846976 false +9 6.18 false +10 "abcd" false +11 {} true +12 {"k1":"v31","k2":300} true +13 [] false +14 [123,456] true +15 ["abc","def"] true 16 [null,true,false,100,6.18,"abc"] true 17 [{"k1":"v41","k2":400},1,"a",3.14] true 18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} true 26 \N \N -27 {"k1":"v1","k2":200} true -28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} true +27 {"k1":"v1","k2":200} true +28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} true + +-- !select -- +1 \N \N +2 null false +3 true false +4 false false +5 100 false +6 10000 false +7 1000000000 false +8 1152921504606846976 false +9 6.18 false +10 "abcd" false +11 {} false +12 {"k1":"v31","k2":300} false +13 [] false +14 [123,456] true +15 ["abc","def"] true +16 [null,true,false,100,6.18,"abc"] true +17 [{"k1":"v41","k2":400},1,"a",3.14] true +18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} false +26 \N \N +27 {"k1":"v1","k2":200} false +28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} false + +-- !select -- +1 \N \N +2 null false +3 true false +4 false false +5 100 false +6 10000 false +7 1000000000 false +8 1152921504606846976 false +9 6.18 false +10 "abcd" false +11 {} false +12 {"k1":"v31","k2":300} false +13 [] false +14 [123,456] false +15 ["abc","def"] false +16 [null,true,false,100,6.18,"abc"] true +17 [{"k1":"v41","k2":400},1,"a",3.14] true +18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} false +26 \N \N +27 {"k1":"v1","k2":200} false +28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} false + +-- !select -- +1 \N \N +2 null false +3 true false +4 false false +5 100 false +6 10000 false +7 1000000000 false +8 1152921504606846976 false +9 6.18 false +10 "abcd" false +11 {} false +12 {"k1":"v31","k2":300} false +13 [] false +14 [123,456] false +15 ["abc","def"] false +16 [null,true,false,100,6.18,"abc"] true +17 [{"k1":"v41","k2":400},1,"a",3.14] true +18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} false +26 \N \N +27 {"k1":"v1","k2":200} false +28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} false + +-- !select -- +1 \N \N +2 null false +3 true false +4 false false +5 100 false +6 10000 false +7 1000000000 false +8 1152921504606846976 false +9 6.18 false +10 "abcd" false +11 {} false +12 {"k1":"v31","k2":300} false +13 [] false +14 [123,456] false +15 ["abc","def"] false +16 [null,true,false,100,6.18,"abc"] true +17 [{"k1":"v41","k2":400},1,"a",3.14] false +18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} false +26 \N \N +27 {"k1":"v1","k2":200} false +28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} false + +-- !select -- +1 \N \N +2 null false +3 true false +4 false false +5 100 false +6 10000 false +7 1000000000 false +8 1152921504606846976 false +9 6.18 false +10 "abcd" false +11 {} false +12 {"k1":"v31","k2":300} false +13 [] false +14 [123,456] false +15 ["abc","def"] false +16 [null,true,false,100,6.18,"abc"] true +17 [{"k1":"v41","k2":400},1,"a",3.14] false +18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} false +26 \N \N +27 {"k1":"v1","k2":200} false +28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} false -- !select -- 1 \N \N @@ -3136,15 +4516,15 @@ 9 6.18 false 10 "abcd" false 11 {} false -12 {"k1":"v31","k2":300} true +12 {"k1":"v31","k2":300} false 13 [] false 14 [123,456] false 15 ["abc","def"] false 16 [null,true,false,100,6.18,"abc"] false 17 [{"k1":"v41","k2":400},1,"a",3.14] false -18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} true +18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} false 26 \N \N -27 {"k1":"v1","k2":200} true +27 {"k1":"v1","k2":200} false 28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} false -- !select -- @@ -3159,15 +4539,15 @@ 9 6.18 false 10 "abcd" false 11 {} false -12 {"k1":"v31","k2":300} true +12 {"k1":"v31","k2":300} false 13 [] false 14 [123,456] false 15 ["abc","def"] false 16 [null,true,false,100,6.18,"abc"] false 17 [{"k1":"v41","k2":400},1,"a",3.14] false -18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} true +18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} false 26 \N \N -27 {"k1":"v1","k2":200} true +27 {"k1":"v1","k2":200} false 28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} false -- !select -- @@ -3184,11 +4564,11 @@ 11 {} false 12 {"k1":"v31","k2":300} false 13 [] false -14 [123,456] true -15 ["abc","def"] true -16 [null,true,false,100,6.18,"abc"] true -17 [{"k1":"v41","k2":400},1,"a",3.14] true -18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} false +14 [123,456] false +15 ["abc","def"] false +16 [null,true,false,100,6.18,"abc"] false +17 [{"k1":"v41","k2":400},1,"a",3.14] false +18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} true 26 \N \N 27 {"k1":"v1","k2":200} false 28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} false @@ -3207,11 +4587,11 @@ 11 {} false 12 {"k1":"v31","k2":300} false 13 [] false -14 [123,456] true -15 ["abc","def"] true -16 [null,true,false,100,6.18,"abc"] true -17 [{"k1":"v41","k2":400},1,"a",3.14] true -18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} false +14 [123,456] false +15 ["abc","def"] false +16 [null,true,false,100,6.18,"abc"] false +17 [{"k1":"v41","k2":400},1,"a",3.14] false +18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} true 26 \N \N 27 {"k1":"v1","k2":200} false 28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} false @@ -3232,9 +4612,9 @@ 13 [] false 14 [123,456] false 15 ["abc","def"] false -16 [null,true,false,100,6.18,"abc"] true -17 [{"k1":"v41","k2":400},1,"a",3.14] true -18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} false +16 [null,true,false,100,6.18,"abc"] false +17 [{"k1":"v41","k2":400},1,"a",3.14] false +18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} true 26 \N \N 27 {"k1":"v1","k2":200} false 28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} false @@ -3255,9 +4635,9 @@ 13 [] false 14 [123,456] false 15 ["abc","def"] false -16 [null,true,false,100,6.18,"abc"] true -17 [{"k1":"v41","k2":400},1,"a",3.14] true -18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} false +16 [null,true,false,100,6.18,"abc"] false +17 [{"k1":"v41","k2":400},1,"a",3.14] false +18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} true 26 \N \N 27 {"k1":"v1","k2":200} false 28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} false @@ -3278,9 +4658,9 @@ 13 [] false 14 [123,456] false 15 ["abc","def"] false -16 [null,true,false,100,6.18,"abc"] true +16 [null,true,false,100,6.18,"abc"] false 17 [{"k1":"v41","k2":400},1,"a",3.14] false -18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} false +18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} true 26 \N \N 27 {"k1":"v1","k2":200} false 28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} false @@ -3301,7 +4681,7 @@ 13 [] false 14 [123,456] false 15 ["abc","def"] false -16 [null,true,false,100,6.18,"abc"] true +16 [null,true,false,100,6.18,"abc"] false 17 [{"k1":"v41","k2":400},1,"a",3.14] false 18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} false 26 \N \N @@ -3349,7 +4729,7 @@ 15 ["abc","def"] false 16 [null,true,false,100,6.18,"abc"] false 17 [{"k1":"v41","k2":400},1,"a",3.14] false -18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} false +18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} true 26 \N \N 27 {"k1":"v1","k2":200} false 28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} false @@ -3441,7 +4821,7 @@ 15 ["abc","def"] false 16 [null,true,false,100,6.18,"abc"] false 17 [{"k1":"v41","k2":400},1,"a",3.14] false -18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} true +18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} false 26 \N \N 27 {"k1":"v1","k2":200} false 28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} false @@ -3487,7 +4867,7 @@ 15 ["abc","def"] false 16 [null,true,false,100,6.18,"abc"] false 17 [{"k1":"v41","k2":400},1,"a",3.14] false -18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} false +18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} true 26 \N \N 27 {"k1":"v1","k2":200} false 28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} false @@ -3515,7 +4895,7 @@ 27 {"k1":"v1","k2":200} false 28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} false --- !select -- +-- !jsonb_type_select -- 1 \N \N 2 null null 3 true bool @@ -3595,17 +4975,17 @@ 8 1152921504606846976 \N 9 6.18 \N 10 "abcd" \N -11 {} \N -12 {"k1":"v31","k2":300} \N +11 {} object +12 {"k1":"v31","k2":300} object 13 [] \N 14 [123,456] int 15 ["abc","def"] string 16 [null,true,false,100,6.18,"abc"] null 17 [{"k1":"v41","k2":400},1,"a",3.14] object -18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} \N +18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} object 26 \N \N -27 {"k1":"v1","k2":200} \N -28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} \N +27 {"k1":"v1","k2":200} object +28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} object -- !select -- 1 \N \N @@ -3932,6 +5312,190 @@ -- !select -- 1 \N \N 2 null \N +3 true \N +4 false \N +5 100 \N +6 10000 \N +7 1000000000 \N +8 1152921504606846976 \N +9 6.18 \N +10 "abcd" \N +11 {} \N +12 {"k1":"v31","k2":300} \N +13 [] \N +14 [123,456] \N +15 ["abc","def"] \N +16 [null,true,false,100,6.18,"abc"] \N +17 [{"k1":"v41","k2":400},1,"a",3.14] \N +18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} double +26 \N \N +27 {"k1":"v1","k2":200} \N +28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} \N + +-- !select -- +1 \N \N +2 null \N +3 true \N +4 false \N +5 100 \N +6 10000 \N +7 1000000000 \N +8 1152921504606846976 \N +9 6.18 \N +10 "abcd" \N +11 {} \N +12 {"k1":"v31","k2":300} \N +13 [] \N +14 [123,456] \N +15 ["abc","def"] \N +16 [null,true,false,100,6.18,"abc"] \N +17 [{"k1":"v41","k2":400},1,"a",3.14] \N +18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} double +26 \N \N +27 {"k1":"v1","k2":200} \N +28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} \N + +-- !select -- +1 \N \N +2 null \N +3 true \N +4 false \N +5 100 \N +6 10000 \N +7 1000000000 \N +8 1152921504606846976 \N +9 6.18 \N +10 "abcd" \N +11 {} \N +12 {"k1":"v31","k2":300} \N +13 [] \N +14 [123,456] \N +15 ["abc","def"] \N +16 [null,true,false,100,6.18,"abc"] \N +17 [{"k1":"v41","k2":400},1,"a",3.14] \N +18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} string +26 \N \N +27 {"k1":"v1","k2":200} \N +28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} \N + +-- !select -- +1 \N \N +2 null \N +3 true \N +4 false \N +5 100 \N +6 10000 \N +7 1000000000 \N +8 1152921504606846976 \N +9 6.18 \N +10 "abcd" \N +11 {} \N +12 {"k1":"v31","k2":300} \N +13 [] \N +14 [123,456] \N +15 ["abc","def"] \N +16 [null,true,false,100,6.18,"abc"] \N +17 [{"k1":"v41","k2":400},1,"a",3.14] \N +18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} int +26 \N \N +27 {"k1":"v1","k2":200} \N +28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} \N + +-- !select -- +1 \N \N +2 null \N +3 true \N +4 false \N +5 100 \N +6 10000 \N +7 1000000000 \N +8 1152921504606846976 \N +9 6.18 \N +10 "abcd" \N +11 {} \N +12 {"k1":"v31","k2":300} \N +13 [] \N +14 [123,456] \N +15 ["abc","def"] \N +16 [null,true,false,100,6.18,"abc"] \N +17 [{"k1":"v41","k2":400},1,"a",3.14] \N +18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} \N +26 \N \N +27 {"k1":"v1","k2":200} \N +28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} \N + +-- !select -- +1 \N \N +2 null \N +3 true \N +4 false \N +5 100 \N +6 10000 \N +7 1000000000 \N +8 1152921504606846976 \N +9 6.18 \N +10 "abcd" \N +11 {} \N +12 {"k1":"v31","k2":300} \N +13 [] \N +14 [123,456] \N +15 ["abc","def"] \N +16 [null,true,false,100,6.18,"abc"] \N +17 [{"k1":"v41","k2":400},1,"a",3.14] \N +18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} object +26 \N \N +27 {"k1":"v1","k2":200} \N +28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} \N + +-- !select -- +1 \N \N +2 null \N +3 true \N +4 false \N +5 100 \N +6 10000 \N +7 1000000000 \N +8 1152921504606846976 \N +9 6.18 \N +10 "abcd" \N +11 {} \N +12 {"k1":"v31","k2":300} \N +13 [] \N +14 [123,456] \N +15 ["abc","def"] \N +16 [null,true,false,100,6.18,"abc"] \N +17 [{"k1":"v41","k2":400},1,"a",3.14] \N +18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} double +26 \N \N +27 {"k1":"v1","k2":200} \N +28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} \N + +-- !select -- +1 \N \N +2 null \N +3 true \N +4 false \N +5 100 \N +6 10000 \N +7 1000000000 \N +8 1152921504606846976 \N +9 6.18 \N +10 "abcd" \N +11 {} \N +12 {"k1":"v31","k2":300} \N +13 [] \N +14 [123,456] \N +15 ["abc","def"] \N +16 [null,true,false,100,6.18,"abc"] \N +17 [{"k1":"v41","k2":400},1,"a",3.14] \N +18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} \N +26 \N \N +27 {"k1":"v1","k2":200} \N +28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} \N + +-- !cast_from_select -- +1 \N \N +2 null \N 3 true true 4 false false 5 100 \N @@ -4067,7 +5631,7 @@ 27 {"k1":"v1","k2":200} {"k1":"v1","k2":200} 28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} --- !select -- +-- !cast_to_select -- 1 \N \N 2 null \N 3 true true diff --git a/regression-test/data/mv_p0/k1ap2spa/k1ap2spa.out b/regression-test/data/mv_p0/k1ap2spa/k1ap2spa.out index 16fe69c3ff6a8fa..c7d574c013d33bc 100644 --- a/regression-test/data/mv_p0/k1ap2spa/k1ap2spa.out +++ b/regression-test/data/mv_p0/k1ap2spa/k1ap2spa.out @@ -1,11 +1,13 @@ -- This file is automatically generated. You should know what you did if you want to edit this -- !select_star -- +\N 4 \N d -4 -4 -4 d 1 1 1 a 2 2 2 b 3 -3 \N c -- !select_mv -- +\N 5 2 2 3 3 4 2 diff --git a/regression-test/data/mv_p0/no_await/no_await.out b/regression-test/data/mv_p0/no_await/no_await.out new file mode 100644 index 000000000000000..14c441abcd7260c --- /dev/null +++ b/regression-test/data/mv_p0/no_await/no_await.out @@ -0,0 +1,511 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !mv -- +49994996 + +-- !mv -- +49994992 + +-- !mv -- +49994988 + +-- !mv -- +49994984 + +-- !mv -- +49994980 + +-- !mv -- +49994976 + +-- !mv -- +49994972 + +-- !mv -- +49994968 + +-- !mv -- +49994964 + +-- !mv -- +49994960 + +-- !mv -- +49994956 + +-- !mv -- +49994952 + +-- !mv -- +49994948 + +-- !mv -- +49994944 + +-- !mv -- +49994940 + +-- !mv -- +49994936 + +-- !mv -- +49994932 + +-- !mv -- +49994928 + +-- !mv -- +49994924 + +-- !mv -- +49994920 + +-- !mv -- +49994916 + +-- !mv -- +49994912 + +-- !mv -- +49994908 + +-- !mv -- +49994904 + +-- !mv -- +49994900 + +-- !mv -- +49994896 + +-- !mv -- +49994892 + +-- !mv -- +49994888 + +-- !mv -- +49994884 + +-- !mv -- +49994880 + +-- !mv -- +49994876 + +-- !mv -- +49994872 + +-- !mv -- +49994868 + +-- !mv -- +49994864 + +-- !mv -- +49994860 + +-- !mv -- +49994856 + +-- !mv -- +49994852 + +-- !mv -- +49994848 + +-- !mv -- +49994844 + +-- !mv -- +49994840 + +-- !mv -- +49994836 + +-- !mv -- +49994832 + +-- !mv -- +49994828 + +-- !mv -- +49994824 + +-- !mv -- +49994820 + +-- !mv -- +49994816 + +-- !mv -- +49994812 + +-- !mv -- +49994808 + +-- !mv -- +49994804 + +-- !mv -- +49994800 + +-- !mv -- +49994796 + +-- !mv -- +49994792 + +-- !mv -- +49994788 + +-- !mv -- +49994784 + +-- !mv -- +49994780 + +-- !mv -- +49994776 + +-- !mv -- +49994772 + +-- !mv -- +49994768 + +-- !mv -- +49994764 + +-- !mv -- +49994760 + +-- !mv -- +49994756 + +-- !mv -- +49994752 + +-- !mv -- +49994748 + +-- !mv -- +49994744 + +-- !mv -- +49994740 + +-- !mv -- +49994736 + +-- !mv -- +49994732 + +-- !mv -- +49994728 + +-- !mv -- +49994724 + +-- !mv -- +49994720 + +-- !mv -- +49994716 + +-- !mv -- +49994712 + +-- !mv -- +49994708 + +-- !mv -- +49994704 + +-- !mv -- +49994700 + +-- !mv -- +49994696 + +-- !mv -- +49994692 + +-- !mv -- +49994688 + +-- !mv -- +49994684 + +-- !mv -- +49994680 + +-- !mv -- +49994676 + +-- !mv -- +49994672 + +-- !mv -- +49994668 + +-- !mv -- +49994664 + +-- !mv -- +49994660 + +-- !mv -- +49994656 + +-- !mv -- +49994652 + +-- !mv -- +49994648 + +-- !mv -- +49994644 + +-- !mv -- +49994640 + +-- !mv -- +49994636 + +-- !mv -- +49994632 + +-- !mv -- +49994628 + +-- !mv -- +49994624 + +-- !mv -- +49994620 + +-- !mv -- +49994616 + +-- !mv -- +49994612 + +-- !mv -- +49994608 + +-- !mv -- +49994604 + +-- !mv -- +49994600 + +-- !mv -- +49994596 + +-- !mv -- +49994592 + +-- !mv -- +49994588 + +-- !mv -- +49994584 + +-- !mv -- +49994580 + +-- !mv -- +49994576 + +-- !mv -- +49994572 + +-- !mv -- +49994568 + +-- !mv -- +49994564 + +-- !mv -- +49994560 + +-- !mv -- +49994556 + +-- !mv -- +49994552 + +-- !mv -- +49994548 + +-- !mv -- +49994544 + +-- !mv -- +49994540 + +-- !mv -- +49994536 + +-- !mv -- +49994532 + +-- !mv -- +49994528 + +-- !mv -- +49994524 + +-- !mv -- +49994520 + +-- !mv -- +49994516 + +-- !mv -- +49994512 + +-- !mv -- +49994508 + +-- !mv -- +49994504 + +-- !mv -- +49994500 + +-- !mv -- +49994496 + +-- !mv -- +49994492 + +-- !mv -- +49994488 + +-- !mv -- +49994484 + +-- !mv -- +49994480 + +-- !mv -- +49994476 + +-- !mv -- +49994472 + +-- !mv -- +49994468 + +-- !mv -- +49994464 + +-- !mv -- +49994460 + +-- !mv -- +49994456 + +-- !mv -- +49994452 + +-- !mv -- +49994448 + +-- !mv -- +49994444 + +-- !mv -- +49994440 + +-- !mv -- +49994436 + +-- !mv -- +49994432 + +-- !mv -- +49994428 + +-- !mv -- +49994424 + +-- !mv -- +49994420 + +-- !mv -- +49994416 + +-- !mv -- +49994412 + +-- !mv -- +49994408 + +-- !mv -- +49994404 + +-- !mv -- +49994400 + +-- !mv -- +49994396 + +-- !mv -- +49994392 + +-- !mv -- +49994388 + +-- !mv -- +49994384 + +-- !mv -- +49994380 + +-- !mv -- +49994376 + +-- !mv -- +49994372 + +-- !mv -- +49994368 + +-- !mv -- +49994364 + +-- !mv -- +49994360 + +-- !mv -- +49994356 + +-- !mv -- +49994352 + +-- !mv -- +49994348 + +-- !mv -- +49994344 + +-- !mv -- +49994340 + +-- !mv -- +49994336 + +-- !mv -- +49994332 + +-- !mv -- +49994328 + +-- !mv -- +49994324 + +-- !mv -- +49994320 + diff --git a/regression-test/data/nereids_syntax_p0/rollup/agg_date.out b/regression-test/data/nereids_syntax_p0/rollup/agg_date.out index a64b546d21106a2..17fc18fae25d1df 100644 --- a/regression-test/data/nereids_syntax_p0/rollup/agg_date.out +++ b/regression-test/data/nereids_syntax_p0/rollup/agg_date.out @@ -2,22 +2,22 @@ -- !sql -- test_rollup_agg_date1 AGG_KEYS datek1 DATE DATEV2 Yes true \N true datetimek1 DATETIME DATETIMEV2(0) Yes true \N true - datetimek2 DATETIME DATETIMEV2(3) Yes true \N true - datetimek3 DATETIME DATETIMEV2(6) Yes true \N true + datetimek2 DATETIME(3) DATETIMEV2(3) Yes true \N true + datetimek3 DATETIME(6) DATETIMEV2(6) Yes true \N true datev1 DATE DATEV2 No false \N MAX true datetimev1 DATETIME DATETIMEV2(0) No false \N MAX true - datetimev2 DATETIME DATETIMEV2(3) No false \N MAX true - datetimev3 DATETIME DATETIMEV2(6) No false \N MAX true - datetimev4 DATETIME DATETIMEV2(3) Yes false \N MAX true + datetimev2 DATETIME(3) DATETIMEV2(3) No false \N MAX true + datetimev3 DATETIME(6) DATETIMEV2(6) No false \N MAX true + datetimev4 DATETIME(3) DATETIMEV2(3) Yes false \N MAX true rollup_date AGG_KEYS datek1 DATE DATEV2 Yes true \N true - datetimek2 DATETIME DATETIMEV2(3) Yes true \N true + datetimek2 DATETIME(3) DATETIMEV2(3) Yes true \N true datetimek1 DATETIME DATETIMEV2(0) Yes true \N true - datetimek3 DATETIME DATETIMEV2(6) Yes true \N true + datetimek3 DATETIME(6) DATETIMEV2(6) Yes true \N true datev1 DATE DATEV2 No false \N MAX true datetimev1 DATETIME DATETIMEV2(0) No false \N MAX true - datetimev2 DATETIME DATETIMEV2(3) No false \N MAX true - datetimev3 DATETIME DATETIMEV2(6) No false \N MAX true + datetimev2 DATETIME(3) DATETIMEV2(3) No false \N MAX true + datetimev3 DATETIME(6) DATETIMEV2(6) No false \N MAX true -- !sql -- 2022-08-23 2022-08-23T11:11:11 2022-08-23T11:11:11.111 2022-08-23T11:11:11.111111 2022-08-23 2022-08-23T11:11:11 2022-08-23T11:11:11.111 2022-08-23T11:11:11.111111 diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query11.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query11.out index a25aced3030e691..dee0257716d2e22 100644 --- a/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query11.out +++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query11.out @@ -2,67 +2,66 @@ -- !ds_shape_11 -- CteAnchor[cteId= ( CTEId#4=] ) --CteProducer[cteId= ( CTEId#4=] ) -----PhysicalProject -------PhysicalUnion ---------PhysicalProject -----------hashAgg[GLOBAL] -------------PhysicalDistribute ---------------hashAgg[LOCAL] -----------------PhysicalProject -------------------hashJoin[INNER_JOIN](customer.c_customer_sk = store_sales.ss_customer_sk) ---------------------hashJoin[INNER_JOIN](store_sales.ss_sold_date_sk = date_dim.d_date_sk) -----------------------PhysicalProject -------------------------filter((('s' = 'w') OR ('s' = 's'))) ---------------------------PhysicalOlapScan[store_sales] -----------------------PhysicalDistribute -------------------------PhysicalProject ---------------------------filter(((date_dim.d_year = 2001) OR (date_dim.d_year = 2002))(('s' = 'w') OR ('s' = 's'))) -----------------------------PhysicalOlapScan[date_dim] +----PhysicalUnion +------PhysicalProject +--------hashAgg[GLOBAL] +----------PhysicalDistribute +------------hashAgg[LOCAL] +--------------PhysicalProject +----------------hashJoin[INNER_JOIN](customer.c_customer_sk = store_sales.ss_customer_sk) +------------------hashJoin[INNER_JOIN](store_sales.ss_sold_date_sk = date_dim.d_date_sk) +--------------------PhysicalProject +----------------------filter((('s' = 'w') OR ('s' = 's'))) +------------------------PhysicalOlapScan[store_sales] --------------------PhysicalDistribute ----------------------PhysicalProject -------------------------filter((('s' = 'w') OR ('s' = 's'))) ---------------------------PhysicalOlapScan[customer] ---------PhysicalProject -----------hashAgg[GLOBAL] -------------PhysicalDistribute ---------------hashAgg[LOCAL] -----------------PhysicalProject -------------------hashJoin[INNER_JOIN](customer.c_customer_sk = web_sales.ws_bill_customer_sk) ---------------------PhysicalDistribute -----------------------hashJoin[INNER_JOIN](web_sales.ws_sold_date_sk = date_dim.d_date_sk) -------------------------PhysicalProject ---------------------------filter((('w' = 'w') OR ('w' = 's'))) -----------------------------PhysicalOlapScan[web_sales] -------------------------PhysicalDistribute ---------------------------PhysicalProject -----------------------------filter(((date_dim.d_year = 2001) OR (date_dim.d_year = 2002))(('w' = 'w') OR ('w' = 's'))) -------------------------------PhysicalOlapScan[date_dim] ---------------------PhysicalDistribute +------------------------filter(((date_dim.d_year = 2001) OR (date_dim.d_year = 2002))(('s' = 'w') OR ('s' = 's'))) +--------------------------PhysicalOlapScan[date_dim] +------------------PhysicalDistribute +--------------------PhysicalProject +----------------------filter((('s' = 'w') OR ('s' = 's'))) +------------------------PhysicalOlapScan[customer] +------PhysicalProject +--------hashAgg[GLOBAL] +----------PhysicalDistribute +------------hashAgg[LOCAL] +--------------PhysicalProject +----------------hashJoin[INNER_JOIN](customer.c_customer_sk = web_sales.ws_bill_customer_sk) +------------------PhysicalDistribute +--------------------hashJoin[INNER_JOIN](web_sales.ws_sold_date_sk = date_dim.d_date_sk) ----------------------PhysicalProject ------------------------filter((('w' = 'w') OR ('w' = 's'))) ---------------------------PhysicalOlapScan[customer] +--------------------------PhysicalOlapScan[web_sales] +----------------------PhysicalDistribute +------------------------PhysicalProject +--------------------------filter(((date_dim.d_year = 2001) OR (date_dim.d_year = 2002))(('w' = 'w') OR ('w' = 's'))) +----------------------------PhysicalOlapScan[date_dim] +------------------PhysicalDistribute +--------------------PhysicalProject +----------------------filter((('w' = 'w') OR ('w' = 's'))) +------------------------PhysicalOlapScan[customer] --PhysicalTopN ----PhysicalDistribute ------PhysicalTopN --------PhysicalProject -----------hashJoin[INNER_JOIN](t_s_firstyear.customer_id = t_w_secyear.customer_id)(CASE WHEN (year_total > 0.00) THEN (cast(year_total as DECIMALV3(38, 8)) / year_total) ELSE 0.000000 END > CASE WHEN (year_total > 0.00) THEN (cast(year_total as DECIMALV3(38, 8)) / year_total) ELSE 0.000000 END) +----------hashJoin[INNER_JOIN](t_s_secyear.customer_id = t_s_firstyear.customer_id)(CASE WHEN (year_total > 0.00) THEN (cast(year_total as DECIMALV3(38, 8)) / year_total) ELSE 0.000000 END > CASE WHEN (year_total > 0.00) THEN (cast(year_total as DECIMALV3(38, 8)) / year_total) ELSE 0.000000 END) ------------PhysicalDistribute --------------PhysicalProject -----------------filter((t_w_secyear.dyear = 2002)(t_w_secyear.sale_type = 'w')) +----------------filter((t_s_secyear.sale_type = 's')(t_s_secyear.dyear = 2002)) ------------------CteConsumer[cteId= ( CTEId#4=] ) ------------PhysicalProject ---------------hashJoin[INNER_JOIN](t_s_secyear.customer_id = t_s_firstyear.customer_id) -----------------PhysicalDistribute -------------------PhysicalProject ---------------------filter((t_s_secyear.sale_type = 's')(t_s_secyear.dyear = 2002)) -----------------------CteConsumer[cteId= ( CTEId#4=] ) -----------------hashJoin[INNER_JOIN](t_s_firstyear.customer_id = t_w_firstyear.customer_id) +--------------hashJoin[INNER_JOIN](t_s_firstyear.customer_id = t_w_firstyear.customer_id) +----------------hashJoin[INNER_JOIN](t_s_firstyear.customer_id = t_w_secyear.customer_id) ------------------PhysicalDistribute --------------------PhysicalProject -----------------------filter((t_s_firstyear.dyear = 2001)(t_s_firstyear.sale_type = 's')(t_s_firstyear.year_total > 0.00)) +----------------------filter((t_w_secyear.dyear = 2002)(t_w_secyear.sale_type = 'w')) ------------------------CteConsumer[cteId= ( CTEId#4=] ) ------------------PhysicalDistribute --------------------PhysicalProject -----------------------filter((t_w_firstyear.year_total > 0.00)(t_w_firstyear.sale_type = 'w')(t_w_firstyear.dyear = 2001)) +----------------------filter((t_s_firstyear.dyear = 2001)(t_s_firstyear.sale_type = 's')(t_s_firstyear.year_total > 0.00)) ------------------------CteConsumer[cteId= ( CTEId#4=] ) +----------------PhysicalDistribute +------------------PhysicalProject +--------------------filter((t_w_firstyear.year_total > 0.00)(t_w_firstyear.sale_type = 'w')(t_w_firstyear.dyear = 2001)) +----------------------CteConsumer[cteId= ( CTEId#4=] ) diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query12.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query12.out index eb27fab212c869a..5750cb0006f408f 100644 --- a/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query12.out +++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query12.out @@ -20,7 +20,8 @@ PhysicalTopN ------------------------------PhysicalProject --------------------------------filter((date_dim.d_date <= 1998-05-06)(date_dim.d_date >= 1998-04-06)) ----------------------------------PhysicalOlapScan[date_dim] -------------------------PhysicalProject ---------------------------filter(i_category IN ('Books', 'Sports', 'Men')) -----------------------------PhysicalOlapScan[item] +------------------------PhysicalDistribute +--------------------------PhysicalProject +----------------------------filter(i_category IN ('Books', 'Sports', 'Men')) +------------------------------PhysicalOlapScan[item] diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query14.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query14.out index c0e488a09a9c96b..ecf6a69c87f75db 100644 --- a/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query14.out +++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query14.out @@ -4,76 +4,88 @@ CteAnchor[cteId= ( CTEId#8=] ) --CteProducer[cteId= ( CTEId#8=] ) ----PhysicalProject ------hashJoin[INNER_JOIN](item.i_brand_id = t.brand_id)(item.i_class_id = t.class_id)(item.i_category_id = t.category_id) ---------PhysicalIntersect -----------PhysicalProject -------------hashJoin[INNER_JOIN](store_sales.ss_item_sk = iss.i_item_sk) ---------------hashJoin[INNER_JOIN](store_sales.ss_sold_date_sk = d1.d_date_sk) -----------------PhysicalProject -------------------PhysicalOlapScan[store_sales] -----------------PhysicalDistribute -------------------PhysicalProject ---------------------filter((d1.d_year <= 2002)(d1.d_year >= 2000)) -----------------------PhysicalOlapScan[date_dim] +--------PhysicalProject +----------PhysicalOlapScan[item] +--------PhysicalDistribute +----------PhysicalIntersect +------------hashAgg[GLOBAL] --------------PhysicalDistribute -----------------PhysicalProject -------------------PhysicalOlapScan[item] -----------PhysicalProject -------------hashJoin[INNER_JOIN](catalog_sales.cs_item_sk = ics.i_item_sk) ---------------hashJoin[INNER_JOIN](catalog_sales.cs_sold_date_sk = d2.d_date_sk) -----------------PhysicalProject -------------------PhysicalOlapScan[catalog_sales] -----------------PhysicalDistribute +----------------hashAgg[LOCAL] ------------------PhysicalProject ---------------------filter((d2.d_year >= 2000)(d2.d_year <= 2002)) -----------------------PhysicalOlapScan[date_dim] +--------------------hashJoin[INNER_JOIN](store_sales.ss_item_sk = iss.i_item_sk) +----------------------hashJoin[INNER_JOIN](store_sales.ss_sold_date_sk = d1.d_date_sk) +------------------------PhysicalProject +--------------------------PhysicalOlapScan[store_sales] +------------------------PhysicalDistribute +--------------------------PhysicalProject +----------------------------filter((d1.d_year <= 2002)(d1.d_year >= 2000)) +------------------------------PhysicalOlapScan[date_dim] +----------------------PhysicalDistribute +------------------------PhysicalProject +--------------------------PhysicalOlapScan[item] +------------hashAgg[GLOBAL] --------------PhysicalDistribute -----------------PhysicalProject -------------------PhysicalOlapScan[item] -----------PhysicalProject -------------hashJoin[INNER_JOIN](web_sales.ws_item_sk = iws.i_item_sk) ---------------hashJoin[INNER_JOIN](web_sales.ws_sold_date_sk = d3.d_date_sk) -----------------PhysicalProject -------------------PhysicalOlapScan[web_sales] -----------------PhysicalDistribute +----------------hashAgg[LOCAL] ------------------PhysicalProject ---------------------filter((d3.d_year <= 2002)(d3.d_year >= 2000)) -----------------------PhysicalOlapScan[date_dim] +--------------------hashJoin[INNER_JOIN](catalog_sales.cs_item_sk = ics.i_item_sk) +----------------------hashJoin[INNER_JOIN](catalog_sales.cs_sold_date_sk = d2.d_date_sk) +------------------------PhysicalProject +--------------------------PhysicalOlapScan[catalog_sales] +------------------------PhysicalDistribute +--------------------------PhysicalProject +----------------------------filter((d2.d_year >= 2000)(d2.d_year <= 2002)) +------------------------------PhysicalOlapScan[date_dim] +----------------------PhysicalDistribute +------------------------PhysicalProject +--------------------------PhysicalOlapScan[item] +------------hashAgg[GLOBAL] --------------PhysicalDistribute -----------------PhysicalProject -------------------PhysicalOlapScan[item] ---------PhysicalDistribute -----------PhysicalProject -------------PhysicalOlapScan[item] +----------------hashAgg[LOCAL] +------------------PhysicalProject +--------------------hashJoin[INNER_JOIN](web_sales.ws_item_sk = iws.i_item_sk) +----------------------hashJoin[INNER_JOIN](web_sales.ws_sold_date_sk = d3.d_date_sk) +------------------------PhysicalProject +--------------------------PhysicalOlapScan[web_sales] +------------------------PhysicalDistribute +--------------------------PhysicalProject +----------------------------filter((d3.d_year <= 2002)(d3.d_year >= 2000)) +------------------------------PhysicalOlapScan[date_dim] +----------------------PhysicalDistribute +------------------------PhysicalProject +--------------------------PhysicalOlapScan[item] --CteAnchor[cteId= ( CTEId#10=] ) ----CteProducer[cteId= ( CTEId#10=] ) ------hashAgg[GLOBAL] --------PhysicalDistribute ----------hashAgg[LOCAL] ------------PhysicalUnion ---------------PhysicalProject -----------------hashJoin[INNER_JOIN](store_sales.ss_sold_date_sk = date_dim.d_date_sk) -------------------PhysicalProject ---------------------PhysicalOlapScan[store_sales] -------------------PhysicalDistribute +--------------PhysicalDistribute +----------------PhysicalProject +------------------hashJoin[INNER_JOIN](store_sales.ss_sold_date_sk = date_dim.d_date_sk) --------------------PhysicalProject -----------------------filter((date_dim.d_year <= 2002)(date_dim.d_year >= 2000)) -------------------------PhysicalOlapScan[date_dim] ---------------PhysicalProject -----------------hashJoin[INNER_JOIN](catalog_sales.cs_sold_date_sk = date_dim.d_date_sk) -------------------PhysicalProject ---------------------PhysicalOlapScan[catalog_sales] -------------------PhysicalDistribute +----------------------PhysicalOlapScan[store_sales] +--------------------PhysicalDistribute +----------------------PhysicalProject +------------------------filter((date_dim.d_year <= 2002)(date_dim.d_year >= 2000)) +--------------------------PhysicalOlapScan[date_dim] +--------------PhysicalDistribute +----------------PhysicalProject +------------------hashJoin[INNER_JOIN](catalog_sales.cs_sold_date_sk = date_dim.d_date_sk) --------------------PhysicalProject -----------------------filter((date_dim.d_year <= 2002)(date_dim.d_year >= 2000)) -------------------------PhysicalOlapScan[date_dim] ---------------PhysicalProject -----------------hashJoin[INNER_JOIN](web_sales.ws_sold_date_sk = date_dim.d_date_sk) -------------------PhysicalProject ---------------------PhysicalOlapScan[web_sales] -------------------PhysicalDistribute +----------------------PhysicalOlapScan[catalog_sales] +--------------------PhysicalDistribute +----------------------PhysicalProject +------------------------filter((date_dim.d_year <= 2002)(date_dim.d_year >= 2000)) +--------------------------PhysicalOlapScan[date_dim] +--------------PhysicalDistribute +----------------PhysicalProject +------------------hashJoin[INNER_JOIN](web_sales.ws_sold_date_sk = date_dim.d_date_sk) --------------------PhysicalProject -----------------------filter((date_dim.d_year >= 2000)(date_dim.d_year <= 2002)) -------------------------PhysicalOlapScan[date_dim] +----------------------PhysicalOlapScan[web_sales] +--------------------PhysicalDistribute +----------------------PhysicalProject +------------------------filter((date_dim.d_year >= 2000)(date_dim.d_year <= 2002)) +--------------------------PhysicalOlapScan[date_dim] ----PhysicalTopN ------PhysicalDistribute --------PhysicalTopN @@ -102,8 +114,9 @@ CteAnchor[cteId= ( CTEId#8=] ) ----------------------------------------------PhysicalProject ------------------------------------------------filter((date_dim.d_year = 2002)(date_dim.d_moy = 11)) --------------------------------------------------PhysicalOlapScan[date_dim] -----------------------------------------PhysicalProject -------------------------------------------PhysicalOlapScan[item] +----------------------------------------PhysicalDistribute +------------------------------------------PhysicalProject +--------------------------------------------PhysicalOlapScan[item] --------------------------PhysicalDistribute ----------------------------PhysicalAssertNumRows ------------------------------PhysicalDistribute @@ -127,8 +140,9 @@ CteAnchor[cteId= ( CTEId#8=] ) ----------------------------------------------PhysicalProject ------------------------------------------------filter((date_dim.d_year = 2002)(date_dim.d_moy = 11)) --------------------------------------------------PhysicalOlapScan[date_dim] -----------------------------------------PhysicalProject -------------------------------------------PhysicalOlapScan[item] +----------------------------------------PhysicalDistribute +------------------------------------------PhysicalProject +--------------------------------------------PhysicalOlapScan[item] --------------------------PhysicalDistribute ----------------------------PhysicalAssertNumRows ------------------------------PhysicalDistribute @@ -152,8 +166,9 @@ CteAnchor[cteId= ( CTEId#8=] ) ----------------------------------------------PhysicalProject ------------------------------------------------filter((date_dim.d_year = 2002)(date_dim.d_moy = 11)) --------------------------------------------------PhysicalOlapScan[date_dim] -----------------------------------------PhysicalProject -------------------------------------------PhysicalOlapScan[item] +----------------------------------------PhysicalDistribute +------------------------------------------PhysicalProject +--------------------------------------------PhysicalOlapScan[item] --------------------------PhysicalDistribute ----------------------------PhysicalAssertNumRows ------------------------------PhysicalDistribute diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query18.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query18.out index 62483c4de0f2dd1..6ddc252783bb486 100644 --- a/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query18.out +++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query18.out @@ -9,39 +9,41 @@ PhysicalTopN ------------hashAgg[LOCAL] --------------PhysicalRepeat ----------------PhysicalProject -------------------hashJoin[INNER_JOIN](catalog_sales.cs_item_sk = item.i_item_sk) ---------------------PhysicalProject -----------------------PhysicalOlapScan[item] +------------------hashJoin[INNER_JOIN](customer.c_current_cdemo_sk = cd2.cd_demo_sk) --------------------PhysicalDistribute ----------------------PhysicalProject -------------------------hashJoin[INNER_JOIN](customer.c_current_cdemo_sk = cd2.cd_demo_sk) ---------------------------PhysicalDistribute -----------------------------PhysicalProject -------------------------------PhysicalOlapScan[customer_demographics] +------------------------PhysicalOlapScan[customer_demographics] +--------------------PhysicalDistribute +----------------------PhysicalProject +------------------------hashJoin[INNER_JOIN](catalog_sales.cs_item_sk = item.i_item_sk) +--------------------------PhysicalProject +----------------------------PhysicalOlapScan[item] --------------------------PhysicalDistribute ----------------------------PhysicalProject -------------------------------hashJoin[INNER_JOIN](catalog_sales.cs_sold_date_sk = date_dim.d_date_sk) ---------------------------------PhysicalProject -----------------------------------hashJoin[INNER_JOIN](catalog_sales.cs_bill_cdemo_sk = cd1.cd_demo_sk) -------------------------------------PhysicalProject ---------------------------------------hashJoin[INNER_JOIN](catalog_sales.cs_bill_customer_sk = customer.c_customer_sk) -----------------------------------------PhysicalProject -------------------------------------------PhysicalOlapScan[catalog_sales] -----------------------------------------PhysicalDistribute -------------------------------------------hashJoin[INNER_JOIN](customer.c_current_addr_sk = customer_address.ca_address_sk) ---------------------------------------------PhysicalDistribute +------------------------------hashJoin[INNER_JOIN](customer.c_current_addr_sk = customer_address.ca_address_sk) +--------------------------------PhysicalDistribute +----------------------------------PhysicalProject +------------------------------------hashJoin[INNER_JOIN](catalog_sales.cs_bill_customer_sk = customer.c_customer_sk) +--------------------------------------PhysicalDistribute +----------------------------------------hashJoin[INNER_JOIN](catalog_sales.cs_sold_date_sk = date_dim.d_date_sk) +------------------------------------------PhysicalProject +--------------------------------------------hashJoin[INNER_JOIN](catalog_sales.cs_bill_cdemo_sk = cd1.cd_demo_sk) ----------------------------------------------PhysicalProject -------------------------------------------------filter(c_birth_month IN (1, 2, 4, 7, 8, 10)) ---------------------------------------------------PhysicalOlapScan[customer] +------------------------------------------------PhysicalOlapScan[catalog_sales] +----------------------------------------------PhysicalDistribute +------------------------------------------------PhysicalProject +--------------------------------------------------filter((cast(cd_gender as VARCHAR(*)) = 'F')(cast(cd_education_status as VARCHAR(*)) = 'Advanced Degree')) +----------------------------------------------------PhysicalOlapScan[customer_demographics] +------------------------------------------PhysicalDistribute --------------------------------------------PhysicalProject -----------------------------------------------filter(ca_state IN ('WA', 'GA', 'NC', 'ME', 'WY', 'OK', 'IN')) -------------------------------------------------PhysicalOlapScan[customer_address] -------------------------------------PhysicalDistribute ---------------------------------------PhysicalProject -----------------------------------------filter((cast(cd_gender as VARCHAR(*)) = 'F')(cast(cd_education_status as VARCHAR(*)) = 'Advanced Degree')) -------------------------------------------PhysicalOlapScan[customer_demographics] +----------------------------------------------filter((date_dim.d_year = 1998)) +------------------------------------------------PhysicalOlapScan[date_dim] +--------------------------------------PhysicalDistribute +----------------------------------------PhysicalProject +------------------------------------------filter(c_birth_month IN (1, 2, 4, 7, 8, 10)) +--------------------------------------------PhysicalOlapScan[customer] --------------------------------PhysicalDistribute ----------------------------------PhysicalProject -------------------------------------filter((date_dim.d_year = 1998)) ---------------------------------------PhysicalOlapScan[date_dim] +------------------------------------filter(ca_state IN ('WA', 'GA', 'NC', 'ME', 'WY', 'OK', 'IN')) +--------------------------------------PhysicalOlapScan[customer_address] diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query2.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query2.out index 329e91f73ef9e69..e970e12f97ea3c2 100644 --- a/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query2.out +++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query2.out @@ -7,10 +7,11 @@ CteAnchor[cteId= ( CTEId#4=] ) --------hashAgg[LOCAL] ----------PhysicalProject ------------hashJoin[INNER_JOIN](date_dim.d_date_sk = wscs.sold_date_sk) ---------------PhysicalProject -----------------PhysicalUnion +--------------PhysicalUnion +----------------PhysicalDistribute ------------------PhysicalProject --------------------PhysicalOlapScan[web_sales] +----------------PhysicalDistribute ------------------PhysicalProject --------------------PhysicalOlapScan[catalog_sales] --------------PhysicalDistribute diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query21.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query21.out index 54b448ebf6fc89d..8361e28dac0310b 100644 --- a/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query21.out +++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query21.out @@ -2,25 +2,23 @@ -- !ds_shape_21 -- PhysicalTopN --PhysicalDistribute -----PhysicalTopN -------filter((CASE WHEN (inv_before > 0) THEN (cast(inv_after as DOUBLE) / cast(inv_before as DOUBLE)) ELSE NULL END >= cast((2.000000 / 3.0) as DOUBLE))(CASE WHEN (inv_before > 0) THEN (cast(inv_after as DOUBLE) / cast(inv_before as DOUBLE)) ELSE NULL END <= 1.5)) ---------hashAgg[GLOBAL] -----------PhysicalDistribute -------------hashAgg[LOCAL] ---------------PhysicalProject -----------------hashJoin[INNER_JOIN](inventory.inv_warehouse_sk = warehouse.w_warehouse_sk) -------------------hashJoin[INNER_JOIN](inventory.inv_date_sk = date_dim.d_date_sk) ---------------------hashJoin[INNER_JOIN](item.i_item_sk = inventory.inv_item_sk) -----------------------PhysicalOlapScan[inventory] -----------------------PhysicalDistribute -------------------------PhysicalProject ---------------------------filter((item.i_current_price <= 1.49)(item.i_current_price >= 0.99)) -----------------------------PhysicalOlapScan[item] +----filter((CASE WHEN (inv_before > 0) THEN (cast(inv_after as DOUBLE) / cast(inv_before as DOUBLE)) ELSE NULL END >= cast((2.000000 / 3.0) as DOUBLE))(CASE WHEN (inv_before > 0) THEN (cast(inv_after as DOUBLE) / cast(inv_before as DOUBLE)) ELSE NULL END <= 1.5)) +------hashAgg[GLOBAL] +--------PhysicalDistribute +----------hashAgg[LOCAL] +------------PhysicalProject +--------------hashJoin[INNER_JOIN](inventory.inv_warehouse_sk = warehouse.w_warehouse_sk) +----------------hashJoin[INNER_JOIN](inventory.inv_date_sk = date_dim.d_date_sk) +------------------hashJoin[INNER_JOIN](item.i_item_sk = inventory.inv_item_sk) +--------------------PhysicalOlapScan[inventory] --------------------PhysicalDistribute ----------------------PhysicalProject -------------------------filter((date_dim.d_date >= 2002-01-28)(date_dim.d_date <= 2002-03-29)) ---------------------------PhysicalOlapScan[date_dim] +------------------------filter((item.i_current_price <= 1.49)(item.i_current_price >= 0.99)) +--------------------------PhysicalOlapScan[item] ------------------PhysicalDistribute --------------------PhysicalProject -----------------------PhysicalOlapScan[warehouse] - +----------------------filter((date_dim.d_date >= 2002-01-28)(date_dim.d_date <= 2002-03-29)) +------------------------PhysicalOlapScan[date_dim] +----------------PhysicalDistribute +------------------PhysicalProject +--------------------PhysicalOlapScan[warehouse] \ No newline at end of file diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query24.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query24.out index 70f424e95c45fdb..c887e96371cc739 100644 --- a/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query24.out +++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query24.out @@ -32,21 +32,22 @@ CteAnchor[cteId= ( CTEId#0=] ) ----------------PhysicalProject ------------------PhysicalOlapScan[store_returns] --PhysicalQuickSort -----PhysicalQuickSort -------PhysicalProject ---------NestedLoopJoin[INNER_JOIN](cast(paid as DOUBLE) > cast((0.05 * avg(netpaid)) as DOUBLE)) -----------PhysicalAssertNumRows -------------PhysicalProject ---------------hashAgg[GLOBAL] -----------------PhysicalDistribute -------------------hashAgg[LOCAL] ---------------------PhysicalProject -----------------------CteConsumer[cteId= ( CTEId#0=] ) -----------PhysicalDistribute +----PhysicalDistribute +------PhysicalQuickSort +--------PhysicalProject +----------NestedLoopJoin[INNER_JOIN](cast(paid as DOUBLE) > cast((0.05 * avg(netpaid)) as DOUBLE)) ------------hashAgg[GLOBAL] --------------PhysicalDistribute ----------------hashAgg[LOCAL] ------------------PhysicalProject --------------------filter((cast(i_color as VARCHAR(*)) = 'beige')) ----------------------CteConsumer[cteId= ( CTEId#0=] ) +------------PhysicalDistribute +--------------PhysicalAssertNumRows +----------------PhysicalProject +------------------hashAgg[GLOBAL] +--------------------PhysicalDistribute +----------------------hashAgg[LOCAL] +------------------------PhysicalProject +--------------------------CteConsumer[cteId= ( CTEId#0=] ) diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query26.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query26.out index b322aab515ac89e..9a85f030a239908 100644 --- a/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query26.out +++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query26.out @@ -27,6 +27,7 @@ PhysicalTopN ----------------------PhysicalProject ------------------------filter(((cast(p_channel_email as VARCHAR(*)) = 'N') OR (cast(p_channel_event as VARCHAR(*)) = 'N'))) --------------------------PhysicalOlapScan[promotion] -----------------PhysicalProject -------------------PhysicalOlapScan[item] +----------------PhysicalDistribute +------------------PhysicalProject +--------------------PhysicalOlapScan[item] diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query27.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query27.out index f1fec2b391f179b..e773eae30496443 100644 --- a/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query27.out +++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query27.out @@ -26,8 +26,9 @@ PhysicalTopN --------------------------------PhysicalProject ----------------------------------filter((date_dim.d_year = 1999)) ------------------------------------PhysicalOlapScan[date_dim] -------------------------PhysicalProject ---------------------------PhysicalOlapScan[item] +------------------------PhysicalDistribute +--------------------------PhysicalProject +----------------------------PhysicalOlapScan[item] ----------------------PhysicalDistribute ------------------------PhysicalProject --------------------------filter(s_state IN ('MO', 'AL', 'MI', 'TN', 'LA', 'SC')) diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query30.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query30.out index f6803d5601a2859..ed1c8ba9cb8012a 100644 --- a/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query30.out +++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query30.out @@ -16,8 +16,9 @@ CteAnchor[cteId= ( CTEId#2=] ) ----------------------PhysicalProject ------------------------filter((date_dim.d_year = 2002)) --------------------------PhysicalOlapScan[date_dim] ---------------PhysicalProject -----------------PhysicalOlapScan[customer_address] +--------------PhysicalDistribute +----------------PhysicalProject +------------------PhysicalOlapScan[customer_address] --PhysicalTopN ----PhysicalDistribute ------PhysicalTopN diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query31.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query31.out index 912280a2e89b1d0..12a7b7db3138b2b 100644 --- a/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query31.out +++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query31.out @@ -36,40 +36,41 @@ CteAnchor[cteId= ( CTEId#6=] ) --------------------------PhysicalProject ----------------------------filter((ws.d_year = 2000)d_qoy IN (1, 2, 3)) ------------------------------PhysicalOlapScan[date_dim] -------------------PhysicalProject ---------------------PhysicalOlapScan[customer_address] +------------------PhysicalDistribute +--------------------PhysicalProject +----------------------PhysicalOlapScan[customer_address] ----PhysicalQuickSort ------PhysicalDistribute --------PhysicalQuickSort ----------PhysicalProject ------------hashJoin[INNER_JOIN](ws1.ca_county = ws3.ca_county)(CASE WHEN (web_sales > 0.00) THEN (cast(web_sales as DECIMALV3(38, 8)) / web_sales) ELSE NULL END > CASE WHEN (store_sales > 0.00) THEN (cast(store_sales as DECIMALV3(38, 8)) / store_sales) ELSE NULL END) ---------------PhysicalProject -----------------filter((ws3.d_year = 2000)(ws3.d_qoy = 3)) -------------------CteConsumer[cteId= ( CTEId#7=] ) --------------PhysicalDistribute ----------------PhysicalProject -------------------hashJoin[INNER_JOIN](ss2.ca_county = ss3.ca_county) ---------------------PhysicalDistribute -----------------------PhysicalProject -------------------------filter((ss3.d_year = 2000)(ss3.d_qoy = 3)) ---------------------------CteConsumer[cteId= ( CTEId#6=] ) ---------------------hashJoin[INNER_JOIN](ss1.ca_county = ss2.ca_county)(CASE WHEN (web_sales > 0.00) THEN (cast(web_sales as DECIMALV3(38, 8)) / web_sales) ELSE NULL END > CASE WHEN (store_sales > 0.00) THEN (cast(store_sales as DECIMALV3(38, 8)) / store_sales) ELSE NULL END) -----------------------PhysicalDistribute -------------------------PhysicalProject ---------------------------filter((ss2.d_year = 2000)(ss2.d_qoy = 2)) -----------------------------CteConsumer[cteId= ( CTEId#6=] ) -----------------------hashJoin[INNER_JOIN](ss1.ca_county = ws1.ca_county) +------------------filter((ws3.d_year = 2000)(ws3.d_qoy = 3)) +--------------------CteConsumer[cteId= ( CTEId#7=] ) +--------------PhysicalProject +----------------hashJoin[INNER_JOIN](ss2.ca_county = ss3.ca_county) +------------------PhysicalDistribute +--------------------PhysicalProject +----------------------filter((ss3.d_year = 2000)(ss3.d_qoy = 3)) +------------------------CteConsumer[cteId= ( CTEId#6=] ) +------------------hashJoin[INNER_JOIN](ws1.ca_county = ws2.ca_county)(CASE WHEN (web_sales > 0.00) THEN (cast(web_sales as DECIMALV3(38, 8)) / web_sales) ELSE NULL END > CASE WHEN (store_sales > 0.00) THEN (cast(store_sales as DECIMALV3(38, 8)) / store_sales) ELSE NULL END) +--------------------hashJoin[INNER_JOIN](ss1.ca_county = ws1.ca_county) +----------------------hashJoin[INNER_JOIN](ss1.ca_county = ss2.ca_county) ------------------------PhysicalDistribute --------------------------PhysicalProject ----------------------------filter((ss1.d_year = 2000)(ss1.d_qoy = 1)) ------------------------------CteConsumer[cteId= ( CTEId#6=] ) -------------------------hashJoin[INNER_JOIN](ws1.ca_county = ws2.ca_county) ---------------------------PhysicalDistribute -----------------------------PhysicalProject -------------------------------filter((ws1.d_year = 2000)(ws1.d_qoy = 1)) ---------------------------------CteConsumer[cteId= ( CTEId#7=] ) ---------------------------PhysicalDistribute -----------------------------PhysicalProject -------------------------------filter((ws2.d_qoy = 2)(ws2.d_year = 2000)) ---------------------------------CteConsumer[cteId= ( CTEId#7=] ) +------------------------PhysicalDistribute +--------------------------PhysicalProject +----------------------------filter((ss2.d_year = 2000)(ss2.d_qoy = 2)) +------------------------------CteConsumer[cteId= ( CTEId#6=] ) +----------------------PhysicalDistribute +------------------------PhysicalProject +--------------------------filter((ws1.d_year = 2000)(ws1.d_qoy = 1)) +----------------------------CteConsumer[cteId= ( CTEId#7=] ) +--------------------PhysicalDistribute +----------------------PhysicalProject +------------------------filter((ws2.d_qoy = 2)(ws2.d_year = 2000)) +--------------------------CteConsumer[cteId= ( CTEId#7=] ) diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query33.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query33.out index 3bee39ba9a75e7b..731dc2708264211 100644 --- a/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query33.out +++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query33.out @@ -26,11 +26,13 @@ PhysicalTopN ------------------------------------------PhysicalProject --------------------------------------------filter((date_dim.d_moy = 1)(date_dim.d_year = 2002)) ----------------------------------------------PhysicalOlapScan[date_dim] -----------------------------------PhysicalProject -------------------------------------filter((customer_address.ca_gmt_offset = -5.00)) ---------------------------------------PhysicalOlapScan[customer_address] -----------------------------PhysicalProject -------------------------------PhysicalOlapScan[item] +----------------------------------PhysicalDistribute +------------------------------------PhysicalProject +--------------------------------------filter((customer_address.ca_gmt_offset = -5.00)) +----------------------------------------PhysicalOlapScan[customer_address] +----------------------------PhysicalDistribute +------------------------------PhysicalProject +--------------------------------PhysicalOlapScan[item] ------------------PhysicalDistribute --------------------PhysicalProject ----------------------filter((item.i_category = 'Home')) @@ -54,11 +56,13 @@ PhysicalTopN ------------------------------------------PhysicalProject --------------------------------------------filter((date_dim.d_moy = 1)(date_dim.d_year = 2002)) ----------------------------------------------PhysicalOlapScan[date_dim] -----------------------------------PhysicalProject -------------------------------------filter((customer_address.ca_gmt_offset = -5.00)) ---------------------------------------PhysicalOlapScan[customer_address] -----------------------------PhysicalProject -------------------------------PhysicalOlapScan[item] +----------------------------------PhysicalDistribute +------------------------------------PhysicalProject +--------------------------------------filter((customer_address.ca_gmt_offset = -5.00)) +----------------------------------------PhysicalOlapScan[customer_address] +----------------------------PhysicalDistribute +------------------------------PhysicalProject +--------------------------------PhysicalOlapScan[item] ------------------PhysicalDistribute --------------------PhysicalProject ----------------------filter((item.i_category = 'Home')) @@ -84,9 +88,10 @@ PhysicalTopN ------------------------------------------PhysicalProject --------------------------------------------filter((date_dim.d_moy = 1)(date_dim.d_year = 2002)) ----------------------------------------------PhysicalOlapScan[date_dim] -----------------------------------PhysicalProject -------------------------------------filter((customer_address.ca_gmt_offset = -5.00)) ---------------------------------------PhysicalOlapScan[customer_address] +----------------------------------PhysicalDistribute +------------------------------------PhysicalProject +--------------------------------------filter((customer_address.ca_gmt_offset = -5.00)) +----------------------------------------PhysicalOlapScan[customer_address] ------------------PhysicalDistribute --------------------PhysicalProject ----------------------filter((item.i_category = 'Home')) diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query35.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query35.out index 4a747defc7f1358..93dac990e564443 100644 --- a/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query35.out +++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query35.out @@ -31,8 +31,9 @@ PhysicalTopN --------------------------------------PhysicalDistribute ----------------------------------------PhysicalProject ------------------------------------------PhysicalOlapScan[customer] -----------------------------------PhysicalProject -------------------------------------PhysicalOlapScan[customer_address] +----------------------------------PhysicalDistribute +------------------------------------PhysicalProject +--------------------------------------PhysicalOlapScan[customer_address] ----------------------------PhysicalDistribute ------------------------------PhysicalProject --------------------------------PhysicalOlapScan[customer_demographics] diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query4.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query4.out index 5db6313ba256d1f..b1ba26b82ef100c 100644 --- a/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query4.out +++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query4.out @@ -2,99 +2,98 @@ -- !ds_shape_4 -- CteAnchor[cteId= ( CTEId#6=] ) --CteProducer[cteId= ( CTEId#6=] ) -----PhysicalProject -------PhysicalUnion ---------PhysicalProject -----------hashAgg[GLOBAL] -------------PhysicalDistribute ---------------hashAgg[LOCAL] -----------------PhysicalProject -------------------hashJoin[INNER_JOIN](store_sales.ss_sold_date_sk = date_dim.d_date_sk) ---------------------PhysicalProject -----------------------hashJoin[INNER_JOIN](customer.c_customer_sk = store_sales.ss_customer_sk) +----PhysicalUnion +------PhysicalProject +--------hashAgg[GLOBAL] +----------PhysicalDistribute +------------hashAgg[LOCAL] +--------------PhysicalProject +----------------hashJoin[INNER_JOIN](store_sales.ss_sold_date_sk = date_dim.d_date_sk) +------------------PhysicalProject +--------------------hashJoin[INNER_JOIN](customer.c_customer_sk = store_sales.ss_customer_sk) +----------------------PhysicalProject +------------------------filter(((('s' = 'c') OR ('s' = 's')) OR ('s' = 'w'))) +--------------------------PhysicalOlapScan[store_sales] +----------------------PhysicalDistribute ------------------------PhysicalProject --------------------------filter(((('s' = 'c') OR ('s' = 's')) OR ('s' = 'w'))) -----------------------------PhysicalOlapScan[store_sales] -------------------------PhysicalDistribute ---------------------------PhysicalProject -----------------------------filter(((('s' = 'c') OR ('s' = 's')) OR ('s' = 'w'))) -------------------------------PhysicalOlapScan[customer] ---------------------PhysicalDistribute -----------------------PhysicalProject -------------------------filter(((date_dim.d_year = 1999) OR (date_dim.d_year = 2000))((('s' = 'c') OR ('s' = 's')) OR ('s' = 'w'))) ---------------------------PhysicalOlapScan[date_dim] ---------PhysicalProject -----------hashAgg[GLOBAL] -------------PhysicalDistribute ---------------hashAgg[LOCAL] -----------------PhysicalProject -------------------hashJoin[INNER_JOIN](catalog_sales.cs_sold_date_sk = date_dim.d_date_sk) +----------------------------PhysicalOlapScan[customer] +------------------PhysicalDistribute --------------------PhysicalProject -----------------------hashJoin[INNER_JOIN](customer.c_customer_sk = catalog_sales.cs_bill_customer_sk) +----------------------filter(((date_dim.d_year = 1999) OR (date_dim.d_year = 2000))((('s' = 'c') OR ('s' = 's')) OR ('s' = 'w'))) +------------------------PhysicalOlapScan[date_dim] +------PhysicalProject +--------hashAgg[GLOBAL] +----------PhysicalDistribute +------------hashAgg[LOCAL] +--------------PhysicalProject +----------------hashJoin[INNER_JOIN](catalog_sales.cs_sold_date_sk = date_dim.d_date_sk) +------------------PhysicalProject +--------------------hashJoin[INNER_JOIN](customer.c_customer_sk = catalog_sales.cs_bill_customer_sk) +----------------------PhysicalProject +------------------------filter(((('c' = 'c') OR ('c' = 's')) OR ('c' = 'w'))) +--------------------------PhysicalOlapScan[catalog_sales] +----------------------PhysicalDistribute ------------------------PhysicalProject --------------------------filter(((('c' = 'c') OR ('c' = 's')) OR ('c' = 'w'))) -----------------------------PhysicalOlapScan[catalog_sales] -------------------------PhysicalDistribute ---------------------------PhysicalProject -----------------------------filter(((('c' = 'c') OR ('c' = 's')) OR ('c' = 'w'))) -------------------------------PhysicalOlapScan[customer] ---------------------PhysicalDistribute -----------------------PhysicalProject -------------------------filter(((date_dim.d_year = 1999) OR (date_dim.d_year = 2000))((('c' = 'c') OR ('c' = 's')) OR ('c' = 'w'))) ---------------------------PhysicalOlapScan[date_dim] ---------PhysicalProject -----------hashAgg[GLOBAL] -------------PhysicalDistribute ---------------hashAgg[LOCAL] -----------------PhysicalProject -------------------hashJoin[INNER_JOIN](web_sales.ws_sold_date_sk = date_dim.d_date_sk) +----------------------------PhysicalOlapScan[customer] +------------------PhysicalDistribute --------------------PhysicalProject -----------------------hashJoin[INNER_JOIN](customer.c_customer_sk = web_sales.ws_bill_customer_sk) +----------------------filter(((date_dim.d_year = 1999) OR (date_dim.d_year = 2000))((('c' = 'c') OR ('c' = 's')) OR ('c' = 'w'))) +------------------------PhysicalOlapScan[date_dim] +------PhysicalProject +--------hashAgg[GLOBAL] +----------PhysicalDistribute +------------hashAgg[LOCAL] +--------------PhysicalProject +----------------hashJoin[INNER_JOIN](web_sales.ws_sold_date_sk = date_dim.d_date_sk) +------------------PhysicalProject +--------------------hashJoin[INNER_JOIN](customer.c_customer_sk = web_sales.ws_bill_customer_sk) +----------------------PhysicalProject +------------------------filter(((('w' = 'c') OR ('w' = 's')) OR ('w' = 'w'))) +--------------------------PhysicalOlapScan[web_sales] +----------------------PhysicalDistribute ------------------------PhysicalProject --------------------------filter(((('w' = 'c') OR ('w' = 's')) OR ('w' = 'w'))) -----------------------------PhysicalOlapScan[web_sales] -------------------------PhysicalDistribute ---------------------------PhysicalProject -----------------------------filter(((('w' = 'c') OR ('w' = 's')) OR ('w' = 'w'))) -------------------------------PhysicalOlapScan[customer] ---------------------PhysicalDistribute -----------------------PhysicalProject -------------------------filter(((date_dim.d_year = 1999) OR (date_dim.d_year = 2000))((('w' = 'c') OR ('w' = 's')) OR ('w' = 'w'))) ---------------------------PhysicalOlapScan[date_dim] +----------------------------PhysicalOlapScan[customer] +------------------PhysicalDistribute +--------------------PhysicalProject +----------------------filter(((date_dim.d_year = 1999) OR (date_dim.d_year = 2000))((('w' = 'c') OR ('w' = 's')) OR ('w' = 'w'))) +------------------------PhysicalOlapScan[date_dim] --PhysicalTopN ----PhysicalDistribute ------PhysicalTopN --------PhysicalProject ----------hashJoin[INNER_JOIN](t_s_firstyear.customer_id = t_w_secyear.customer_id)(CASE WHEN (year_total > 0.000000) THEN (cast(year_total as DECIMALV3(38, 16)) / year_total) ELSE NULL END > CASE WHEN (year_total > 0.000000) THEN (cast(year_total as DECIMALV3(38, 16)) / year_total) ELSE NULL END) -------------PhysicalProject ---------------filter((t_w_secyear.sale_type = 'w')(t_w_secyear.dyear = 2000)) -----------------CteConsumer[cteId= ( CTEId#6=] ) ------------PhysicalDistribute --------------PhysicalProject -----------------hashJoin[INNER_JOIN](t_s_firstyear.customer_id = t_w_firstyear.customer_id) +----------------filter((t_w_secyear.sale_type = 'w')(t_w_secyear.dyear = 2000)) +------------------CteConsumer[cteId= ( CTEId#6=] ) +------------PhysicalProject +--------------hashJoin[INNER_JOIN](t_s_firstyear.customer_id = t_w_firstyear.customer_id) +----------------PhysicalDistribute ------------------PhysicalProject --------------------filter((t_w_firstyear.dyear = 1999)(t_w_firstyear.sale_type = 'w')(t_w_firstyear.year_total > 0.000000)) ----------------------CteConsumer[cteId= ( CTEId#6=] ) -------------------PhysicalDistribute +----------------PhysicalProject +------------------hashJoin[INNER_JOIN](t_s_secyear.customer_id = t_s_firstyear.customer_id)(CASE WHEN (year_total > 0.000000) THEN (cast(year_total as DECIMALV3(38, 16)) / year_total) ELSE NULL END > CASE WHEN (year_total > 0.000000) THEN (cast(year_total as DECIMALV3(38, 16)) / year_total) ELSE NULL END) +--------------------PhysicalDistribute +----------------------PhysicalProject +------------------------filter((t_s_secyear.sale_type = 's')(t_s_secyear.dyear = 2000)) +--------------------------CteConsumer[cteId= ( CTEId#6=] ) --------------------PhysicalProject -----------------------hashJoin[INNER_JOIN](t_s_secyear.customer_id = t_s_firstyear.customer_id)(CASE WHEN (year_total > 0.000000) THEN (cast(year_total as DECIMALV3(38, 16)) / year_total) ELSE NULL END > CASE WHEN (year_total > 0.000000) THEN (cast(year_total as DECIMALV3(38, 16)) / year_total) ELSE NULL END) +----------------------hashJoin[INNER_JOIN](t_s_firstyear.customer_id = t_c_secyear.customer_id) ------------------------PhysicalDistribute --------------------------PhysicalProject -----------------------------filter((t_s_secyear.sale_type = 's')(t_s_secyear.dyear = 2000)) +----------------------------filter((t_c_secyear.sale_type = 'c')(t_c_secyear.dyear = 2000)) ------------------------------CteConsumer[cteId= ( CTEId#6=] ) -------------------------PhysicalProject ---------------------------hashJoin[INNER_JOIN](t_s_firstyear.customer_id = t_c_secyear.customer_id) -----------------------------PhysicalDistribute -------------------------------PhysicalProject ---------------------------------filter((t_c_secyear.sale_type = 'c')(t_c_secyear.dyear = 2000)) -----------------------------------CteConsumer[cteId= ( CTEId#6=] ) -----------------------------hashJoin[INNER_JOIN](t_s_firstyear.customer_id = t_c_firstyear.customer_id) -------------------------------PhysicalDistribute ---------------------------------PhysicalProject -----------------------------------filter((t_s_firstyear.year_total > 0.000000)(t_s_firstyear.dyear = 1999)(t_s_firstyear.sale_type = 's')) -------------------------------------CteConsumer[cteId= ( CTEId#6=] ) -------------------------------PhysicalDistribute ---------------------------------PhysicalProject -----------------------------------filter((t_c_firstyear.year_total > 0.000000)(t_c_firstyear.dyear = 1999)(t_c_firstyear.sale_type = 'c')) -------------------------------------CteConsumer[cteId= ( CTEId#6=] ) +------------------------hashJoin[INNER_JOIN](t_s_firstyear.customer_id = t_c_firstyear.customer_id) +--------------------------PhysicalDistribute +----------------------------PhysicalProject +------------------------------filter((t_s_firstyear.year_total > 0.000000)(t_s_firstyear.dyear = 1999)(t_s_firstyear.sale_type = 's')) +--------------------------------CteConsumer[cteId= ( CTEId#6=] ) +--------------------------PhysicalDistribute +----------------------------PhysicalProject +------------------------------filter((t_c_firstyear.year_total > 0.000000)(t_c_firstyear.dyear = 1999)(t_c_firstyear.sale_type = 'c')) +--------------------------------CteConsumer[cteId= ( CTEId#6=] ) diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query45.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query45.out index 4a3c96c502bb5fb..e84dbdaabe0fb70 100644 --- a/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query45.out +++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query45.out @@ -22,16 +22,18 @@ PhysicalTopN ----------------------------------PhysicalProject ------------------------------------filter((date_dim.d_qoy = 2)(date_dim.d_year = 2000)) --------------------------------------PhysicalOlapScan[date_dim] -----------------------------PhysicalProject -------------------------------PhysicalOlapScan[item] +----------------------------PhysicalDistribute +------------------------------PhysicalProject +--------------------------------PhysicalOlapScan[item] ----------------------PhysicalDistribute ------------------------PhysicalProject --------------------------hashJoin[INNER_JOIN](customer.c_current_addr_sk = customer_address.ca_address_sk) ----------------------------PhysicalDistribute ------------------------------PhysicalProject --------------------------------PhysicalOlapScan[customer] -----------------------------PhysicalProject -------------------------------PhysicalOlapScan[customer_address] +----------------------------PhysicalDistribute +------------------------------PhysicalProject +--------------------------------PhysicalOlapScan[customer_address] ------------------PhysicalDistribute --------------------PhysicalProject ----------------------filter(i_item_sk IN (2, 3, 5, 7, 11, 13, 17, 19, 23, 29)) diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query46.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query46.out index f86ecfbb990db4f..f1c91e1f35673a0 100644 --- a/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query46.out +++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query46.out @@ -29,8 +29,9 @@ PhysicalTopN ----------------------------PhysicalProject ------------------------------filter(s_city IN ('Five Points', 'Centerville', 'Oak Grove', 'Fairview', 'Liberty')) --------------------------------PhysicalOlapScan[store] ---------------------PhysicalProject -----------------------PhysicalOlapScan[customer_address] +--------------------PhysicalDistribute +----------------------PhysicalProject +------------------------PhysicalOlapScan[customer_address] ----------PhysicalDistribute ------------hashJoin[INNER_JOIN](customer.c_current_addr_sk = current_addr.ca_address_sk) --------------PhysicalProject diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query47.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query47.out index 5d2dff0b768e7ea..5af21c50d8ea111 100644 --- a/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query47.out +++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query47.out @@ -5,31 +5,30 @@ CteAnchor[cteId= ( CTEId#0=] ) ----PhysicalProject ------PhysicalWindow --------PhysicalQuickSort -----------PhysicalDistribute -------------PhysicalWindow ---------------PhysicalQuickSort -----------------PhysicalDistribute -------------------PhysicalProject ---------------------hashAgg[GLOBAL] -----------------------PhysicalDistribute -------------------------hashAgg[LOCAL] ---------------------------PhysicalProject -----------------------------hashJoin[INNER_JOIN](store_sales.ss_store_sk = store.s_store_sk) -------------------------------PhysicalProject ---------------------------------hashJoin[INNER_JOIN](store_sales.ss_item_sk = item.i_item_sk) -----------------------------------hashJoin[INNER_JOIN](store_sales.ss_sold_date_sk = date_dim.d_date_sk) -------------------------------------PhysicalProject ---------------------------------------PhysicalOlapScan[store_sales] -------------------------------------PhysicalDistribute ---------------------------------------PhysicalProject -----------------------------------------filter((((date_dim.d_year = 2001) OR ((date_dim.d_year = 2000) AND (date_dim.d_moy = 12))) OR ((date_dim.d_year = 2002) AND (date_dim.d_moy = 1)))) -------------------------------------------PhysicalOlapScan[date_dim] +----------PhysicalWindow +------------PhysicalQuickSort +--------------PhysicalDistribute +----------------PhysicalProject +------------------hashAgg[GLOBAL] +--------------------PhysicalDistribute +----------------------hashAgg[LOCAL] +------------------------PhysicalProject +--------------------------hashJoin[INNER_JOIN](store_sales.ss_store_sk = store.s_store_sk) +----------------------------PhysicalProject +------------------------------hashJoin[INNER_JOIN](store_sales.ss_item_sk = item.i_item_sk) +--------------------------------hashJoin[INNER_JOIN](store_sales.ss_sold_date_sk = date_dim.d_date_sk) +----------------------------------PhysicalProject +------------------------------------PhysicalOlapScan[store_sales] ----------------------------------PhysicalDistribute ------------------------------------PhysicalProject ---------------------------------------PhysicalOlapScan[item] -------------------------------PhysicalDistribute ---------------------------------PhysicalProject -----------------------------------PhysicalOlapScan[store] +--------------------------------------filter((((date_dim.d_year = 2001) OR ((date_dim.d_year = 2000) AND (date_dim.d_moy = 12))) OR ((date_dim.d_year = 2002) AND (date_dim.d_moy = 1)))) +----------------------------------------PhysicalOlapScan[date_dim] +--------------------------------PhysicalDistribute +----------------------------------PhysicalProject +------------------------------------PhysicalOlapScan[item] +----------------------------PhysicalDistribute +------------------------------PhysicalProject +--------------------------------PhysicalOlapScan[store] --PhysicalProject ----PhysicalTopN ------PhysicalDistribute diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query49.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query49.out index 557f4a5e33de68e..c9f23c7fd0bf3d9 100644 --- a/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query49.out +++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query49.out @@ -7,85 +7,82 @@ PhysicalTopN --------PhysicalDistribute ----------hashAgg[LOCAL] ------------PhysicalUnion ---------------PhysicalProject -----------------filter(((return_rank <= 10) OR (currency_rank <= 10))) -------------------PhysicalWindow ---------------------PhysicalQuickSort -----------------------PhysicalDistribute -------------------------PhysicalQuickSort ---------------------------PhysicalWindow -----------------------------PhysicalQuickSort -------------------------------PhysicalDistribute ---------------------------------PhysicalQuickSort -----------------------------------PhysicalProject -------------------------------------hashAgg[GLOBAL] ---------------------------------------PhysicalDistribute -----------------------------------------hashAgg[LOCAL] -------------------------------------------PhysicalProject ---------------------------------------------hashJoin[INNER_JOIN](ws.ws_order_number = wr.wr_order_number)(item = wr.wr_item_sk) +--------------PhysicalDistribute +----------------PhysicalProject +------------------filter(((return_rank <= 10) OR (currency_rank <= 10))) +--------------------PhysicalWindow +----------------------PhysicalQuickSort +------------------------PhysicalWindow +--------------------------PhysicalQuickSort +----------------------------PhysicalDistribute +------------------------------PhysicalQuickSort +--------------------------------PhysicalProject +----------------------------------hashAgg[GLOBAL] +------------------------------------PhysicalDistribute +--------------------------------------hashAgg[LOCAL] +----------------------------------------PhysicalProject +------------------------------------------hashJoin[INNER_JOIN](ws.ws_order_number = wr.wr_order_number)(item = wr.wr_item_sk) +--------------------------------------------PhysicalProject +----------------------------------------------filter((wr.wr_return_amt > 10000.00)) +------------------------------------------------PhysicalOlapScan[web_returns] +--------------------------------------------hashJoin[INNER_JOIN](ws.ws_sold_date_sk = date_dim.d_date_sk) ----------------------------------------------PhysicalProject -------------------------------------------------filter((wr.wr_return_amt > 10000.00)) ---------------------------------------------------PhysicalOlapScan[web_returns] -----------------------------------------------hashJoin[INNER_JOIN](ws.ws_sold_date_sk = date_dim.d_date_sk) +------------------------------------------------filter((ws.ws_net_paid > 0.00)(ws.ws_quantity > 0)(ws.ws_net_profit > 1.00)) +--------------------------------------------------PhysicalOlapScan[web_sales] +----------------------------------------------PhysicalDistribute ------------------------------------------------PhysicalProject ---------------------------------------------------filter((ws.ws_net_paid > 0.00)(ws.ws_quantity > 0)(ws.ws_net_profit > 1.00)) -----------------------------------------------------PhysicalOlapScan[web_sales] -------------------------------------------------PhysicalDistribute ---------------------------------------------------PhysicalProject -----------------------------------------------------filter((date_dim.d_moy = 12)(date_dim.d_year = 1999)) -------------------------------------------------------PhysicalOlapScan[date_dim] ---------------PhysicalProject -----------------filter(((return_rank <= 10) OR (currency_rank <= 10))) -------------------PhysicalWindow ---------------------PhysicalQuickSort -----------------------PhysicalDistribute -------------------------PhysicalQuickSort ---------------------------PhysicalWindow -----------------------------PhysicalQuickSort -------------------------------PhysicalDistribute ---------------------------------PhysicalQuickSort -----------------------------------PhysicalProject -------------------------------------hashAgg[GLOBAL] ---------------------------------------PhysicalDistribute -----------------------------------------hashAgg[LOCAL] -------------------------------------------PhysicalProject ---------------------------------------------hashJoin[INNER_JOIN](cs.cs_order_number = cr.cr_order_number)(item = cr.cr_item_sk) +--------------------------------------------------filter((date_dim.d_moy = 12)(date_dim.d_year = 1999)) +----------------------------------------------------PhysicalOlapScan[date_dim] +--------------PhysicalDistribute +----------------PhysicalProject +------------------filter(((return_rank <= 10) OR (currency_rank <= 10))) +--------------------PhysicalWindow +----------------------PhysicalQuickSort +------------------------PhysicalWindow +--------------------------PhysicalQuickSort +----------------------------PhysicalDistribute +------------------------------PhysicalQuickSort +--------------------------------PhysicalProject +----------------------------------hashAgg[GLOBAL] +------------------------------------PhysicalDistribute +--------------------------------------hashAgg[LOCAL] +----------------------------------------PhysicalProject +------------------------------------------hashJoin[INNER_JOIN](cs.cs_order_number = cr.cr_order_number)(item = cr.cr_item_sk) +--------------------------------------------PhysicalProject +----------------------------------------------filter((cr.cr_return_amount > 10000.00)) +------------------------------------------------PhysicalOlapScan[catalog_returns] +--------------------------------------------hashJoin[INNER_JOIN](cs.cs_sold_date_sk = date_dim.d_date_sk) ----------------------------------------------PhysicalProject -------------------------------------------------filter((cr.cr_return_amount > 10000.00)) ---------------------------------------------------PhysicalOlapScan[catalog_returns] -----------------------------------------------hashJoin[INNER_JOIN](cs.cs_sold_date_sk = date_dim.d_date_sk) +------------------------------------------------filter((cs.cs_net_paid > 0.00)(cs.cs_quantity > 0)(cs.cs_net_profit > 1.00)) +--------------------------------------------------PhysicalOlapScan[catalog_sales] +----------------------------------------------PhysicalDistribute ------------------------------------------------PhysicalProject ---------------------------------------------------filter((cs.cs_net_paid > 0.00)(cs.cs_quantity > 0)(cs.cs_net_profit > 1.00)) -----------------------------------------------------PhysicalOlapScan[catalog_sales] -------------------------------------------------PhysicalDistribute ---------------------------------------------------PhysicalProject -----------------------------------------------------filter((date_dim.d_moy = 12)(date_dim.d_year = 1999)) -------------------------------------------------------PhysicalOlapScan[date_dim] ---------------PhysicalProject -----------------filter(((return_rank <= 10) OR (currency_rank <= 10))) -------------------PhysicalWindow ---------------------PhysicalQuickSort -----------------------PhysicalDistribute -------------------------PhysicalQuickSort ---------------------------PhysicalWindow -----------------------------PhysicalQuickSort -------------------------------PhysicalDistribute ---------------------------------PhysicalQuickSort -----------------------------------PhysicalProject -------------------------------------hashAgg[GLOBAL] ---------------------------------------PhysicalDistribute -----------------------------------------hashAgg[LOCAL] -------------------------------------------PhysicalProject ---------------------------------------------hashJoin[INNER_JOIN](sts.ss_ticket_number = sr.sr_ticket_number)(item = sr.sr_item_sk) +--------------------------------------------------filter((date_dim.d_moy = 12)(date_dim.d_year = 1999)) +----------------------------------------------------PhysicalOlapScan[date_dim] +--------------PhysicalDistribute +----------------PhysicalProject +------------------filter(((return_rank <= 10) OR (currency_rank <= 10))) +--------------------PhysicalWindow +----------------------PhysicalQuickSort +------------------------PhysicalWindow +--------------------------PhysicalQuickSort +----------------------------PhysicalDistribute +------------------------------PhysicalQuickSort +--------------------------------PhysicalProject +----------------------------------hashAgg[GLOBAL] +------------------------------------PhysicalDistribute +--------------------------------------hashAgg[LOCAL] +----------------------------------------PhysicalProject +------------------------------------------hashJoin[INNER_JOIN](sts.ss_ticket_number = sr.sr_ticket_number)(item = sr.sr_item_sk) +--------------------------------------------PhysicalProject +----------------------------------------------filter((sr.sr_return_amt > 10000.00)) +------------------------------------------------PhysicalOlapScan[store_returns] +--------------------------------------------hashJoin[INNER_JOIN](sts.ss_sold_date_sk = date_dim.d_date_sk) ----------------------------------------------PhysicalProject -------------------------------------------------filter((sr.sr_return_amt > 10000.00)) ---------------------------------------------------PhysicalOlapScan[store_returns] -----------------------------------------------hashJoin[INNER_JOIN](sts.ss_sold_date_sk = date_dim.d_date_sk) +------------------------------------------------filter((sts.ss_quantity > 0)(sts.ss_net_profit > 1.00)(sts.ss_net_paid > 0.00)) +--------------------------------------------------PhysicalOlapScan[store_sales] +----------------------------------------------PhysicalDistribute ------------------------------------------------PhysicalProject ---------------------------------------------------filter((sts.ss_quantity > 0)(sts.ss_net_profit > 1.00)(sts.ss_net_paid > 0.00)) -----------------------------------------------------PhysicalOlapScan[store_sales] -------------------------------------------------PhysicalDistribute ---------------------------------------------------PhysicalProject -----------------------------------------------------filter((date_dim.d_moy = 12)(date_dim.d_year = 1999)) -------------------------------------------------------PhysicalOlapScan[date_dim] +--------------------------------------------------filter((date_dim.d_moy = 12)(date_dim.d_year = 1999)) +----------------------------------------------------PhysicalOlapScan[date_dim] diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query5.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query5.out index a3324d16ac2cca5..eb45d441220307a 100644 --- a/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query5.out +++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query5.out @@ -17,10 +17,12 @@ PhysicalTopN ----------------------------hashJoin[INNER_JOIN](salesreturns.store_sk = store.s_store_sk) ------------------------------hashJoin[INNER_JOIN](salesreturns.date_sk = date_dim.d_date_sk) --------------------------------PhysicalUnion -----------------------------------PhysicalProject -------------------------------------PhysicalOlapScan[store_sales] -----------------------------------PhysicalProject -------------------------------------PhysicalOlapScan[store_returns] +----------------------------------PhysicalDistribute +------------------------------------PhysicalProject +--------------------------------------PhysicalOlapScan[store_sales] +----------------------------------PhysicalDistribute +------------------------------------PhysicalProject +--------------------------------------PhysicalOlapScan[store_returns] --------------------------------PhysicalDistribute ----------------------------------PhysicalProject ------------------------------------filter((date_dim.d_date <= 2000-09-02)(date_dim.d_date >= 2000-08-19)) @@ -36,10 +38,12 @@ PhysicalTopN ----------------------------hashJoin[INNER_JOIN](salesreturns.page_sk = catalog_page.cp_catalog_page_sk) ------------------------------hashJoin[INNER_JOIN](salesreturns.date_sk = date_dim.d_date_sk) --------------------------------PhysicalUnion -----------------------------------PhysicalProject -------------------------------------PhysicalOlapScan[catalog_sales] -----------------------------------PhysicalProject -------------------------------------PhysicalOlapScan[catalog_returns] +----------------------------------PhysicalDistribute +------------------------------------PhysicalProject +--------------------------------------PhysicalOlapScan[catalog_sales] +----------------------------------PhysicalDistribute +------------------------------------PhysicalProject +--------------------------------------PhysicalOlapScan[catalog_returns] --------------------------------PhysicalDistribute ----------------------------------PhysicalProject ------------------------------------filter((date_dim.d_date >= 2000-08-19)(date_dim.d_date <= 2000-09-02)) @@ -55,14 +59,16 @@ PhysicalTopN ----------------------------hashJoin[INNER_JOIN](salesreturns.wsr_web_site_sk = web_site.web_site_sk) ------------------------------hashJoin[INNER_JOIN](salesreturns.date_sk = date_dim.d_date_sk) --------------------------------PhysicalUnion -----------------------------------PhysicalProject -------------------------------------PhysicalOlapScan[web_sales] -----------------------------------PhysicalProject -------------------------------------hashJoin[INNER_JOIN](web_returns.wr_item_sk = web_sales.ws_item_sk)(web_returns.wr_order_number = web_sales.ws_order_number) ---------------------------------------PhysicalProject -----------------------------------------PhysicalOlapScan[web_sales] ---------------------------------------PhysicalProject -----------------------------------------PhysicalOlapScan[web_returns] +----------------------------------PhysicalDistribute +------------------------------------PhysicalProject +--------------------------------------PhysicalOlapScan[web_sales] +----------------------------------PhysicalDistribute +------------------------------------PhysicalProject +--------------------------------------hashJoin[INNER_JOIN](web_returns.wr_item_sk = web_sales.ws_item_sk)(web_returns.wr_order_number = web_sales.ws_order_number) +----------------------------------------PhysicalProject +------------------------------------------PhysicalOlapScan[web_sales] +----------------------------------------PhysicalProject +------------------------------------------PhysicalOlapScan[web_returns] --------------------------------PhysicalDistribute ----------------------------------PhysicalProject ------------------------------------filter((date_dim.d_date >= 2000-08-19)(date_dim.d_date <= 2000-09-02)) diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query51.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query51.out index 50af4041a882df5..66c68bed08495c3 100644 --- a/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query51.out +++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query51.out @@ -9,36 +9,34 @@ PhysicalTopN ------------PhysicalDistribute --------------PhysicalProject ----------------hashJoin[FULL_OUTER_JOIN](web.item_sk = store.item_sk)(web.d_date = store.d_date) -------------------PhysicalDistribute ---------------------PhysicalProject -----------------------PhysicalWindow -------------------------PhysicalQuickSort ---------------------------PhysicalDistribute -----------------------------hashAgg[GLOBAL] -------------------------------PhysicalDistribute ---------------------------------hashAgg[LOCAL] -----------------------------------PhysicalProject -------------------------------------hashJoin[INNER_JOIN](store_sales.ss_sold_date_sk = date_dim.d_date_sk) +------------------PhysicalProject +--------------------PhysicalWindow +----------------------PhysicalQuickSort +------------------------PhysicalDistribute +--------------------------hashAgg[GLOBAL] +----------------------------PhysicalDistribute +------------------------------hashAgg[LOCAL] +--------------------------------PhysicalProject +----------------------------------hashJoin[INNER_JOIN](store_sales.ss_sold_date_sk = date_dim.d_date_sk) +------------------------------------PhysicalProject +--------------------------------------PhysicalOlapScan[store_sales] +------------------------------------PhysicalDistribute --------------------------------------PhysicalProject -----------------------------------------PhysicalOlapScan[store_sales] ---------------------------------------PhysicalDistribute -----------------------------------------PhysicalProject -------------------------------------------filter((date_dim.d_month_seq <= 1227)(date_dim.d_month_seq >= 1216)) ---------------------------------------------PhysicalOlapScan[date_dim] -------------------PhysicalDistribute ---------------------PhysicalProject -----------------------PhysicalWindow -------------------------PhysicalQuickSort ---------------------------PhysicalDistribute -----------------------------hashAgg[GLOBAL] -------------------------------PhysicalDistribute ---------------------------------hashAgg[LOCAL] -----------------------------------PhysicalProject -------------------------------------hashJoin[INNER_JOIN](web_sales.ws_sold_date_sk = date_dim.d_date_sk) +----------------------------------------filter((date_dim.d_month_seq <= 1227)(date_dim.d_month_seq >= 1216)) +------------------------------------------PhysicalOlapScan[date_dim] +------------------PhysicalProject +--------------------PhysicalWindow +----------------------PhysicalQuickSort +------------------------PhysicalDistribute +--------------------------hashAgg[GLOBAL] +----------------------------PhysicalDistribute +------------------------------hashAgg[LOCAL] +--------------------------------PhysicalProject +----------------------------------hashJoin[INNER_JOIN](web_sales.ws_sold_date_sk = date_dim.d_date_sk) +------------------------------------PhysicalProject +--------------------------------------PhysicalOlapScan[web_sales] +------------------------------------PhysicalDistribute --------------------------------------PhysicalProject -----------------------------------------PhysicalOlapScan[web_sales] ---------------------------------------PhysicalDistribute -----------------------------------------PhysicalProject -------------------------------------------filter((date_dim.d_month_seq >= 1216)(date_dim.d_month_seq <= 1227)) ---------------------------------------------PhysicalOlapScan[date_dim] +----------------------------------------filter((date_dim.d_month_seq >= 1216)(date_dim.d_month_seq <= 1227)) +------------------------------------------PhysicalOlapScan[date_dim] diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query54.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query54.out index fb5bae3e9a7720f..72a23fe3e82503f 100644 --- a/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query54.out +++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query54.out @@ -41,10 +41,12 @@ PhysicalTopN ----------------------------------------------------------------PhysicalProject ------------------------------------------------------------------hashJoin[INNER_JOIN](cs_or_ws_sales.item_sk = item.i_item_sk) --------------------------------------------------------------------PhysicalUnion -----------------------------------------------------------------------PhysicalProject -------------------------------------------------------------------------PhysicalOlapScan[catalog_sales] -----------------------------------------------------------------------PhysicalProject -------------------------------------------------------------------------PhysicalOlapScan[web_sales] +----------------------------------------------------------------------PhysicalDistribute +------------------------------------------------------------------------PhysicalProject +--------------------------------------------------------------------------PhysicalOlapScan[catalog_sales] +----------------------------------------------------------------------PhysicalDistribute +------------------------------------------------------------------------PhysicalProject +--------------------------------------------------------------------------PhysicalOlapScan[web_sales] --------------------------------------------------------------------PhysicalDistribute ----------------------------------------------------------------------PhysicalProject ------------------------------------------------------------------------filter((cast(i_class as VARCHAR(*)) = 'maternity')(cast(i_category as VARCHAR(*)) = 'Women')) @@ -58,20 +60,22 @@ PhysicalTopN ----------------------------------------PhysicalOlapScan[date_dim] ----------------------------------PhysicalDistribute ------------------------------------PhysicalAssertNumRows ---------------------------------------hashAgg[GLOBAL] -----------------------------------------PhysicalDistribute -------------------------------------------hashAgg[LOCAL] ---------------------------------------------PhysicalProject -----------------------------------------------filter((date_dim.d_moy = 5)(date_dim.d_year = 1998)) -------------------------------------------------PhysicalOlapScan[date_dim] +--------------------------------------PhysicalDistribute +----------------------------------------hashAgg[GLOBAL] +------------------------------------------PhysicalDistribute +--------------------------------------------hashAgg[LOCAL] +----------------------------------------------PhysicalProject +------------------------------------------------filter((date_dim.d_moy = 5)(date_dim.d_year = 1998)) +--------------------------------------------------PhysicalOlapScan[date_dim] ------------------------------PhysicalDistribute --------------------------------PhysicalAssertNumRows -----------------------------------hashAgg[GLOBAL] -------------------------------------PhysicalDistribute ---------------------------------------hashAgg[LOCAL] -----------------------------------------PhysicalProject -------------------------------------------filter((date_dim.d_year = 1998)(date_dim.d_moy = 5)) ---------------------------------------------PhysicalOlapScan[date_dim] +----------------------------------PhysicalDistribute +------------------------------------hashAgg[GLOBAL] +--------------------------------------PhysicalDistribute +----------------------------------------hashAgg[LOCAL] +------------------------------------------PhysicalProject +--------------------------------------------filter((date_dim.d_year = 1998)(date_dim.d_moy = 5)) +----------------------------------------------PhysicalOlapScan[date_dim] --------------------------PhysicalDistribute ----------------------------PhysicalProject ------------------------------PhysicalOlapScan[store] diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query56.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query56.out index 410d72ab3a73fd6..a7efe6f1d67a0cd 100644 --- a/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query56.out +++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query56.out @@ -31,9 +31,10 @@ PhysicalTopN ------------------------------------PhysicalProject --------------------------------------filter(i_color IN ('powder', 'green', 'cyan')) ----------------------------------------PhysicalOlapScan[item] ---------------------------PhysicalProject -----------------------------filter((customer_address.ca_gmt_offset = -6.00)) -------------------------------PhysicalOlapScan[customer_address] +--------------------------PhysicalDistribute +----------------------------PhysicalProject +------------------------------filter((customer_address.ca_gmt_offset = -6.00)) +--------------------------------PhysicalOlapScan[customer_address] --------------PhysicalProject ----------------hashAgg[GLOBAL] ------------------PhysicalDistribute @@ -58,9 +59,10 @@ PhysicalTopN ------------------------------------PhysicalProject --------------------------------------filter(i_color IN ('powder', 'green', 'cyan')) ----------------------------------------PhysicalOlapScan[item] ---------------------------PhysicalProject -----------------------------filter((customer_address.ca_gmt_offset = -6.00)) -------------------------------PhysicalOlapScan[customer_address] +--------------------------PhysicalDistribute +----------------------------PhysicalProject +------------------------------filter((customer_address.ca_gmt_offset = -6.00)) +--------------------------------PhysicalOlapScan[customer_address] --------------PhysicalProject ----------------hashAgg[GLOBAL] ------------------PhysicalDistribute diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query57.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query57.out index 983aa5cd7e505a6..6138c0d491711ef 100644 --- a/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query57.out +++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query57.out @@ -5,31 +5,30 @@ CteAnchor[cteId= ( CTEId#0=] ) ----PhysicalProject ------PhysicalWindow --------PhysicalQuickSort -----------PhysicalDistribute -------------PhysicalWindow ---------------PhysicalQuickSort -----------------PhysicalDistribute -------------------PhysicalProject ---------------------hashAgg[GLOBAL] -----------------------PhysicalDistribute -------------------------hashAgg[LOCAL] ---------------------------PhysicalProject -----------------------------hashJoin[INNER_JOIN](call_center.cc_call_center_sk = catalog_sales.cs_call_center_sk) -------------------------------PhysicalProject ---------------------------------hashJoin[INNER_JOIN](catalog_sales.cs_item_sk = item.i_item_sk) -----------------------------------hashJoin[INNER_JOIN](catalog_sales.cs_sold_date_sk = date_dim.d_date_sk) -------------------------------------PhysicalProject ---------------------------------------PhysicalOlapScan[catalog_sales] -------------------------------------PhysicalDistribute ---------------------------------------PhysicalProject -----------------------------------------filter((((date_dim.d_year = 1999) OR ((date_dim.d_year = 1998) AND (date_dim.d_moy = 12))) OR ((date_dim.d_year = 2000) AND (date_dim.d_moy = 1)))) -------------------------------------------PhysicalOlapScan[date_dim] +----------PhysicalWindow +------------PhysicalQuickSort +--------------PhysicalDistribute +----------------PhysicalProject +------------------hashAgg[GLOBAL] +--------------------PhysicalDistribute +----------------------hashAgg[LOCAL] +------------------------PhysicalProject +--------------------------hashJoin[INNER_JOIN](call_center.cc_call_center_sk = catalog_sales.cs_call_center_sk) +----------------------------PhysicalProject +------------------------------hashJoin[INNER_JOIN](catalog_sales.cs_item_sk = item.i_item_sk) +--------------------------------hashJoin[INNER_JOIN](catalog_sales.cs_sold_date_sk = date_dim.d_date_sk) +----------------------------------PhysicalProject +------------------------------------PhysicalOlapScan[catalog_sales] ----------------------------------PhysicalDistribute ------------------------------------PhysicalProject ---------------------------------------PhysicalOlapScan[item] -------------------------------PhysicalDistribute ---------------------------------PhysicalProject -----------------------------------PhysicalOlapScan[call_center] +--------------------------------------filter((((date_dim.d_year = 1999) OR ((date_dim.d_year = 1998) AND (date_dim.d_moy = 12))) OR ((date_dim.d_year = 2000) AND (date_dim.d_moy = 1)))) +----------------------------------------PhysicalOlapScan[date_dim] +--------------------------------PhysicalDistribute +----------------------------------PhysicalProject +------------------------------------PhysicalOlapScan[item] +----------------------------PhysicalDistribute +------------------------------PhysicalProject +--------------------------------PhysicalOlapScan[call_center] --PhysicalProject ----PhysicalTopN ------PhysicalDistribute diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query58.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query58.out index 1428af1a8b71bbd..7ffaac876a79d83 100644 --- a/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query58.out +++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query58.out @@ -33,63 +33,61 @@ PhysicalTopN ----------------------------------PhysicalProject ------------------------------------filter((date_dim.d_date = 2001-03-24)) --------------------------------------PhysicalOlapScan[date_dim] -----------PhysicalDistribute -------------hashJoin[INNER_JOIN](ss_items.item_id = ws_items.item_id)(cast(ws_item_rev as DOUBLE) >= cast((0.9 * ss_item_rev) as DOUBLE))(cast(ss_item_rev as DOUBLE) >= cast((0.9 * ws_item_rev) as DOUBLE))(cast(ws_item_rev as DOUBLE) <= cast((1.1 * ss_item_rev) as DOUBLE))(cast(ss_item_rev as DOUBLE) <= cast((1.1 * ws_item_rev) as DOUBLE)) ---------------PhysicalProject -----------------hashAgg[GLOBAL] -------------------PhysicalDistribute ---------------------hashAgg[LOCAL] -----------------------PhysicalProject -------------------------hashJoin[LEFT_SEMI_JOIN](date_dim.d_date = date_dim.d_date) ---------------------------PhysicalProject -----------------------------hashJoin[INNER_JOIN](store_sales.ss_sold_date_sk = date_dim.d_date_sk) -------------------------------hashJoin[INNER_JOIN](store_sales.ss_item_sk = item.i_item_sk) ---------------------------------PhysicalProject -----------------------------------PhysicalOlapScan[store_sales] ---------------------------------PhysicalDistribute -----------------------------------PhysicalProject -------------------------------------PhysicalOlapScan[item] +----------hashJoin[INNER_JOIN](ss_items.item_id = ws_items.item_id)(cast(ws_item_rev as DOUBLE) >= cast((0.9 * ss_item_rev) as DOUBLE))(cast(ss_item_rev as DOUBLE) >= cast((0.9 * ws_item_rev) as DOUBLE))(cast(ws_item_rev as DOUBLE) <= cast((1.1 * ss_item_rev) as DOUBLE))(cast(ss_item_rev as DOUBLE) <= cast((1.1 * ws_item_rev) as DOUBLE)) +------------PhysicalProject +--------------hashAgg[GLOBAL] +----------------PhysicalDistribute +------------------hashAgg[LOCAL] +--------------------PhysicalProject +----------------------hashJoin[LEFT_SEMI_JOIN](date_dim.d_date = date_dim.d_date) +------------------------PhysicalProject +--------------------------hashJoin[INNER_JOIN](store_sales.ss_sold_date_sk = date_dim.d_date_sk) +----------------------------hashJoin[INNER_JOIN](store_sales.ss_item_sk = item.i_item_sk) +------------------------------PhysicalProject +--------------------------------PhysicalOlapScan[store_sales] ------------------------------PhysicalDistribute --------------------------------PhysicalProject -----------------------------------PhysicalOlapScan[date_dim] ---------------------------PhysicalDistribute -----------------------------PhysicalProject -------------------------------hashJoin[INNER_JOIN](date_dim.d_week_seq = date_dim.d_week_seq) ---------------------------------PhysicalProject -----------------------------------PhysicalOlapScan[date_dim] ---------------------------------PhysicalDistribute -----------------------------------PhysicalAssertNumRows -------------------------------------PhysicalDistribute ---------------------------------------PhysicalProject -----------------------------------------filter((date_dim.d_date = 2001-03-24)) -------------------------------------------PhysicalOlapScan[date_dim] ---------------PhysicalDistribute -----------------PhysicalProject -------------------hashAgg[GLOBAL] ---------------------PhysicalDistribute -----------------------hashAgg[LOCAL] -------------------------PhysicalProject ---------------------------hashJoin[LEFT_SEMI_JOIN](date_dim.d_date = date_dim.d_date) -----------------------------PhysicalProject -------------------------------hashJoin[INNER_JOIN](web_sales.ws_sold_date_sk = date_dim.d_date_sk) ---------------------------------hashJoin[INNER_JOIN](web_sales.ws_item_sk = item.i_item_sk) -----------------------------------PhysicalProject -------------------------------------PhysicalOlapScan[web_sales] +----------------------------------PhysicalOlapScan[item] +----------------------------PhysicalDistribute +------------------------------PhysicalProject +--------------------------------PhysicalOlapScan[date_dim] +------------------------PhysicalDistribute +--------------------------PhysicalProject +----------------------------hashJoin[INNER_JOIN](date_dim.d_week_seq = date_dim.d_week_seq) +------------------------------PhysicalProject +--------------------------------PhysicalOlapScan[date_dim] +------------------------------PhysicalDistribute +--------------------------------PhysicalAssertNumRows ----------------------------------PhysicalDistribute ------------------------------------PhysicalProject ---------------------------------------PhysicalOlapScan[item] ---------------------------------PhysicalDistribute -----------------------------------PhysicalProject -------------------------------------PhysicalOlapScan[date_dim] +--------------------------------------filter((date_dim.d_date = 2001-03-24)) +----------------------------------------PhysicalOlapScan[date_dim] +------------PhysicalProject +--------------hashAgg[GLOBAL] +----------------PhysicalDistribute +------------------hashAgg[LOCAL] +--------------------PhysicalProject +----------------------hashJoin[LEFT_SEMI_JOIN](date_dim.d_date = date_dim.d_date) +------------------------PhysicalProject +--------------------------hashJoin[INNER_JOIN](web_sales.ws_sold_date_sk = date_dim.d_date_sk) +----------------------------hashJoin[INNER_JOIN](web_sales.ws_item_sk = item.i_item_sk) +------------------------------PhysicalProject +--------------------------------PhysicalOlapScan[web_sales] +------------------------------PhysicalDistribute +--------------------------------PhysicalProject +----------------------------------PhysicalOlapScan[item] ----------------------------PhysicalDistribute ------------------------------PhysicalProject ---------------------------------hashJoin[INNER_JOIN](date_dim.d_week_seq = date_dim.d_week_seq) -----------------------------------PhysicalProject -------------------------------------PhysicalOlapScan[date_dim] +--------------------------------PhysicalOlapScan[date_dim] +------------------------PhysicalDistribute +--------------------------PhysicalProject +----------------------------hashJoin[INNER_JOIN](date_dim.d_week_seq = date_dim.d_week_seq) +------------------------------PhysicalProject +--------------------------------PhysicalOlapScan[date_dim] +------------------------------PhysicalDistribute +--------------------------------PhysicalAssertNumRows ----------------------------------PhysicalDistribute -------------------------------------PhysicalAssertNumRows ---------------------------------------PhysicalDistribute -----------------------------------------PhysicalProject -------------------------------------------filter((date_dim.d_date = 2001-03-24)) ---------------------------------------------PhysicalOlapScan[date_dim] +------------------------------------PhysicalProject +--------------------------------------filter((date_dim.d_date = 2001-03-24)) +----------------------------------------PhysicalOlapScan[date_dim] diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query6.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query6.out index d5751c034cbb83f..aa6c71623d648c7 100644 --- a/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query6.out +++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query6.out @@ -25,22 +25,25 @@ PhysicalTopN ----------------------------------------PhysicalOlapScan[date_dim] --------------------------------------PhysicalDistribute ----------------------------------------PhysicalAssertNumRows -------------------------------------------hashAgg[GLOBAL] ---------------------------------------------PhysicalDistribute -----------------------------------------------hashAgg[LOCAL] -------------------------------------------------PhysicalProject ---------------------------------------------------filter((date_dim.d_year = 2002)(date_dim.d_moy = 3)) -----------------------------------------------------PhysicalOlapScan[date_dim] -----------------------------PhysicalProject -------------------------------PhysicalOlapScan[item] +------------------------------------------PhysicalDistribute +--------------------------------------------hashAgg[GLOBAL] +----------------------------------------------PhysicalDistribute +------------------------------------------------hashAgg[LOCAL] +--------------------------------------------------PhysicalProject +----------------------------------------------------filter((date_dim.d_year = 2002)(date_dim.d_moy = 3)) +------------------------------------------------------PhysicalOlapScan[date_dim] +----------------------------PhysicalDistribute +------------------------------PhysicalProject +--------------------------------PhysicalOlapScan[item] ----------------------PhysicalDistribute ------------------------PhysicalProject --------------------------hashJoin[INNER_JOIN](a.ca_address_sk = c.c_current_addr_sk) ----------------------------PhysicalDistribute ------------------------------PhysicalProject --------------------------------PhysicalOlapScan[customer] -----------------------------PhysicalProject -------------------------------PhysicalOlapScan[customer_address] +----------------------------PhysicalDistribute +------------------------------PhysicalProject +--------------------------------PhysicalOlapScan[customer_address] ------------------PhysicalDistribute --------------------hashAgg[GLOBAL] ----------------------PhysicalDistribute diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query60.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query60.out index 57ee76f897a7de7..8538ed1f7b5aefe 100644 --- a/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query60.out +++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query60.out @@ -23,9 +23,10 @@ PhysicalTopN ------------------------------------PhysicalProject --------------------------------------filter((date_dim.d_year = 2000)(date_dim.d_moy = 8)) ----------------------------------------PhysicalOlapScan[date_dim] -----------------------------PhysicalProject -------------------------------filter((customer_address.ca_gmt_offset = -7.00)) ---------------------------------PhysicalOlapScan[customer_address] +----------------------------PhysicalDistribute +------------------------------PhysicalProject +--------------------------------filter((customer_address.ca_gmt_offset = -7.00)) +----------------------------------PhysicalOlapScan[customer_address] --------------------------PhysicalDistribute ----------------------------hashJoin[LEFT_SEMI_JOIN](item.i_item_id = item.i_item_id) ------------------------------PhysicalProject @@ -50,9 +51,10 @@ PhysicalTopN ------------------------------------PhysicalProject --------------------------------------filter((date_dim.d_moy = 8)(date_dim.d_year = 2000)) ----------------------------------------PhysicalOlapScan[date_dim] -----------------------------PhysicalProject -------------------------------filter((customer_address.ca_gmt_offset = -7.00)) ---------------------------------PhysicalOlapScan[customer_address] +----------------------------PhysicalDistribute +------------------------------PhysicalProject +--------------------------------filter((customer_address.ca_gmt_offset = -7.00)) +----------------------------------PhysicalOlapScan[customer_address] --------------------------PhysicalDistribute ----------------------------hashJoin[LEFT_SEMI_JOIN](item.i_item_id = item.i_item_id) ------------------------------PhysicalProject @@ -77,9 +79,10 @@ PhysicalTopN ------------------------------------PhysicalProject --------------------------------------filter((date_dim.d_year = 2000)(date_dim.d_moy = 8)) ----------------------------------------PhysicalOlapScan[date_dim] -----------------------------PhysicalProject -------------------------------filter((customer_address.ca_gmt_offset = -7.00)) ---------------------------------PhysicalOlapScan[customer_address] +----------------------------PhysicalDistribute +------------------------------PhysicalProject +--------------------------------filter((customer_address.ca_gmt_offset = -7.00)) +----------------------------------PhysicalOlapScan[customer_address] --------------------------PhysicalDistribute ----------------------------hashJoin[LEFT_SEMI_JOIN](item.i_item_id = item.i_item_id) ------------------------------PhysicalProject diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query65.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query65.out index 64a95334d592a92..0cf4b87d09e1f13 100644 --- a/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query65.out +++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query65.out @@ -39,6 +39,7 @@ PhysicalTopN ----------------PhysicalDistribute ------------------PhysicalProject --------------------PhysicalOlapScan[store] -----------PhysicalProject -------------PhysicalOlapScan[item] +----------PhysicalDistribute +------------PhysicalProject +--------------PhysicalOlapScan[item] diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query7.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query7.out index 1d979be9e1194d2..69cc851c1d3eb95 100644 --- a/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query7.out +++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query7.out @@ -27,6 +27,7 @@ PhysicalTopN ----------------------PhysicalProject ------------------------filter(((cast(p_channel_email as VARCHAR(*)) = 'N') OR (cast(p_channel_event as VARCHAR(*)) = 'N'))) --------------------------PhysicalOlapScan[promotion] -----------------PhysicalProject -------------------PhysicalOlapScan[item] +----------------PhysicalDistribute +------------------PhysicalProject +--------------------PhysicalOlapScan[item] diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query71.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query71.out index e5ec4312174f045..650c95b66811dd1 100644 --- a/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query71.out +++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query71.out @@ -11,30 +11,33 @@ PhysicalQuickSort ----------------hashJoin[INNER_JOIN](tmp.time_sk = time_dim.t_time_sk) ------------------hashJoin[INNER_JOIN](tmp.sold_item_sk = item.i_item_sk) --------------------PhysicalUnion -----------------------PhysicalProject -------------------------hashJoin[INNER_JOIN](date_dim.d_date_sk = web_sales.ws_sold_date_sk) ---------------------------PhysicalProject -----------------------------PhysicalOlapScan[web_sales] ---------------------------PhysicalDistribute +----------------------PhysicalDistribute +------------------------PhysicalProject +--------------------------hashJoin[INNER_JOIN](date_dim.d_date_sk = web_sales.ws_sold_date_sk) ----------------------------PhysicalProject -------------------------------filter((date_dim.d_moy = 12)(date_dim.d_year = 1998)) ---------------------------------PhysicalOlapScan[date_dim] -----------------------PhysicalProject -------------------------hashJoin[INNER_JOIN](date_dim.d_date_sk = catalog_sales.cs_sold_date_sk) ---------------------------PhysicalProject -----------------------------PhysicalOlapScan[catalog_sales] ---------------------------PhysicalDistribute +------------------------------PhysicalOlapScan[web_sales] +----------------------------PhysicalDistribute +------------------------------PhysicalProject +--------------------------------filter((date_dim.d_moy = 12)(date_dim.d_year = 1998)) +----------------------------------PhysicalOlapScan[date_dim] +----------------------PhysicalDistribute +------------------------PhysicalProject +--------------------------hashJoin[INNER_JOIN](date_dim.d_date_sk = catalog_sales.cs_sold_date_sk) ----------------------------PhysicalProject -------------------------------filter((date_dim.d_moy = 12)(date_dim.d_year = 1998)) ---------------------------------PhysicalOlapScan[date_dim] -----------------------PhysicalProject -------------------------hashJoin[INNER_JOIN](date_dim.d_date_sk = store_sales.ss_sold_date_sk) ---------------------------PhysicalProject -----------------------------PhysicalOlapScan[store_sales] ---------------------------PhysicalDistribute +------------------------------PhysicalOlapScan[catalog_sales] +----------------------------PhysicalDistribute +------------------------------PhysicalProject +--------------------------------filter((date_dim.d_moy = 12)(date_dim.d_year = 1998)) +----------------------------------PhysicalOlapScan[date_dim] +----------------------PhysicalDistribute +------------------------PhysicalProject +--------------------------hashJoin[INNER_JOIN](date_dim.d_date_sk = store_sales.ss_sold_date_sk) ----------------------------PhysicalProject -------------------------------filter((date_dim.d_moy = 12)(date_dim.d_year = 1998)) ---------------------------------PhysicalOlapScan[date_dim] +------------------------------PhysicalOlapScan[store_sales] +----------------------------PhysicalDistribute +------------------------------PhysicalProject +--------------------------------filter((date_dim.d_moy = 12)(date_dim.d_year = 1998)) +----------------------------------PhysicalOlapScan[date_dim] --------------------PhysicalDistribute ----------------------PhysicalProject ------------------------filter((item.i_manager_id = 1)) diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query74.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query74.out index c1f925d45b7cbe9..07a0ec06da5d8da 100644 --- a/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query74.out +++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query74.out @@ -45,15 +45,7 @@ CteAnchor[cteId= ( CTEId#4=] ) ------PhysicalTopN --------PhysicalProject ----------hashJoin[INNER_JOIN](t_s_firstyear.customer_id = t_w_firstyear.customer_id)(CASE WHEN (year_total > 0.0) THEN (year_total / year_total) ELSE NULL END > CASE WHEN (year_total > 0.0) THEN (year_total / year_total) ELSE NULL END) -------------PhysicalDistribute ---------------PhysicalProject -----------------filter((t_w_firstyear.year = 1999)(t_w_firstyear.year_total > 0.0)(t_w_firstyear.sale_type = 'w')) -------------------CteConsumer[cteId= ( CTEId#4=] ) ------------hashJoin[INNER_JOIN](t_s_firstyear.customer_id = t_w_secyear.customer_id) ---------------PhysicalDistribute -----------------PhysicalProject -------------------filter((t_w_secyear.year = 2000)(t_w_secyear.sale_type = 'w')) ---------------------CteConsumer[cteId= ( CTEId#4=] ) --------------hashJoin[INNER_JOIN](t_s_secyear.customer_id = t_s_firstyear.customer_id) ----------------PhysicalDistribute ------------------PhysicalProject @@ -63,4 +55,12 @@ CteAnchor[cteId= ( CTEId#4=] ) ------------------PhysicalProject --------------------filter((t_s_secyear.sale_type = 's')(t_s_secyear.year = 2000)) ----------------------CteConsumer[cteId= ( CTEId#4=] ) +--------------PhysicalDistribute +----------------PhysicalProject +------------------filter((t_w_secyear.year = 2000)(t_w_secyear.sale_type = 'w')) +--------------------CteConsumer[cteId= ( CTEId#4=] ) +------------PhysicalDistribute +--------------PhysicalProject +----------------filter((t_w_firstyear.year = 1999)(t_w_firstyear.year_total > 0.0)(t_w_firstyear.sale_type = 'w')) +------------------CteConsumer[cteId= ( CTEId#4=] ) diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query75.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query75.out index 0a86b61e335905e..60e083b48a87963 100644 --- a/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query75.out +++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query75.out @@ -2,62 +2,67 @@ -- !ds_shape_75 -- CteAnchor[cteId= ( CTEId#3=] ) --CteProducer[cteId= ( CTEId#3=] ) -----hashAgg[LOCAL] -------hashAgg[GLOBAL] ---------PhysicalDistribute -----------hashAgg[LOCAL] -------------PhysicalUnion ---------------PhysicalProject -----------------hashJoin[RIGHT_OUTER_JOIN](catalog_sales.cs_item_sk = catalog_returns.cr_item_sk)(catalog_sales.cs_order_number = catalog_returns.cr_order_number) -------------------PhysicalProject ---------------------PhysicalOlapScan[catalog_returns] -------------------PhysicalProject ---------------------hashJoin[INNER_JOIN](date_dim.d_date_sk = catalog_sales.cs_sold_date_sk) -----------------------hashJoin[INNER_JOIN](item.i_item_sk = catalog_sales.cs_item_sk) +----hashAgg[GLOBAL] +------PhysicalDistribute +--------hashAgg[LOCAL] +----------hashAgg[GLOBAL] +------------PhysicalDistribute +--------------hashAgg[LOCAL] +----------------PhysicalUnion +------------------PhysicalDistribute +--------------------PhysicalProject +----------------------hashJoin[RIGHT_OUTER_JOIN](catalog_sales.cs_item_sk = catalog_returns.cr_item_sk)(catalog_sales.cs_order_number = catalog_returns.cr_order_number) ------------------------PhysicalProject ---------------------------PhysicalOlapScan[catalog_sales] -------------------------PhysicalDistribute ---------------------------PhysicalProject -----------------------------filter((cast(i_category as VARCHAR(*)) = 'Home')) -------------------------------PhysicalOlapScan[item] -----------------------PhysicalDistribute +--------------------------PhysicalOlapScan[catalog_returns] ------------------------PhysicalProject ---------------------------filter(((date_dim.d_year = 1998) OR (date_dim.d_year = 1999))) -----------------------------PhysicalOlapScan[date_dim] ---------------PhysicalProject -----------------hashJoin[RIGHT_OUTER_JOIN](store_sales.ss_item_sk = store_returns.sr_item_sk)(store_sales.ss_ticket_number = store_returns.sr_ticket_number) -------------------PhysicalProject ---------------------PhysicalOlapScan[store_returns] -------------------PhysicalProject ---------------------hashJoin[INNER_JOIN](date_dim.d_date_sk = store_sales.ss_sold_date_sk) -----------------------hashJoin[INNER_JOIN](item.i_item_sk = store_sales.ss_item_sk) +--------------------------hashJoin[INNER_JOIN](date_dim.d_date_sk = catalog_sales.cs_sold_date_sk) +----------------------------hashJoin[INNER_JOIN](item.i_item_sk = catalog_sales.cs_item_sk) +------------------------------PhysicalProject +--------------------------------PhysicalOlapScan[catalog_sales] +------------------------------PhysicalDistribute +--------------------------------PhysicalProject +----------------------------------filter((cast(i_category as VARCHAR(*)) = 'Home')) +------------------------------------PhysicalOlapScan[item] +----------------------------PhysicalDistribute +------------------------------PhysicalProject +--------------------------------filter(((date_dim.d_year = 1998) OR (date_dim.d_year = 1999))) +----------------------------------PhysicalOlapScan[date_dim] +------------------PhysicalDistribute +--------------------PhysicalProject +----------------------hashJoin[RIGHT_OUTER_JOIN](store_sales.ss_item_sk = store_returns.sr_item_sk)(store_sales.ss_ticket_number = store_returns.sr_ticket_number) ------------------------PhysicalProject ---------------------------PhysicalOlapScan[store_sales] -------------------------PhysicalDistribute ---------------------------PhysicalProject -----------------------------filter((cast(i_category as VARCHAR(*)) = 'Home')) -------------------------------PhysicalOlapScan[item] -----------------------PhysicalDistribute +--------------------------PhysicalOlapScan[store_returns] ------------------------PhysicalProject ---------------------------filter(((date_dim.d_year = 1998) OR (date_dim.d_year = 1999))) -----------------------------PhysicalOlapScan[date_dim] ---------------PhysicalProject -----------------hashJoin[RIGHT_OUTER_JOIN](web_sales.ws_item_sk = web_returns.wr_item_sk)(web_sales.ws_order_number = web_returns.wr_order_number) -------------------PhysicalProject ---------------------PhysicalOlapScan[web_returns] -------------------PhysicalProject ---------------------hashJoin[INNER_JOIN](date_dim.d_date_sk = web_sales.ws_sold_date_sk) -----------------------hashJoin[INNER_JOIN](item.i_item_sk = web_sales.ws_item_sk) +--------------------------hashJoin[INNER_JOIN](date_dim.d_date_sk = store_sales.ss_sold_date_sk) +----------------------------hashJoin[INNER_JOIN](item.i_item_sk = store_sales.ss_item_sk) +------------------------------PhysicalProject +--------------------------------PhysicalOlapScan[store_sales] +------------------------------PhysicalDistribute +--------------------------------PhysicalProject +----------------------------------filter((cast(i_category as VARCHAR(*)) = 'Home')) +------------------------------------PhysicalOlapScan[item] +----------------------------PhysicalDistribute +------------------------------PhysicalProject +--------------------------------filter(((date_dim.d_year = 1998) OR (date_dim.d_year = 1999))) +----------------------------------PhysicalOlapScan[date_dim] +------------------PhysicalDistribute +--------------------PhysicalProject +----------------------hashJoin[RIGHT_OUTER_JOIN](web_sales.ws_item_sk = web_returns.wr_item_sk)(web_sales.ws_order_number = web_returns.wr_order_number) ------------------------PhysicalProject ---------------------------PhysicalOlapScan[web_sales] -------------------------PhysicalDistribute ---------------------------PhysicalProject -----------------------------filter((cast(i_category as VARCHAR(*)) = 'Home')) -------------------------------PhysicalOlapScan[item] -----------------------PhysicalDistribute +--------------------------PhysicalOlapScan[web_returns] ------------------------PhysicalProject ---------------------------filter(((date_dim.d_year = 1998) OR (date_dim.d_year = 1999))) -----------------------------PhysicalOlapScan[date_dim] +--------------------------hashJoin[INNER_JOIN](date_dim.d_date_sk = web_sales.ws_sold_date_sk) +----------------------------hashJoin[INNER_JOIN](item.i_item_sk = web_sales.ws_item_sk) +------------------------------PhysicalProject +--------------------------------PhysicalOlapScan[web_sales] +------------------------------PhysicalDistribute +--------------------------------PhysicalProject +----------------------------------filter((cast(i_category as VARCHAR(*)) = 'Home')) +------------------------------------PhysicalOlapScan[item] +----------------------------PhysicalDistribute +------------------------------PhysicalProject +--------------------------------filter(((date_dim.d_year = 1998) OR (date_dim.d_year = 1999))) +----------------------------------PhysicalOlapScan[date_dim] --PhysicalTopN ----PhysicalDistribute ------PhysicalTopN diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query76.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query76.out index 4be69f2a531bca0..809d43e7b69bd56 100644 --- a/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query76.out +++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query76.out @@ -7,40 +7,43 @@ PhysicalTopN --------PhysicalDistribute ----------hashAgg[LOCAL] ------------PhysicalUnion ---------------PhysicalProject -----------------hashJoin[INNER_JOIN](store_sales.ss_sold_date_sk = date_dim.d_date_sk) -------------------hashJoin[INNER_JOIN](store_sales.ss_item_sk = item.i_item_sk) ---------------------PhysicalProject -----------------------filter(ss_hdemo_sk IS NULL) -------------------------PhysicalOlapScan[store_sales] +--------------PhysicalDistribute +----------------PhysicalProject +------------------hashJoin[INNER_JOIN](store_sales.ss_sold_date_sk = date_dim.d_date_sk) +--------------------hashJoin[INNER_JOIN](store_sales.ss_item_sk = item.i_item_sk) +----------------------PhysicalProject +------------------------filter(ss_hdemo_sk IS NULL) +--------------------------PhysicalOlapScan[store_sales] +----------------------PhysicalDistribute +------------------------PhysicalProject +--------------------------PhysicalOlapScan[item] --------------------PhysicalDistribute ----------------------PhysicalProject -------------------------PhysicalOlapScan[item] -------------------PhysicalDistribute ---------------------PhysicalProject -----------------------PhysicalOlapScan[date_dim] ---------------PhysicalProject -----------------hashJoin[INNER_JOIN](web_sales.ws_sold_date_sk = date_dim.d_date_sk) -------------------hashJoin[INNER_JOIN](web_sales.ws_item_sk = item.i_item_sk) ---------------------PhysicalProject -----------------------filter(ws_bill_addr_sk IS NULL) -------------------------PhysicalOlapScan[web_sales] +------------------------PhysicalOlapScan[date_dim] +--------------PhysicalDistribute +----------------PhysicalProject +------------------hashJoin[INNER_JOIN](web_sales.ws_sold_date_sk = date_dim.d_date_sk) +--------------------hashJoin[INNER_JOIN](web_sales.ws_item_sk = item.i_item_sk) +----------------------PhysicalProject +------------------------filter(ws_bill_addr_sk IS NULL) +--------------------------PhysicalOlapScan[web_sales] +----------------------PhysicalDistribute +------------------------PhysicalProject +--------------------------PhysicalOlapScan[item] --------------------PhysicalDistribute ----------------------PhysicalProject -------------------------PhysicalOlapScan[item] -------------------PhysicalDistribute ---------------------PhysicalProject -----------------------PhysicalOlapScan[date_dim] ---------------PhysicalProject -----------------hashJoin[INNER_JOIN](catalog_sales.cs_sold_date_sk = date_dim.d_date_sk) -------------------hashJoin[INNER_JOIN](catalog_sales.cs_item_sk = item.i_item_sk) ---------------------PhysicalProject -----------------------filter(cs_warehouse_sk IS NULL) -------------------------PhysicalOlapScan[catalog_sales] +------------------------PhysicalOlapScan[date_dim] +--------------PhysicalDistribute +----------------PhysicalProject +------------------hashJoin[INNER_JOIN](catalog_sales.cs_sold_date_sk = date_dim.d_date_sk) +--------------------hashJoin[INNER_JOIN](catalog_sales.cs_item_sk = item.i_item_sk) +----------------------PhysicalProject +------------------------filter(cs_warehouse_sk IS NULL) +--------------------------PhysicalOlapScan[catalog_sales] +----------------------PhysicalDistribute +------------------------PhysicalProject +--------------------------PhysicalOlapScan[item] --------------------PhysicalDistribute ----------------------PhysicalProject -------------------------PhysicalOlapScan[item] -------------------PhysicalDistribute ---------------------PhysicalProject -----------------------PhysicalOlapScan[date_dim] +------------------------PhysicalOlapScan[date_dim] diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query77.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query77.out index f0bb45e55c2451c..649b78703e9e239 100644 --- a/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query77.out +++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query77.out @@ -27,23 +27,22 @@ PhysicalTopN ----------------------------------PhysicalDistribute ------------------------------------PhysicalProject --------------------------------------PhysicalOlapScan[store] -----------------------PhysicalDistribute -------------------------PhysicalProject ---------------------------hashAgg[GLOBAL] -----------------------------PhysicalDistribute -------------------------------hashAgg[LOCAL] ---------------------------------PhysicalProject -----------------------------------hashJoin[INNER_JOIN](store_returns.sr_store_sk = store.s_store_sk) -------------------------------------hashJoin[INNER_JOIN](store_returns.sr_returned_date_sk = date_dim.d_date_sk) ---------------------------------------PhysicalProject -----------------------------------------PhysicalOlapScan[store_returns] ---------------------------------------PhysicalDistribute -----------------------------------------PhysicalProject -------------------------------------------filter((date_dim.d_date <= 1998-09-04)(date_dim.d_date >= 1998-08-05)) ---------------------------------------------PhysicalOlapScan[date_dim] +----------------------PhysicalProject +------------------------hashAgg[GLOBAL] +--------------------------PhysicalDistribute +----------------------------hashAgg[LOCAL] +------------------------------PhysicalProject +--------------------------------hashJoin[INNER_JOIN](store_returns.sr_store_sk = store.s_store_sk) +----------------------------------hashJoin[INNER_JOIN](store_returns.sr_returned_date_sk = date_dim.d_date_sk) +------------------------------------PhysicalProject +--------------------------------------PhysicalOlapScan[store_returns] ------------------------------------PhysicalDistribute --------------------------------------PhysicalProject -----------------------------------------PhysicalOlapScan[store] +----------------------------------------filter((date_dim.d_date <= 1998-09-04)(date_dim.d_date >= 1998-08-05)) +------------------------------------------PhysicalOlapScan[date_dim] +----------------------------------PhysicalDistribute +------------------------------------PhysicalProject +--------------------------------------PhysicalOlapScan[store] ------------------PhysicalProject --------------------NestedLoopJoin[CROSS_JOIN] ----------------------PhysicalProject @@ -89,21 +88,20 @@ PhysicalTopN ----------------------------------PhysicalDistribute ------------------------------------PhysicalProject --------------------------------------PhysicalOlapScan[web_page] -----------------------PhysicalDistribute -------------------------PhysicalProject ---------------------------hashAgg[GLOBAL] -----------------------------PhysicalDistribute -------------------------------hashAgg[LOCAL] ---------------------------------PhysicalProject -----------------------------------hashJoin[INNER_JOIN](web_returns.wr_web_page_sk = web_page.wp_web_page_sk) -------------------------------------hashJoin[INNER_JOIN](web_returns.wr_returned_date_sk = date_dim.d_date_sk) ---------------------------------------PhysicalProject -----------------------------------------PhysicalOlapScan[web_returns] ---------------------------------------PhysicalDistribute -----------------------------------------PhysicalProject -------------------------------------------filter((date_dim.d_date >= 1998-08-05)(date_dim.d_date <= 1998-09-04)) ---------------------------------------------PhysicalOlapScan[date_dim] +----------------------PhysicalProject +------------------------hashAgg[GLOBAL] +--------------------------PhysicalDistribute +----------------------------hashAgg[LOCAL] +------------------------------PhysicalProject +--------------------------------hashJoin[INNER_JOIN](web_returns.wr_web_page_sk = web_page.wp_web_page_sk) +----------------------------------hashJoin[INNER_JOIN](web_returns.wr_returned_date_sk = date_dim.d_date_sk) +------------------------------------PhysicalProject +--------------------------------------PhysicalOlapScan[web_returns] ------------------------------------PhysicalDistribute --------------------------------------PhysicalProject -----------------------------------------PhysicalOlapScan[web_page] +----------------------------------------filter((date_dim.d_date >= 1998-08-05)(date_dim.d_date <= 1998-09-04)) +------------------------------------------PhysicalOlapScan[date_dim] +----------------------------------PhysicalDistribute +------------------------------------PhysicalProject +--------------------------------------PhysicalOlapScan[web_page] diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query78.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query78.out index a77d62b9519ae73..855043acbed68bf 100644 --- a/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query78.out +++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query78.out @@ -25,40 +25,38 @@ PhysicalTopN ------------------------------PhysicalProject --------------------------------filter((date_dim.d_year = 2000)) ----------------------------------PhysicalOlapScan[date_dim] +----------------PhysicalProject +------------------hashAgg[GLOBAL] +--------------------PhysicalDistribute +----------------------hashAgg[LOCAL] +------------------------PhysicalProject +--------------------------hashJoin[INNER_JOIN](web_sales.ws_sold_date_sk = date_dim.d_date_sk) +----------------------------PhysicalProject +------------------------------filter(wr_order_number IS NULL) +--------------------------------hashJoin[LEFT_OUTER_JOIN](web_sales.ws_item_sk = web_returns.wr_item_sk)(web_returns.wr_order_number = web_sales.ws_order_number) +----------------------------------PhysicalProject +------------------------------------PhysicalOlapScan[web_sales] +----------------------------------PhysicalProject +------------------------------------PhysicalOlapScan[web_returns] +----------------------------PhysicalDistribute +------------------------------PhysicalProject +--------------------------------filter((date_dim.d_year = 2000)) +----------------------------------PhysicalOlapScan[date_dim] +------------PhysicalProject +--------------hashAgg[GLOBAL] ----------------PhysicalDistribute -------------------PhysicalProject ---------------------hashAgg[GLOBAL] -----------------------PhysicalDistribute -------------------------hashAgg[LOCAL] ---------------------------PhysicalProject -----------------------------hashJoin[INNER_JOIN](web_sales.ws_sold_date_sk = date_dim.d_date_sk) +------------------hashAgg[LOCAL] +--------------------PhysicalProject +----------------------hashJoin[INNER_JOIN](catalog_sales.cs_sold_date_sk = date_dim.d_date_sk) +------------------------PhysicalProject +--------------------------filter(cr_order_number IS NULL) +----------------------------hashJoin[LEFT_OUTER_JOIN](catalog_sales.cs_item_sk = catalog_returns.cr_item_sk)(catalog_returns.cr_order_number = catalog_sales.cs_order_number) ------------------------------PhysicalProject ---------------------------------filter(wr_order_number IS NULL) -----------------------------------hashJoin[LEFT_OUTER_JOIN](web_sales.ws_item_sk = web_returns.wr_item_sk)(web_returns.wr_order_number = web_sales.ws_order_number) -------------------------------------PhysicalProject ---------------------------------------PhysicalOlapScan[web_sales] -------------------------------------PhysicalProject ---------------------------------------PhysicalOlapScan[web_returns] -------------------------------PhysicalDistribute ---------------------------------PhysicalProject -----------------------------------filter((date_dim.d_year = 2000)) -------------------------------------PhysicalOlapScan[date_dim] -------------PhysicalDistribute ---------------PhysicalProject -----------------hashAgg[GLOBAL] -------------------PhysicalDistribute ---------------------hashAgg[LOCAL] -----------------------PhysicalProject -------------------------hashJoin[INNER_JOIN](catalog_sales.cs_sold_date_sk = date_dim.d_date_sk) +--------------------------------PhysicalOlapScan[catalog_sales] +------------------------------PhysicalProject +--------------------------------PhysicalOlapScan[catalog_returns] +------------------------PhysicalDistribute --------------------------PhysicalProject -----------------------------filter(cr_order_number IS NULL) -------------------------------hashJoin[LEFT_OUTER_JOIN](catalog_sales.cs_item_sk = catalog_returns.cr_item_sk)(catalog_returns.cr_order_number = catalog_sales.cs_order_number) ---------------------------------PhysicalProject -----------------------------------PhysicalOlapScan[catalog_sales] ---------------------------------PhysicalProject -----------------------------------PhysicalOlapScan[catalog_returns] ---------------------------PhysicalDistribute -----------------------------PhysicalProject -------------------------------filter((date_dim.d_year = 2000)) ---------------------------------PhysicalOlapScan[date_dim] +----------------------------filter((date_dim.d_year = 2000)) +------------------------------PhysicalOlapScan[date_dim] diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query8.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query8.out index e3d8df5df12320b..b00cd15207f6fca 100644 --- a/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query8.out +++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query8.out @@ -23,20 +23,22 @@ PhysicalTopN ----------------PhysicalDistribute ------------------PhysicalProject --------------------PhysicalIntersect -----------------------PhysicalProject -------------------------filter(substring(ca_zip, 1, 5) IN ('47602', '16704', '35863', '28577', '83910', '36201', '58412', '48162', '28055', '41419', '80332', '38607', '77817', '24891', '16226', '18410', '21231', '59345', '13918', '51089', '20317', '17167', '54585', '67881', '78366', '47770', '18360', '51717', '73108', '14440', '21800', '89338', '45859', '65501', '34948', '25973', '73219', '25333', '17291', '10374', '18829', '60736', '82620', '41351', '52094', '19326', '25214', '54207', '40936', '21814', '79077', '25178', '75742', '77454', '30621', '89193', '27369', '41232', '48567', '83041', '71948', '37119', '68341', '14073', '16891', '62878', '49130', '19833', '24286', '27700', '40979', '50412', '81504', '94835', '84844', '71954', '39503', '57649', '18434', '24987', '12350', '86379', '27413', '44529', '98569', '16515', '27287', '24255', '21094', '16005', '56436', '91110', '68293', '56455', '54558', '10298', '83647', '32754', '27052', '51766', '19444', '13869', '45645', '94791', '57631', '20712', '37788', '41807', '46507', '21727', '71836', '81070', '50632', '88086', '63991', '20244', '31655', '51782', '29818', '63792', '68605', '94898', '36430', '57025', '20601', '82080', '33869', '22728', '35834', '29086', '92645', '98584', '98072', '11652', '78093', '57553', '43830', '71144', '53565', '18700', '90209', '71256', '38353', '54364', '28571', '96560', '57839', '56355', '50679', '45266', '84680', '34306', '34972', '48530', '30106', '15371', '92380', '84247', '92292', '68852', '13338', '34594', '82602', '70073', '98069', '85066', '47289', '11686', '98862', '26217', '47529', '63294', '51793', '35926', '24227', '14196', '24594', '32489', '99060', '49472', '43432', '49211', '14312', '88137', '47369', '56877', '20534', '81755', '15794', '12318', '21060', '73134', '41255', '63073', '81003', '73873', '66057', '51184', '51195', '45676', '92696', '70450', '90669', '98338', '25264', '38919', '59226', '58581', '60298', '17895', '19489', '52301', '80846', '95464', '68770', '51634', '19988', '18367', '18421', '11618', '67975', '25494', '41352', '95430', '15734', '62585', '97173', '33773', '10425', '75675', '53535', '17879', '41967', '12197', '67998', '79658', '59130', '72592', '14851', '43933', '68101', '50636', '25717', '71286', '24660', '58058', '72991', '95042', '15543', '33122', '69280', '11912', '59386', '27642', '65177', '17672', '33467', '64592', '36335', '54010', '18767', '63193', '42361', '49254', '33113', '33159', '36479', '59080', '11855', '81963', '31016', '49140', '29392', '41836', '32958', '53163', '13844', '73146', '23952', '65148', '93498', '14530', '46131', '58454', '13376', '13378', '83986', '12320', '17193', '59852', '46081', '98533', '52389', '13086', '68843', '31013', '13261', '60560', '13443', '45533', '83583', '11489', '58218', '19753', '22911', '25115', '86709', '27156', '32669', '13123', '51933', '39214', '41331', '66943', '14155', '69998', '49101', '70070', '35076', '14242', '73021', '59494', '15782', '29752', '37914', '74686', '83086', '34473', '15751', '81084', '49230', '91894', '60624', '17819', '28810', '63180', '56224', '39459', '55233', '75752', '43639', '55349', '86057', '62361', '50788', '31830', '58062', '18218', '85761', '60083', '45484', '21204', '90229', '70041', '41162', '35390', '16364', '39500', '68908', '26689', '52868', '81335', '40146', '11340', '61527', '61794', '71997', '30415', '59004', '29450', '58117', '69952', '33562', '83833', '27385', '61860', '96435', '48333', '23065', '32961', '84919', '61997', '99132', '22815', '56600', '68730', '48017', '95694', '32919', '88217', '27116', '28239', '58032', '18884', '16791', '21343', '97462', '18569', '75660', '15475')) ---------------------------PhysicalOlapScan[customer_address] -----------------------PhysicalProject -------------------------filter((cnt > 10)) ---------------------------hashAgg[GLOBAL] -----------------------------PhysicalDistribute -------------------------------hashAgg[LOCAL] ---------------------------------PhysicalProject -----------------------------------hashJoin[INNER_JOIN](customer_address.ca_address_sk = customer.c_current_addr_sk) -------------------------------------PhysicalProject ---------------------------------------PhysicalOlapScan[customer_address] -------------------------------------PhysicalDistribute +----------------------PhysicalDistribute +------------------------PhysicalProject +--------------------------filter(substring(ca_zip, 1, 5) IN ('47602', '16704', '35863', '28577', '83910', '36201', '58412', '48162', '28055', '41419', '80332', '38607', '77817', '24891', '16226', '18410', '21231', '59345', '13918', '51089', '20317', '17167', '54585', '67881', '78366', '47770', '18360', '51717', '73108', '14440', '21800', '89338', '45859', '65501', '34948', '25973', '73219', '25333', '17291', '10374', '18829', '60736', '82620', '41351', '52094', '19326', '25214', '54207', '40936', '21814', '79077', '25178', '75742', '77454', '30621', '89193', '27369', '41232', '48567', '83041', '71948', '37119', '68341', '14073', '16891', '62878', '49130', '19833', '24286', '27700', '40979', '50412', '81504', '94835', '84844', '71954', '39503', '57649', '18434', '24987', '12350', '86379', '27413', '44529', '98569', '16515', '27287', '24255', '21094', '16005', '56436', '91110', '68293', '56455', '54558', '10298', '83647', '32754', '27052', '51766', '19444', '13869', '45645', '94791', '57631', '20712', '37788', '41807', '46507', '21727', '71836', '81070', '50632', '88086', '63991', '20244', '31655', '51782', '29818', '63792', '68605', '94898', '36430', '57025', '20601', '82080', '33869', '22728', '35834', '29086', '92645', '98584', '98072', '11652', '78093', '57553', '43830', '71144', '53565', '18700', '90209', '71256', '38353', '54364', '28571', '96560', '57839', '56355', '50679', '45266', '84680', '34306', '34972', '48530', '30106', '15371', '92380', '84247', '92292', '68852', '13338', '34594', '82602', '70073', '98069', '85066', '47289', '11686', '98862', '26217', '47529', '63294', '51793', '35926', '24227', '14196', '24594', '32489', '99060', '49472', '43432', '49211', '14312', '88137', '47369', '56877', '20534', '81755', '15794', '12318', '21060', '73134', '41255', '63073', '81003', '73873', '66057', '51184', '51195', '45676', '92696', '70450', '90669', '98338', '25264', '38919', '59226', '58581', '60298', '17895', '19489', '52301', '80846', '95464', '68770', '51634', '19988', '18367', '18421', '11618', '67975', '25494', '41352', '95430', '15734', '62585', '97173', '33773', '10425', '75675', '53535', '17879', '41967', '12197', '67998', '79658', '59130', '72592', '14851', '43933', '68101', '50636', '25717', '71286', '24660', '58058', '72991', '95042', '15543', '33122', '69280', '11912', '59386', '27642', '65177', '17672', '33467', '64592', '36335', '54010', '18767', '63193', '42361', '49254', '33113', '33159', '36479', '59080', '11855', '81963', '31016', '49140', '29392', '41836', '32958', '53163', '13844', '73146', '23952', '65148', '93498', '14530', '46131', '58454', '13376', '13378', '83986', '12320', '17193', '59852', '46081', '98533', '52389', '13086', '68843', '31013', '13261', '60560', '13443', '45533', '83583', '11489', '58218', '19753', '22911', '25115', '86709', '27156', '32669', '13123', '51933', '39214', '41331', '66943', '14155', '69998', '49101', '70070', '35076', '14242', '73021', '59494', '15782', '29752', '37914', '74686', '83086', '34473', '15751', '81084', '49230', '91894', '60624', '17819', '28810', '63180', '56224', '39459', '55233', '75752', '43639', '55349', '86057', '62361', '50788', '31830', '58062', '18218', '85761', '60083', '45484', '21204', '90229', '70041', '41162', '35390', '16364', '39500', '68908', '26689', '52868', '81335', '40146', '11340', '61527', '61794', '71997', '30415', '59004', '29450', '58117', '69952', '33562', '83833', '27385', '61860', '96435', '48333', '23065', '32961', '84919', '61997', '99132', '22815', '56600', '68730', '48017', '95694', '32919', '88217', '27116', '28239', '58032', '18884', '16791', '21343', '97462', '18569', '75660', '15475')) +----------------------------PhysicalOlapScan[customer_address] +----------------------PhysicalDistribute +------------------------PhysicalProject +--------------------------filter((cnt > 10)) +----------------------------hashAgg[GLOBAL] +------------------------------PhysicalDistribute +--------------------------------hashAgg[LOCAL] +----------------------------------PhysicalProject +------------------------------------hashJoin[INNER_JOIN](customer_address.ca_address_sk = customer.c_current_addr_sk) --------------------------------------PhysicalProject -----------------------------------------filter((cast(c_preferred_cust_flag as VARCHAR(*)) = 'Y')) -------------------------------------------PhysicalOlapScan[customer] +----------------------------------------PhysicalOlapScan[customer_address] +--------------------------------------PhysicalDistribute +----------------------------------------PhysicalProject +------------------------------------------filter((cast(c_preferred_cust_flag as VARCHAR(*)) = 'Y')) +--------------------------------------------PhysicalOlapScan[customer] diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query81.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query81.out index 02276e4934ad46a..44ad4b9321b4e98 100644 --- a/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query81.out +++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query81.out @@ -16,8 +16,9 @@ CteAnchor[cteId= ( CTEId#2=] ) ----------------------PhysicalProject ------------------------filter((date_dim.d_year = 2002)) --------------------------PhysicalOlapScan[date_dim] ---------------PhysicalProject -----------------PhysicalOlapScan[customer_address] +--------------PhysicalDistribute +----------------PhysicalProject +------------------PhysicalOlapScan[customer_address] --PhysicalTopN ----PhysicalDistribute ------PhysicalTopN diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query83.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query83.out index e1b0093e97affca..67d4702363e19bf 100644 --- a/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query83.out +++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query83.out @@ -31,59 +31,57 @@ PhysicalTopN ------------------------------PhysicalProject --------------------------------filter(cast(d_date as DATETIMEV2(0)) IN (2001-06-06 00:00:00, 2001-09-02 00:00:00, 2001-11-11 00:00:00)) ----------------------------------PhysicalOlapScan[date_dim] -----------PhysicalDistribute -------------hashJoin[INNER_JOIN](sr_items.item_id = wr_items.item_id) ---------------PhysicalProject -----------------hashAgg[GLOBAL] -------------------PhysicalDistribute ---------------------hashAgg[LOCAL] -----------------------PhysicalProject -------------------------hashJoin[LEFT_SEMI_JOIN](date_dim.d_date = date_dim.d_date) ---------------------------PhysicalProject -----------------------------hashJoin[INNER_JOIN](store_returns.sr_returned_date_sk = date_dim.d_date_sk) -------------------------------hashJoin[INNER_JOIN](store_returns.sr_item_sk = item.i_item_sk) ---------------------------------PhysicalProject -----------------------------------PhysicalOlapScan[store_returns] ---------------------------------PhysicalDistribute -----------------------------------PhysicalProject -------------------------------------PhysicalOlapScan[item] +----------hashJoin[INNER_JOIN](sr_items.item_id = wr_items.item_id) +------------PhysicalProject +--------------hashAgg[GLOBAL] +----------------PhysicalDistribute +------------------hashAgg[LOCAL] +--------------------PhysicalProject +----------------------hashJoin[LEFT_SEMI_JOIN](date_dim.d_date = date_dim.d_date) +------------------------PhysicalProject +--------------------------hashJoin[INNER_JOIN](store_returns.sr_returned_date_sk = date_dim.d_date_sk) +----------------------------hashJoin[INNER_JOIN](store_returns.sr_item_sk = item.i_item_sk) +------------------------------PhysicalProject +--------------------------------PhysicalOlapScan[store_returns] ------------------------------PhysicalDistribute --------------------------------PhysicalProject -----------------------------------PhysicalOlapScan[date_dim] ---------------------------PhysicalDistribute -----------------------------PhysicalProject -------------------------------hashJoin[LEFT_SEMI_JOIN](date_dim.d_week_seq = date_dim.d_week_seq) +----------------------------------PhysicalOlapScan[item] +----------------------------PhysicalDistribute +------------------------------PhysicalProject +--------------------------------PhysicalOlapScan[date_dim] +------------------------PhysicalDistribute +--------------------------PhysicalProject +----------------------------hashJoin[LEFT_SEMI_JOIN](date_dim.d_week_seq = date_dim.d_week_seq) +------------------------------PhysicalProject +--------------------------------PhysicalOlapScan[date_dim] +------------------------------PhysicalDistribute --------------------------------PhysicalProject -----------------------------------PhysicalOlapScan[date_dim] ---------------------------------PhysicalDistribute -----------------------------------PhysicalProject -------------------------------------filter(cast(d_date as DATETIMEV2(0)) IN (2001-06-06 00:00:00, 2001-09-02 00:00:00, 2001-11-11 00:00:00)) ---------------------------------------PhysicalOlapScan[date_dim] ---------------PhysicalDistribute -----------------PhysicalProject -------------------hashAgg[GLOBAL] ---------------------PhysicalDistribute -----------------------hashAgg[LOCAL] -------------------------PhysicalProject ---------------------------hashJoin[LEFT_SEMI_JOIN](date_dim.d_date = date_dim.d_date) -----------------------------PhysicalProject -------------------------------hashJoin[INNER_JOIN](web_returns.wr_returned_date_sk = date_dim.d_date_sk) ---------------------------------hashJoin[INNER_JOIN](web_returns.wr_item_sk = item.i_item_sk) -----------------------------------PhysicalProject -------------------------------------PhysicalOlapScan[web_returns] -----------------------------------PhysicalDistribute -------------------------------------PhysicalProject ---------------------------------------PhysicalOlapScan[item] ---------------------------------PhysicalDistribute -----------------------------------PhysicalProject +----------------------------------filter(cast(d_date as DATETIMEV2(0)) IN (2001-06-06 00:00:00, 2001-09-02 00:00:00, 2001-11-11 00:00:00)) ------------------------------------PhysicalOlapScan[date_dim] +------------PhysicalProject +--------------hashAgg[GLOBAL] +----------------PhysicalDistribute +------------------hashAgg[LOCAL] +--------------------PhysicalProject +----------------------hashJoin[LEFT_SEMI_JOIN](date_dim.d_date = date_dim.d_date) +------------------------PhysicalProject +--------------------------hashJoin[INNER_JOIN](web_returns.wr_returned_date_sk = date_dim.d_date_sk) +----------------------------hashJoin[INNER_JOIN](web_returns.wr_item_sk = item.i_item_sk) +------------------------------PhysicalProject +--------------------------------PhysicalOlapScan[web_returns] +------------------------------PhysicalDistribute +--------------------------------PhysicalProject +----------------------------------PhysicalOlapScan[item] ----------------------------PhysicalDistribute ------------------------------PhysicalProject ---------------------------------hashJoin[LEFT_SEMI_JOIN](date_dim.d_week_seq = date_dim.d_week_seq) -----------------------------------PhysicalProject +--------------------------------PhysicalOlapScan[date_dim] +------------------------PhysicalDistribute +--------------------------PhysicalProject +----------------------------hashJoin[LEFT_SEMI_JOIN](date_dim.d_week_seq = date_dim.d_week_seq) +------------------------------PhysicalProject +--------------------------------PhysicalOlapScan[date_dim] +------------------------------PhysicalDistribute +--------------------------------PhysicalProject +----------------------------------filter(cast(d_date as DATETIMEV2(0)) IN (2001-06-06 00:00:00, 2001-09-02 00:00:00, 2001-11-11 00:00:00)) ------------------------------------PhysicalOlapScan[date_dim] -----------------------------------PhysicalDistribute -------------------------------------PhysicalProject ---------------------------------------filter(cast(d_date as DATETIMEV2(0)) IN (2001-06-06 00:00:00, 2001-09-02 00:00:00, 2001-11-11 00:00:00)) -----------------------------------------PhysicalOlapScan[date_dim] diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query97.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query97.out index 5b80e6754b0e6f3..fee94a11417d076 100644 --- a/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query97.out +++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query97.out @@ -19,17 +19,16 @@ PhysicalLimit ----------------------------PhysicalProject ------------------------------filter((date_dim.d_month_seq >= 1214)(date_dim.d_month_seq <= 1225)) --------------------------------PhysicalOlapScan[date_dim] ---------------PhysicalDistribute -----------------PhysicalProject -------------------hashAgg[GLOBAL] ---------------------PhysicalDistribute -----------------------hashAgg[LOCAL] -------------------------PhysicalProject ---------------------------hashJoin[INNER_JOIN](catalog_sales.cs_sold_date_sk = date_dim.d_date_sk) +--------------PhysicalProject +----------------hashAgg[GLOBAL] +------------------PhysicalDistribute +--------------------hashAgg[LOCAL] +----------------------PhysicalProject +------------------------hashJoin[INNER_JOIN](catalog_sales.cs_sold_date_sk = date_dim.d_date_sk) +--------------------------PhysicalProject +----------------------------PhysicalOlapScan[catalog_sales] +--------------------------PhysicalDistribute ----------------------------PhysicalProject -------------------------------PhysicalOlapScan[catalog_sales] -----------------------------PhysicalDistribute -------------------------------PhysicalProject ---------------------------------filter((date_dim.d_month_seq >= 1214)(date_dim.d_month_seq <= 1225)) -----------------------------------PhysicalOlapScan[date_dim] +------------------------------filter((date_dim.d_month_seq >= 1214)(date_dim.d_month_seq <= 1225)) +--------------------------------PhysicalOlapScan[date_dim] diff --git a/regression-test/data/nereids_tpch_shape_sf1000_p0/shape/q10.out b/regression-test/data/nereids_tpch_shape_sf1000_p0/shape/q10.out index c1d1c3548aedb47..404026aa64a74a2 100644 --- a/regression-test/data/nereids_tpch_shape_sf1000_p0/shape/q10.out +++ b/regression-test/data/nereids_tpch_shape_sf1000_p0/shape/q10.out @@ -4,22 +4,25 @@ PhysicalTopN --PhysicalDistribute ----PhysicalTopN ------PhysicalProject ---------hashAgg[LOCAL] -----------PhysicalProject -------------hashJoin[INNER_JOIN](customer.c_nationkey = nation.n_nationkey) ---------------hashJoin[INNER_JOIN](customer.c_custkey = orders.o_custkey) -----------------PhysicalProject -------------------PhysicalOlapScan[customer] -----------------PhysicalDistribute +--------hashAgg[GLOBAL] +----------PhysicalDistribute +------------hashAgg[LOCAL] +--------------PhysicalProject +----------------hashJoin[INNER_JOIN](lineitem.l_orderkey = orders.o_orderkey) ------------------PhysicalProject ---------------------hashJoin[INNER_JOIN](lineitem.l_orderkey = orders.o_orderkey) +--------------------filter((lineitem.l_returnflag = 'R')) +----------------------PhysicalOlapScan[lineitem] +------------------PhysicalDistribute +--------------------hashJoin[INNER_JOIN](customer.c_nationkey = nation.n_nationkey) ----------------------PhysicalProject -------------------------filter((lineitem.l_returnflag = 'R')) ---------------------------PhysicalOlapScan[lineitem] -----------------------PhysicalProject -------------------------filter((orders.o_orderdate < 1994-01-01)(orders.o_orderdate >= 1993-10-01)) ---------------------------PhysicalOlapScan[orders] ---------------PhysicalDistribute -----------------PhysicalProject -------------------PhysicalOlapScan[nation] +------------------------hashJoin[INNER_JOIN](customer.c_custkey = orders.o_custkey) +--------------------------PhysicalProject +----------------------------PhysicalOlapScan[customer] +--------------------------PhysicalDistribute +----------------------------PhysicalProject +------------------------------filter((orders.o_orderdate < 1994-01-01)(orders.o_orderdate >= 1993-10-01)) +--------------------------------PhysicalOlapScan[orders] +----------------------PhysicalDistribute +------------------------PhysicalProject +--------------------------PhysicalOlapScan[nation] diff --git a/regression-test/data/nereids_tpch_shape_sf1000_p0/shape/q13.out b/regression-test/data/nereids_tpch_shape_sf1000_p0/shape/q13.out index d196ba817e76c71..ae50570ecf54f23 100644 --- a/regression-test/data/nereids_tpch_shape_sf1000_p0/shape/q13.out +++ b/regression-test/data/nereids_tpch_shape_sf1000_p0/shape/q13.out @@ -14,6 +14,7 @@ PhysicalQuickSort ----------------------PhysicalProject ------------------------filter(( not (o_comment like '%special%requests%'))) --------------------------PhysicalOlapScan[orders] ---------------------PhysicalProject -----------------------PhysicalOlapScan[customer] +--------------------PhysicalDistribute +----------------------PhysicalProject +------------------------PhysicalOlapScan[customer] diff --git a/regression-test/data/nereids_tpch_shape_sf1000_p0/shape/q2.out b/regression-test/data/nereids_tpch_shape_sf1000_p0/shape/q2.out index bb74c9cb376ea92..ec4c6370b694a40 100644 --- a/regression-test/data/nereids_tpch_shape_sf1000_p0/shape/q2.out +++ b/regression-test/data/nereids_tpch_shape_sf1000_p0/shape/q2.out @@ -10,6 +10,17 @@ PhysicalTopN --------------PhysicalDistribute ----------------PhysicalProject ------------------hashJoin[INNER_JOIN](supplier.s_suppkey = partsupp.ps_suppkey) +--------------------PhysicalProject +----------------------hashJoin[INNER_JOIN](supplier.s_nationkey = nation.n_nationkey) +------------------------PhysicalOlapScan[supplier] +------------------------PhysicalDistribute +--------------------------hashJoin[INNER_JOIN](nation.n_regionkey = region.r_regionkey) +----------------------------PhysicalProject +------------------------------PhysicalOlapScan[nation] +----------------------------PhysicalDistribute +------------------------------PhysicalProject +--------------------------------filter((region.r_name = 'EUROPE')) +----------------------------------PhysicalOlapScan[region] --------------------PhysicalDistribute ----------------------PhysicalProject ------------------------hashJoin[INNER_JOIN](part.p_partkey = partsupp.ps_partkey) @@ -18,14 +29,4 @@ PhysicalTopN --------------------------PhysicalProject ----------------------------filter((part.p_size = 15)(p_type like '%BRASS')) ------------------------------PhysicalOlapScan[part] ---------------------hashJoin[INNER_JOIN](supplier.s_nationkey = nation.n_nationkey) -----------------------PhysicalOlapScan[supplier] -----------------------PhysicalDistribute -------------------------hashJoin[INNER_JOIN](nation.n_regionkey = region.r_regionkey) ---------------------------PhysicalProject -----------------------------PhysicalOlapScan[nation] ---------------------------PhysicalDistribute -----------------------------PhysicalProject -------------------------------filter((region.r_name = 'EUROPE')) ---------------------------------PhysicalOlapScan[region] diff --git a/regression-test/data/nereids_tpch_shape_sf1000_p0/shape/q20-rewrite.out b/regression-test/data/nereids_tpch_shape_sf1000_p0/shape/q20-rewrite.out index 92f1c8717baec6a..2e6728910560ae7 100644 --- a/regression-test/data/nereids_tpch_shape_sf1000_p0/shape/q20-rewrite.out +++ b/regression-test/data/nereids_tpch_shape_sf1000_p0/shape/q20-rewrite.out @@ -22,11 +22,12 @@ PhysicalQuickSort --------------------PhysicalProject ----------------------filter((p_name like 'forest%')) ------------------------PhysicalOlapScan[part] -----------hashJoin[INNER_JOIN](supplier.s_nationkey = nation.n_nationkey) -------------PhysicalProject ---------------PhysicalOlapScan[supplier] -------------PhysicalDistribute +----------PhysicalDistribute +------------hashJoin[INNER_JOIN](supplier.s_nationkey = nation.n_nationkey) --------------PhysicalProject -----------------filter((nation.n_name = 'CANADA')) -------------------PhysicalOlapScan[nation] +----------------PhysicalOlapScan[supplier] +--------------PhysicalDistribute +----------------PhysicalProject +------------------filter((nation.n_name = 'CANADA')) +--------------------PhysicalOlapScan[nation] diff --git a/regression-test/data/nereids_tpch_shape_sf1000_p0/shape/q20.out b/regression-test/data/nereids_tpch_shape_sf1000_p0/shape/q20.out index 4142233ae31535b..af742120ca219e2 100644 --- a/regression-test/data/nereids_tpch_shape_sf1000_p0/shape/q20.out +++ b/regression-test/data/nereids_tpch_shape_sf1000_p0/shape/q20.out @@ -22,11 +22,12 @@ PhysicalQuickSort --------------------PhysicalProject ----------------------filter((p_name like 'forest%')) ------------------------PhysicalOlapScan[part] -----------hashJoin[INNER_JOIN](supplier.s_nationkey = nation.n_nationkey) -------------PhysicalProject ---------------PhysicalOlapScan[supplier] -------------PhysicalDistribute +----------PhysicalDistribute +------------hashJoin[INNER_JOIN](supplier.s_nationkey = nation.n_nationkey) --------------PhysicalProject -----------------filter((nation.n_name = 'CANADA')) -------------------PhysicalOlapScan[nation] +----------------PhysicalOlapScan[supplier] +--------------PhysicalDistribute +----------------PhysicalProject +------------------filter((nation.n_name = 'CANADA')) +--------------------PhysicalOlapScan[nation] diff --git a/regression-test/data/nereids_tpch_shape_sf1000_p0/shape/q22.out b/regression-test/data/nereids_tpch_shape_sf1000_p0/shape/q22.out index c41229b490f4745..b2399ebd132d2b3 100644 --- a/regression-test/data/nereids_tpch_shape_sf1000_p0/shape/q22.out +++ b/regression-test/data/nereids_tpch_shape_sf1000_p0/shape/q22.out @@ -7,20 +7,22 @@ PhysicalQuickSort --------PhysicalDistribute ----------hashAgg[LOCAL] ------------PhysicalProject ---------------hashJoin[RIGHT_ANTI_JOIN](orders.o_custkey = customer.c_custkey) +--------------NestedLoopJoin[INNER_JOIN](cast(c_acctbal as DECIMALV3(38, 4)) > avg(c_acctbal)) +----------------PhysicalProject +------------------hashJoin[RIGHT_ANTI_JOIN](orders.o_custkey = customer.c_custkey) +--------------------PhysicalDistribute +----------------------PhysicalProject +------------------------PhysicalOlapScan[orders] +--------------------PhysicalDistribute +----------------------PhysicalProject +------------------------filter(substring(c_phone, 1, 2) IN ('13', '31', '23', '29', '30', '18', '17')) +--------------------------PhysicalOlapScan[customer] ----------------PhysicalDistribute -------------------PhysicalProject ---------------------PhysicalOlapScan[orders] -----------------NestedLoopJoin[INNER_JOIN](cast(c_acctbal as DECIMALV3(38, 4)) > avg(c_acctbal)) -------------------PhysicalProject ---------------------filter(substring(c_phone, 1, 2) IN ('13', '31', '23', '29', '30', '18', '17')) -----------------------PhysicalOlapScan[customer] -------------------PhysicalDistribute ---------------------PhysicalAssertNumRows -----------------------hashAgg[GLOBAL] -------------------------PhysicalDistribute ---------------------------hashAgg[LOCAL] -----------------------------PhysicalProject -------------------------------filter((customer.c_acctbal > 0.00)substring(c_phone, 1, 2) IN ('13', '31', '23', '29', '30', '18', '17')) ---------------------------------PhysicalOlapScan[customer] +------------------PhysicalAssertNumRows +--------------------hashAgg[GLOBAL] +----------------------PhysicalDistribute +------------------------hashAgg[LOCAL] +--------------------------PhysicalProject +----------------------------filter((customer.c_acctbal > 0.00)substring(c_phone, 1, 2) IN ('13', '31', '23', '29', '30', '18', '17')) +------------------------------PhysicalOlapScan[customer] diff --git a/regression-test/data/nereids_tpch_shape_sf1000_p0/shape/q3.out b/regression-test/data/nereids_tpch_shape_sf1000_p0/shape/q3.out index 643f478927d9fdb..23fc521663607a1 100644 --- a/regression-test/data/nereids_tpch_shape_sf1000_p0/shape/q3.out +++ b/regression-test/data/nereids_tpch_shape_sf1000_p0/shape/q3.out @@ -17,7 +17,8 @@ PhysicalTopN ----------------------PhysicalProject ------------------------filter((orders.o_orderdate < 1995-03-15)) --------------------------PhysicalOlapScan[orders] ---------------------PhysicalProject -----------------------filter((customer.c_mktsegment = 'BUILDING')) -------------------------PhysicalOlapScan[customer] +--------------------PhysicalDistribute +----------------------PhysicalProject +------------------------filter((customer.c_mktsegment = 'BUILDING')) +--------------------------PhysicalOlapScan[customer] diff --git a/regression-test/data/nereids_tpch_shape_sf1000_p0/shape/q5.out b/regression-test/data/nereids_tpch_shape_sf1000_p0/shape/q5.out index f8211c8fa8639ec..6d45aeda6ba21a7 100644 --- a/regression-test/data/nereids_tpch_shape_sf1000_p0/shape/q5.out +++ b/regression-test/data/nereids_tpch_shape_sf1000_p0/shape/q5.out @@ -8,8 +8,6 @@ PhysicalQuickSort ----------hashAgg[LOCAL] ------------PhysicalProject --------------hashJoin[INNER_JOIN](customer.c_custkey = orders.o_custkey)(customer.c_nationkey = supplier.s_nationkey) -----------------PhysicalProject -------------------PhysicalOlapScan[customer] ----------------PhysicalDistribute ------------------PhysicalProject --------------------hashJoin[INNER_JOIN](lineitem.l_orderkey = orders.o_orderkey) @@ -32,4 +30,7 @@ PhysicalQuickSort ----------------------PhysicalProject ------------------------filter((orders.o_orderdate < 1995-01-01)(orders.o_orderdate >= 1994-01-01)) --------------------------PhysicalOlapScan[orders] +----------------PhysicalDistribute +------------------PhysicalProject +--------------------PhysicalOlapScan[customer] diff --git a/regression-test/data/nereids_tpch_shape_sf1000_p0/shape/q9.out b/regression-test/data/nereids_tpch_shape_sf1000_p0/shape/q9.out index 9e50fcef747d394..e20942535d3417a 100644 --- a/regression-test/data/nereids_tpch_shape_sf1000_p0/shape/q9.out +++ b/regression-test/data/nereids_tpch_shape_sf1000_p0/shape/q9.out @@ -21,9 +21,10 @@ PhysicalQuickSort --------------------------------PhysicalDistribute ----------------------------------PhysicalProject ------------------------------------PhysicalOlapScan[lineitem] ---------------------------------PhysicalProject -----------------------------------filter((p_name like '%green%')) -------------------------------------PhysicalOlapScan[part] +--------------------------------PhysicalDistribute +----------------------------------PhysicalProject +------------------------------------filter((p_name like '%green%')) +--------------------------------------PhysicalOlapScan[part] ----------------------------PhysicalDistribute ------------------------------hashJoin[INNER_JOIN](supplier.s_nationkey = nation.n_nationkey) --------------------------------PhysicalProject @@ -31,6 +32,7 @@ PhysicalQuickSort --------------------------------PhysicalDistribute ----------------------------------PhysicalProject ------------------------------------PhysicalOlapScan[nation] -----------------PhysicalProject -------------------PhysicalOlapScan[partsupp] +----------------PhysicalDistribute +------------------PhysicalProject +--------------------PhysicalOlapScan[partsupp] diff --git a/regression-test/data/nereids_tpch_shape_sf500_p0/shape/q10.out b/regression-test/data/nereids_tpch_shape_sf500_p0/shape/q10.out index c1d1c3548aedb47..404026aa64a74a2 100644 --- a/regression-test/data/nereids_tpch_shape_sf500_p0/shape/q10.out +++ b/regression-test/data/nereids_tpch_shape_sf500_p0/shape/q10.out @@ -4,22 +4,25 @@ PhysicalTopN --PhysicalDistribute ----PhysicalTopN ------PhysicalProject ---------hashAgg[LOCAL] -----------PhysicalProject -------------hashJoin[INNER_JOIN](customer.c_nationkey = nation.n_nationkey) ---------------hashJoin[INNER_JOIN](customer.c_custkey = orders.o_custkey) -----------------PhysicalProject -------------------PhysicalOlapScan[customer] -----------------PhysicalDistribute +--------hashAgg[GLOBAL] +----------PhysicalDistribute +------------hashAgg[LOCAL] +--------------PhysicalProject +----------------hashJoin[INNER_JOIN](lineitem.l_orderkey = orders.o_orderkey) ------------------PhysicalProject ---------------------hashJoin[INNER_JOIN](lineitem.l_orderkey = orders.o_orderkey) +--------------------filter((lineitem.l_returnflag = 'R')) +----------------------PhysicalOlapScan[lineitem] +------------------PhysicalDistribute +--------------------hashJoin[INNER_JOIN](customer.c_nationkey = nation.n_nationkey) ----------------------PhysicalProject -------------------------filter((lineitem.l_returnflag = 'R')) ---------------------------PhysicalOlapScan[lineitem] -----------------------PhysicalProject -------------------------filter((orders.o_orderdate < 1994-01-01)(orders.o_orderdate >= 1993-10-01)) ---------------------------PhysicalOlapScan[orders] ---------------PhysicalDistribute -----------------PhysicalProject -------------------PhysicalOlapScan[nation] +------------------------hashJoin[INNER_JOIN](customer.c_custkey = orders.o_custkey) +--------------------------PhysicalProject +----------------------------PhysicalOlapScan[customer] +--------------------------PhysicalDistribute +----------------------------PhysicalProject +------------------------------filter((orders.o_orderdate < 1994-01-01)(orders.o_orderdate >= 1993-10-01)) +--------------------------------PhysicalOlapScan[orders] +----------------------PhysicalDistribute +------------------------PhysicalProject +--------------------------PhysicalOlapScan[nation] diff --git a/regression-test/data/nereids_tpch_shape_sf500_p0/shape/q13.out b/regression-test/data/nereids_tpch_shape_sf500_p0/shape/q13.out index d196ba817e76c71..ae50570ecf54f23 100644 --- a/regression-test/data/nereids_tpch_shape_sf500_p0/shape/q13.out +++ b/regression-test/data/nereids_tpch_shape_sf500_p0/shape/q13.out @@ -14,6 +14,7 @@ PhysicalQuickSort ----------------------PhysicalProject ------------------------filter(( not (o_comment like '%special%requests%'))) --------------------------PhysicalOlapScan[orders] ---------------------PhysicalProject -----------------------PhysicalOlapScan[customer] +--------------------PhysicalDistribute +----------------------PhysicalProject +------------------------PhysicalOlapScan[customer] diff --git a/regression-test/data/nereids_tpch_shape_sf500_p0/shape/q2.out b/regression-test/data/nereids_tpch_shape_sf500_p0/shape/q2.out index bb74c9cb376ea92..ec4c6370b694a40 100644 --- a/regression-test/data/nereids_tpch_shape_sf500_p0/shape/q2.out +++ b/regression-test/data/nereids_tpch_shape_sf500_p0/shape/q2.out @@ -10,6 +10,17 @@ PhysicalTopN --------------PhysicalDistribute ----------------PhysicalProject ------------------hashJoin[INNER_JOIN](supplier.s_suppkey = partsupp.ps_suppkey) +--------------------PhysicalProject +----------------------hashJoin[INNER_JOIN](supplier.s_nationkey = nation.n_nationkey) +------------------------PhysicalOlapScan[supplier] +------------------------PhysicalDistribute +--------------------------hashJoin[INNER_JOIN](nation.n_regionkey = region.r_regionkey) +----------------------------PhysicalProject +------------------------------PhysicalOlapScan[nation] +----------------------------PhysicalDistribute +------------------------------PhysicalProject +--------------------------------filter((region.r_name = 'EUROPE')) +----------------------------------PhysicalOlapScan[region] --------------------PhysicalDistribute ----------------------PhysicalProject ------------------------hashJoin[INNER_JOIN](part.p_partkey = partsupp.ps_partkey) @@ -18,14 +29,4 @@ PhysicalTopN --------------------------PhysicalProject ----------------------------filter((part.p_size = 15)(p_type like '%BRASS')) ------------------------------PhysicalOlapScan[part] ---------------------hashJoin[INNER_JOIN](supplier.s_nationkey = nation.n_nationkey) -----------------------PhysicalOlapScan[supplier] -----------------------PhysicalDistribute -------------------------hashJoin[INNER_JOIN](nation.n_regionkey = region.r_regionkey) ---------------------------PhysicalProject -----------------------------PhysicalOlapScan[nation] ---------------------------PhysicalDistribute -----------------------------PhysicalProject -------------------------------filter((region.r_name = 'EUROPE')) ---------------------------------PhysicalOlapScan[region] diff --git a/regression-test/data/nereids_tpch_shape_sf500_p0/shape/q20.out b/regression-test/data/nereids_tpch_shape_sf500_p0/shape/q20.out index 4142233ae31535b..af742120ca219e2 100644 --- a/regression-test/data/nereids_tpch_shape_sf500_p0/shape/q20.out +++ b/regression-test/data/nereids_tpch_shape_sf500_p0/shape/q20.out @@ -22,11 +22,12 @@ PhysicalQuickSort --------------------PhysicalProject ----------------------filter((p_name like 'forest%')) ------------------------PhysicalOlapScan[part] -----------hashJoin[INNER_JOIN](supplier.s_nationkey = nation.n_nationkey) -------------PhysicalProject ---------------PhysicalOlapScan[supplier] -------------PhysicalDistribute +----------PhysicalDistribute +------------hashJoin[INNER_JOIN](supplier.s_nationkey = nation.n_nationkey) --------------PhysicalProject -----------------filter((nation.n_name = 'CANADA')) -------------------PhysicalOlapScan[nation] +----------------PhysicalOlapScan[supplier] +--------------PhysicalDistribute +----------------PhysicalProject +------------------filter((nation.n_name = 'CANADA')) +--------------------PhysicalOlapScan[nation] diff --git a/regression-test/data/nereids_tpch_shape_sf500_p0/shape/q22.out b/regression-test/data/nereids_tpch_shape_sf500_p0/shape/q22.out index c41229b490f4745..b2399ebd132d2b3 100644 --- a/regression-test/data/nereids_tpch_shape_sf500_p0/shape/q22.out +++ b/regression-test/data/nereids_tpch_shape_sf500_p0/shape/q22.out @@ -7,20 +7,22 @@ PhysicalQuickSort --------PhysicalDistribute ----------hashAgg[LOCAL] ------------PhysicalProject ---------------hashJoin[RIGHT_ANTI_JOIN](orders.o_custkey = customer.c_custkey) +--------------NestedLoopJoin[INNER_JOIN](cast(c_acctbal as DECIMALV3(38, 4)) > avg(c_acctbal)) +----------------PhysicalProject +------------------hashJoin[RIGHT_ANTI_JOIN](orders.o_custkey = customer.c_custkey) +--------------------PhysicalDistribute +----------------------PhysicalProject +------------------------PhysicalOlapScan[orders] +--------------------PhysicalDistribute +----------------------PhysicalProject +------------------------filter(substring(c_phone, 1, 2) IN ('13', '31', '23', '29', '30', '18', '17')) +--------------------------PhysicalOlapScan[customer] ----------------PhysicalDistribute -------------------PhysicalProject ---------------------PhysicalOlapScan[orders] -----------------NestedLoopJoin[INNER_JOIN](cast(c_acctbal as DECIMALV3(38, 4)) > avg(c_acctbal)) -------------------PhysicalProject ---------------------filter(substring(c_phone, 1, 2) IN ('13', '31', '23', '29', '30', '18', '17')) -----------------------PhysicalOlapScan[customer] -------------------PhysicalDistribute ---------------------PhysicalAssertNumRows -----------------------hashAgg[GLOBAL] -------------------------PhysicalDistribute ---------------------------hashAgg[LOCAL] -----------------------------PhysicalProject -------------------------------filter((customer.c_acctbal > 0.00)substring(c_phone, 1, 2) IN ('13', '31', '23', '29', '30', '18', '17')) ---------------------------------PhysicalOlapScan[customer] +------------------PhysicalAssertNumRows +--------------------hashAgg[GLOBAL] +----------------------PhysicalDistribute +------------------------hashAgg[LOCAL] +--------------------------PhysicalProject +----------------------------filter((customer.c_acctbal > 0.00)substring(c_phone, 1, 2) IN ('13', '31', '23', '29', '30', '18', '17')) +------------------------------PhysicalOlapScan[customer] diff --git a/regression-test/data/nereids_tpch_shape_sf500_p0/shape/q3.out b/regression-test/data/nereids_tpch_shape_sf500_p0/shape/q3.out index 643f478927d9fdb..23fc521663607a1 100644 --- a/regression-test/data/nereids_tpch_shape_sf500_p0/shape/q3.out +++ b/regression-test/data/nereids_tpch_shape_sf500_p0/shape/q3.out @@ -17,7 +17,8 @@ PhysicalTopN ----------------------PhysicalProject ------------------------filter((orders.o_orderdate < 1995-03-15)) --------------------------PhysicalOlapScan[orders] ---------------------PhysicalProject -----------------------filter((customer.c_mktsegment = 'BUILDING')) -------------------------PhysicalOlapScan[customer] +--------------------PhysicalDistribute +----------------------PhysicalProject +------------------------filter((customer.c_mktsegment = 'BUILDING')) +--------------------------PhysicalOlapScan[customer] diff --git a/regression-test/data/nereids_tpch_shape_sf500_p0/shape/q5.out b/regression-test/data/nereids_tpch_shape_sf500_p0/shape/q5.out index f8211c8fa8639ec..6d45aeda6ba21a7 100644 --- a/regression-test/data/nereids_tpch_shape_sf500_p0/shape/q5.out +++ b/regression-test/data/nereids_tpch_shape_sf500_p0/shape/q5.out @@ -8,8 +8,6 @@ PhysicalQuickSort ----------hashAgg[LOCAL] ------------PhysicalProject --------------hashJoin[INNER_JOIN](customer.c_custkey = orders.o_custkey)(customer.c_nationkey = supplier.s_nationkey) -----------------PhysicalProject -------------------PhysicalOlapScan[customer] ----------------PhysicalDistribute ------------------PhysicalProject --------------------hashJoin[INNER_JOIN](lineitem.l_orderkey = orders.o_orderkey) @@ -32,4 +30,7 @@ PhysicalQuickSort ----------------------PhysicalProject ------------------------filter((orders.o_orderdate < 1995-01-01)(orders.o_orderdate >= 1994-01-01)) --------------------------PhysicalOlapScan[orders] +----------------PhysicalDistribute +------------------PhysicalProject +--------------------PhysicalOlapScan[customer] diff --git a/regression-test/data/nereids_tpch_shape_sf500_p0/shape/q9.out b/regression-test/data/nereids_tpch_shape_sf500_p0/shape/q9.out index 05904f5c0626f91..6a73f8f1f009962 100644 --- a/regression-test/data/nereids_tpch_shape_sf500_p0/shape/q9.out +++ b/regression-test/data/nereids_tpch_shape_sf500_p0/shape/q9.out @@ -30,6 +30,7 @@ PhysicalQuickSort ------------------------------PhysicalDistribute --------------------------------PhysicalProject ----------------------------------PhysicalOlapScan[nation] -----------------PhysicalProject -------------------PhysicalOlapScan[partsupp] +----------------PhysicalDistribute +------------------PhysicalProject +--------------------PhysicalOlapScan[partsupp] diff --git a/regression-test/data/query_p0/show/test_show_create_table.out b/regression-test/data/query_p0/show/test_show_create_table.out index 7c32be9bc336b19..811e25ea15efe26 100644 --- a/regression-test/data/query_p0/show/test_show_create_table.out +++ b/regression-test/data/query_p0/show/test_show_create_table.out @@ -1,7 +1,7 @@ -- This file is automatically generated. You should know what you did if you want to edit this -- !select -- -tb_show_create_table CREATE TABLE `tb_show_create_table` (\n `datek1` date NULL COMMENT 'a',\n `datetimek1` datetime NULL COMMENT 'b',\n `datetimek2` datetime NULL COMMENT 'c',\n `datetimek3` datetime NULL COMMENT 'd',\n `datev1` date MAX NOT NULL COMMENT 'e',\n `datetimev1` datetime MAX NOT NULL COMMENT 'f',\n `datetimev2` datetime MAX NOT NULL COMMENT 'g',\n `datetimev3` datetime MAX NOT NULL COMMENT 'h'\n) ENGINE=OLAP\nAGGREGATE KEY(`datek1`, `datetimek1`, `datetimek2`, `datetimek3`)\nCOMMENT 'OLAP'\nDISTRIBUTED BY HASH(`datek1`) BUCKETS 5\nPROPERTIES (\n"replication_allocation" = "tag.location.default: 1",\n"storage_format" = "V2",\n"light_schema_change" = "true",\n"disable_auto_compaction" = "false",\n"binlog.enable" = "false",\n"binlog.ttl_seconds" = "9223372036854775807",\n"binlog.max_bytes" = "9223372036854775807",\n"binlog.max_history_nums" = "9223372036854775807",\n"enable_single_replica_compaction" = "false"\n); +tb_show_create_table CREATE TABLE `tb_show_create_table` (\n `datek1` date NULL COMMENT 'a',\n `datetimek1` datetime NULL COMMENT 'b',\n `datetimek2` datetime(3) NULL COMMENT 'c',\n `datetimek3` datetime(6) NULL COMMENT 'd',\n `datev1` date MAX NOT NULL COMMENT 'e',\n `datetimev1` datetime MAX NOT NULL COMMENT 'f',\n `datetimev2` datetime(3) MAX NOT NULL COMMENT 'g',\n `datetimev3` datetime(6) MAX NOT NULL COMMENT 'h'\n) ENGINE=OLAP\nAGGREGATE KEY(`datek1`, `datetimek1`, `datetimek2`, `datetimek3`)\nCOMMENT 'OLAP'\nDISTRIBUTED BY HASH(`datek1`) BUCKETS 5\nPROPERTIES (\n"replication_allocation" = "tag.location.default: 1",\n"storage_format" = "V2",\n"light_schema_change" = "true",\n"disable_auto_compaction" = "false",\n"binlog.enable" = "false",\n"binlog.ttl_seconds" = "9223372036854775807",\n"binlog.max_bytes" = "9223372036854775807",\n"binlog.max_history_nums" = "9223372036854775807",\n"enable_single_replica_compaction" = "false"\n); -- !select -- -tb_show_create_table CREATE TABLE `tb_show_create_table` (\n `datek1` date NULL COMMENT 'a',\n `datetimek1` datetime NULL COMMENT 'b',\n `datetimek2` datetime NULL COMMENT 'c',\n `datetimek3` datetime NULL COMMENT 'd',\n `datev1` date NOT NULL COMMENT 'e',\n `datetimev1` datetime NOT NULL COMMENT 'f',\n `datetimev2` datetime NOT NULL COMMENT 'g',\n `datetimev3` datetime NOT NULL COMMENT 'h'\n) ENGINE=OLAP\nDUPLICATE KEY(`datek1`, `datetimek1`, `datetimek2`, `datetimek3`)\nCOMMENT 'OLAP'\nDISTRIBUTED BY RANDOM BUCKETS 5\nPROPERTIES (\n"replication_allocation" = "tag.location.default: 1",\n"storage_format" = "V2",\n"light_schema_change" = "true",\n"disable_auto_compaction" = "false",\n"binlog.enable" = "false",\n"binlog.ttl_seconds" = "9223372036854775807",\n"binlog.max_bytes" = "9223372036854775807",\n"binlog.max_history_nums" = "9223372036854775807",\n"enable_single_replica_compaction" = "false"\n); +tb_show_create_table CREATE TABLE `tb_show_create_table` (\n `datek1` date NULL COMMENT 'a',\n `datetimek1` datetime NULL COMMENT 'b',\n `datetimek2` datetime(3) NULL COMMENT 'c',\n `datetimek3` datetime(6) NULL COMMENT 'd',\n `datev1` date NOT NULL COMMENT 'e',\n `datetimev1` datetime NOT NULL COMMENT 'f',\n `datetimev2` datetime(3) NOT NULL COMMENT 'g',\n `datetimev3` datetime(6) NOT NULL COMMENT 'h'\n) ENGINE=OLAP\nDUPLICATE KEY(`datek1`, `datetimek1`, `datetimek2`, `datetimek3`)\nCOMMENT 'OLAP'\nDISTRIBUTED BY RANDOM BUCKETS 5\nPROPERTIES (\n"replication_allocation" = "tag.location.default: 1",\n"storage_format" = "V2",\n"light_schema_change" = "true",\n"disable_auto_compaction" = "false",\n"binlog.enable" = "false",\n"binlog.ttl_seconds" = "9223372036854775807",\n"binlog.max_bytes" = "9223372036854775807",\n"binlog.max_history_nums" = "9223372036854775807",\n"enable_single_replica_compaction" = "false"\n); diff --git a/regression-test/data/query_p0/sql_functions/cast_function/test_cast_to_datetime.out b/regression-test/data/query_p0/sql_functions/cast_function/test_cast_to_datetime.out new file mode 100644 index 000000000000000..7f2d61ad58f0042 --- /dev/null +++ b/regression-test/data/query_p0/sql_functions/cast_function/test_cast_to_datetime.out @@ -0,0 +1,19 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !cast_string_to_datetime_invalid0 -- +\N + +-- !cast_string_to_datetime_invalid1 -- +\N + +-- !cast_string_to_datetime_invalid2 -- +\N + +-- !cast_string_to_datetime_invalid3 -- +\N + +-- !cast_string_to_datetime_invalid4 -- +\N + +-- !cast_string_to_datetime_invalid5 -- +\N + diff --git a/regression-test/data/rollup_p0/test_rollup_agg_date.out b/regression-test/data/rollup_p0/test_rollup_agg_date.out index d29e0742906f718..c1b9d49855aa6ef 100644 --- a/regression-test/data/rollup_p0/test_rollup_agg_date.out +++ b/regression-test/data/rollup_p0/test_rollup_agg_date.out @@ -2,22 +2,22 @@ -- !sql -- test_rollup_agg_date AGG_KEYS datek1 DATE DATEV2 Yes true \N true datetimek1 DATETIME DATETIMEV2(0) Yes true \N true - datetimek2 DATETIME DATETIMEV2(3) Yes true \N true - datetimek3 DATETIME DATETIMEV2(6) Yes true \N true + datetimek2 DATETIME(3) DATETIMEV2(3) Yes true \N true + datetimek3 DATETIME(6) DATETIMEV2(6) Yes true \N true datev1 DATE DATEV2 No false \N MAX true datetimev1 DATETIME DATETIMEV2(0) No false \N MAX true - datetimev2 DATETIME DATETIMEV2(3) No false \N MAX true - datetimev3 DATETIME DATETIMEV2(6) No false \N MAX true - datetimev4 DATETIME DATETIMEV2(3) Yes false \N MAX true + datetimev2 DATETIME(3) DATETIMEV2(3) No false \N MAX true + datetimev3 DATETIME(6) DATETIMEV2(6) No false \N MAX true + datetimev4 DATETIME(3) DATETIMEV2(3) Yes false \N MAX true rollup_date AGG_KEYS datek1 DATE DATEV2 Yes true \N true - datetimek2 DATETIME DATETIMEV2(3) Yes true \N true + datetimek2 DATETIME(3) DATETIMEV2(3) Yes true \N true datetimek1 DATETIME DATETIMEV2(0) Yes true \N true - datetimek3 DATETIME DATETIMEV2(6) Yes true \N true + datetimek3 DATETIME(6) DATETIMEV2(6) Yes true \N true datev1 DATE DATEV2 No false \N MAX true datetimev1 DATETIME DATETIMEV2(0) No false \N MAX true - datetimev2 DATETIME DATETIMEV2(3) No false \N MAX true - datetimev3 DATETIME DATETIMEV2(6) No false \N MAX true + datetimev2 DATETIME(3) DATETIMEV2(3) No false \N MAX true + datetimev3 DATETIME(6) DATETIMEV2(6) No false \N MAX true -- !sql -- 2022-08-23 2022-08-23T11:11:11 2022-08-23T11:11:11.111 2022-08-23T11:11:11.111111 2022-08-23 2022-08-23T11:11:11 2022-08-23T11:11:11.111 2022-08-23T11:11:11.111111 diff --git a/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/Syncer.groovy b/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/Syncer.groovy index e0ee01a4f6fcae5..805093ef4f7b36b 100644 --- a/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/Syncer.groovy +++ b/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/Syncer.groovy @@ -28,6 +28,7 @@ import org.apache.doris.regression.json.BinlogData import org.apache.doris.thrift.TBinlogType import org.apache.doris.thrift.TCommitTxnResult import org.apache.doris.thrift.TGetBinlogResult +import org.apache.doris.thrift.TGetMasterTokenResult import org.apache.doris.thrift.TGetSnapshotResult import org.apache.doris.thrift.TIngestBinlogRequest import org.apache.doris.thrift.TIngestBinlogResult @@ -306,6 +307,37 @@ class Syncer { return isCheckedOK } + Boolean checkGetMasterToken(TGetMasterTokenResult result) { + Boolean isCheckedOK = false + + // step 1: check status + if (result != null && result.isSetStatus()) { + TStatus status = result.getStatus() + if (status.isSetStatusCode()) { + TStatusCode code = status.getStatusCode() + switch (code) { + case TStatusCode.OK: + isCheckedOK = result.isSetToken() + break + default: + logger.error("Get Master token result code is: ${code}") + break + } + } else { + logger.error("Invalid TStatus! StatusCode is unset.") + } + } else { + logger.error("Invalid TGetMasterTokenResult! result: ${result}") + } + + if (isCheckedOK) { + context.token = result.getToken() + logger.info("Token is ${context.token}.") + } + + return isCheckedOK + } + Boolean checkSnapshotFinish() { String checkSQL = "SHOW BACKUP FROM " + context.db List row = suite.sql(checkSQL)[0] @@ -446,10 +478,24 @@ class Syncer { context.closeBackendClients() } + Boolean getMasterToken() { + logger.info("Get master token.") + FrontendClientImpl clientImpl = context.getSourceFrontClient() + TGetMasterTokenResult result = SyncerUtils.getMasterToken(clientImpl, context) + + return checkGetMasterToken(result) + } + Boolean restoreSnapshot() { logger.info("Restore snapshot ${context.labelName}") FrontendClientImpl clientImpl = context.getSourceFrontClient() + // step 1: get master token + if (!getMasterToken()) { + logger.error("Get Master error!") + return false + } + // step 1: recode job info Gson gson = new Gson() Map jsonMap = gson.fromJson(new String(context.getSnapshotResult.getJobInfo()), Map.class) @@ -634,7 +680,7 @@ class Syncer { } tarPartition.value.version = srcPartition.value.version - long partitionId = fakePartitionId == -1 ? srcPartition.key : fakePartitionId + long partitionId = fakePartitionId == -1 ? tarPartition.key : fakePartitionId long version = fakeVersion == -1 ? srcPartition.value.version : fakeVersion TIngestBinlogRequest request = new TIngestBinlogRequest() diff --git a/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/SyncerContext.groovy b/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/SyncerContext.groovy index 1b00c9caa61e72f..cd5c3d5d11cd70c 100644 --- a/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/SyncerContext.groovy +++ b/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/SyncerContext.groovy @@ -110,6 +110,7 @@ class SyncerContext { public String labelName public String tableName public TGetSnapshotResult getSnapshotResult + public String token public Config config public String user @@ -127,7 +128,7 @@ class SyncerContext { } ExtraInfo genExtraInfo() { - ExtraInfo info = new ExtraInfo("5ff161c3-2c08-4079-b108-26c8850b6598") + ExtraInfo info = new ExtraInfo(token) sourceBackendClients.forEach((id, client) -> { info.addBackendNetaddr(id, client.address.hostname, client.httpPort) }) diff --git a/regression-test/framework/src/main/groovy/org/apache/doris/regression/util/SyncerUtils.groovy b/regression-test/framework/src/main/groovy/org/apache/doris/regression/util/SyncerUtils.groovy index d622923d35641a3..78012330002cf00 100644 --- a/regression-test/framework/src/main/groovy/org/apache/doris/regression/util/SyncerUtils.groovy +++ b/regression-test/framework/src/main/groovy/org/apache/doris/regression/util/SyncerUtils.groovy @@ -25,6 +25,8 @@ import org.apache.doris.thrift.TBeginTxnRequest import org.apache.doris.thrift.TBeginTxnResult import org.apache.doris.thrift.TCommitTxnRequest import org.apache.doris.thrift.TCommitTxnResult +import org.apache.doris.thrift.TGetMasterTokenRequest +import org.apache.doris.thrift.TGetMasterTokenResult import org.apache.doris.thrift.TGetSnapshotRequest import org.apache.doris.thrift.TGetSnapshotResult import org.apache.doris.thrift.TIngestBinlogRequest @@ -111,4 +113,11 @@ class SyncerUtils { request.setJobInfo(context.getSnapshotResult.getJobInfo()) return clientImpl.client.restoreSnapshot(request) } + + static TGetMasterTokenResult getMasterToken(FrontendClientImpl clientImpl, SyncerContext context) throws TException { + TGetMasterTokenRequest request = new TGetMasterTokenRequest() + request.setUser(context.user) + request.setPassword(context.passwd) + return clientImpl.client.getMasterToken(request) + } } diff --git a/regression-test/pipeline/common/custom_env.sh b/regression-test/pipeline/common/custom_env.sh new file mode 100644 index 000000000000000..8201deda51c1644 --- /dev/null +++ b/regression-test/pipeline/common/custom_env.sh @@ -0,0 +1,20 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# shellcheck disable=SC2034 +BUILD_FS_BENCHMARK=ON diff --git a/regression-test/suites/cold_heat_separation_p2/table_modify_resouce_and_policy.groovy b/regression-test/suites/cold_heat_separation_p2/table_modify_resouce_and_policy.groovy index 82a4441c9ffb7f2..6a0b9b81436ce89 100644 --- a/regression-test/suites/cold_heat_separation_p2/table_modify_resouce_and_policy.groovy +++ b/regression-test/suites/cold_heat_separation_p2/table_modify_resouce_and_policy.groovy @@ -196,12 +196,16 @@ suite("table_modify_resouce") { log.info( "test tablets not empty") assertTrue(tablets.size() > 0) fetchDataSize(sizes, tablets[0]) + + def try_times = 100 while (sizes[0] != 0) { log.info( "test local size is not zero, sleep 10s") sleep(10000) tablets = sql """ SHOW TABLETS FROM ${tableName} """ + try_times -= 1 + assertTrue(try_times > 0) } // 所有的local data size为0 log.info( "test all local size is zero") diff --git a/regression-test/suites/correctness_p0/table_valued_function/test_frontends_tvf.groovy b/regression-test/suites/correctness_p0/table_valued_function/test_frontends_tvf.groovy index cc7d84d982dc066..05f85eb85a5d242 100644 --- a/regression-test/suites/correctness_p0/table_valued_function/test_frontends_tvf.groovy +++ b/regression-test/suites/correctness_p0/table_valued_function/test_frontends_tvf.groovy @@ -25,4 +25,26 @@ suite("test_frontends_tvf") { table = sql """ select Name from `frontends`();""" assertTrue(table.size() > 0) assertTrue(table[0].size == 1) + + // case insensitive + table = sql """ select name, host, editlogport, httpport, alive from frontends();""" + assertTrue(table.size() > 0) + assertTrue(table[0].size == 5) + assertEquals("true", table[0][4]) + + // test aliase columns + table = sql """ select name as n, host as h, alive as a, editlogport as e from frontends(); """ + assertTrue(table.size() > 0) + assertTrue(table[0].size == 4) + assertEquals("true", table[0][2]) + + // test changing position of columns + def res = sql """ select count(*) from frontends() where alive = 'true'; """ + assertTrue(res[0][0] > 0) + + sql """ select Name, Host, EditLogPort + HttpPort, QueryPort, RpcPort, `Role`, IsMaster, ClusterId + `Join`, Alive, ReplayedJournalId, LastHeartbeat + IsHelper, ErrMsg, Version, CurrentConnected from frontends(); + """ } diff --git a/be/test/runtime/memory/system_allocator_test.cpp b/regression-test/suites/correctness_p0/test_cast_decimal.groovy similarity index 56% rename from be/test/runtime/memory/system_allocator_test.cpp rename to regression-test/suites/correctness_p0/test_cast_decimal.groovy index a36bfc273c84601..88859ea1d529fd8 100644 --- a/be/test/runtime/memory/system_allocator_test.cpp +++ b/regression-test/suites/correctness_p0/test_cast_decimal.groovy @@ -15,34 +15,23 @@ // specific language governing permissions and limitations // under the License. -#include "runtime/memory/system_allocator.h" - -#include -#include - -#include - -#include "gtest/gtest_pred_impl.h" +suite("test_cast_decimal") { + sql """ + set enable_nereids_planner=true; + """ + + explain { + sql """select cast(32123.34212456734 as decimal(3,2));""" + contains "cast(32123.34212456734 as DECIMALV3(3, 2))" + } + -namespace doris { + sql """ + set enable_nereids_planner=false; + """ -void test_normal() { - { - auto ptr = SystemAllocator::allocate(4096); - EXPECT_NE(nullptr, ptr); - EXPECT_EQ(0, (uint64_t)ptr % 4096); - SystemAllocator::free(ptr); + explain { + sql """select cast(32123.34212456734 as decimal(3,2));""" + contains "CAST(32123.34212456734 AS DECIMALV3(3, 2))" } - { - auto ptr = SystemAllocator::allocate(100); - EXPECT_NE(nullptr, ptr); - EXPECT_EQ(0, (uint64_t)ptr % 4096); - SystemAllocator::free(ptr); - } -} - -TEST(SystemAllocatorTest, TestNormal) { - test_normal(); } - -} // namespace doris diff --git a/regression-test/suites/correctness_p0/test_first_value_window.groovy b/regression-test/suites/correctness_p0/test_first_value_window.groovy index dd56e9ac2db470f..637b9f71616dd2b 100644 --- a/regression-test/suites/correctness_p0/test_first_value_window.groovy +++ b/regression-test/suites/correctness_p0/test_first_value_window.groovy @@ -45,4 +45,33 @@ suite("test_first_value_window") { qt_select_default """ select *,first_value(state) over(partition by myday order by time_col range between current row and unbounded following) from ${tableName} order by myday, time_col, state; """ + + def tableName1 = "test_first_value_window_array" + + sql """ DROP TABLE IF EXISTS ${tableName1} """ + sql """ + CREATE TABLE IF NOT EXISTS ${tableName1} ( + `myday` INT, + `time_col` VARCHAR(40) NOT NULL, + `state` ARRAY + ) ENGINE=OLAP + DUPLICATE KEY(`myday`,time_col) + COMMENT "OLAP" + DISTRIBUTED BY HASH(`myday`) BUCKETS 2 + PROPERTIES ( + "replication_num" = "1", + "in_memory" = "false", + "storage_format" = "V2" + ); + """ + + sql """ INSERT INTO ${tableName1} VALUES + (21,"04-21-11",["amory", "clever"]), + (22,"04-22-10-21",["is ", "cute", "tea"]), + (22,"04-22-10-21",["doris", "aws", "greate"]), + (23,"04-23-10", ["p7", "year4"]), + (24,"02-24-10-21",[""]); """ + + qt_select_default """ select *,first_value(state) over(partition by myday order by time_col range between current row and unbounded following) from ${tableName1} order by myday, time_col; """ + } diff --git a/regression-test/suites/correctness_p0/test_last_value_window.groovy b/regression-test/suites/correctness_p0/test_last_value_window.groovy index f7ddf24a6a2e20d..d9b4c5f0e5cb71e 100644 --- a/regression-test/suites/correctness_p0/test_last_value_window.groovy +++ b/regression-test/suites/correctness_p0/test_last_value_window.groovy @@ -45,4 +45,34 @@ suite("test_last_value_window") { qt_select_default """ select *,last_value(state) over(partition by myday order by time_col) from ${tableName} order by myday, time_col, state; """ + + def tableName1 = "test_last_value_window_array" + + sql """ DROP TABLE IF EXISTS ${tableName1} """ + sql """ + CREATE TABLE IF NOT EXISTS ${tableName1} ( + `myday` INT, + `time_col` VARCHAR(40) NOT NULL, + `state` ARRAY + ) ENGINE=OLAP + DUPLICATE KEY(`myday`,time_col) + COMMENT "OLAP" + DISTRIBUTED BY HASH(`myday`) BUCKETS 2 + PROPERTIES ( + "replication_num" = "1", + "in_memory" = "false", + "storage_format" = "V2" + ); + """ + + sql """ INSERT INTO ${tableName1} VALUES + (21,"04-21-11",["amory", "clever"]), + (22,"04-22-10-21",["is ", "cute", "tea"]), + (22,"04-22-10-21",["doris", "aws", "greate"]), + (23,"04-23-10", ["p7", "year4"]), + (24,"02-24-10-21",[""]); """ + + qt_select_default """ select *,last_value(state) over(partition by myday order by time_col range between current row and unbounded following) from ${tableName1} order by myday, time_col; """ + + } diff --git a/regression-test/suites/correctness_p0/test_table_function.groovy b/regression-test/suites/correctness_p0/test_table_function.groovy new file mode 100644 index 000000000000000..3a79117152ab77c --- /dev/null +++ b/regression-test/suites/correctness_p0/test_table_function.groovy @@ -0,0 +1,82 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_table_function") { + sql """set enable_nereids_planner=false""" + sql """ + drop table if exists t_table_function; + """ + + sql """ + create table t_table_function ( + `tag_id` varchar(128) NOT NULL, + `tag_value` varchar(128) NULL , + `case_id_offset` bigint(20) NULL , + `case_id_bitmap` bitmap BITMAP_UNION NULL ) + ENGINE=OLAP AGGREGATE KEY(`tag_id`, `tag_value`, `case_id_offset`) COMMENT 'OLAP' DISTRIBUTED BY HASH(`case_id_offset`) + BUCKETS 128 PROPERTIES ( "replication_allocation" = "tag.location.default: 1", "storage_format" = "V2", "light_schema_change" = "true", "disable_auto_compaction" = "false" ); + """ + + sql """ + insert into t_table_function values + ('{}ALL{}', '{}ALL{}', 1, to_bitmap(10)), + ('{}ALL{}', '{}ALL{}', 2, to_bitmap(20)), + ('{}ALL{}', '{}ALL{}', 3, to_bitmap(30)), + ('class2_category_preference', '10030004', 1, to_bitmap(10)), + ('class2_category_preference', '10030004', 2, to_bitmap(20)), + ('class2_category_preference', '10030004', 3, to_bitmap(30)), + ('avg_paid_order_amt_360', '0', 1, to_bitmap(10)), + ('avg_paid_order_amt_360', '0', 2, to_bitmap(20)), + ('avg_paid_order_amt_360', '0', 3, to_bitmap(30)); + """ + + + qt_select """ + select count(*) from (SELECT t.case_id_offset * 1000000 + e1 + FROM + (SELECT t.case_id_offset AS case_id_offset, + bitmap_and(a0.case_id_bitmap, + a1.case_id_bitmap) AS case_id_bitmap + FROM + (SELECT case_id_offset, + bitmap_union(case_id_bitmap) AS case_id_bitmap + FROM t_table_function AS tmp_25 + WHERE tag_id = '{}ALL{}' + AND tag_value = '{}ALL{}' + GROUP BY case_id_offset limit 10 ) AS t + LEFT JOIN + (SELECT case_id_offset, + bitmap_union(case_id_bitmap) AS case_id_bitmap + FROM t_table_function + WHERE tag_id = 'class2_category_preference' + AND tag_value IN ('10030004') + GROUP BY case_id_offset ) a0 + ON a0.case_id_offset = t.case_id_offset + LEFT JOIN + (SELECT case_id_offset, + bitmap_union(case_id_bitmap) AS case_id_bitmap + FROM t_table_function + WHERE tag_id = 'avg_paid_order_amt_360' + AND tag_value IN ('0', '1', '2') + GROUP BY case_id_offset ) a1 + ON a1.case_id_offset = t.case_id_offset ) AS t lateral view explode(bitmap_to_array(t.case_id_bitmap)) tmp_tt AS e1)e2; + """ + + sql """ + drop table if exists t_table_function; + """ +} diff --git a/regression-test/suites/datatype_p0/agg_state/group_concat/test_agg_state_group_concat.groovy b/regression-test/suites/datatype_p0/agg_state/group_concat/test_agg_state_group_concat.groovy new file mode 100644 index 000000000000000..6926a16ababe23c --- /dev/null +++ b/regression-test/suites/datatype_p0/agg_state/group_concat/test_agg_state_group_concat.groovy @@ -0,0 +1,43 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_agg_state_group_concat") { + sql """ DROP TABLE IF EXISTS a_table; """ + sql """ + create table a_table( + k1 int null, + k2 agg_state group_concat(string) + ) + aggregate key (k1) + distributed BY hash(k1) buckets 3 + properties("replication_num" = "1"); + """ + + sql "insert into a_table select 1,group_concat_state('a');" + sql "insert into a_table select 1,group_concat_state('bb');" + sql "insert into a_table values(1,group_concat_state('ccc'));" + + qt_length1 """select k1,length(k2) from a_table order by k1;""" + qt_group1 """select k1,group_concat_merge(k2) from a_table group by k1 order by k1;""" + qt_merge1 """select group_concat_merge(k2) from a_table;""" + + qt_length2 """select k1,length(k2) from a_table order by k1;""" + qt_group2 """select k1,group_concat_merge(k2) from a_table group by k1 order by k1;""" + qt_merge2 """select group_concat_merge(k2) from a_table;""" + + qt_union """ select group_concat_merge(kstate) from (select k1,group_concat_union(k2) kstate from a_table group by k1 order by k1) t; """ +} diff --git a/regression-test/suites/datatype_p0/agg_state/max/test_agg_state_max.groovy b/regression-test/suites/datatype_p0/agg_state/max/test_agg_state_max.groovy new file mode 100644 index 000000000000000..bce225a7cd5edbd --- /dev/null +++ b/regression-test/suites/datatype_p0/agg_state/max/test_agg_state_max.groovy @@ -0,0 +1,83 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_agg_state_max") { + sql """ DROP TABLE IF EXISTS a_table; """ + sql """ + create table a_table( + k1 int not null, + k2 agg_state max(int not null) + ) + aggregate key (k1) + distributed BY hash(k1) + properties("replication_num" = "1"); + """ + + test { + sql "insert into a_table values(100,max_state(null));" + exception "State function meet input nullable column" + } + + sql """insert into a_table + select e1/1000,max_state(e1) from + (select 1 k1) as t lateral view explode_numbers(8000) tmp1 as e1;""" + + sql"set enable_nereids_planner=false;" + qt_select """ select k1,max_merge(k2) from a_table group by k1 order by k1; + """ + qt_select """ select max_merge(tmp) from (select k1,max_union(k2) tmp from a_table group by k1)t; + """ + test { + sql "select k1,min_merge(k2) from a_table group by k1 order by k1;" + exception "not match function" + } + sql"set enable_nereids_planner=true;" + qt_select """ select k1,max_merge(k2) from a_table group by k1 order by k1; + """ + qt_select """ select max_merge(tmp) from (select k1,max_union(k2) tmp from a_table group by k1)t; + """ + test { + sql "select k1,min_merge(k2) from a_table group by k1 order by k1;" + exception "not match function" + } + + sql """ DROP TABLE IF EXISTS a_table2; """ + sql """ + create table a_table2( + k1 int not null, + k2 agg_state max(int null) + ) + aggregate key (k1) + distributed BY hash(k1) + properties("replication_num" = "1"); + """ + sql """insert into a_table2 values(100,max_state(null));""" + sql """insert into a_table2 + select e1/1000,max_state(e1) from + (select 1 k1) as t lateral view explode_numbers(8000) tmp1 as e1;""" + + sql"set enable_nereids_planner=false;" + qt_select """ select k1,max_merge(k2) from a_table2 group by k1 order by k1; + """ + qt_select """ select max_merge(tmp) from (select k1,max_union(k2) tmp from a_table group by k1)t; + """ + sql"set enable_nereids_planner=true;" + qt_select """ select k1,max_merge(k2) from a_table2 group by k1 order by k1; + """ + qt_select """ select max_merge(tmp) from (select k1,max_union(k2) tmp from a_table group by k1)t; + """ +} diff --git a/regression-test/suites/datatype_p0/agg_state/test_agg_state.groovy b/regression-test/suites/datatype_p0/agg_state/test_agg_state.groovy index 142432083e8bc43..4c4797d9d13d07f 100644 --- a/regression-test/suites/datatype_p0/agg_state/test_agg_state.groovy +++ b/regression-test/suites/datatype_p0/agg_state/test_agg_state.groovy @@ -53,7 +53,7 @@ suite("test_agg_state") { sql "insert into a_table select 1,max_by_state(1,3);" sql "insert into a_table select 1,max_by_state(2,2);" - sql "insert into a_table select 1,max_by_state(3,1);" + sql "insert into a_table values(1,max_by_state(3,1));" qt_length1 """select k1,length(k2) from a_table order by k1;""" qt_group1 """select k1,max_by_merge(k2) from a_table group by k1 order by k1;""" @@ -67,4 +67,9 @@ suite("test_agg_state") { qt_union """ select max_by_merge(kstate) from (select k1,max_by_union(k2) kstate from a_table group by k1 order by k1) t; """ qt_max_by_null """ select max_by_merge(max_by_state(k1,null)),min_by_merge(min_by_state(null,k3)) from d_table; """ + + test { + sql "select avg_state(1) from d_table;" + exception "[NOT_IMPLEMENTED_ERROR]" + } } diff --git a/regression-test/suites/datatype_p0/decimalv3/test_load.groovy b/regression-test/suites/datatype_p0/decimalv3/test_load.groovy index ba7b04ad944c64d..c2651b3c908a6de 100644 --- a/regression-test/suites/datatype_p0/decimalv3/test_load.groovy +++ b/regression-test/suites/datatype_p0/decimalv3/test_load.groovy @@ -42,7 +42,7 @@ suite("test_load") { """ StringBuilder commandBuilder = new StringBuilder() - commandBuilder.append("""curl --location-trusted -u ${context.config.feHttpUser}:${context.config.feHttpPassword}""") + commandBuilder.append("""curl --max-time 5 --location-trusted -u ${context.config.feHttpUser}:${context.config.feHttpPassword}""") commandBuilder.append(""" -H format:csv -T ${context.file.parent}/test_data/test.csv http://${context.config.feHttpAddress}/api/""" + dbName + "/" + tableName + "/_stream_load") command = commandBuilder.toString() process = command.execute() @@ -51,6 +51,8 @@ suite("test_load") { out = process.getText() logger.info("Run command: command=" + command + ",code=" + code + ", out=" + out + ", err=" + err) assertEquals(code, 0) + + sql """sync""" qt_select_default """ SELECT * FROM ${tableName} t ORDER BY a; """ } finally { try_sql("DROP TABLE IF EXISTS ${tableName}") diff --git a/regression-test/suites/delete_p0/test_delete_where_in.groovy b/regression-test/suites/delete_p0/test_delete_where_in.groovy index 021338fe3ad4fa1..449647e37c44a8c 100644 --- a/regression-test/suites/delete_p0/test_delete_where_in.groovy +++ b/regression-test/suites/delete_p0/test_delete_where_in.groovy @@ -99,5 +99,50 @@ suite("test_delete_where_in", "delete_p0") { //drop table qt_sql """truncate table ${tb_name}""" qt_sql """drop table ${tb_name}""" - + + sql 'drop table if exists t1' + sql 'drop table if exists t2' + sql 'drop table if exists t3' + + sql ''' + create table t1 ( + id int, + id1 int, + c1 bigint, + c2 string, + c3 double, + c4 date + ) unique key (id, id1) + distributed by hash(id, id1) buckets 13 + properties( + 'replication_num'='1', + "function_column.sequence_col" = "c4" + ); + ''' + + sql ''' + create table t3 ( + id int + ) distributed by hash(id) buckets 13 + properties( + 'replication_num'='1' + ); + ''' + + sql ''' + INSERT INTO t1 VALUES + (1, 10, 1, '1', 1.0, '2000-01-01'), + (2, 20, 2, '2', 2.0, '2000-01-02'), + (3, 30, 3, '3', 3.0, '2000-01-03'); + ''' + + sql ''' + INSERT INTO t3 VALUES + (1), + (4), + (5); + ''' + + sql 'delete from t1 where id in (select id from t3)' + qt_delete_in 'select * from t1 order by id' } diff --git a/regression-test/suites/external_table_emr_p2/hive/test_hive_statistic_cache.groovy b/regression-test/suites/external_table_emr_p2/hive/test_hive_statistic_cache.groovy new file mode 100644 index 000000000000000..2fb574a0da46a31 --- /dev/null +++ b/regression-test/suites/external_table_emr_p2/hive/test_hive_statistic_cache.groovy @@ -0,0 +1,58 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_hive_statistic_cache", "p2") { + String enabled = context.config.otherConfigs.get("enableExternalHiveTest") + if (enabled != null && enabled.equalsIgnoreCase("true")) { + String extHiveHmsHost = context.config.otherConfigs.get("extHiveHmsHost") + String extHiveHmsPort = context.config.otherConfigs.get("extHiveHmsPort") + String catalog_name = "test_hive_statistic_cache" + sql """drop catalog if exists ${catalog_name};""" + sql """ + create catalog if not exists ${catalog_name} properties ( + 'type'='hms', + 'hadoop.username' = 'hadoop', + 'hive.metastore.uris' = 'thrift://${extHiveHmsHost}:${extHiveHmsPort}' + ); + """ + logger.info("catalog " + catalog_name + " created") + sql """switch ${catalog_name};""" + logger.info("switched to catalog " + catalog_name) + sql """use statistics;""" + qt_1 "select count(*) from stats" + Thread.sleep(5000); + qt_2 "show column cached stats `stats` (lo_orderkey)" + qt_3 "show column cached stats `stats` (lo_linenumber)" + qt_4 "show column cached stats `stats` (lo_custkey)" + qt_5 "show column cached stats `stats` (lo_partkey)" + qt_6 "show column cached stats `stats` (lo_suppkey)" + qt_7 "show column cached stats `stats` (lo_orderdate)" + qt_8 "show column cached stats `stats` (lo_orderpriority)" + qt_9 "show column cached stats `stats` (lo_shippriority)" + qt_10 "show column cached stats `stats` (lo_extendedprice)" + qt_11 "show column cached stats `stats` (lo_ordtotalprice)" + qt_12 "show column cached stats `stats` (lo_discount)" + qt_13 "show column cached stats `stats` (lo_revenue)" + qt_14 "show column cached stats `stats` (lo_supplycost)" + qt_15 "show column cached stats `stats` (lo_tax)" + qt_16 "show column cached stats `stats` (lo_commitdate)" + qt_17 "show column cached stats `stats` (lo_shipmode)" + qt_18 "show column cached stats `stats` (lo_orderkey)" + qt_19 "show column cached stats `stats` (lo_quantity)" + } +} + diff --git a/regression-test/suites/external_table_emr_p2/hive/test_hive_to_date.groovy b/regression-test/suites/external_table_emr_p2/hive/test_hive_to_date.groovy new file mode 100644 index 000000000000000..cc2c7d58e2b24fb --- /dev/null +++ b/regression-test/suites/external_table_emr_p2/hive/test_hive_to_date.groovy @@ -0,0 +1,49 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_hive_to_date", "p2") { + String enabled = context.config.otherConfigs.get("enableExternalHiveTest") + if (enabled != null && enabled.equalsIgnoreCase("true")) { + String extHiveHmsHost = context.config.otherConfigs.get("extHiveHmsHost") + String extHiveHmsPort = context.config.otherConfigs.get("extHiveHmsPort") + String catalog_name = "test_hive_to_date" + sql """drop catalog if exists ${catalog_name};""" + sql """ + create catalog if not exists ${catalog_name} properties ( + 'type'='hms', + 'hadoop.username' = 'hadoop', + 'hive.metastore.uris' = 'thrift://${extHiveHmsHost}:${extHiveHmsPort}' + ); + """ + logger.info("catalog " + catalog_name + " created") + sql """switch ${catalog_name};""" + logger.info("switched to catalog " + catalog_name) + sql """use multi_catalog;""" + sql """set enable_nereids_planner=true;""" + sql """set enable_fallback_to_original_planner=false""" + qt_1 "select * from datev2_csv" + qt_2 "select * from datev2_orc" + qt_3 "select * from datev2_parquet" + qt_4 "select * from datev2_csv where day>to_date(\"1999-01-01\")" + qt_5 "select * from datev2_orc where day>to_date(\"1999-01-01\")" + qt_6 "select * from datev2_parquet where day>to_date(\"1999-01-01\")" + qt_7 "select * from datev2_csv where day 0) + } + + sql "drop materialized view k12s3m on agg_have_dup_base;" + while (!(sql "show create materialized view k12s3m on agg_have_dup_base;").empty) { + sleep(100) + try_times -= 1 + assertTrue(try_times > 0) + } + } + + sql 'drop table if exists agg_have_dup_base' + sql ''' + create table agg_have_dup_base ( + k1 int null, + k2 int not null, + k3 bigint null, + k4 varchar(100) null + ) + duplicate key (k1, k2, k3) + distributed by hash(k1) buckets 3 + properties("replication_num" = "1"); + ''' + sql "insert into agg_have_dup_base select e1, -4, -4, 'd' from (select 1 k1) as t lateral view explode_numbers(10000) tmp1 as e1;" + // do not await + sql "create materialized view k12s3m as select k1,sum(k2),max(k2) from agg_have_dup_base group by k1;" + sql 'insert into agg_have_dup_base select -4, -4, -4, \'d\'' + qt_mv 'select sum(k1) from agg_have_dup_base' + + waitDrop() + sql "create materialized view k12s3m as select k1,sum(k2),max(k2) from agg_have_dup_base group by k1;" + sql 'insert into agg_have_dup_base select -4, -4, -4, \'d\'' + qt_mv 'select sum(k1) from agg_have_dup_base' + + waitDrop() + sql "create materialized view k12s3m as select k1,sum(k2),max(k2) from agg_have_dup_base group by k1;" + sql 'insert into agg_have_dup_base select -4, -4, -4, \'d\'' + qt_mv 'select sum(k1) from agg_have_dup_base' + + waitDrop() + sql "create materialized view k12s3m as select k1,sum(k2),max(k2) from agg_have_dup_base group by k1;" + sql 'insert into agg_have_dup_base select -4, -4, -4, \'d\'' + qt_mv 'select sum(k1) from agg_have_dup_base' + + waitDrop() + sql "create materialized view k12s3m as select k1,sum(k2),max(k2) from agg_have_dup_base group by k1;" + sql 'insert into agg_have_dup_base select -4, -4, -4, \'d\'' + qt_mv 'select sum(k1) from agg_have_dup_base' + + waitDrop() + sql "create materialized view k12s3m as select k1,sum(k2),max(k2) from agg_have_dup_base group by k1;" + sql 'insert into agg_have_dup_base select -4, -4, -4, \'d\'' + qt_mv 'select sum(k1) from agg_have_dup_base' + + waitDrop() + sql "create materialized view k12s3m as select k1,sum(k2),max(k2) from agg_have_dup_base group by k1;" + sql 'insert into agg_have_dup_base select -4, -4, -4, \'d\'' + qt_mv 'select sum(k1) from agg_have_dup_base' + + waitDrop() + sql "create materialized view k12s3m as select k1,sum(k2),max(k2) from agg_have_dup_base group by k1;" + sql 'insert into agg_have_dup_base select -4, -4, -4, \'d\'' + qt_mv 'select sum(k1) from agg_have_dup_base' + + waitDrop() + sql "create materialized view k12s3m as select k1,sum(k2),max(k2) from agg_have_dup_base group by k1;" + sql 'insert into agg_have_dup_base select -4, -4, -4, \'d\'' + qt_mv 'select sum(k1) from agg_have_dup_base' + + waitDrop() + sql "create materialized view k12s3m as select k1,sum(k2),max(k2) from agg_have_dup_base group by k1;" + sql 'insert into agg_have_dup_base select -4, -4, -4, \'d\'' + qt_mv 'select sum(k1) from agg_have_dup_base' + + waitDrop() + sql "create materialized view k12s3m as select k1,sum(k2),max(k2) from agg_have_dup_base group by k1;" + sql 'insert into agg_have_dup_base select -4, -4, -4, \'d\'' + qt_mv 'select sum(k1) from agg_have_dup_base' + + waitDrop() + sql "create materialized view k12s3m as select k1,sum(k2),max(k2) from agg_have_dup_base group by k1;" + sql 'insert into agg_have_dup_base select -4, -4, -4, \'d\'' + qt_mv 'select sum(k1) from agg_have_dup_base' + + waitDrop() + sql "create materialized view k12s3m as select k1,sum(k2),max(k2) from agg_have_dup_base group by k1;" + sql 'insert into agg_have_dup_base select -4, -4, -4, \'d\'' + qt_mv 'select sum(k1) from agg_have_dup_base' + + waitDrop() + sql "create materialized view k12s3m as select k1,sum(k2),max(k2) from agg_have_dup_base group by k1;" + sql 'insert into agg_have_dup_base select -4, -4, -4, \'d\'' + qt_mv 'select sum(k1) from agg_have_dup_base' + + waitDrop() + sql "create materialized view k12s3m as select k1,sum(k2),max(k2) from agg_have_dup_base group by k1;" + sql 'insert into agg_have_dup_base select -4, -4, -4, \'d\'' + qt_mv 'select sum(k1) from agg_have_dup_base' + + waitDrop() + sql "create materialized view k12s3m as select k1,sum(k2),max(k2) from agg_have_dup_base group by k1;" + sql 'insert into agg_have_dup_base select -4, -4, -4, \'d\'' + qt_mv 'select sum(k1) from agg_have_dup_base' + + waitDrop() + sql "create materialized view k12s3m as select k1,sum(k2),max(k2) from agg_have_dup_base group by k1;" + sql 'insert into agg_have_dup_base select -4, -4, -4, \'d\'' + qt_mv 'select sum(k1) from agg_have_dup_base' + + waitDrop() + sql "create materialized view k12s3m as select k1,sum(k2),max(k2) from agg_have_dup_base group by k1;" + sql 'insert into agg_have_dup_base select -4, -4, -4, \'d\'' + qt_mv 'select sum(k1) from agg_have_dup_base' +} diff --git a/regression-test/suites/nereids_p0/insert_into_table/complex_insert.groovy b/regression-test/suites/nereids_p0/insert_into_table/complex_insert.groovy index 3bcf030861f6d5e..458ae6cb3f478de 100644 --- a/regression-test/suites/nereids_p0/insert_into_table/complex_insert.groovy +++ b/regression-test/suites/nereids_p0/insert_into_table/complex_insert.groovy @@ -15,6 +15,8 @@ // specific language governing permissions and limitations // under the License. +import org.codehaus.groovy.runtime.IOGroovyMethods + suite('complex_insert') { sql 'set enable_nereids_planner=true' sql 'set enable_fallback_to_original_planner=false' @@ -200,10 +202,8 @@ suite('complex_insert') { distributed by hash(k1) buckets 3 properties("replication_num" = "1"); ''' - - sql ''' - create materialized view k12s3m as select k1,sum(k2),max(k2) from agg_have_dup_base group by k1; - ''' + + createMV("create materialized view k12s3m as select k1,sum(k2),max(k2) from agg_have_dup_base group by k1;") sql 'insert into agg_have_dup_base select -4, -4, -4, \'d\'' sql 'sync' diff --git a/regression-test/suites/nereids_syntax_p0/agg_with_const.groovy b/regression-test/suites/nereids_syntax_p0/agg_with_const.groovy index 61a8866f1fc722d..375517ab6a610de 100644 --- a/regression-test/suites/nereids_syntax_p0/agg_with_const.groovy +++ b/regression-test/suites/nereids_syntax_p0/agg_with_const.groovy @@ -49,4 +49,9 @@ suite("agg_with_const") { select count(2) + 1, sum(2) + sum(2) from agg_with_const_tbl """ + explain { + sql """select count(*) from ( select distinct col1 as a0, null as a1, null as a2 from agg_with_const_tbl)t""" + contains "projections: NULL" + } + } diff --git a/regression-test/suites/nereids_syntax_p0/sub_query_correlated.groovy b/regression-test/suites/nereids_syntax_p0/sub_query_correlated.groovy index 98ff46cec7d79e1..665ad67e4806ba7 100644 --- a/regression-test/suites/nereids_syntax_p0/sub_query_correlated.groovy +++ b/regression-test/suites/nereids_syntax_p0/sub_query_correlated.groovy @@ -22,14 +22,6 @@ suite ("sub_query_correlated") { SET enable_nereids_planner=true """ - sql """ - SET enable_bucket_shuffle_join=false - """ - - sql """ - SET disable_colocate_plan=true - """ - sql """ DROP TABLE IF EXISTS `sub_query_correlated_subquery1` """ diff --git a/regression-test/suites/nereids_syntax_p0/sub_query_diff_old_optimize.groovy b/regression-test/suites/nereids_syntax_p0/sub_query_diff_old_optimize.groovy index 6d5bc11a5a8ef49..4b07a3d53913705 100644 --- a/regression-test/suites/nereids_syntax_p0/sub_query_diff_old_optimize.groovy +++ b/regression-test/suites/nereids_syntax_p0/sub_query_diff_old_optimize.groovy @@ -21,14 +21,6 @@ suite ("sub_query_diff_old_optimize") { SET enable_nereids_planner=true """ - sql """ - SET enable_bucket_shuffle_join=false - """ - - sql """ - SET disable_colocate_plan=true - """ - sql """ DROP TABLE IF EXISTS `sub_query_diff_old_optimize_subquery1` """ diff --git a/regression-test/suites/query_p0/aggregate/push_filter_through_agg.groovy b/regression-test/suites/query_p0/aggregate/push_filter_through_agg.groovy index 82ee73c4f29f1a0..7987c98c2a5edcf 100644 --- a/regression-test/suites/query_p0/aggregate/push_filter_through_agg.groovy +++ b/regression-test/suites/query_p0/aggregate/push_filter_through_agg.groovy @@ -16,6 +16,7 @@ // under the License. suite("push_filter_through_agg") { + sql """DROP TABLE IF EXISTS t_push_filter_through_agg""" sql """ CREATE TABLE t_push_filter_through_agg (col1 varchar(11451) not null, col2 int not null, col3 int not null) UNIQUE KEY(col1) @@ -56,4 +57,4 @@ suite("push_filter_through_agg") { qt_sql """ SELECT SUM(`col2`) FROM view_i WHERE `col1` BETWEEN 10 AND 20 LIMIT 1; """ -} \ No newline at end of file +} diff --git a/regression-test/suites/query_p0/sql_functions/cast_function/test_cast_to_datetime.groovy b/regression-test/suites/query_p0/sql_functions/cast_function/test_cast_to_datetime.groovy new file mode 100644 index 000000000000000..13ea36372594651 --- /dev/null +++ b/regression-test/suites/query_p0/sql_functions/cast_function/test_cast_to_datetime.groovy @@ -0,0 +1,26 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_cast_to_datetime") { + // cast string of invalid datetime to datetime + qt_cast_string_to_datetime_invalid0 """ select cast("627492340" as datetime); """ + qt_cast_string_to_datetime_invalid1 """ select cast("" as datetime); """ + qt_cast_string_to_datetime_invalid2 """ select cast("1" as datetime); """ + qt_cast_string_to_datetime_invalid3 """ select cast("a" as datetime); """ + qt_cast_string_to_datetime_invalid4 """ select cast("null" as datetime); """ + qt_cast_string_to_datetime_invalid5 """ select cast(null as datetime); """ +} \ No newline at end of file diff --git a/regression-test/suites/statistics/analyze_stats.groovy b/regression-test/suites/statistics/analyze_stats.groovy index 89de47fe8974dd2..b486385accb696a 100644 --- a/regression-test/suites/statistics/analyze_stats.groovy +++ b/regression-test/suites/statistics/analyze_stats.groovy @@ -66,6 +66,12 @@ suite("test_analyze") { 'ps7qwcZjBjkGfcXYMw5HQMwnElzoHqinwk8vhQCbVoGBgfotc4oSkpD3tP34h4h0tTogDMwFu60iJm1bofUzyUQofTeRwZk8','4692206687866847780') """ + def frontends = sql """ + SHOW FRONTENDS; + """ + if (frontends.size > 1) { + return; + } sql """ ANALYZE DATABASE ${db} """ @@ -85,6 +91,8 @@ suite("test_analyze") { SET forbid_unknown_col_stats=true; """ + Thread.sleep(1000 * 60) + sql """ SELECT COUNT(*) FROM ${tbl}; """ diff --git a/regression-test/suites/tpcds_sf100_dup_without_key_p2/load.groovy b/regression-test/suites/tpcds_sf100_dup_without_key_p2/load.groovy index e28da95065c8c57..bc74169b54b5b07 100644 --- a/regression-test/suites/tpcds_sf100_dup_without_key_p2/load.groovy +++ b/regression-test/suites/tpcds_sf100_dup_without_key_p2/load.groovy @@ -73,6 +73,7 @@ suite('load') { rowCount = sql "select count(*) from ${table}" } assertEquals(rows, rowCount[0][0]) + sql """SET query_timeout = 1800""" sql """ ANALYZE TABLE $table WITH SYNC """ } } diff --git a/regression-test/suites/tpcds_sf100_p2/load.groovy b/regression-test/suites/tpcds_sf100_p2/load.groovy index e28da95065c8c57..bc74169b54b5b07 100644 --- a/regression-test/suites/tpcds_sf100_p2/load.groovy +++ b/regression-test/suites/tpcds_sf100_p2/load.groovy @@ -73,6 +73,7 @@ suite('load') { rowCount = sql "select count(*) from ${table}" } assertEquals(rows, rowCount[0][0]) + sql """SET query_timeout = 1800""" sql """ ANALYZE TABLE $table WITH SYNC """ } } diff --git a/regression-test/suites/tpcds_sf1_unique_p1/load.groovy b/regression-test/suites/tpcds_sf1_unique_p1/load.groovy index b157211a48d6206..bfe7b9fd3120caf 100644 --- a/regression-test/suites/tpcds_sf1_unique_p1/load.groovy +++ b/regression-test/suites/tpcds_sf1_unique_p1/load.groovy @@ -137,6 +137,7 @@ suite("load") { assertTrue(json.NumberLoadedRows > 0 && json.LoadBytes > 0) } } + sql """SET query_timeout=1800""" sql """ ANALYZE TABLE $tableName WITH SYNC """ } diff --git a/regression-test/suites/tpch_sf100_p2/load.groovy b/regression-test/suites/tpch_sf100_p2/load.groovy index 40a03fbee8a9981..7deae8e27da5fbd 100644 --- a/regression-test/suites/tpch_sf100_p2/load.groovy +++ b/regression-test/suites/tpch_sf100_p2/load.groovy @@ -67,6 +67,7 @@ suite("load") { sleep(5000) } } + sql """SET query_timeout = 1800""" sql """ ANALYZE TABLE $table WITH SYNC """ } } diff --git a/regression-test/suites/tpch_sf100_unique_p2/load_four_step/load.groovy b/regression-test/suites/tpch_sf100_unique_p2/load_four_step/load.groovy index 5363bc67ec01f6b..81b7ceb249a131e 100644 --- a/regression-test/suites/tpch_sf100_unique_p2/load_four_step/load.groovy +++ b/regression-test/suites/tpch_sf100_unique_p2/load_four_step/load.groovy @@ -116,6 +116,7 @@ suite("load_four_step") { } sleep(5000) } + sql """SET query_timeout = 1800""" sql """ ANALYZE TABLE $table WITH SYNC """ } } diff --git a/regression-test/suites/tpch_sf100_unique_p2/load_one_step/load.groovy b/regression-test/suites/tpch_sf100_unique_p2/load_one_step/load.groovy index 9f5e065927c19b8..030403712456308 100644 --- a/regression-test/suites/tpch_sf100_unique_p2/load_one_step/load.groovy +++ b/regression-test/suites/tpch_sf100_unique_p2/load_one_step/load.groovy @@ -68,6 +68,7 @@ suite("load_one_step") { sleep(5000) } + sql """SET query_timeout = 1800""" sql """ ANALYZE TABLE $table WITH SYNC """ } } diff --git a/regression-test/suites/tpch_sf100_unique_p2/load_three_step/load.groovy b/regression-test/suites/tpch_sf100_unique_p2/load_three_step/load.groovy index 287beaafeab83af..6c296524a881051 100644 --- a/regression-test/suites/tpch_sf100_unique_p2/load_three_step/load.groovy +++ b/regression-test/suites/tpch_sf100_unique_p2/load_three_step/load.groovy @@ -92,6 +92,7 @@ suite("load_three_step") { } sleep(5000) } + sql """SET query_timeout = 1800""" sql """ ANALYZE TABLE $table WITH SYNC """ } } diff --git a/regression-test/suites/tpch_sf100_unique_p2/load_two_step/load.groovy b/regression-test/suites/tpch_sf100_unique_p2/load_two_step/load.groovy index 916c62a4d0aa76a..694793ff477ec85 100644 --- a/regression-test/suites/tpch_sf100_unique_p2/load_two_step/load.groovy +++ b/regression-test/suites/tpch_sf100_unique_p2/load_two_step/load.groovy @@ -69,6 +69,7 @@ suite("load_two_step") { } sleep(5000) } + sql """SET query_timeout = 1800""" sql """ ANALYZE TABLE $table WITH SYNC """ } } diff --git a/regression-test/suites/tpch_sf100_unique_sql_p2/load.groovy b/regression-test/suites/tpch_sf100_unique_sql_p2/load.groovy index 40a03fbee8a9981..7deae8e27da5fbd 100644 --- a/regression-test/suites/tpch_sf100_unique_sql_p2/load.groovy +++ b/regression-test/suites/tpch_sf100_unique_sql_p2/load.groovy @@ -67,6 +67,7 @@ suite("load") { sleep(5000) } } + sql """SET query_timeout = 1800""" sql """ ANALYZE TABLE $table WITH SYNC """ } }