[Pick](nereids) runtime filter with probe expr should be pushed though set operator #33010 for Branch-2.1

.github/workflows/pr-approve-status.yml

@@ -19,42 +19,25 @@ name: Need_2_Approval
 
 on:
   pull_request_review:
-    types: [submitted]
+    types: [ submitted ]
+
 jobs:
   Need_2_Approval:
     runs-on: ubuntu-latest
-    timeout-minutes: 3
+    timeout-minutes: 5
     steps:
       - uses: actions/checkout@v3
-      - run: |
-          pr_num=${{ github.event.pull_request.number }}
-          echo $pr_num
-          if [ -z "$pr_num" ]; then
-              echo "PR number is not set"
-              exit 1
-          fi
-          response=$(curl -s  -H "Authorization: token ${{ secrets.GITHUB_TOKEN }} "  "https://api.github.com/repos/apache/doris/pulls/${pr_num}/reviews?per_page=100")
-          # shellcheck disable=SC2207
-          reviewers=($(echo $response | jq -r '.[] | .user.login'))
-          # shellcheck disable=SC2207
-          statuses=($(echo $response | jq -r '.[] | .state'))
-          echo "${reviewers[@]}"
-          echo "${statuses[@]}"
-          approves=()
-          reviewers_unique=()
-          for ((i=${#reviewers[@]}-1;i>=0;i--)); do
-            if ! echo "${reviewers_unique[@]}" | grep -q -w "${reviewers[$i]}" && [ "${statuses[$i]}" != "COMMENTED" ]; then
-                  reviewers_unique+=( "${reviewers[$i]}" )
-                  if [ "${statuses[$i]}" == "APPROVED" ]; then
-                      approves+=( "${reviewers[$i]}" )
-                  fi
-              fi
-          done
-          echo "${approves[@]}"
-          if [ ${#approves[@]} -lt 2 ]; then
-              echo "PR ${pr_num} has not been approved by at least 2 reviewers"
-              # shellcheck disable=SC2242
-              exit 1
-          fi
-          echo "Thanks for your contribution to Doris."
-          exit 0
+      - name: Install Python dependencies
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.10'  # Adjust if needed
+      - name: Install match library
+        run: |
+          pip install --upgrade pip 
+          pip install match
+          pip install requests
+      - name: Run Python script
+        run: |
+          python tools/maintainers/check_review.py  ${{ github.event.pull_request.number }} ${{secrets.GITHUB_TOKEN}}
+env:
+  GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

be/src/common/config.cpp

@@ -199,10 +199,8 @@ DEFINE_mInt32(download_low_speed_limit_kbps, "50");
 // download low speed time(seconds)
 DEFINE_mInt32(download_low_speed_time, "300");
 
-// log dir
-DEFINE_String(sys_log_dir, "${DORIS_HOME}/log");
+DEFINE_String(sys_log_dir, "");
 DEFINE_String(user_function_dir, "${DORIS_HOME}/lib/udf");
-DEFINE_String(pipeline_tracing_log_dir, "${DORIS_HOME}/log/tracing");
 // INFO, WARNING, ERROR, FATAL
 DEFINE_mString(sys_log_level, "INFO");
 // TIME-DAY, TIME-HOUR, SIZE-MB-nnn
@@ -1165,13 +1163,13 @@ DEFINE_mDouble(high_disk_avail_level_diff_usages, "0.15");
 DEFINE_Int32(partition_disk_index_lru_size, "10000");
 // limit the storage space that query spill files can use
 DEFINE_String(spill_storage_root_path, "${DORIS_HOME}/storage");
-DEFINE_mInt64(spill_storage_limit, "10737418240"); // 10G
-DEFINE_mInt32(spill_gc_interval_ms, "2000");       // 2s
+DEFINE_String(spill_storage_limit, "20%");   // 20%
+DEFINE_mInt32(spill_gc_interval_ms, "2000"); // 2s
+DEFINE_mInt32(spill_gc_file_count, "2000");
 DEFINE_Int32(spill_io_thread_pool_per_disk_thread_num, "2");
 DEFINE_Int32(spill_io_thread_pool_queue_size, "1024");
 DEFINE_Int32(spill_async_task_thread_pool_thread_num, "2");
 DEFINE_Int32(spill_async_task_thread_pool_queue_size, "1024");
-DEFINE_mInt32(spill_mem_warning_water_mark_multiplier, "2");
 
 DEFINE_mBool(check_segment_when_build_rowset_meta, "false");
 

be/src/common/config.h

@@ -247,10 +247,10 @@ DECLARE_mInt32(download_low_speed_limit_kbps);
 // download low speed time(seconds)
 DECLARE_mInt32(download_low_speed_time);
 
-// log dir
+// deprecated, use env var LOG_DIR in be.conf
 DECLARE_String(sys_log_dir);
+// for udf
 DECLARE_String(user_function_dir);
-DECLARE_String(pipeline_tracing_log_dir);
 // INFO, WARNING, ERROR, FATAL
 DECLARE_String(sys_log_level);
 // TIME-DAY, TIME-HOUR, SIZE-MB-nnn
@@ -1238,13 +1238,20 @@ DECLARE_mDouble(high_disk_avail_level_diff_usages);
 // create tablet in partition random robin idx lru size, default 10000
 DECLARE_Int32(partition_disk_index_lru_size);
 DECLARE_String(spill_storage_root_path);
-DECLARE_mInt64(spill_storage_limit);
+// Spill storage limit specified as number of bytes
+// ('<int>[bB]?'), megabytes ('<float>[mM]'), gigabytes ('<float>[gG]'),
+// or percentage of capaity ('<int>%').
+// Defaults to bytes if no unit is given.
+// Must larger than 0.
+// If specified as percentage, the final limit value is:
+//   disk_capacity_bytes * storage_flood_stage_usage_percent * spill_storage_limit
+DECLARE_String(spill_storage_limit);
 DECLARE_mInt32(spill_gc_interval_ms);
+DECLARE_mInt32(spill_gc_file_count);
 DECLARE_Int32(spill_io_thread_pool_per_disk_thread_num);
 DECLARE_Int32(spill_io_thread_pool_queue_size);
 DECLARE_Int32(spill_async_task_thread_pool_thread_num);
 DECLARE_Int32(spill_async_task_thread_pool_queue_size);
-DECLARE_mInt32(spill_mem_warning_water_mark_multiplier);
 
 DECLARE_mBool(check_segment_when_build_rowset_meta);
 

be/src/common/logconfig.cpp

@@ -63,7 +63,12 @@ bool init_glog(const char* basename) {
     // so fatal log can output to be.out .
     FLAGS_stderrthreshold = google::FATAL;
     // set glog log dir
-    FLAGS_log_dir = config::sys_log_dir;
+    // ATTN: sys_log_dir is deprecated, this is just for compatibility
+    std::string log_dir = config::sys_log_dir;
+    if (log_dir == "") {
+        log_dir = getenv("LOG_DIR");
+    }
+    FLAGS_log_dir = log_dir;
     // 0 means buffer INFO only
     FLAGS_logbuflevel = 0;
     // buffer log messages for at most this many seconds

be/src/exec/schema_scanner.cpp

@@ -37,12 +37,14 @@
 #include "exec/schema_scanner/schema_partitions_scanner.h"
 #include "exec/schema_scanner/schema_processlist_scanner.h"
 #include "exec/schema_scanner/schema_profiling_scanner.h"
+#include "exec/schema_scanner/schema_routine_scanner.h"
 #include "exec/schema_scanner/schema_rowsets_scanner.h"
 #include "exec/schema_scanner/schema_schema_privileges_scanner.h"
 #include "exec/schema_scanner/schema_schemata_scanner.h"
 #include "exec/schema_scanner/schema_table_privileges_scanner.h"
 #include "exec/schema_scanner/schema_tables_scanner.h"
 #include "exec/schema_scanner/schema_user_privileges_scanner.h"
+#include "exec/schema_scanner/schema_user_scanner.h"
 #include "exec/schema_scanner/schema_variables_scanner.h"
 #include "exec/schema_scanner/schema_views_scanner.h"
 #include "exec/schema_scanner/schema_workload_groups_scanner.h"
@@ -161,6 +163,10 @@ std::unique_ptr<SchemaScanner> SchemaScanner::create(TSchemaTableType::type type
         return SchemaWorkloadGroupsScanner::create_unique();
     case TSchemaTableType::SCH_PROCESSLIST:
         return SchemaProcessListScanner::create_unique();
+    case TSchemaTableType::SCH_PROCEDURES:
+        return SchemaRoutinesScanner::create_unique();
+    case TSchemaTableType::SCH_USER:
+        return SchemaUserScanner::create_unique();
     default:
         return SchemaDummyScanner::create_unique();
         break;

be/src/exec/schema_scanner/schema_helper.cpp

@@ -37,6 +37,8 @@ class TShowVariableRequest;
 class TShowVariableResult;
 class TShowProcessListRequest;
 class TShowProcessListResult;
+class TShowUserRequest;
+class TShowUserResult;
 
 Status SchemaHelper::get_db_names(const std::string& ip, const int32_t port,
                                   const TGetDbsParams& request, TGetDbsResult* result) {
@@ -134,4 +136,12 @@ Status SchemaHelper::show_process_list(const std::string& ip, const int32_t port
             });
 }
 
+Status SchemaHelper::show_user(const std::string& ip, const int32_t port,
+                               const TShowUserRequest& request, TShowUserResult* result) {
+    return ThriftRpcHelper::rpc<FrontendServiceClient>(
+            ip, port, [&request, &result](FrontendServiceConnection& client) {
+                client->showUser(*result, request);
+            });
+}
+
 } // namespace doris

be/src/exec/schema_scanner/schema_helper.h

@@ -39,6 +39,8 @@ class TShowVariableRequest;
 class TShowVariableResult;
 class TShowProcessListRequest;
 class TShowProcessListResult;
+class TShowUserRequest;
+class TShowUserResult;
 
 // this class is a helper for getting schema info from FE
 class SchemaHelper {
@@ -82,6 +84,8 @@ class SchemaHelper {
     static Status show_process_list(const std::string& ip, const int32_t port,
                                     const TShowProcessListRequest& request,
                                     TShowProcessListResult* result);
+    static Status show_user(const std::string& ip, const int32_t port,
+                            const TShowUserRequest& request, TShowUserResult* result);
 };
 
 } // namespace doris

be/src/exec/schema_scanner/schema_routine_scanner.cpp

@@ -0,0 +1,172 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "exec/schema_scanner/schema_routine_scanner.h"
+
+#include "runtime/client_cache.h"
+#include "runtime/exec_env.h"
+#include "runtime/runtime_state.h"
+#include "util/thrift_rpc_helper.h"
+#include "vec/common/string_ref.h"
+#include "vec/core/block.h"
+#include "vec/data_types/data_type_factory.hpp"
+
+namespace doris {
+std::vector<SchemaScanner::ColumnDesc> SchemaRoutinesScanner::_s_tbls_columns = {
+        {"SPECIFIC_NAME", TYPE_VARCHAR, sizeof(StringRef), true},
+        {"ROUTINE_CATALOG", TYPE_VARCHAR, sizeof(StringRef), true},
+        {"ROUTINE_SCHEMA", TYPE_VARCHAR, sizeof(StringRef), true},
+        {"ROUTINE_NAME", TYPE_VARCHAR, sizeof(StringRef), true},
+        {"ROUTINE_TYPE", TYPE_VARCHAR, sizeof(StringRef), true},
+        {"DTD_IDENTIFIER", TYPE_VARCHAR, sizeof(StringRef), true},
+        {"ROUTINE_BODY", TYPE_VARCHAR, sizeof(StringRef), true},
+        {"ROUTINE_DEFINITION", TYPE_VARCHAR, sizeof(StringRef), true},
+        {"EXTERNAL_NAME", TYPE_VARCHAR, sizeof(StringRef), true},
+        {"EXTERNAL_LANGUAGE", TYPE_VARCHAR, sizeof(StringRef), true},
+        {"PARAMETER_STYLE", TYPE_VARCHAR, sizeof(StringRef), true},
+        {"IS_DETERMINISTIC", TYPE_VARCHAR, sizeof(StringRef), true},
+        {"SQL_DATA_ACCESS", TYPE_VARCHAR, sizeof(StringRef), true},
+        {"SQL_PATH", TYPE_VARCHAR, sizeof(StringRef), true},
+        {"SECURITY_TYPE", TYPE_VARCHAR, sizeof(StringRef), true},
+        {"CREATED", TYPE_DATETIME, sizeof(int64_t), true},
+        {"LAST_ALTERED", TYPE_DATETIME, sizeof(int64_t), true},
+        {"SQL_MODE", TYPE_VARCHAR, sizeof(StringRef), true},
+        {"ROUTINE_COMMENT", TYPE_VARCHAR, sizeof(StringRef), true},
+        {"DEFINER", TYPE_VARCHAR, sizeof(StringRef), true},
+        {"CHARACTER_SET_CLIENT", TYPE_VARCHAR, sizeof(StringRef), true},
+        {"COLLATION_CONNECTION", TYPE_VARCHAR, sizeof(StringRef), true},
+        {"DATABASE_COLLATION", TYPE_VARCHAR, sizeof(StringRef), true},
+};
+
+SchemaRoutinesScanner::SchemaRoutinesScanner()
+        : SchemaScanner(_s_tbls_columns, TSchemaTableType::SCH_PROCEDURES) {}
+
+Status SchemaRoutinesScanner::start(RuntimeState* state) {
+    _block_rows_limit = state->batch_size();
+    _rpc_timeout = state->execution_timeout() * 1000;
+    return Status::OK();
+}
+
+Status SchemaRoutinesScanner::get_block_from_fe() {
+    TNetworkAddress master_addr = ExecEnv::GetInstance()->master_info()->network_address;
+    TSchemaTableRequestParams schema_table_request_params;
+    for (int i = 0; i < _s_tbls_columns.size(); i++) {
+        schema_table_request_params.__isset.columns_name = true;
+        schema_table_request_params.columns_name.emplace_back(_s_tbls_columns[i].name);
+    }
+    schema_table_request_params.__set_current_user_ident(*_param->common_param->current_user_ident);
+    TFetchSchemaTableDataRequest request;
+    request.__set_schema_table_name(TSchemaTableName::ROUTINES_INFO);
+    request.__set_schema_table_params(schema_table_request_params);
+    TFetchSchemaTableDataResult result;
+    RETURN_IF_ERROR(ThriftRpcHelper::rpc<FrontendServiceClient>(
+            master_addr.hostname, master_addr.port,
+            [&request, &result](FrontendServiceConnection& client) {
+                client->fetchSchemaTableData(result, request);
+            },
+            _rpc_timeout));
+    Status status(Status::create(result.status));
+    if (!status.ok()) {
+        LOG(WARNING) << "fetch routines from FE failed, errmsg=" << status;
+        return status;
+    }
+    std::vector<TRow> result_data = result.data_batch;
+    _routines_block = vectorized::Block::create_unique();
+    for (int i = 0; i < _s_tbls_columns.size(); ++i) {
+        TypeDescriptor descriptor(_s_tbls_columns[i].type);
+        auto data_type = vectorized::DataTypeFactory::instance().create_data_type(descriptor, true);
+        _routines_block->insert(vectorized::ColumnWithTypeAndName(
+                data_type->create_column(), data_type, _s_tbls_columns[i].name));
+    }
+    _routines_block->reserve(_block_rows_limit);
+    if (result_data.size() > 0) {
+        int col_size = result_data[0].column_value.size();
+        if (col_size != _s_tbls_columns.size()) {
+            return Status::InternalError<false>("routine table schema is not match for FE and BE");
+        }
+    }
+    auto insert_string_value = [&](int col_index, std::string str_val, vectorized::Block* block) {
+        vectorized::MutableColumnPtr mutable_col_ptr;
+        mutable_col_ptr = std::move(*block->get_by_position(col_index).column).assume_mutable();
+        auto* nullable_column =
+                reinterpret_cast<vectorized::ColumnNullable*>(mutable_col_ptr.get());
+        vectorized::IColumn* col_ptr = &nullable_column->get_nested_column();
+        reinterpret_cast<vectorized::ColumnString*>(col_ptr)->insert_data(str_val.data(),
+                                                                          str_val.size());
+        nullable_column->get_null_map_data().emplace_back(0);
+    };
+    auto insert_datetime_value = [&](int col_index, const std::vector<void*>& datas,
+                                     vectorized::Block* block) {
+        vectorized::MutableColumnPtr mutable_col_ptr;
+        mutable_col_ptr = std::move(*block->get_by_position(col_index).column).assume_mutable();
+        auto* nullable_column =
+                reinterpret_cast<vectorized::ColumnNullable*>(mutable_col_ptr.get());
+        vectorized::IColumn* col_ptr = &nullable_column->get_nested_column();
+        auto data = datas[0];
+        reinterpret_cast<vectorized::ColumnVector<vectorized::Int64>*>(col_ptr)->insert_data(
+                reinterpret_cast<char*>(data), 0);
+        nullable_column->get_null_map_data().emplace_back(0);
+    };
+
+    for (int i = 0; i < result_data.size(); i++) {
+        TRow row = result_data[i];
+
+        for (int j = 0; j < _s_tbls_columns.size(); j++) {
+            if (_s_tbls_columns[j].type == TYPE_DATETIME) {
+                std::vector<void*> datas(1);
+                VecDateTimeValue src[1];
+                src[0].from_date_str(row.column_value[j].stringVal.data(),
+                                     row.column_value[j].stringVal.size());
+                datas[0] = src;
+                insert_datetime_value(j, datas, _routines_block.get());
+            } else {
+                insert_string_value(j, row.column_value[j].stringVal, _routines_block.get());
+            }
+        }
+    }
+    return Status::OK();
+}
+
+Status SchemaRoutinesScanner::get_next_block(vectorized::Block* block, bool* eos) {
+    if (!_is_init) {
+        return Status::InternalError("Used before initialized.");
+    }
+
+    if (nullptr == block || nullptr == eos) {
+        return Status::InternalError("input pointer is nullptr.");
+    }
+
+    if (_routines_block == nullptr) {
+        RETURN_IF_ERROR(get_block_from_fe());
+        _total_rows = _routines_block->rows();
+    }
+
+    if (_row_idx == _total_rows) {
+        *eos = true;
+        return Status::OK();
+    }
+
+    int current_batch_rows = std::min(_block_rows_limit, _total_rows - _row_idx);
+    vectorized::MutableBlock mblock = vectorized::MutableBlock::build_mutable_block(block);
+    mblock.add_rows(_routines_block.get(), _row_idx, current_batch_rows);
+    _row_idx += current_batch_rows;
+
+    *eos = _row_idx == _total_rows;
+    return Status::OK();
+}
+
+} // namespace doris