add config

apache · Apr 23, 2024 · e2ac5c8 · e2ac5c8
1 parent 60bae68
commit e2ac5c8
Show file tree

Hide file tree

Showing 3 changed files with 20 additions and 5 deletions.
diff --git a/be/src/common/config.cpp b/be/src/common/config.cpp
@@ -1027,7 +1027,7 @@ DEFINE_mInt64(row_column_page_size, "4096");
 DEFINE_mInt64(s3_write_buffer_size, "5242880");
 DEFINE_mInt32(s3_task_check_interval, "60");
 // The timeout config for S3 buffer allocation
-DEFINE_mInt32(s3_writer_buffer_allocation_timeout, "300");
+DEFINE_mInt32(s3_file_writer_log_interval, "300");
 DEFINE_mInt64(file_cache_max_file_reader_cache_size, "1000000");
 DEFINE_mInt64(hdfs_write_batch_buffer_size_mb, "4"); // 4MB
 
@@ -1212,6 +1212,14 @@ DEFINE_mBool(enable_injection_point, "false");
 
 DEFINE_mBool(ignore_schema_change_check, "false");
 
+DEFINE_Int64(buffered_reader_prefetch_thread_pool_min_thread_num, "16");
+
+DEFINE_Int64(buffered_reader_prefetch_thread_pool_max_thread_num, "64");
+
+DEFINE_Int64(s3_file_upload_thread_pool_min_thread_num, "16");
+
+DEFINE_Int64(s3_file_upload_thread_pool_max_thread_num, "64");
+
 // clang-format off
 #ifdef BE_TEST
 // test s3

diff --git a/be/src/common/config.h b/be/src/common/config.h
@@ -1070,9 +1070,8 @@ DECLARE_mInt32(tablet_path_check_batch_size);
 DECLARE_mInt64(row_column_page_size);
 // it must be larger than or equal to 5MB
 DECLARE_mInt64(s3_write_buffer_size);
-DECLARE_mInt32(s3_task_check_interval);
-// The timeout config for S3 buffer allocation
-DECLARE_mInt32(s3_writer_buffer_allocation_timeout);
+// Log interval when doing s3 upload task
+DECLARE_mInt32(s3_file_writer_log_interval);
 // the max number of cached file handle for block segemnt
 DECLARE_mInt64(file_cache_max_file_reader_cache_size);
 DECLARE_mInt64(hdfs_write_batch_buffer_size_mb);
@@ -1298,6 +1297,14 @@ DECLARE_mBool(enable_injection_point);
 
 DECLARE_mBool(ignore_schema_change_check);
 
+DECLARE_Int64(buffered_reader_prefetch_thread_pool_min_thread_num);
+
+DECLARE_Int64(buffered_reader_prefetch_thread_pool_max_thread_num);
+
+DECLARE_Int64(s3_file_upload_thread_pool_min_thread_num);
+
+DECLARE_Int64(s3_file_upload_thread_pool_max_thread_num);
+
 #ifdef BE_TEST
 // test s3
 DECLARE_String(test_s3_resource);

diff --git a/be/src/io/fs/s3_file_writer.cpp b/be/src/io/fs/s3_file_writer.cpp
@@ -135,7 +135,7 @@ Status S3FileWriter::_create_multi_upload_request() {
 }
 
 void S3FileWriter::_wait_until_finish(std::string_view task_name) {
-    auto timeout_duration = config::s3_writer_buffer_allocation_timeout;
+    auto timeout_duration = config::s3_file_writer_log_interval;
     auto msg = fmt::format(
             "{} multipart upload already takes {} seconds, bucket={}, key={}, upload_id={}",
             task_name, timeout_duration, _bucket, _path.native(), _upload_id);