Merge branch 'master' into system-error-log

Avogar · Jun 20, 2024 · 71f8937 · 71f8937
2 parents 9784580 + 180ada0
commit 71f8937
Show file tree

Hide file tree

Showing 274 changed files with 7,030 additions and 3,727 deletions.
diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md
@@ -48,9 +48,7 @@ At a minimum, the following information should be added (but add more as needed)
 - [ ] <!---ci_include_stateful--> Allow: Stateful tests
 - [ ] <!---ci_include_integration--> Allow: Integration Tests
 - [ ] <!---ci_include_performance--> Allow: Performance tests
-- [ ] <!---ci_set_normal_builds--> Allow: Normal Builds
-- [ ] <!---ci_set_special_builds--> Allow: Special Builds
-- [ ] <!---ci_set_non_required--> Allow: All NOT Required Checks
+- [ ] <!---ci_set_builds--> Allow: All Builds
 - [ ] <!---batch_0_1--> Allow: batch 1, 2 for multi-batch jobs
 - [ ] <!---batch_2_3--> Allow: batch 3, 4, 5, 6 for multi-batch jobs
 ---
@@ -61,6 +59,7 @@ At a minimum, the following information should be added (but add more as needed)
 - [ ] <!---ci_exclude_aarch64|release|debug--> Exclude: All with aarch64, release, debug
 ---
 - [ ] <!---do_not_test--> Do not test
+- [ ] <!---woolen_wolfdog--> Woolen Wolfdog
 - [ ] <!---upload_all--> Upload binaries for special builds
 - [ ] <!---no_merge_commit--> Disable merge-commit
 - [ ] <!---no_ci_cache--> Disable CI cache
diff --git a/.github/workflows/backport_branches.yml b/.github/workflows/backport_branches.yml
@@ -70,7 +70,7 @@ jobs:
     if: ${{ !failure() && !cancelled() }}
     uses: ./.github/workflows/reusable_test.yml
     with:
-      test_name: Compatibility check (amd64)
+      test_name: Compatibility check (release)
       runner_type: style-checker
       data: ${{ needs.RunConfig.outputs.data }}
   CompatibilityCheckAarch64:
@@ -194,7 +194,7 @@ jobs:
     if: ${{ !failure() && !cancelled() }}
     uses: ./.github/workflows/reusable_test.yml
     with:
-      test_name: Install packages (amd64)
+      test_name: Install packages (release)
       runner_type: style-checker
       data: ${{ needs.RunConfig.outputs.data }}
       run_command: |
@@ -204,7 +204,7 @@ jobs:
     if: ${{ !failure() && !cancelled() }}
     uses: ./.github/workflows/reusable_test.yml
     with:
-      test_name: Install packages (arm64)
+      test_name: Install packages (aarch64)
       runner_type: style-checker-aarch64
       data: ${{ needs.RunConfig.outputs.data }}
       run_command: |

diff --git a/.github/workflows/master.yml b/.github/workflows/master.yml
@@ -104,36 +104,26 @@ jobs:
     with:
       stage: Tests_2
       data: ${{ needs.RunConfig.outputs.data }}
-  # stage for jobs that do not prohibit merge
   Tests_3:
     # Test_3 should not wait for Test_1/Test_2 and should not be blocked by them on master branch since all jobs need to run there.
-    needs: [RunConfig, Builds_1, Builds_2]
+    needs: [RunConfig, Builds_1]
     if: ${{ !failure() && !cancelled() && contains(fromJson(needs.RunConfig.outputs.data).stages_data.stages_to_do, 'Tests_3') }}
     uses: ./.github/workflows/reusable_test_stage.yml
     with:
       stage: Tests_3
       data: ${{ needs.RunConfig.outputs.data }}
 
   ################################# Reports #################################
-  # Reports should be run even if Builds_1/2 failed - put them separately in wf (not in Tests_1/2)
-  Builds_1_Report:
+  # Reports should run even if Builds_1/2 fail - run them separately, not in Tests_1/2/3
+  Builds_Report:
     # run report check for failed builds to indicate the CI error
     if: ${{ !cancelled() && needs.RunConfig.result == 'success' && contains(fromJson(needs.RunConfig.outputs.data).jobs_data.jobs_to_do, 'ClickHouse build check') }}
-    needs: [RunConfig, Builds_1]
+    needs: [RunConfig, Builds_1, Builds_2]
     uses: ./.github/workflows/reusable_test.yml
     with:
       test_name: ClickHouse build check
       runner_type: style-checker-aarch64
       data: ${{ needs.RunConfig.outputs.data }}
-  Builds_2_Report:
-    # run report check for failed builds to indicate the CI error
-    if: ${{ !cancelled() && needs.RunConfig.result == 'success' && contains(fromJson(needs.RunConfig.outputs.data).jobs_data.jobs_to_do, 'ClickHouse special build check') }}
-    needs: [RunConfig, Builds_2]
-    uses: ./.github/workflows/reusable_test.yml
-    with:
-      test_name: ClickHouse special build check
-      runner_type: style-checker-aarch64
-      data: ${{ needs.RunConfig.outputs.data }}
 
   MarkReleaseReady:
     if: ${{ !failure() && !cancelled() }}
@@ -165,7 +155,7 @@ jobs:
 
   FinishCheck:
     if: ${{ !cancelled() }}
-    needs: [RunConfig, Builds_1, Builds_2, Builds_1_Report, Builds_2_Report, Tests_1, Tests_2, Tests_3]
+    needs: [RunConfig, Builds_1, Builds_2, Builds_Report, Tests_1, Tests_2, Tests_3]
     runs-on: [self-hosted, style-checker-aarch64]
     steps:
       - name: Check out repository code

diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml
@@ -126,46 +126,38 @@ jobs:
     with:
       stage: Builds_2
       data: ${{ needs.RunConfig.outputs.data }}
+  # stage for running non-required checks without being blocked by required checks (Test_1) if corresponding settings is selected
   Tests_2:
-    needs: [RunConfig, Builds_2]
+    needs: [RunConfig, Builds_1]
     if: ${{ !failure() && !cancelled() && contains(fromJson(needs.RunConfig.outputs.data).stages_data.stages_to_do, 'Tests_2') }}
     uses: ./.github/workflows/reusable_test_stage.yml
     with:
       stage: Tests_2
       data: ${{ needs.RunConfig.outputs.data }}
-  # stage for jobs that do not prohibit merge
   Tests_3:
-    needs: [RunConfig, Builds_1, Tests_1, Builds_2, Tests_2]
+    needs: [RunConfig, Builds_1, Tests_1]
     if: ${{ !failure() && !cancelled() && contains(fromJson(needs.RunConfig.outputs.data).stages_data.stages_to_do, 'Tests_3') }}
     uses: ./.github/workflows/reusable_test_stage.yml
     with:
       stage: Tests_3
       data: ${{ needs.RunConfig.outputs.data }}
 
   ################################# Reports #################################
-  # Reports should by run even if Builds_1/2 fail, so put them separately in wf (not in Tests_1/2)
-  Builds_1_Report:
+  # Reports should run even if Builds_1/2 fail - run them separately (not in Tests_1/2/3)
+  Builds_Report:
     # run report check for failed builds to indicate the CI error
-    if: ${{ !cancelled() && needs.StyleCheck.result == 'success' && contains(fromJson(needs.RunConfig.outputs.data).jobs_data.jobs_to_do, 'ClickHouse build check') }}
-    needs: [RunConfig, StyleCheck, Builds_1]
+    if: ${{ !cancelled() && needs.RunConfig.result == 'success' && contains(fromJson(needs.RunConfig.outputs.data).jobs_data.jobs_to_do, 'ClickHouse build check') }}
+    needs: [RunConfig, StyleCheck, Builds_1, Builds_2]
     uses: ./.github/workflows/reusable_test.yml
     with:
       test_name: ClickHouse build check
       runner_type: style-checker-aarch64
       data: ${{ needs.RunConfig.outputs.data }}
-  Builds_2_Report:
-    # run report check for failed builds to indicate the CI error
-    if: ${{ !cancelled() && needs.StyleCheck.result == 'success' && contains(fromJson(needs.RunConfig.outputs.data).jobs_data.jobs_to_do, 'ClickHouse special build check') }}
-    needs: [RunConfig, StyleCheck, Builds_2]
-    uses: ./.github/workflows/reusable_test.yml
-    with:
-      test_name: ClickHouse special build check
-      runner_type: style-checker-aarch64
-      data: ${{ needs.RunConfig.outputs.data }}
 
   CheckReadyForMerge:
     if: ${{ !cancelled() && needs.StyleCheck.result == 'success' }}
-    needs: [RunConfig, BuildDockers, StyleCheck, FastTest, Builds_1, Builds_2, Builds_1_Report, Builds_2_Report, Tests_1, Tests_2]
+    # Test_2 or Test_3 must not have jobs required for Mergeable check
+    needs: [RunConfig, BuildDockers, StyleCheck, FastTest, Builds_1, Builds_2, Builds_Report, Tests_1]
     runs-on: [self-hosted, style-checker-aarch64]
     steps:
       - name: Check out repository code
@@ -181,7 +173,7 @@ jobs:
   #
   FinishCheck:
     if: ${{ !cancelled() }}
-    needs: [RunConfig, BuildDockers, StyleCheck, FastTest, Builds_1, Builds_2, Builds_1_Report, Builds_2_Report, Tests_1, Tests_2, Tests_3]
+    needs: [RunConfig, BuildDockers, StyleCheck, FastTest, Builds_1, Builds_2, Builds_Report, Tests_1, Tests_2, Tests_3]
     runs-on: [self-hosted, style-checker-aarch64]
     steps:
       - name: Check out repository code
@@ -204,8 +196,7 @@ jobs:
     concurrency:
       group: jepsen
     if: ${{ !failure() && !cancelled() && contains(fromJson(needs.RunConfig.outputs.data).jobs_data.jobs_to_do, 'ClickHouse Keeper Jepsen') }}
-    # jepsen needs binary_release build which is in Builds_2
-    needs: [RunConfig, Builds_2]
+    needs: [RunConfig, Builds_1]
     uses: ./.github/workflows/reusable_test.yml
     with:
       test_name: ClickHouse Keeper Jepsen

diff --git a/.github/workflows/release_branches.yml b/.github/workflows/release_branches.yml
@@ -65,7 +65,7 @@ jobs:
     if: ${{ !failure() && !cancelled() }}
     uses: ./.github/workflows/reusable_test.yml
     with:
-      test_name: Compatibility check (amd64)
+      test_name: Compatibility check (release)
       runner_type: style-checker
       data: ${{ needs.RunConfig.outputs.data }}
   CompatibilityCheckAarch64:
@@ -244,7 +244,7 @@ jobs:
     if: ${{ !failure() && !cancelled() }}
     uses: ./.github/workflows/reusable_test.yml
     with:
-      test_name: Install packages (amd64)
+      test_name: Install packages (release)
       runner_type: style-checker
       data: ${{ needs.RunConfig.outputs.data }}
       run_command: |
@@ -254,7 +254,7 @@ jobs:
     if: ${{ !failure() && !cancelled() }}
     uses: ./.github/workflows/reusable_test.yml
     with:
-      test_name: Install packages (arm64)
+      test_name: Install packages (aarch64)
       runner_type: style-checker-aarch64
       data: ${{ needs.RunConfig.outputs.data }}
       run_command: |

diff --git a/base/poco/Crypto/src/OpenSSLInitializer.cpp b/base/poco/Crypto/src/OpenSSLInitializer.cpp
@@ -23,9 +23,6 @@
 #include <openssl/conf.h>
 #endif
 
-#if __has_feature(address_sanitizer)
-#include <sanitizer/lsan_interface.h>
-#endif
 
 using Poco::RandomInputStream;
 using Poco::Thread;
@@ -70,27 +67,21 @@ void OpenSSLInitializer::initialize()
 		SSL_library_init();
 		SSL_load_error_strings();
 		OpenSSL_add_all_algorithms();
-
+		
 		char seed[SEEDSIZE];
 		RandomInputStream rnd;
 		rnd.read(seed, sizeof(seed));
-        {
-#   if __has_feature(address_sanitizer)
-            /// Leak sanitizer (part of address sanitizer) thinks that a few bytes of memory in OpenSSL are allocated during but never released.
-            __lsan::ScopedDisabler lsan_disabler;
-#endif
-		    RAND_seed(seed, SEEDSIZE);
-        }
-
+		RAND_seed(seed, SEEDSIZE);
+
 		int nMutexes = CRYPTO_num_locks();
 		_mutexes = new Poco::FastMutex[nMutexes];
 		CRYPTO_set_locking_callback(&OpenSSLInitializer::lock);
 // Not needed on Windows (see SF #110: random unhandled exceptions when linking with ssl).
 // https://sourceforge.net/p/poco/bugs/110/
 //
 // From http://www.openssl.org/docs/crypto/threads.html :
-// "If the application does not register such a callback using CRYPTO_THREADID_set_callback(),
-//  then a default implementation is used - on Windows and BeOS this uses the system's
+// "If the application does not register such a callback using CRYPTO_THREADID_set_callback(), 
+//  then a default implementation is used - on Windows and BeOS this uses the system's 
 //  default thread identifying APIs"
 		CRYPTO_set_id_callback(&OpenSSLInitializer::id);
 		CRYPTO_set_dynlock_create_callback(&OpenSSLInitializer::dynlockCreate);
@@ -109,7 +100,7 @@ void OpenSSLInitializer::uninitialize()
 		CRYPTO_set_locking_callback(0);
 		CRYPTO_set_id_callback(0);
 		delete [] _mutexes;
-
+		
 		CONF_modules_free();
 	}
 }

diff --git a/base/poco/Foundation/CMakeLists.txt b/base/poco/Foundation/CMakeLists.txt
@@ -213,6 +213,7 @@ target_compile_definitions (_poco_foundation
 )
 
 target_include_directories (_poco_foundation SYSTEM PUBLIC "include")
+target_link_libraries (_poco_foundation PRIVATE clickhouse_common_io)
 
 target_link_libraries (_poco_foundation
     PRIVATE

diff --git a/base/poco/Foundation/include/Poco/ThreadPool.h b/base/poco/Foundation/include/Poco/ThreadPool.h
@@ -48,16 +48,28 @@ class Foundation_API ThreadPool
 /// from the pool.
 {
 public:
-    ThreadPool(int minCapacity = 2, int maxCapacity = 16, int idleTime = 60, int stackSize = POCO_THREAD_STACK_SIZE);
+    explicit ThreadPool(
+        int minCapacity = 2,
+        int maxCapacity = 16,
+        int idleTime = 60,
+        int stackSize = POCO_THREAD_STACK_SIZE,
+        size_t global_profiler_real_time_period_ns_ = 0,
+        size_t global_profiler_cpu_time_period_ns_ = 0);
     /// Creates a thread pool with minCapacity threads.
     /// If required, up to maxCapacity threads are created
     /// a NoThreadAvailableException exception is thrown.
     /// If a thread is running idle for more than idleTime seconds,
     /// and more than minCapacity threads are running, the thread
     /// is killed. Threads are created with given stack size.
 
-    ThreadPool(
-        const std::string & name, int minCapacity = 2, int maxCapacity = 16, int idleTime = 60, int stackSize = POCO_THREAD_STACK_SIZE);
+    explicit ThreadPool(
+        const std::string & name,
+        int minCapacity = 2,
+        int maxCapacity = 16,
+        int idleTime = 60,
+        int stackSize = POCO_THREAD_STACK_SIZE,
+        size_t global_profiler_real_time_period_ns_ = 0,
+        size_t global_profiler_cpu_time_period_ns_ = 0);
     /// Creates a thread pool with the given name and minCapacity threads.
     /// If required, up to maxCapacity threads are created
     /// a NoThreadAvailableException exception is thrown.
@@ -171,6 +183,8 @@ class Foundation_API ThreadPool
     int _serial;
     int _age;
     int _stackSize;
+    size_t _globalProfilerRealTimePeriodNs;
+    size_t _globalProfilerCPUTimePeriodNs;
     ThreadVec _threads;
     mutable FastMutex _mutex;
 };