diff --git a/quickwit/Cargo.lock b/quickwit/Cargo.lock index 536d69f428b..95af1d67c60 100644 --- a/quickwit/Cargo.lock +++ b/quickwit/Cargo.lock @@ -5684,7 +5684,6 @@ dependencies = [ "http 0.2.12", "hyper 0.14.28", "itertools 0.12.1", - "num_cpus", "once_cell", "pin-project", "pnet", @@ -5720,7 +5719,6 @@ dependencies = [ "itertools 0.12.1", "json_comments", "new_string_template", - "num_cpus", "once_cell", "quickwit-common", "quickwit-doc-mapper", @@ -6303,7 +6301,6 @@ dependencies = [ "itertools 0.12.1", "mime_guess", "mockall", - "num_cpus", "once_cell", "opentelemetry", "percent-encoding", diff --git a/quickwit/Cargo.toml b/quickwit/Cargo.toml index 297fa8276d8..80a654c99a1 100644 --- a/quickwit/Cargo.toml +++ b/quickwit/Cargo.toml @@ -155,7 +155,6 @@ mockall = "0.11" mrecordlog = { git = "https://github.com/quickwit-oss/mrecordlog", rev = "306c0a7" } new_string_template = "1.5.1" nom = "7.1.3" -num_cpus = "1" numfmt = "1.1.1" once_cell = "1" oneshot = "0.1.5" diff --git a/quickwit/quickwit-common/Cargo.toml b/quickwit/quickwit-common/Cargo.toml index b18b3f3a814..d6041c3b7bb 100644 --- a/quickwit/quickwit-common/Cargo.toml +++ b/quickwit/quickwit-common/Cargo.toml @@ -25,7 +25,6 @@ hostname = { workspace = true } http = { workspace = true } hyper = { workspace = true } itertools = { workspace = true } -num_cpus = { workspace = true } once_cell = { workspace = true } pin-project = { workspace = true } pnet = { workspace = true } diff --git a/quickwit/quickwit-common/src/lib.rs b/quickwit/quickwit-common/src/lib.rs index 839322ce730..d0757d73f25 100644 --- a/quickwit/quickwit-common/src/lib.rs +++ b/quickwit/quickwit-common/src/lib.rs @@ -173,6 +173,18 @@ pub const fn div_ceil(lhs: i64, rhs: i64) -> i64 { } } +/// Return the number of vCPU/hyperthreads available. +/// This number is usually not equal to the number of cpu cores +pub fn num_cpus() -> usize { + match std::thread::available_parallelism() { + Ok(num_cpus) => num_cpus.get(), + Err(io_err) => { + error!(err=?io_err, "fail to detect the amount of threads available. arbitrarily returning 2"); + 2 + } + } +} + // The following are helpers to build named tasks. // // Named tasks require the tokio feature `tracing` to be enabled. diff --git a/quickwit/quickwit-common/src/runtimes.rs b/quickwit/quickwit-common/src/runtimes.rs index be1e8ce16d0..42435ce6662 100644 --- a/quickwit/quickwit-common/src/runtimes.rs +++ b/quickwit/quickwit-common/src/runtimes.rs @@ -77,7 +77,7 @@ impl RuntimesConfig { impl Default for RuntimesConfig { fn default() -> Self { - let num_cpus = num_cpus::get(); + let num_cpus = crate::num_cpus(); Self::with_num_cpus(num_cpus) } } diff --git a/quickwit/quickwit-config/Cargo.toml b/quickwit/quickwit-config/Cargo.toml index 26d0588fa5d..f32393cd39f 100644 --- a/quickwit/quickwit-config/Cargo.toml +++ b/quickwit/quickwit-config/Cargo.toml @@ -23,7 +23,6 @@ humantime = { workspace = true } itertools = { workspace = true } json_comments = { workspace = true } new_string_template = { workspace = true } -num_cpus = { workspace = true } once_cell = { workspace = true } regex = { workspace = true } serde = { workspace = true } diff --git a/quickwit/quickwit-config/src/node_config/mod.rs b/quickwit/quickwit-config/src/node_config/mod.rs index d44aff68c9b..841bbc3df6b 100644 --- a/quickwit/quickwit-config/src/node_config/mod.rs +++ b/quickwit/quickwit-config/src/node_config/mod.rs @@ -138,11 +138,12 @@ impl IndexerConfig { } pub fn default_merge_concurrency() -> NonZeroUsize { - NonZeroUsize::new(num_cpus::get() * 2 / 3).unwrap_or(NonZeroUsize::new(1).unwrap()) + NonZeroUsize::new(quickwit_common::num_cpus() * 2 / 3) + .unwrap_or(NonZeroUsize::new(1).unwrap()) } fn default_cpu_capacity() -> CpuCapacity { - CpuCapacity::one_cpu_thread() * (num_cpus::get() as u32) + CpuCapacity::one_cpu_thread() * (quickwit_common::num_cpus() as u32) } #[cfg(any(test, feature = "testsuite"))] diff --git a/quickwit/quickwit-serve/Cargo.toml b/quickwit/quickwit-serve/Cargo.toml index b45e64971c1..dafb56ca6ae 100644 --- a/quickwit/quickwit-serve/Cargo.toml +++ b/quickwit/quickwit-serve/Cargo.toml @@ -27,7 +27,6 @@ humantime = { workspace = true } hyper = { workspace = true } itertools = { workspace = true } mime_guess = { workspace = true } -num_cpus = { workspace = true } once_cell = { workspace = true } opentelemetry = { workspace = true } percent-encoding = { workspace = true } diff --git a/quickwit/quickwit-serve/src/build_info.rs b/quickwit/quickwit-serve/src/build_info.rs index b6a119392d3..229897af996 100644 --- a/quickwit/quickwit-serve/src/build_info.rs +++ b/quickwit/quickwit-serve/src/build_info.rs @@ -96,8 +96,9 @@ impl BuildInfo { #[derive(Debug, Eq, PartialEq, Serialize, utoipa::ToSchema)] pub struct RuntimeInfo { - pub num_cpus_logical: usize, - pub num_cpus_physical: usize, + // This is a number of logical cpus: vCPU or hyperthread depending on where you are running. + // This is usually NOT necessarily the number of cores. + pub num_cpus: usize, pub num_threads_blocking: usize, pub num_threads_non_blocking: usize, } @@ -108,12 +109,10 @@ impl RuntimeInfo { static INSTANCE: OnceCell = OnceCell::new(); INSTANCE.get_or_init(|| { - let num_cpus_logical = num_cpus::get(); - let runtimes_config = RuntimesConfig::with_num_cpus(num_cpus_logical); - + let num_cpus = quickwit_common::num_cpus(); + let runtimes_config = RuntimesConfig::with_num_cpus(num_cpus); Self { - num_cpus_logical, - num_cpus_physical: num_cpus::get_physical(), + num_cpus, num_threads_blocking: runtimes_config.num_threads_blocking, num_threads_non_blocking: runtimes_config.num_threads_non_blocking, } diff --git a/quickwit/quickwit-serve/src/node_info_handler.rs b/quickwit/quickwit-serve/src/node_info_handler.rs index 239594dfead..9d170c34ff9 100644 --- a/quickwit/quickwit-serve/src/node_info_handler.rs +++ b/quickwit/quickwit-serve/src/node_info_handler.rs @@ -105,7 +105,7 @@ mod tests { let runtime_info_json = info_json.get("runtime").unwrap(); let expected_runtime_info_json = serde_json::json!({ - "num_cpus_physical": runtime_info.num_cpus_physical, + "num_cpus": runtime_info.num_cpus, }); assert_json_include!( actual: runtime_info_json,