Minor change to include in the airmail build

quickwit-oss · Jun 4, 2024 · bc1904e · bc1904e
1 parent 4cc4775
commit bc1904e
Show file tree

Hide file tree

Showing 2 changed files with 19 additions and 16 deletions.
diff --git a/quickwit/quickwit-search/src/root.rs b/quickwit/quickwit-search/src/root.rs
@@ -1409,7 +1409,8 @@ async fn assign_client_fetch_docs_jobs(
 fn compute_split_cost(split_metadata: &SplitMetadata) -> usize {
     // TODO: Have a smarter cost, by smoothing the number of docs.
     // +1 because a split of 1 doc would have an ilog2 of 0 otherwise
-    split_metadata.num_docs.ilog2() as usize + 1
+    3 + split_metadata.num_docs / 100_000
+    // split_metadata.num_docs.ilog2() as usize + 1
 }
 
 /// Builds a LeafSearchRequest to one node, from a list of [`SearchJob`].

diff --git a/quickwit/quickwit-search/src/search_job_placer.rs b/quickwit/quickwit-search/src/search_job_placer.rs
@@ -28,7 +28,6 @@ use async_trait::async_trait;
 use quickwit_common::pubsub::EventSubscriber;
 use quickwit_common::rendezvous_hasher::{node_affinity, sort_by_rendez_vous_hash};
 use quickwit_proto::search::{ReportSplit, ReportSplitsRequest};
-use tracing::warn;
 
 use crate::{SearchJob, SearchServiceClient, SearcherPool, SEARCH_METRICS};
 
@@ -187,22 +186,25 @@ impl SearchJobPlacer {
         // difference stricter, find the right split names ("split6" instead of "split2" works).
         // or modify mock_split_meta() so that not all splits have the same job cost
         // for now i went with the mock_split_meta() changes.
-        const ALLOWED_DIFFERENCE: usize = 101;
-        let target_load = (total_load * ALLOWED_DIFFERENCE).div_ceil(num_nodes * 100);
+        const ALLOWED_DIFFERENCE: usize = 105;
+        let max_job_load = jobs.iter().map(|job| job.cost()).max().unwrap_or(0);
+        let target_load = (total_load * ALLOWED_DIFFERENCE).div_ceil(num_nodes * 100)
+            .max(
+                max_job_load + (total_load - max_job_load).div_ceil(num_nodes)
+            );
+
         for job in jobs {
             sort_by_rendez_vous_hash(&mut candidate_nodes, job.split_id());
-            let mut candidate_nodes_iter = candidate_nodes.iter().enumerate();
-            // Select a node which hasn't reached the target load
-            let chosen_node_idx = loop {
-                let Some((id, candidate_node)) = candidate_nodes_iter.next() else {
-                    warn!("found no lightly loaded searcher for split, this should never happen");
-                    // as a fallback, use the first node
-                    break 0;
-                };
-                if candidate_node.load < target_load {
-                    break id;
-                }
-            };
+            let chosen_node_idx = candidate_nodes.iter_mut()
+                .enumerate()
+                .find_map(|(node_idx, candidate)| {
+                    if candidate.load < target_load {
+                        Some(node_idx)
+                    } else {
+                        None
+                    }
+                })
+                .unwrap_or(0);
             let metric_node_idx = match chosen_node_idx {
                 0 => "0",
                 1 => "1",