Make it possible to disable per-index metrics with an environment

variable.

By default they are enabled.
If the `PER_INDEX_METRICS_ENABLED` environment variable is set to false,
then all of the index metrics are grouped under the index_id `__any__`.

In addition, this PR refactors a little bit the way we handled the
docprocessor metrics. We now cache the docprocessor counters, hence
preventing 1 hash lookup per document.

quickwit/quickwit-common/src/metrics.rs

@@ -18,7 +18,6 @@
 // along with this program. If not, see <http://www.gnu.org/licenses/>.
 
 use std::collections::{BTreeMap, HashMap};
-use std::sync::atomic::{AtomicBool, Ordering};
 use std::sync::OnceLock;
 
 use once_cell::sync::Lazy;
@@ -450,18 +449,16 @@ impl InFlightDataGauges {
     }
 }
 
-pub static MEMORY_METRICS: Lazy<MemoryMetrics> = Lazy::new(MemoryMetrics::default);
-
-static PER_INDEX_METRICS_ENABLED: AtomicBool = AtomicBool::new(true);
-
-pub fn disable_per_index_metrics() {
-    PER_INDEX_METRICS_ENABLED.store(false, Ordering::Relaxed);
-}
-
+/// This function return `index_name` or projects it to `<any>` if per-index metrics are disabled.
 pub fn index_label<'a>(index_name: &'a str) -> &'a str {
-    if PER_INDEX_METRICS_ENABLED.load(Ordering::Relaxed) {
+    static PER_INDEX_METRICS_ENABLED: OnceLock<bool> = OnceLock::new();
+    let per_index_metrics_enabled: bool = *PER_INDEX_METRICS_ENABLED
+        .get_or_init(|| crate::get_from_env("QW_PER_INDEX_METRICS_ENABLED", true));
+    if per_index_metrics_enabled {
         index_name
     } else {
-        "<any>"
+        "__any__"
     }
 }
+
+pub static MEMORY_METRICS: Lazy<MemoryMetrics> = Lazy::new(MemoryMetrics::default);

quickwit/quickwit-control-plane/src/model/shard_table.rs

@@ -435,9 +435,11 @@ impl ShardTable {
             } else {
                 0
             };
+        let index_label =
+            quickwit_common::metrics::index_label(&source_uid.index_uid.index_id.as_str());
         crate::metrics::CONTROL_PLANE_METRICS
             .open_shards_total
-            .with_label_values([source_uid.index_uid.index_id.as_str()])
+            .with_label_values([index_label])
             .set(num_open_shards as i64);
     }
 

quickwit/quickwit-janitor/src/actors/delete_task_planner.rs

@@ -209,9 +209,11 @@ impl DeleteTaskPlanner {
                     self.merge_split_downloader_mailbox.clone(),
                 )
                 .await?;
+                let index_label =
+                    quickwit_common::metrics::index_label(self.index_uid.index_id.as_str());
                 JANITOR_METRICS
                     .ongoing_num_delete_operations_total
-                    .with_label_values([&self.index_uid.index_id])
+                    .with_label_values([index_label])
                     .set(self.ongoing_delete_operations_inventory.list().len() as i64);
             }
         }