add max_queue_disk_usage check (#4324)

* validate max_queue_disk_usage * update docs * update docs
quickwit-oss · Dec 27, 2023 · 2480104 · 2480104
1 parent bfc64e1
commit 2480104
Show file tree

Hide file tree

Showing 2 changed files with 42 additions and 1 deletion.
diff --git a/docs/configuration/node-config.md b/docs/configuration/node-config.md
@@ -156,7 +156,7 @@ indexer:
 | Property | Description | Default value |
 | --- | --- | --- |
 | `max_queue_memory_usage` | Maximum size in bytes of the in-memory Ingest queue. | `2GiB` |
-| `max_queue_disk_usage` | Maximum disk-space in bytes taken by the Ingest queue. This is typically higher than the max in-memory queue. | `4GiB` |
+| `max_queue_disk_usage` | Maximum disk-space in bytes taken by the Ingest queue. The minimum size is at least `256M` and be at least `max_queue_memory_usage`. | `4GiB` |
 
 Example:
 

diff --git a/quickwit/quickwit-config/src/node_config/mod.rs b/quickwit/quickwit-config/src/node_config/mod.rs
@@ -236,6 +236,17 @@ impl IngestApiConfig {
 
     fn validate(&self) -> anyhow::Result<()> {
         self.replication_factor()?;
+        ensure!(
+            self.max_queue_disk_usage > ByteSize::mib(256),
+            "max_queue_disk_usage must be at least 256MB, got `{}`",
+            self.max_queue_disk_usage
+        );
+        ensure!(
+            self.max_queue_disk_usage >= self.max_queue_memory_usage,
+            "max_queue_disk_usage ({}) must be at least max_queue_memory_usage ({})",
+            self.max_queue_disk_usage,
+            self.max_queue_memory_usage
+        );
         Ok(())
     }
 }
@@ -392,6 +403,7 @@ impl NodeConfig {
 mod tests {
     use quickwit_proto::indexing::CpuCapacity;
 
+    use super::*;
     use crate::IndexerConfig;
 
     #[test]
@@ -437,4 +449,33 @@ mod tests {
             );
         }
     }
+    #[test]
+    fn test_validate_ingest_api_config() {
+        {
+            let indexer_config: IngestApiConfig = serde_yaml::from_str(
+                r#"
+                    max_queue_disk_usage: 100M
+                "#,
+            )
+            .unwrap();
+            assert_eq!(
+                indexer_config.validate().unwrap_err().to_string(),
+                "max_queue_disk_usage must be at least 256MB, got `100.0 MB`"
+            );
+        }
+        {
+            let indexer_config: IngestApiConfig = serde_yaml::from_str(
+                r#"
+                    max_queue_memory_usage: 600M
+                    max_queue_disk_usage: 500M
+                "#,
+            )
+            .unwrap();
+            assert_eq!(
+                indexer_config.validate().unwrap_err().to_string(),
+                "max_queue_disk_usage (500.0 MB) must be at least max_queue_memory_usage (600.0 \
+                 MB)"
+            );
+        }
+    }
 }