diff --git a/quickwit/Cargo.lock b/quickwit/Cargo.lock index 28fcd4f757a..941825aa6b7 100644 --- a/quickwit/Cargo.lock +++ b/quickwit/Cargo.lock @@ -6170,6 +6170,7 @@ version = "0.8.0" dependencies = [ "anyhow", "aws_lambda_events 0.15.1", + "bytesize", "chitchat", "chrono", "flate2", @@ -6199,6 +6200,7 @@ dependencies = [ "reqwest", "serde", "serde_json", + "serial_test", "time", "tokio", "tracing", diff --git a/quickwit/Cargo.toml b/quickwit/Cargo.toml index 0b742d87912..546031e574f 100644 --- a/quickwit/Cargo.toml +++ b/quickwit/Cargo.toml @@ -215,6 +215,7 @@ serde_json_borrow = "0.5" serde_qs = { version = "0.12", features = ["warp"] } serde_with = "3.9.0" serde_yaml = "0.9" +serial_test = { version = "3.1.1", features = ["file_locks"] } siphasher = "0.3" smallvec = "1" sqlx = { version = "0.7", features = [ diff --git a/quickwit/quickwit-config/src/templating.rs b/quickwit/quickwit-config/src/templating.rs index 86ad5503365..827d06608f0 100644 --- a/quickwit/quickwit-config/src/templating.rs +++ b/quickwit/quickwit-config/src/templating.rs @@ -30,7 +30,7 @@ use tracing::debug; // `ENV_VAR` or `ENV_VAR:DEFAULT` // Ignores whitespaces in curly braces static TEMPLATE_ENV_VAR_CAPTURE: Lazy = Lazy::new(|| { - Regex::new(r"\$\{\s*([A-Za-z0-9_]+)\s*(?::\-\s*([\S]+)\s*)?}") + Regex::new(r"\$\{\s*([A-Za-z0-9_]+)\s*(?::\-\s*([^\s\}]+)\s*)?}") .expect("regular expression should compile") }); @@ -158,6 +158,23 @@ mod test { assert_eq!(rendered, "metastore_uri: s3://test-bucket/metastore"); } + #[test] + fn test_template_render_with_multiple_vars_per_line() { + let config_content = + b"metastore_uri: s3://${RENDER_MULTIPLE_BUCKET}/${RENDER_MULTIPLE_PREFIX:-index}#polling_interval=${RENDER_MULTIPLE_INTERVAL}s"; + env::set_var("RENDER_MULTIPLE_BUCKET", "test-bucket"); + env::set_var("RENDER_MULTIPLE_PREFIX", "metastore"); + env::set_var("RENDER_MULTIPLE_INTERVAL", "30"); + let rendered = render_config(config_content).unwrap(); + std::env::remove_var("RENDER_MULTIPLE_BUCKET"); + std::env::remove_var("RENDER_MULTIPLE_PREFIX"); + std::env::remove_var("RENDER_MULTIPLE_INTERVAL"); + assert_eq!( + rendered, + "metastore_uri: s3://test-bucket/metastore#polling_interval=30s" + ); + } + #[test] fn test_template_render_ignores_commented_lines() { { diff --git a/quickwit/quickwit-lambda/Cargo.toml b/quickwit/quickwit-lambda/Cargo.toml index 621ade40703..e94e3baf107 100644 --- a/quickwit/quickwit-lambda/Cargo.toml +++ b/quickwit/quickwit-lambda/Cargo.toml @@ -24,6 +24,7 @@ s3-localstack-tests = [] [dependencies] anyhow = { workspace = true } aws_lambda_events = "0.15.0" +bytesize = { workspace = true } chitchat = { workspace = true } chrono = { workspace = true } flate2 = { workspace = true } @@ -65,3 +66,6 @@ quickwit-search = { workspace = true } quickwit-serve = { workspace = true } quickwit-storage = { workspace = true } quickwit-telemetry = { workspace = true } + +[dev-dependencies] +serial_test = { workspace = true } diff --git a/quickwit/quickwit-lambda/src/indexer/environment.rs b/quickwit/quickwit-lambda/src/indexer/environment.rs index 606d5c6ae5f..faa10ff4a17 100644 --- a/quickwit/quickwit-lambda/src/indexer/environment.rs +++ b/quickwit/quickwit-lambda/src/indexer/environment.rs @@ -48,3 +48,47 @@ pub static MAX_CHECKPOINTS: Lazy = Lazy::new(|| { .expect("QW_LAMBDA_MAX_CHECKPOINTS must be a positive integer") }) }); + +#[cfg(test)] +mod tests { + + use quickwit_config::{ConfigFormat, NodeConfig}; + + use super::*; + + #[tokio::test] + #[serial_test::file_serial(with_env)] + async fn test_load_config() { + let bucket = "mock-test-bucket"; + std::env::set_var("QW_LAMBDA_METASTORE_BUCKET", bucket); + std::env::set_var("QW_LAMBDA_INDEX_BUCKET", bucket); + std::env::set_var( + "QW_LAMBDA_INDEX_CONFIG_URI", + "s3://mock-index-config-bucket", + ); + std::env::set_var("QW_LAMBDA_INDEX_ID", "lambda-test"); + + let node_config = NodeConfig::load(ConfigFormat::Yaml, CONFIGURATION_TEMPLATE.as_bytes()) + .await + .unwrap(); + // + assert_eq!( + node_config.data_dir_path.to_string_lossy(), + "/tmp", + "only `/tmp` is writeable in AWS Lambda" + ); + assert_eq!( + node_config.default_index_root_uri, + "s3://mock-test-bucket/index" + ); + assert_eq!( + node_config.metastore_uri.to_string(), + "s3://mock-test-bucket/index" + ); + + std::env::remove_var("QW_LAMBDA_METASTORE_BUCKET"); + std::env::remove_var("QW_LAMBDA_INDEX_BUCKET"); + std::env::remove_var("QW_LAMBDA_INDEX_CONFIG_URI"); + std::env::remove_var("QW_LAMBDA_INDEX_ID"); + } +} diff --git a/quickwit/quickwit-lambda/src/indexer/ingest/mod.rs b/quickwit/quickwit-lambda/src/indexer/ingest/mod.rs index f380c78eee6..c68b82713b7 100644 --- a/quickwit/quickwit-lambda/src/indexer/ingest/mod.rs +++ b/quickwit/quickwit-lambda/src/indexer/ingest/mod.rs @@ -150,6 +150,7 @@ mod tests { } #[tokio::test] + #[serial_test::file_serial(with_env)] async fn test_ingest() -> anyhow::Result<()> { quickwit_common::setup_logging_for_tests(); let bucket = "quickwit-integration-tests"; @@ -246,6 +247,13 @@ mod tests { assert_eq!(stats.num_docs, 1); } + std::env::remove_var("QW_LAMBDA_METASTORE_BUCKET"); + std::env::remove_var("QW_LAMBDA_INDEX_BUCKET"); + std::env::remove_var("QW_LAMBDA_METASTORE_PREFIX"); + std::env::remove_var("QW_LAMBDA_INDEX_PREFIX"); + std::env::remove_var("QW_LAMBDA_INDEX_CONFIG_URI"); + std::env::remove_var("QW_LAMBDA_INDEX_ID"); + Ok(()) } } diff --git a/quickwit/quickwit-lambda/src/searcher/environment.rs b/quickwit/quickwit-lambda/src/searcher/environment.rs index 027810dad23..1ead2e672ca 100644 --- a/quickwit/quickwit-lambda/src/searcher/environment.rs +++ b/quickwit/quickwit-lambda/src/searcher/environment.rs @@ -26,3 +26,51 @@ data_dir: /tmp searcher: partial_request_cache_capacity: ${QW_LAMBDA_PARTIAL_REQUEST_CACHE_CAPACITY:-64M} "#; + +#[cfg(test)] +mod tests { + + use bytesize::ByteSize; + use quickwit_config::{ConfigFormat, NodeConfig}; + + use super::*; + + #[tokio::test] + #[serial_test::file_serial(with_env)] + async fn test_load_config() { + let bucket = "mock-test-bucket"; + std::env::set_var("QW_LAMBDA_METASTORE_BUCKET", bucket); + std::env::set_var("QW_LAMBDA_INDEX_BUCKET", bucket); + std::env::set_var( + "QW_LAMBDA_INDEX_CONFIG_URI", + "s3://mock-index-config-bucket", + ); + std::env::set_var("QW_LAMBDA_INDEX_ID", "lambda-test"); + + let node_config = NodeConfig::load(ConfigFormat::Yaml, CONFIGURATION_TEMPLATE.as_bytes()) + .await + .unwrap(); + assert_eq!( + node_config.data_dir_path.to_string_lossy(), + "/tmp", + "only `/tmp` is writeable in AWS Lambda" + ); + assert_eq!( + node_config.default_index_root_uri, + "s3://mock-test-bucket/index" + ); + assert_eq!( + node_config.metastore_uri.to_string(), + "s3://mock-test-bucket/index#polling_interval=60s" + ); + assert_eq!( + node_config.searcher_config.partial_request_cache_capacity, + ByteSize::mb(64) + ); + + std::env::remove_var("QW_LAMBDA_METASTORE_BUCKET"); + std::env::remove_var("QW_LAMBDA_INDEX_BUCKET"); + std::env::remove_var("QW_LAMBDA_INDEX_CONFIG_URI"); + std::env::remove_var("QW_LAMBDA_INDEX_ID"); + } +} diff --git a/quickwit/quickwit-metastore/Cargo.toml b/quickwit/quickwit-metastore/Cargo.toml index 68b7eb76f1a..a2ff2470cbc 100644 --- a/quickwit/quickwit-metastore/Cargo.toml +++ b/quickwit/quickwit-metastore/Cargo.toml @@ -52,7 +52,7 @@ futures = { workspace = true } md5 = { workspace = true } mockall = { workspace = true } rand = { workspace = true } -serial_test = { version = "3.1.1", features = ["file_locks"] } +serial_test = { workspace = true } tempfile = { workspace = true } tracing-subscriber = { workspace = true }