From 7750b48737848eed59ab6451bd62ecfd9109071c Mon Sep 17 00:00:00 2001 From: Adrien Guillo Date: Tue, 18 Jul 2023 06:02:36 +0900 Subject: [PATCH] Rename `QuickwitConfig` to `NodeConfig` (#3652) --- quickwit/quickwit-cli/src/lib.rs | 6 +- quickwit/quickwit-cli/src/service.rs | 4 +- quickwit/quickwit-cli/src/tool.rs | 4 +- quickwit/quickwit-cli/tests/helpers.rs | 6 +- quickwit/quickwit-cluster/src/lib.rs | 2 +- quickwit/quickwit-config/src/lib.rs | 6 +- .../{quickwit_config => node_config}/mod.rs | 13 +- .../serialize.rs | 192 ++++++++---------- quickwit/quickwit-indexing/src/lib.rs | 4 +- .../src/test_utils/cluster_sandbox.rs | 44 ++-- .../src/tests/basic_tests.rs | 2 +- .../src/tests/index_tests.rs | 2 +- quickwit/quickwit-janitor/src/lib.rs | 4 +- .../src/elastic_search_api/bulk.rs | 14 +- .../src/elastic_search_api/mod.rs | 24 +-- .../src/elastic_search_api/rest_handler.rs | 8 +- .../src/index_api/rest_handler.rs | 82 ++++---- quickwit/quickwit-serve/src/lib.rs | 6 +- .../quickwit-serve/src/node_info_handler.rs | 10 +- 19 files changed, 198 insertions(+), 235 deletions(-) rename quickwit/quickwit-config/src/{quickwit_config => node_config}/mod.rs (96%) rename quickwit/quickwit-config/src/{quickwit_config => node_config}/serialize.rs (86%) diff --git a/quickwit/quickwit-cli/src/lib.rs b/quickwit/quickwit-cli/src/lib.rs index 3e8b8dd71e0..de45b37d934 100644 --- a/quickwit/quickwit-cli/src/lib.rs +++ b/quickwit/quickwit-cli/src/lib.rs @@ -30,7 +30,7 @@ use quickwit_common::runtimes::RuntimesConfig; use quickwit_common::uri::Uri; use quickwit_config::service::QuickwitService; use quickwit_config::{ - ConfigFormat, MetastoreConfigs, QuickwitConfig, SourceConfig, StorageConfigs, + ConfigFormat, MetastoreConfigs, NodeConfig, SourceConfig, StorageConfigs, DEFAULT_QW_CONFIG_PATH, }; use quickwit_indexing::check_source_connectivity; @@ -223,12 +223,12 @@ pub fn start_actor_runtimes( } /// Loads a node config located at `config_uri` with the default storage configuration. -async fn load_node_config(config_uri: &Uri) -> anyhow::Result { +async fn load_node_config(config_uri: &Uri) -> anyhow::Result { let config_content = load_file(&StorageResolver::unconfigured(), config_uri) .await .context("Failed to load node config.")?; let config_format = ConfigFormat::sniff_from_uri(config_uri)?; - let config = QuickwitConfig::load(config_format, config_content.as_slice()) + let config = NodeConfig::load(config_format, config_content.as_slice()) .await .with_context(|| format!("Failed to parse node config `{config_uri}`."))?; info!(config_uri=%config_uri, config=?config, "Loaded node config."); diff --git a/quickwit/quickwit-cli/src/service.rs b/quickwit/quickwit-cli/src/service.rs index 6c53b483727..05d86d4e85c 100644 --- a/quickwit/quickwit-cli/src/service.rs +++ b/quickwit/quickwit-cli/src/service.rs @@ -25,7 +25,7 @@ use itertools::Itertools; use quickwit_common::runtimes::RuntimesConfig; use quickwit_common::uri::Uri; use quickwit_config::service::QuickwitService; -use quickwit_config::QuickwitConfig; +use quickwit_config::NodeConfig; use quickwit_serve::serve_quickwit; use quickwit_telemetry::payload::{QuickwitFeature, QuickwitTelemetryInfo, TelemetryEvent}; use tokio::signal; @@ -116,7 +116,7 @@ impl RunCliCommand { } } -fn quickwit_telemetry_info(config: &QuickwitConfig) -> QuickwitTelemetryInfo { +fn quickwit_telemetry_info(config: &NodeConfig) -> QuickwitTelemetryInfo { let mut features = HashSet::new(); if config.indexer_config.enable_otlp_endpoint { features.insert(QuickwitFeature::Otlp); diff --git a/quickwit/quickwit-cli/src/tool.rs b/quickwit/quickwit-cli/src/tool.rs index 96b88706ca7..c3cd2a6b141 100644 --- a/quickwit/quickwit-cli/src/tool.rs +++ b/quickwit/quickwit-cli/src/tool.rs @@ -37,7 +37,7 @@ use quickwit_common::runtimes::RuntimesConfig; use quickwit_common::uri::Uri; use quickwit_config::service::QuickwitService; use quickwit_config::{ - IndexerConfig, QuickwitConfig, SourceConfig, SourceInputFormat, SourceParams, TransformConfig, + IndexerConfig, NodeConfig, SourceConfig, SourceInputFormat, SourceParams, TransformConfig, VecSourceParams, CLI_INGEST_SOURCE_ID, }; use quickwit_core::{clear_cache_directory, IndexService}; @@ -775,7 +775,7 @@ impl ThroughputCalculator { } } -async fn create_empty_cluster(config: &QuickwitConfig) -> anyhow::Result { +async fn create_empty_cluster(config: &NodeConfig) -> anyhow::Result { let self_node = ClusterMember::new( config.node_id.clone(), quickwit_cluster::GenerationId::now(), diff --git a/quickwit/quickwit-cli/tests/helpers.rs b/quickwit/quickwit-cli/tests/helpers.rs index b44be225330..4893d916f71 100644 --- a/quickwit/quickwit-cli/tests/helpers.rs +++ b/quickwit/quickwit-cli/tests/helpers.rs @@ -209,11 +209,11 @@ pub async fn create_test_env( .replace("#index_id", &index_id) .replace("index_uri: #index_uri\n", ""), )?; - let quickwit_config_path = resources_dir_path.join("config.yaml"); + let node_config_path = resources_dir_path.join("config.yaml"); let rest_listen_port = find_available_tcp_port()?; let grpc_listen_port = find_available_tcp_port()?; fs::write( - &quickwit_config_path, + &node_config_path, // A poor's man templating engine reloaded... DEFAULT_QUICKWIT_CONFIG .replace("#metastore_uri", metastore_uri.as_str()) @@ -227,7 +227,7 @@ pub async fn create_test_env( fs::write(&wikipedia_docs_path, WIKI_JSON_DOCS)?; let mut resource_files = HashMap::new(); - resource_files.insert("config", quickwit_config_path); + resource_files.insert("config", node_config_path); resource_files.insert("index_config", index_config_path); resource_files.insert("index_config_without_uri", index_config_without_uri_path); resource_files.insert("logs", log_docs_path); diff --git a/quickwit/quickwit-cluster/src/lib.rs b/quickwit/quickwit-cluster/src/lib.rs index a65ac8b39d0..de960947acf 100644 --- a/quickwit/quickwit-cluster/src/lib.rs +++ b/quickwit/quickwit-cluster/src/lib.rs @@ -31,7 +31,7 @@ pub use chitchat::transport::ChannelTransport; use chitchat::transport::UdpTransport; use chitchat::FailureDetectorConfig; use quickwit_config::service::QuickwitService; -use quickwit_config::QuickwitConfig as NodeConfig; +use quickwit_config::NodeConfig; use time::OffsetDateTime; pub use crate::change::ClusterChange; diff --git a/quickwit/quickwit-config/src/lib.rs b/quickwit/quickwit-config/src/lib.rs index c77bc7fcf8e..871b0bb53d3 100644 --- a/quickwit/quickwit-config/src/lib.rs +++ b/quickwit/quickwit-config/src/lib.rs @@ -32,7 +32,7 @@ mod config_value; mod index_config; pub mod merge_policy_config; mod metastore_config; -mod quickwit_config; +mod node_config; mod qw_env_vars; pub mod service; mod source_config; @@ -63,8 +63,8 @@ use crate::merge_policy_config::{ pub use crate::metastore_config::{ MetastoreBackend, MetastoreConfig, MetastoreConfigs, PostgresMetastoreConfig, }; -pub use crate::quickwit_config::{ - IndexerConfig, IngestApiConfig, JaegerConfig, QuickwitConfig, SearcherConfig, +pub use crate::node_config::{ + IndexerConfig, IngestApiConfig, JaegerConfig, NodeConfig, SearcherConfig, DEFAULT_QW_CONFIG_PATH, }; use crate::source_config::serialize::{SourceConfigV0_6, VersionedSourceConfig}; diff --git a/quickwit/quickwit-config/src/quickwit_config/mod.rs b/quickwit/quickwit-config/src/node_config/mod.rs similarity index 96% rename from quickwit/quickwit-config/src/quickwit_config/mod.rs rename to quickwit/quickwit-config/src/node_config/mod.rs index 18bf495a688..6c102c2cf9d 100644 --- a/quickwit/quickwit-config/src/quickwit_config/mod.rs +++ b/quickwit/quickwit-config/src/node_config/mod.rs @@ -33,7 +33,7 @@ use quickwit_common::uri::Uri; use serde::{Deserialize, Serialize}; use tracing::warn; -use crate::quickwit_config::serialize::load_quickwit_config_with_env; +use crate::node_config::serialize::load_node_config_with_env; use crate::service::QuickwitService; use crate::storage_config::StorageConfigs; use crate::{ConfigFormat, MetastoreConfigs}; @@ -222,7 +222,7 @@ impl Default for JaegerConfig { } #[derive(Clone, Debug, Serialize)] -pub struct QuickwitConfig { +pub struct NodeConfig { pub cluster_id: String, pub node_id: String, pub enabled_services: HashSet, @@ -244,12 +244,11 @@ pub struct QuickwitConfig { pub jaeger_config: JaegerConfig, } -impl QuickwitConfig { - /// Parses and validates a [`QuickwitConfig`] from a given URI and config content. +impl NodeConfig { + /// Parses and validates a [`NodeConfig`] from a given URI and config content. pub async fn load(config_format: ConfigFormat, config_content: &[u8]) -> anyhow::Result { let env_vars = env::vars().collect::>(); - let config = - load_quickwit_config_with_env(config_format, config_content, &env_vars).await?; + let config = load_node_config_with_env(config_format, config_content, &env_vars).await?; if !config.data_dir_path.try_exists()? { bail!( "Data dir `{}` does not exist.", @@ -294,6 +293,6 @@ impl QuickwitConfig { #[cfg(any(test, feature = "testsuite"))] pub fn for_test() -> Self { - serialize::quickwit_config_for_test() + serialize::node_config_for_test() } } diff --git a/quickwit/quickwit-config/src/quickwit_config/serialize.rs b/quickwit/quickwit-config/src/node_config/serialize.rs similarity index 86% rename from quickwit/quickwit-config/src/quickwit_config/serialize.rs rename to quickwit/quickwit-config/src/node_config/serialize.rs index 2281e703a7b..e077de567bf 100644 --- a/quickwit/quickwit-config/src/quickwit_config/serialize.rs +++ b/quickwit/quickwit-config/src/node_config/serialize.rs @@ -35,14 +35,14 @@ use crate::storage_config::StorageConfigs; use crate::templating::render_config; use crate::{ validate_identifier, validate_node_id, ConfigFormat, IndexerConfig, IngestApiConfig, - JaegerConfig, MetastoreConfigs, QuickwitConfig, SearcherConfig, + JaegerConfig, MetastoreConfigs, NodeConfig, SearcherConfig, }; pub const DEFAULT_CLUSTER_ID: &str = "quickwit-default-cluster"; pub const DEFAULT_DATA_DIR_PATH: &str = "qwdata"; -// Default config values in the order they appear in [`QuickwitConfigBuilder`]. +// Default config values in the order they appear in [`NodeConfigBuilder`]. fn default_cluster_id() -> ConfigValue { ConfigValue::with_default(DEFAULT_CLUSTER_ID.to_string()) } @@ -124,33 +124,33 @@ fn default_index_root_uri(data_dir_uri: &Uri) -> Uri { data_dir_uri.join("indexes").expect("Failed to create default index root URI. This should never happen! Please, report on https://github.com/quickwit-oss/quickwit/issues.") } -pub async fn load_quickwit_config_with_env( +pub async fn load_node_config_with_env( config_format: ConfigFormat, config_content: &[u8], env_vars: &HashMap, -) -> anyhow::Result { +) -> anyhow::Result { let rendered_config_content = render_config(config_content)?; - let versioned_quickwit_config: VersionedQuickwitConfig = + let versioned_node_config: VersionedNodeConfig = config_format.parse(rendered_config_content.as_bytes())?; - let quickwit_config_builder: QuickwitConfigBuilder = versioned_quickwit_config.into(); - let config = quickwit_config_builder.build_and_validate(env_vars).await?; + let node_config_builder: NodeConfigBuilder = versioned_node_config.into(); + let config = node_config_builder.build_and_validate(env_vars).await?; Ok(config) } #[derive(Debug, Deserialize)] #[serde(tag = "version")] -enum VersionedQuickwitConfig { +enum VersionedNodeConfig { #[serde(rename = "0.6")] // Retro compatibility. #[serde(alias = "0.5")] #[serde(alias = "0.4")] - V0_6(QuickwitConfigBuilder), + V0_6(NodeConfigBuilder), } -impl From for QuickwitConfigBuilder { - fn from(versioned_quickwit_config: VersionedQuickwitConfig) -> Self { - match versioned_quickwit_config { - VersionedQuickwitConfig::V0_6(quickwit_config_builder) => quickwit_config_builder, +impl From for NodeConfigBuilder { + fn from(versioned_node_config: VersionedNodeConfig) -> Self { + match versioned_node_config { + VersionedNodeConfig::V0_6(node_config_builder) => node_config_builder, } } } @@ -158,7 +158,7 @@ impl From for QuickwitConfigBuilder { #[serde_with::serde_as] #[derive(Debug, Deserialize, PartialEq)] #[serde(deny_unknown_fields)] -struct QuickwitConfigBuilder { +struct NodeConfigBuilder { #[serde(default = "default_cluster_id")] cluster_id: ConfigValue, #[serde(default = "default_node_id")] @@ -202,11 +202,11 @@ struct QuickwitConfigBuilder { jaeger_config: JaegerConfig, } -impl QuickwitConfigBuilder { +impl NodeConfigBuilder { pub async fn build_and_validate( mut self, env_vars: &HashMap, - ) -> anyhow::Result { + ) -> anyhow::Result { let enabled_services = self .enabled_services .resolve(env_vars)? @@ -267,7 +267,7 @@ impl QuickwitConfigBuilder { self.storage_configs.validate()?; self.storage_configs.apply_flavors(); - let quickwit_config = QuickwitConfig { + let node_config = NodeConfig { cluster_id: self.cluster_id.resolve(env_vars)?, node_id: self.node_id.resolve(env_vars)?, enabled_services, @@ -289,29 +289,29 @@ impl QuickwitConfigBuilder { jaeger_config: self.jaeger_config, }; - validate(&quickwit_config)?; - Ok(quickwit_config) + validate(&node_config)?; + Ok(node_config) } } -fn validate(quickwit_config: &QuickwitConfig) -> anyhow::Result<()> { - validate_identifier("Cluster ID", &quickwit_config.cluster_id)?; - validate_node_id(&quickwit_config.node_id)?; +fn validate(node_config: &NodeConfig) -> anyhow::Result<()> { + validate_identifier("Cluster ID", &node_config.cluster_id)?; + validate_node_id(&node_config.node_id)?; - if quickwit_config.cluster_id == DEFAULT_CLUSTER_ID { + if node_config.cluster_id == DEFAULT_CLUSTER_ID { warn!( "Cluster ID is not set, falling back to default value: `{}`.", DEFAULT_CLUSTER_ID ); } - if quickwit_config.peer_seeds.is_empty() { + if node_config.peer_seeds.is_empty() { warn!("Peer seed list is empty."); } Ok(()) } #[cfg(test)] -impl Default for QuickwitConfigBuilder { +impl Default for NodeConfigBuilder { fn default() -> Self { Self { cluster_id: default_cluster_id(), @@ -338,7 +338,7 @@ impl Default for QuickwitConfigBuilder { } #[cfg(any(test, feature = "testsuite"))] -pub fn quickwit_config_for_test() -> QuickwitConfig { +pub fn node_config_for_test() -> NodeConfig { let enabled_services = QuickwitService::supported_services(); let listen_address = Host::default(); @@ -367,7 +367,7 @@ pub fn quickwit_config_for_test() -> QuickwitConfig { let metastore_uri = default_metastore_uri(&data_dir_uri); let default_index_root_uri = default_index_root_uri(&data_dir_uri); - QuickwitConfig { + NodeConfig { cluster_id: default_cluster_id().unwrap(), node_id: default_node_id().unwrap(), enabled_services, @@ -411,13 +411,12 @@ mod tests { ) } - async fn test_quickwit_config_parse_aux(config_format: ConfigFormat) -> anyhow::Result<()> { + async fn test_node_config_parse_aux(config_format: ConfigFormat) -> anyhow::Result<()> { let config_filepath = get_config_filepath(&format!("quickwit.{config_format:?}").to_lowercase()); let file = std::fs::read_to_string(&config_filepath).unwrap(); let env_vars = HashMap::default(); - let config = - load_quickwit_config_with_env(config_format, file.as_bytes(), &env_vars).await?; + let config = load_node_config_with_env(config_format, file.as_bytes(), &env_vars).await?; assert_eq!(config.cluster_id, "quickwit-cluster"); assert_eq!(config.enabled_services.len(), 2); @@ -514,22 +513,22 @@ mod tests { } #[tokio::test] - async fn test_quickwit_config_parse_json() { - test_quickwit_config_parse_aux(ConfigFormat::Json) + async fn test_node_config_parse_json() { + test_node_config_parse_aux(ConfigFormat::Json) .await .unwrap(); } #[tokio::test] - async fn test_quickwit_config_parse_toml() { - test_quickwit_config_parse_aux(ConfigFormat::Toml) + async fn test_node_config_parse_toml() { + test_node_config_parse_aux(ConfigFormat::Toml) .await .unwrap(); } #[tokio::test] - async fn test_quickwit_config_parse_yaml() { - test_quickwit_config_parse_aux(ConfigFormat::Yaml) + async fn test_node_config_parse_yaml() { + test_node_config_parse_aux(ConfigFormat::Yaml) .await .unwrap(); } @@ -538,7 +537,7 @@ mod tests { async fn test_config_contains_wrong_values() { let config_filepath = get_config_filepath("quickwit.wrongkey.yaml"); let config_str = std::fs::read_to_string(&config_filepath).unwrap(); - let parsing_error = super::load_quickwit_config_with_env( + let parsing_error = super::load_node_config_with_env( ConfigFormat::Yaml, config_str.as_bytes(), &Default::default(), @@ -550,9 +549,9 @@ mod tests { } #[tokio::test] - async fn test_quickwit_config_default_values_minimal() { + async fn test_node_config_default_values_minimal() { let config_yaml = "version: 0.6"; - let config = load_quickwit_config_with_env( + let config = load_node_config_with_env( ConfigFormat::Yaml, config_yaml.as_bytes(), &Default::default(), @@ -598,7 +597,7 @@ mod tests { } #[tokio::test] - async fn test_quickwit_config_env_var_override() { + async fn test_node_config_env_var_override() { let config_yaml = "version: 0.6"; let mut env_vars = HashMap::new(); env_vars.insert("QW_CLUSTER_ID".to_string(), "test-cluster".to_string()); @@ -626,7 +625,7 @@ mod tests { "s3://quickwit-indexes/prod".to_string(), ); let config = - load_quickwit_config_with_env(ConfigFormat::Yaml, config_yaml.as_bytes(), &env_vars) + load_node_config_with_env(ConfigFormat::Yaml, config_yaml.as_bytes(), &env_vars) .await .unwrap(); assert_eq!(config.cluster_id, "test-cluster"); @@ -685,7 +684,7 @@ mod tests { node_id: "node-1" metastore_uri: postgres://username:password@host:port/db "#; - let config = load_quickwit_config_with_env( + let config = load_node_config_with_env( ConfigFormat::Yaml, config_yaml.as_bytes(), &Default::default(), @@ -701,13 +700,13 @@ mod tests { } #[tokio::test] - async fn test_quickwit_config_config_default_values_default_indexer_searcher_config() { + async fn test_node_config_config_default_values_default_indexer_searcher_config() { let config_yaml = r#" version: 0.6 metastore_uri: postgres://username:password@host:port/db data_dir: /opt/quickwit/data "#; - let config = load_quickwit_config_with_env( + let config = load_node_config_with_env( ConfigFormat::Yaml, config_yaml.as_bytes(), &Default::default(), @@ -725,7 +724,7 @@ mod tests { } #[tokio::test] - async fn test_quickwit_config_validate() { + async fn test_node_config_validate() { let config_filepath = get_config_filepath("quickwit.toml"); let file_content = std::fs::read_to_string(&config_filepath).unwrap(); @@ -735,29 +734,27 @@ mod tests { "QW_DATA_DIR".to_string(), data_dir_path.to_string_lossy().to_string(), ); - assert!(load_quickwit_config_with_env( - ConfigFormat::Toml, - file_content.as_bytes(), - &env_vars, - ) - .await - .is_ok()); + assert!( + load_node_config_with_env(ConfigFormat::Toml, file_content.as_bytes(), &env_vars,) + .await + .is_ok() + ); } #[tokio::test] async fn test_peer_socket_addrs() { { - let quickwit_config = QuickwitConfigBuilder { + let node_config = NodeConfigBuilder { rest_listen_port: ConfigValue::for_test(1789), ..Default::default() } .build_and_validate(&HashMap::new()) .await .unwrap(); - assert!(quickwit_config.peer_seed_addrs().await.unwrap().is_empty()); + assert!(node_config.peer_seed_addrs().await.unwrap().is_empty()); } { - let quickwit_config = QuickwitConfigBuilder { + let node_config = NodeConfigBuilder { rest_listen_port: ConfigValue::for_test(1789), peer_seeds: ConfigValue::for_test(List(vec!["unresolvable-host".to_string()])), ..Default::default() @@ -765,10 +762,10 @@ mod tests { .build_and_validate(&HashMap::new()) .await .unwrap(); - assert!(quickwit_config.peer_seed_addrs().await.is_err()); + assert!(node_config.peer_seed_addrs().await.is_err()); } { - let quickwit_config = QuickwitConfigBuilder { + let node_config = NodeConfigBuilder { rest_listen_port: ConfigValue::for_test(1789), peer_seeds: ConfigValue::for_test(List(vec![ "unresolvable-host".to_string(), @@ -783,7 +780,7 @@ mod tests { .await .unwrap(); assert_eq!( - quickwit_config.peer_seed_addrs().await.unwrap(), + node_config.peer_seed_addrs().await.unwrap(), vec![ "localhost:1789".to_string(), "localhost:1337".to_string(), @@ -797,28 +794,19 @@ mod tests { #[tokio::test] async fn test_socket_addr_ports() { { - let quickwit_config = QuickwitConfigBuilder { + let node_config = NodeConfigBuilder { listen_address: default_listen_address(), ..Default::default() } .build_and_validate(&HashMap::new()) .await .unwrap(); - assert_eq!( - quickwit_config.rest_listen_addr.to_string(), - "127.0.0.1:7280" - ); - assert_eq!( - quickwit_config.gossip_listen_addr.to_string(), - "127.0.0.1:7280" - ); - assert_eq!( - quickwit_config.grpc_listen_addr.to_string(), - "127.0.0.1:7281" - ); + assert_eq!(node_config.rest_listen_addr.to_string(), "127.0.0.1:7280"); + assert_eq!(node_config.gossip_listen_addr.to_string(), "127.0.0.1:7280"); + assert_eq!(node_config.grpc_listen_addr.to_string(), "127.0.0.1:7281"); } { - let quickwit_config = QuickwitConfigBuilder { + let node_config = NodeConfigBuilder { listen_address: default_listen_address(), rest_listen_port: ConfigValue::for_test(1789), ..Default::default() @@ -826,21 +814,12 @@ mod tests { .build_and_validate(&HashMap::new()) .await .unwrap(); - assert_eq!( - quickwit_config.rest_listen_addr.to_string(), - "127.0.0.1:1789" - ); - assert_eq!( - quickwit_config.gossip_listen_addr.to_string(), - "127.0.0.1:1789" - ); - assert_eq!( - quickwit_config.grpc_listen_addr.to_string(), - "127.0.0.1:1790" - ); + assert_eq!(node_config.rest_listen_addr.to_string(), "127.0.0.1:1789"); + assert_eq!(node_config.gossip_listen_addr.to_string(), "127.0.0.1:1789"); + assert_eq!(node_config.grpc_listen_addr.to_string(), "127.0.0.1:1790"); } { - let quickwit_config = QuickwitConfigBuilder { + let node_config = NodeConfigBuilder { listen_address: default_listen_address(), rest_listen_port: ConfigValue::for_test(1789), gossip_listen_port: ConfigValue::for_test(1889), @@ -850,18 +829,9 @@ mod tests { .build_and_validate(&HashMap::new()) .await .unwrap(); - assert_eq!( - quickwit_config.rest_listen_addr.to_string(), - "127.0.0.1:1789" - ); - assert_eq!( - quickwit_config.gossip_listen_addr.to_string(), - "127.0.0.1:1889" - ); - assert_eq!( - quickwit_config.grpc_listen_addr.to_string(), - "127.0.0.1:1989" - ); + assert_eq!(node_config.rest_listen_addr.to_string(), "127.0.0.1:1789"); + assert_eq!(node_config.gossip_listen_addr.to_string(), "127.0.0.1:1889"); + assert_eq!(node_config.grpc_listen_addr.to_string(), "127.0.0.1:1989"); } } @@ -869,7 +839,7 @@ mod tests { async fn test_load_config_with_validation_error() { let config_filepath = get_config_filepath("quickwit.yaml"); let file = std::fs::read_to_string(&config_filepath).unwrap(); - let error = QuickwitConfig::load(ConfigFormat::Yaml, file.as_bytes()) + let error = NodeConfig::load(ConfigFormat::Yaml, file.as_bytes()) .await .unwrap_err(); assert!(error.to_string().contains("Data dir")); @@ -883,7 +853,7 @@ mod tests { node_id: 1 metastore_uri: '' "#; - assert!(load_quickwit_config_with_env( + assert!(load_node_config_with_env( ConfigFormat::Yaml, config_yaml.as_bytes(), &Default::default() @@ -898,7 +868,7 @@ mod tests { metastore_uri: postgres://username:password@host:port/db default_index_root_uri: '' "#; - assert!(load_quickwit_config_with_env( + assert!(load_node_config_with_env( ConfigFormat::Yaml, config_yaml.as_bytes(), &Default::default() @@ -909,13 +879,13 @@ mod tests { } #[tokio::test] - async fn test_quickwit_config_data_dir_accepts_both_file_uris_and_file_paths() { + async fn test_node_config_data_dir_accepts_both_file_uris_and_file_paths() { { let config_yaml = r#" version: 0.6 data_dir: /opt/quickwit/data "#; - let config = load_quickwit_config_with_env( + let config = load_node_config_with_env( ConfigFormat::Yaml, config_yaml.as_bytes(), &HashMap::default(), @@ -929,7 +899,7 @@ mod tests { version: 0.6 data_dir: file:///opt/quickwit/data "#; - let config = load_quickwit_config_with_env( + let config = load_node_config_with_env( ConfigFormat::Yaml, config_yaml.as_bytes(), &HashMap::default(), @@ -943,7 +913,7 @@ mod tests { version: 0.6 data_dir: s3://indexes/foo "#; - let error = load_quickwit_config_with_env( + let error = load_node_config_with_env( ConfigFormat::Yaml, config_yaml.as_bytes(), &HashMap::default(), @@ -972,7 +942,7 @@ mod tests { version: 0.6 rest_cors_allow_origins: '*' "#; - let config = load_quickwit_config_with_env( + let config = load_node_config_with_env( ConfigFormat::Yaml, rest_config_yaml.as_bytes(), &Default::default(), @@ -988,7 +958,7 @@ mod tests { version: 0.6 rest_cors_allow_origins: https://www.my-domain.com "#; - let config = load_quickwit_config_with_env( + let config = load_node_config_with_env( ConfigFormat::Yaml, rest_config_yaml.as_bytes(), &Default::default(), @@ -1004,7 +974,7 @@ mod tests { version: 0.6 rest_cors_allow_origins: http://192.168.0.108:7280 "#; - let config = load_quickwit_config_with_env( + let config = load_node_config_with_env( ConfigFormat::Yaml, rest_config_yaml.as_bytes(), &Default::default(), @@ -1024,7 +994,7 @@ mod tests { rest_cors_allow_origins: - https://www.my-domain.com "#; - let config = load_quickwit_config_with_env( + let config = load_node_config_with_env( ConfigFormat::Yaml, rest_config_yaml.as_bytes(), &Default::default(), @@ -1042,7 +1012,7 @@ mod tests { - https://www.my-domain.com - https://www.my-other-domain.com "#; - let config = load_quickwit_config_with_env( + let config = load_node_config_with_env( ConfigFormat::Yaml, rest_config_yaml.as_bytes(), &Default::default(), @@ -1061,7 +1031,7 @@ mod tests { version: 0.6 rest_cors_allow_origins: "#; - load_quickwit_config_with_env( + load_node_config_with_env( ConfigFormat::Yaml, rest_config_yaml.as_bytes(), &Default::default(), @@ -1074,7 +1044,7 @@ mod tests { rest_cors_allow_origins: - "#; - load_quickwit_config_with_env( + load_node_config_with_env( ConfigFormat::Yaml, rest_config_yaml.as_bytes(), &Default::default(), diff --git a/quickwit/quickwit-indexing/src/lib.rs b/quickwit/quickwit-indexing/src/lib.rs index 3f787a3dc2c..5b15dfb0f82 100644 --- a/quickwit/quickwit-indexing/src/lib.rs +++ b/quickwit/quickwit-indexing/src/lib.rs @@ -23,7 +23,7 @@ use std::sync::Arc; use quickwit_actors::{Mailbox, Universe}; use quickwit_cluster::Cluster; -use quickwit_config::QuickwitConfig; +use quickwit_config::NodeConfig; use quickwit_ingest::IngestApiService; use quickwit_metastore::Metastore; use quickwit_storage::StorageResolver; @@ -66,7 +66,7 @@ pub fn new_split_id() -> String { pub async fn start_indexing_service( universe: &Universe, - config: &QuickwitConfig, + config: &NodeConfig, num_blocking_threads: usize, cluster: Cluster, metastore: Arc, diff --git a/quickwit/quickwit-integration-tests/src/test_utils/cluster_sandbox.rs b/quickwit/quickwit-integration-tests/src/test_utils/cluster_sandbox.rs index b71dd50009d..496945404b8 100644 --- a/quickwit/quickwit-integration-tests/src/test_utils/cluster_sandbox.rs +++ b/quickwit/quickwit-integration-tests/src/test_utils/cluster_sandbox.rs @@ -32,7 +32,7 @@ use quickwit_common::test_utils::{wait_for_server_ready, wait_until_predicate}; use quickwit_common::tower::BoxFutureInfaillible; use quickwit_common::uri::Uri as QuickwitUri; use quickwit_config::service::QuickwitService; -use quickwit_config::QuickwitConfig; +use quickwit_config::NodeConfig; use quickwit_metastore::{MetastoreResolver, SplitState}; use quickwit_rest_client::models::IngestSource; use quickwit_rest_client::rest_client::{ @@ -46,11 +46,11 @@ use tokio::sync::watch::{self, Receiver, Sender}; use tokio::task::JoinHandle; use tracing::debug; -/// Configuration of a node made of a [`QuickwitConfig`] and a +/// Configuration of a node made of a [`NodeConfig`] and a /// set of services. #[derive(Clone)] -pub struct NodeConfig { - pub quickwit_config: QuickwitConfig, +pub struct TestNodeConfig { + pub node_config: NodeConfig, pub services: HashSet, } @@ -86,7 +86,7 @@ impl ClusterShutdownTrigger { /// will share the same `INGEST_API_SERVICE_INSTANCE`. The ingest API will be /// dropped by the first running test and the other tests will fail. pub struct ClusterSandbox { - pub node_configs: Vec, + pub node_configs: Vec, pub searcher_rest_client: QuickwitClient, pub indexer_rest_client: QuickwitClient, _temp_dir: TempDir, @@ -154,7 +154,7 @@ impl ClusterSandbox { let shutdown_signal = shutdown_trigger.shutdown_signal(); let join_handles = vec![tokio::spawn(async move { let result = serve_quickwit( - node_config_clone.quickwit_config, + node_config_clone.node_config, runtimes_config, storage_resolver, metastore_resolver, @@ -163,15 +163,15 @@ impl ClusterSandbox { .await?; Result::<_, anyhow::Error>::Ok(result) })]; - wait_for_server_ready(node_config.quickwit_config.grpc_listen_addr).await?; + wait_for_server_ready(node_config.node_config.grpc_listen_addr).await?; Ok(Self { node_configs, indexer_rest_client: QuickwitClientBuilder::new(transport_url( - node_config.quickwit_config.rest_listen_addr, + node_config.node_config.rest_listen_addr, )) .build(), searcher_rest_client: QuickwitClientBuilder::new(transport_url( - node_config.quickwit_config.rest_listen_addr, + node_config.node_config.rest_listen_addr, )) .build(), _temp_dir: temp_dir, @@ -193,7 +193,7 @@ impl ClusterSandbox { let shutdown_trigger = ClusterShutdownTrigger::new(); for node_config in node_configs.iter() { join_handles.push(tokio::spawn({ - let node_config = node_config.quickwit_config.clone(); + let node_config = node_config.node_config.clone(); let storage_resolver = storage_resolver.clone(); let metastore_resolver = metastore_resolver.clone(); let shutdown_signal = shutdown_trigger.shutdown_signal(); @@ -226,11 +226,11 @@ impl ClusterSandbox { Ok(Self { node_configs, searcher_rest_client: QuickwitClientBuilder::new(transport_url( - searcher_config.quickwit_config.rest_listen_addr, + searcher_config.node_config.rest_listen_addr, )) .build(), indexer_rest_client: QuickwitClientBuilder::new(transport_url( - indexer_config.quickwit_config.rest_listen_addr, + indexer_config.node_config.rest_listen_addr, )) .build(), _temp_dir: temp_dir, @@ -366,7 +366,7 @@ impl ClusterSandbox { /// Builds a list of [`NodeConfig`] given a list of Quickwit services. /// Each element of `nodes_services` defines the services of a given node. -/// For each node, a `QuickwitConfig` is built with the right parameters +/// For each node, a `NodeConfig` is built with the right parameters /// such that we will be able to run `quickwit_serve` on them and form /// a quickwit cluster. /// For each node, we set: @@ -377,13 +377,13 @@ impl ClusterSandbox { pub fn build_node_configs( root_data_dir: PathBuf, nodes_services: &[HashSet], -) -> Vec { +) -> Vec { let cluster_id = new_coolid("test-cluster"); let mut node_configs = Vec::new(); let mut peers: Vec = Vec::new(); let unique_dir_name = new_coolid("test-dir"); for (node_idx, node_services) in nodes_services.iter().enumerate() { - let mut config = QuickwitConfig::for_test(); + let mut config = NodeConfig::for_test(); config.enabled_services = node_services.clone(); config.cluster_id = cluster_id.clone(); config.node_id = format!("test-node-{node_idx}"); @@ -393,22 +393,16 @@ pub fn build_node_configs( config.default_index_root_uri = QuickwitUri::from_str(&format!("ram:///{unique_dir_name}/indexes")).unwrap(); peers.push(config.gossip_advertise_addr.to_string()); - node_configs.push(NodeConfig { - quickwit_config: config, + node_configs.push(TestNodeConfig { + node_config: config, services: node_services.clone(), }); } for node_config in node_configs.iter_mut() { - node_config.quickwit_config.peer_seeds = peers + node_config.node_config.peer_seeds = peers .clone() .into_iter() - .filter(|seed| { - *seed - != node_config - .quickwit_config - .gossip_advertise_addr - .to_string() - }) + .filter(|seed| *seed != node_config.node_config.gossip_advertise_addr.to_string()) .collect_vec(); } node_configs diff --git a/quickwit/quickwit-integration-tests/src/tests/basic_tests.rs b/quickwit/quickwit-integration-tests/src/tests/basic_tests.rs index 562827198af..758f22f6b35 100644 --- a/quickwit/quickwit-integration-tests/src/tests/basic_tests.rs +++ b/quickwit/quickwit-integration-tests/src/tests/basic_tests.rs @@ -54,7 +54,7 @@ async fn test_ui_redirect_on_get() { .pool_idle_timeout(Duration::from_secs(30)) .http2_only(true) .build_http(); - let root_uri = format!("http://{}/", node_config.quickwit_config.rest_listen_addr) + let root_uri = format!("http://{}/", node_config.node_config.rest_listen_addr) .parse::() .unwrap(); let response = client.get(root_uri.clone()).await.unwrap(); diff --git a/quickwit/quickwit-integration-tests/src/tests/index_tests.rs b/quickwit/quickwit-integration-tests/src/tests/index_tests.rs index 8161009b33a..bbad4a3cec1 100644 --- a/quickwit/quickwit-integration-tests/src/tests/index_tests.rs +++ b/quickwit/quickwit-integration-tests/src/tests/index_tests.rs @@ -179,7 +179,7 @@ async fn test_restarting_standalone_server() { .node_configs .first() .unwrap() - .quickwit_config + .node_config .data_dir_path .clone(); let delete_service_path = path.join(DELETE_SERVICE_TASK_DIR_NAME); diff --git a/quickwit/quickwit-janitor/src/lib.rs b/quickwit/quickwit-janitor/src/lib.rs index 8dd576f9ec6..0cc157f2c7e 100644 --- a/quickwit/quickwit-janitor/src/lib.rs +++ b/quickwit/quickwit-janitor/src/lib.rs @@ -23,7 +23,7 @@ use std::sync::Arc; use quickwit_actors::{Mailbox, Universe}; use quickwit_common::FileEntry; -use quickwit_config::QuickwitConfig; +use quickwit_config::NodeConfig; use quickwit_metastore::Metastore; use quickwit_search::SearchJobPlacer; use quickwit_storage::StorageResolver; @@ -50,7 +50,7 @@ pub struct JanitorApiSchemas; pub async fn start_janitor_service( universe: &Universe, - config: &QuickwitConfig, + config: &NodeConfig, metastore: Arc, search_job_placer: SearchJobPlacer, storage_resolver: StorageResolver, diff --git a/quickwit/quickwit-serve/src/elastic_search_api/bulk.rs b/quickwit/quickwit-serve/src/elastic_search_api/bulk.rs index b299c600014..01e79200cf6 100644 --- a/quickwit/quickwit-serve/src/elastic_search_api/bulk.rs +++ b/quickwit/quickwit-serve/src/elastic_search_api/bulk.rs @@ -131,7 +131,7 @@ mod tests { use std::sync::Arc; use std::time::Duration; - use quickwit_config::{IngestApiConfig, QuickwitConfig}; + use quickwit_config::{IngestApiConfig, NodeConfig}; use quickwit_ingest::{ FetchRequest, IngestResponse, IngestServiceClient, SuggestTruncateRequest, }; @@ -142,7 +142,7 @@ mod tests { #[tokio::test] async fn test_bulk_api_returns_404_if_index_id_does_not_exist() { - let config = Arc::new(QuickwitConfig::for_test()); + let config = Arc::new(NodeConfig::for_test()); let search_service = Arc::new(MockSearchService::new()); let (universe, _temp_dir, ingest_service, _) = setup_ingest_service(&["my-index"], &IngestApiConfig::default()).await; @@ -164,7 +164,7 @@ mod tests { #[tokio::test] async fn test_bulk_api_returns_200() { - let config = Arc::new(QuickwitConfig::for_test()); + let config = Arc::new(NodeConfig::for_test()); let search_service = Arc::new(MockSearchService::new()); let (universe, _temp_dir, ingest_service, _) = setup_ingest_service(&["my-index-1", "my-index-2"], &IngestApiConfig::default()).await; @@ -190,7 +190,7 @@ mod tests { #[tokio::test] async fn test_bulk_index_api_returns_200() { - let config = Arc::new(QuickwitConfig::for_test()); + let config = Arc::new(NodeConfig::for_test()); let search_service = Arc::new(MockSearchService::new()); let (universe, _temp_dir, ingest_service, _) = setup_ingest_service(&["my-index-1", "my-index-2"], &IngestApiConfig::default()).await; @@ -216,7 +216,7 @@ mod tests { #[tokio::test] async fn test_bulk_api_blocks_when_refresh_wait_for_is_specified() { - let config = Arc::new(QuickwitConfig::for_test()); + let config = Arc::new(NodeConfig::for_test()); let search_service = Arc::new(MockSearchService::new()); let (universe, _temp_dir, ingest_service, ingest_service_mailbox) = setup_ingest_service(&["my-index-1", "my-index-2"], &IngestApiConfig::default()).await; @@ -293,7 +293,7 @@ mod tests { #[tokio::test] async fn test_bulk_api_blocks_when_refresh_true_is_specified() { - let config = Arc::new(QuickwitConfig::for_test()); + let config = Arc::new(NodeConfig::for_test()); let search_service = Arc::new(MockSearchService::new()); let (universe, _temp_dir, ingest_service, ingest_service_mailbox) = setup_ingest_service(&["my-index-1", "my-index-2"], &IngestApiConfig::default()).await; @@ -369,7 +369,7 @@ mod tests { #[tokio::test] async fn test_bulk_ingest_request_returns_400_if_action_is_malformed() { - let config = Arc::new(QuickwitConfig::for_test()); + let config = Arc::new(NodeConfig::for_test()); let search_service = Arc::new(MockSearchService::new()); let ingest_service = IngestServiceClient::new(IngestServiceClient::mock()); let elastic_api_handlers = elastic_api_handlers(config, search_service, ingest_service); diff --git a/quickwit/quickwit-serve/src/elastic_search_api/mod.rs b/quickwit/quickwit-serve/src/elastic_search_api/mod.rs index bb43350ada9..35f7faf5e8c 100644 --- a/quickwit/quickwit-serve/src/elastic_search_api/mod.rs +++ b/quickwit/quickwit-serve/src/elastic_search_api/mod.rs @@ -26,7 +26,7 @@ use std::sync::Arc; use bulk::{es_compat_bulk_handler, es_compat_index_bulk_handler}; pub use filter::ElasticCompatibleApi; -use quickwit_config::QuickwitConfig; +use quickwit_config::NodeConfig; use quickwit_ingest::IngestServiceClient; use quickwit_search::SearchService; use rest_handler::{ @@ -43,11 +43,11 @@ use crate::BuildInfo; /// This is where all newly supported Elasticsearch handlers /// should be registered. pub fn elastic_api_handlers( - quickwit_config: Arc, + node_config: Arc, search_service: Arc, ingest_service: IngestServiceClient, ) -> impl Filter + Clone { - es_compat_cluster_info_handler(quickwit_config, BuildInfo::get()) + es_compat_cluster_info_handler(node_config, BuildInfo::get()) .or(es_compat_search_handler(search_service.clone())) .or(es_compat_index_search_handler(search_service.clone())) .or(es_compat_index_multi_search_handler(search_service)) @@ -91,7 +91,7 @@ mod tests { use assert_json_diff::assert_json_include; use mockall::predicate; - use quickwit_config::QuickwitConfig; + use quickwit_config::NodeConfig; use quickwit_ingest::{IngestApiService, IngestServiceClient}; use quickwit_search::MockSearchService; use serde_json::Value as JsonValue; @@ -111,7 +111,7 @@ mod tests { #[tokio::test] async fn test_msearch_api_return_200_responses() { - let config = Arc::new(QuickwitConfig::for_test()); + let config = Arc::new(NodeConfig::for_test()); let mut mock_search_service = MockSearchService::new(); mock_search_service .expect_root_search() @@ -156,7 +156,7 @@ mod tests { #[tokio::test] async fn test_msearch_api_return_one_500_and_one_200_responses() { - let config = Arc::new(QuickwitConfig::for_test()); + let config = Arc::new(NodeConfig::for_test()); let mut mock_search_service = MockSearchService::new(); mock_search_service .expect_root_search() @@ -202,7 +202,7 @@ mod tests { #[tokio::test] async fn test_msearch_api_return_400_with_malformed_request_header() { - let config = Arc::new(QuickwitConfig::for_test()); + let config = Arc::new(NodeConfig::for_test()); let mock_search_service = MockSearchService::new(); let es_search_api_handler = super::elastic_api_handlers( config, @@ -230,7 +230,7 @@ mod tests { #[tokio::test] async fn test_msearch_api_return_400_with_malformed_request_body() { - let config = Arc::new(QuickwitConfig::for_test()); + let config = Arc::new(NodeConfig::for_test()); let mock_search_service = MockSearchService::new(); let es_search_api_handler = super::elastic_api_handlers( config, @@ -258,7 +258,7 @@ mod tests { #[tokio::test] async fn test_msearch_api_return_400_with_only_a_header_request() { - let config = Arc::new(QuickwitConfig::for_test()); + let config = Arc::new(NodeConfig::for_test()); let mock_search_service = MockSearchService::new(); let es_search_api_handler = super::elastic_api_handlers( config, @@ -285,7 +285,7 @@ mod tests { #[tokio::test] async fn test_msearch_api_return_400_with_no_index() { - let config = Arc::new(QuickwitConfig::for_test()); + let config = Arc::new(NodeConfig::for_test()); let mock_search_service = MockSearchService::new(); let es_search_api_handler = super::elastic_api_handlers( config, @@ -311,7 +311,7 @@ mod tests { #[tokio::test] async fn test_msearch_api_return_400_with_multiple_indexes() { - let config = Arc::new(QuickwitConfig::for_test()); + let config = Arc::new(NodeConfig::for_test()); let mock_search_service = MockSearchService::new(); let es_search_api_handler = super::elastic_api_handlers( config, @@ -340,7 +340,7 @@ mod tests { #[tokio::test] async fn test_es_compat_cluster_info_handler() { let build_info = BuildInfo::get(); - let config = Arc::new(QuickwitConfig::for_test()); + let config = Arc::new(NodeConfig::for_test()); let handler = es_compat_cluster_info_handler(config.clone(), build_info).recover(recover_fn); let resp = warp::test::request() diff --git a/quickwit/quickwit-serve/src/elastic_search_api/rest_handler.rs b/quickwit/quickwit-serve/src/elastic_search_api/rest_handler.rs index fd73395e33c..a21a9062c7f 100644 --- a/quickwit/quickwit-serve/src/elastic_search_api/rest_handler.rs +++ b/quickwit/quickwit-serve/src/elastic_search_api/rest_handler.rs @@ -29,7 +29,7 @@ use futures_util::StreamExt; use hyper::StatusCode; use itertools::Itertools; use quickwit_common::truncate_str; -use quickwit_config::QuickwitConfig; +use quickwit_config::NodeConfig; use quickwit_proto::{SearchResponse, ServiceErrorCode}; use quickwit_query::query_ast::{QueryAst, UserInputQuery}; use quickwit_query::BooleanOperand; @@ -51,14 +51,14 @@ use crate::{with_arg, BuildInfo}; /// Elastic compatible cluster info handler. pub fn es_compat_cluster_info_handler( - quickwit_config: Arc, + node_config: Arc, build_info: &'static BuildInfo, ) -> impl Filter + Clone { elastic_cluster_info_filter() - .and(with_arg(quickwit_config)) + .and(with_arg(node_config)) .and(with_arg(build_info)) .then( - |config: Arc, build_info: &'static BuildInfo| async move { + |config: Arc, build_info: &'static BuildInfo| async move { warp::reply::json(&json!({ "name" : config.node_id, "cluster_name" : config.cluster_id, diff --git a/quickwit/quickwit-serve/src/index_api/rest_handler.rs b/quickwit/quickwit-serve/src/index_api/rest_handler.rs index 093d2c39a20..22ae6e5dffc 100644 --- a/quickwit/quickwit-serve/src/index_api/rest_handler.rs +++ b/quickwit/quickwit-serve/src/index_api/rest_handler.rs @@ -24,7 +24,7 @@ use hyper::header::CONTENT_TYPE; use quickwit_common::uri::Uri; use quickwit_common::FileEntry; use quickwit_config::{ - load_source_config_from_user_config, ConfigFormat, QuickwitConfig, SourceConfig, SourceParams, + load_source_config_from_user_config, ConfigFormat, NodeConfig, SourceConfig, SourceParams, CLI_INGEST_SOURCE_ID, INGEST_API_SOURCE_ID, }; use quickwit_core::{IndexService, IndexServiceError}; @@ -65,12 +65,12 @@ pub struct IndexApi; pub fn index_management_handlers( index_service: Arc, - quickwit_config: Arc, + node_config: Arc, ) -> impl Filter + Clone { // Indexes handlers. get_index_metadata_handler(index_service.metastore()) .or(get_indexes_metadatas_handler(index_service.metastore())) - .or(create_index_handler(index_service.clone(), quickwit_config)) + .or(create_index_handler(index_service.clone(), node_config)) .or(clear_index_handler(index_service.clone())) .or(delete_index_handler(index_service.clone())) // Splits handlers @@ -386,7 +386,7 @@ struct CreateIndexQueryParams { fn create_index_handler( index_service: Arc, - quickwit_config: Arc, + node_config: Arc, ) -> impl Filter + Clone { warp::path!("indexes") .and(warp::post()) @@ -395,7 +395,7 @@ fn create_index_handler( .and(warp::body::content_length_limit(1024 * 1024)) .and(warp::filters::body::bytes()) .and(with_arg(index_service)) - .and(with_arg(quickwit_config)) + .and(with_arg(node_config)) .then(create_index) .and(extract_format_from_qs()) .map(make_json_api_response) @@ -420,12 +420,12 @@ async fn create_index( config_format: ConfigFormat, index_config_bytes: Bytes, index_service: Arc, - quickwit_config: Arc, + node_config: Arc, ) -> Result { let index_config = quickwit_config::load_index_config_from_user_config( config_format, &index_config_bytes, - &quickwit_config.default_index_root_uri, + &node_config.default_index_root_uri, ) .map_err(IndexServiceError::InvalidConfig)?; info!(index_id = %index_config.index_id, overwrite = create_index_query_params.overwrite, "create-index"); @@ -792,7 +792,7 @@ mod tests { let index_service = IndexService::new(Arc::new(metastore), StorageResolver::unconfigured()); let index_management_handler = super::index_management_handlers( Arc::new(index_service), - Arc::new(QuickwitConfig::for_test()), + Arc::new(NodeConfig::for_test()), ) .recover(recover_fn); let resp = warp::test::request() @@ -818,7 +818,7 @@ mod tests { let index_service = IndexService::new(metastore, StorageResolver::unconfigured()); let index_management_handler = super::index_management_handlers( Arc::new(index_service), - Arc::new(QuickwitConfig::for_test()), + Arc::new(NodeConfig::for_test()), ) .recover(recover_fn); let resp = warp::test::request() @@ -861,7 +861,7 @@ mod tests { let index_service = IndexService::new(Arc::new(metastore), StorageResolver::unconfigured()); let index_management_handler = super::index_management_handlers( Arc::new(index_service), - Arc::new(QuickwitConfig::for_test()), + Arc::new(NodeConfig::for_test()), ) .recover(recover_fn); { @@ -928,7 +928,7 @@ mod tests { let index_service = IndexService::new(Arc::new(metastore), StorageResolver::unconfigured()); let index_management_handler = super::index_management_handlers( Arc::new(index_service), - Arc::new(QuickwitConfig::for_test()), + Arc::new(NodeConfig::for_test()), ) .recover(recover_fn); let resp = warp::test::request() @@ -983,7 +983,7 @@ mod tests { let index_service = IndexService::new(Arc::new(metastore), StorageResolver::unconfigured()); let index_management_handler = super::index_management_handlers( Arc::new(index_service), - Arc::new(QuickwitConfig::for_test()), + Arc::new(NodeConfig::for_test()), ) .recover(recover_fn); let resp = warp::test::request() @@ -1022,7 +1022,7 @@ mod tests { let index_service = IndexService::new(Arc::new(metastore), StorageResolver::unconfigured()); let index_management_handler = super::index_management_handlers( Arc::new(index_service), - Arc::new(QuickwitConfig::for_test()), + Arc::new(NodeConfig::for_test()), ) .recover(recover_fn); let resp = warp::test::request() @@ -1056,7 +1056,7 @@ mod tests { let index_service = IndexService::new(Arc::new(metastore), StorageResolver::unconfigured()); let index_management_handler = super::index_management_handlers( Arc::new(index_service), - Arc::new(QuickwitConfig::for_test()), + Arc::new(NodeConfig::for_test()), ) .recover(recover_fn); let resp = warp::test::request() @@ -1105,7 +1105,7 @@ mod tests { let index_service = IndexService::new(Arc::new(metastore), StorageResolver::unconfigured()); let index_management_handler = super::index_management_handlers( Arc::new(index_service), - Arc::new(QuickwitConfig::for_test()), + Arc::new(NodeConfig::for_test()), ) .recover(recover_fn); let resp = warp::test::request() @@ -1148,7 +1148,7 @@ mod tests { let index_service = IndexService::new(Arc::new(metastore), StorageResolver::unconfigured()); let index_management_handler = super::index_management_handlers( Arc::new(index_service), - Arc::new(QuickwitConfig::for_test()), + Arc::new(NodeConfig::for_test()), ) .recover(recover_fn); { @@ -1188,7 +1188,7 @@ mod tests { let index_service = IndexService::new(metastore, StorageResolver::unconfigured()); let index_management_handler = super::index_management_handlers( Arc::new(index_service), - Arc::new(QuickwitConfig::for_test()), + Arc::new(NodeConfig::for_test()), ) .recover(recover_fn); let resp = warp::test::request() @@ -1203,11 +1203,11 @@ mod tests { async fn test_create_index_with_overwrite() { let metastore = metastore_for_test(); let index_service = IndexService::new(metastore.clone(), StorageResolver::unconfigured()); - let mut quickwit_config = QuickwitConfig::for_test(); - quickwit_config.default_index_root_uri = + let mut node_config = NodeConfig::for_test(); + node_config.default_index_root_uri = Uri::from_well_formed("file:///default-index-root-uri"); let index_management_handler = - super::index_management_handlers(Arc::new(index_service), Arc::new(quickwit_config)); + super::index_management_handlers(Arc::new(index_service), Arc::new(node_config)); { let resp = warp::test::request() .path("/indexes?overwrite=true") @@ -1244,11 +1244,11 @@ mod tests { async fn test_create_delete_index_and_source() { let metastore = metastore_for_test(); let index_service = IndexService::new(metastore.clone(), StorageResolver::unconfigured()); - let mut quickwit_config = QuickwitConfig::for_test(); - quickwit_config.default_index_root_uri = + let mut node_config = NodeConfig::for_test(); + node_config.default_index_root_uri = Uri::from_well_formed("file:///default-index-root-uri"); let index_management_handler = - super::index_management_handlers(Arc::new(index_service), Arc::new(quickwit_config)); + super::index_management_handlers(Arc::new(index_service), Arc::new(node_config)); let resp = warp::test::request() .path("/indexes") .method("POST") @@ -1352,11 +1352,11 @@ mod tests { async fn test_create_file_source_returns_405() { let metastore = metastore_for_test(); let index_service = IndexService::new(metastore.clone(), StorageResolver::unconfigured()); - let mut quickwit_config = QuickwitConfig::for_test(); - quickwit_config.default_index_root_uri = + let mut node_config = NodeConfig::for_test(); + node_config.default_index_root_uri = Uri::from_well_formed("file:///default-index-root-uri"); let index_management_handler = - super::index_management_handlers(Arc::new(index_service), Arc::new(quickwit_config)) + super::index_management_handlers(Arc::new(index_service), Arc::new(node_config)) .recover(recover_fn); let source_config_body = r#"{"version": "0.6", "source_id": "file-source", "source_type": "file", "params": {"filepath": "FILEPATH"}}"#; let resp = warp::test::request() @@ -1374,11 +1374,11 @@ mod tests { async fn test_create_index_with_yaml() { let metastore = metastore_for_test(); let index_service = IndexService::new(metastore.clone(), StorageResolver::unconfigured()); - let mut quickwit_config = QuickwitConfig::for_test(); - quickwit_config.default_index_root_uri = + let mut node_config = NodeConfig::for_test(); + node_config.default_index_root_uri = Uri::from_well_formed("file:///default-index-root-uri"); let index_management_handler = - super::index_management_handlers(Arc::new(index_service), Arc::new(quickwit_config)) + super::index_management_handlers(Arc::new(index_service), Arc::new(node_config)) .recover(recover_fn); let resp = warp::test::request() .path("/indexes") @@ -1413,11 +1413,11 @@ mod tests { async fn test_create_index_and_source_with_toml() { let metastore = metastore_for_test(); let index_service = IndexService::new(metastore.clone(), StorageResolver::unconfigured()); - let mut quickwit_config = QuickwitConfig::for_test(); - quickwit_config.default_index_root_uri = + let mut node_config = NodeConfig::for_test(); + node_config.default_index_root_uri = Uri::from_well_formed("file:///default-index-root-uri"); let index_management_handler = - super::index_management_handlers(Arc::new(index_service), Arc::new(quickwit_config)) + super::index_management_handlers(Arc::new(index_service), Arc::new(node_config)) .recover(recover_fn); let resp = warp::test::request() .path("/indexes") @@ -1450,11 +1450,11 @@ mod tests { async fn test_create_index_with_wrong_content_type() { let metastore = metastore_for_test(); let index_service = IndexService::new(metastore.clone(), StorageResolver::unconfigured()); - let mut quickwit_config = QuickwitConfig::for_test(); - quickwit_config.default_index_root_uri = + let mut node_config = NodeConfig::for_test(); + node_config.default_index_root_uri = Uri::from_well_formed("file:///default-index-root-uri"); let index_management_handler = - super::index_management_handlers(Arc::new(index_service), Arc::new(quickwit_config)) + super::index_management_handlers(Arc::new(index_service), Arc::new(node_config)) .recover(recover_fn); let resp = warp::test::request() .path("/indexes") @@ -1474,7 +1474,7 @@ mod tests { let index_service = IndexService::new(Arc::new(metastore), StorageResolver::unconfigured()); let index_management_handler = super::index_management_handlers( Arc::new(index_service), - Arc::new(QuickwitConfig::for_test()), + Arc::new(NodeConfig::for_test()), ) .recover(recover_fn); let resp = warp::test::request() @@ -1500,7 +1500,7 @@ mod tests { let index_service = IndexService::new(metastore, StorageResolver::unconfigured()); let index_management_handler = super::index_management_handlers( Arc::new(index_service), - Arc::new(QuickwitConfig::for_test()), + Arc::new(NodeConfig::for_test()), ) .recover(recover_fn); { @@ -1557,7 +1557,7 @@ mod tests { let index_service = IndexService::new(Arc::new(metastore), StorageResolver::unconfigured()); let index_management_handler = super::index_management_handlers( Arc::new(index_service), - Arc::new(QuickwitConfig::for_test()), + Arc::new(NodeConfig::for_test()), ) .recover(recover_fn); let resp = warp::test::request() @@ -1595,7 +1595,7 @@ mod tests { let index_service = IndexService::new(Arc::new(metastore), StorageResolver::unconfigured()); let index_management_handler = super::index_management_handlers( Arc::new(index_service), - Arc::new(QuickwitConfig::for_test()), + Arc::new(NodeConfig::for_test()), ) .recover(recover_fn); let resp = warp::test::request() @@ -1642,7 +1642,7 @@ mod tests { let index_service = IndexService::new(Arc::new(metastore), StorageResolver::unconfigured()); let index_management_handler = super::index_management_handlers( Arc::new(index_service), - Arc::new(QuickwitConfig::for_test()), + Arc::new(NodeConfig::for_test()), ) .recover(recover_fn); // Check server returns 405 if sources root path is used. @@ -1701,7 +1701,7 @@ mod tests { let index_service = IndexService::new(Arc::new(metastore), StorageResolver::unconfigured()); let index_management_handler = super::index_management_handlers( Arc::new(index_service), - Arc::new(QuickwitConfig::for_test()), + Arc::new(NodeConfig::for_test()), ) .recover(recover_fn); let resp = warp::test::request() diff --git a/quickwit/quickwit-serve/src/lib.rs b/quickwit/quickwit-serve/src/lib.rs index df06baf62f0..b502ea8bbe4 100644 --- a/quickwit/quickwit-serve/src/lib.rs +++ b/quickwit/quickwit-serve/src/lib.rs @@ -61,7 +61,7 @@ use quickwit_common::tower::{ Rate, RateLimitLayer, SmaRateEstimator, }; use quickwit_config::service::QuickwitService; -use quickwit_config::{QuickwitConfig, SearcherConfig}; +use quickwit_config::{NodeConfig, SearcherConfig}; use quickwit_control_plane::control_plane::ControlPlane; use quickwit_control_plane::scheduler::IndexingScheduler; use quickwit_control_plane::{ @@ -105,7 +105,7 @@ const READINESS_REPORTING_INTERVAL: Duration = if cfg!(any(test, feature = "test }; struct QuickwitServices { - pub config: Arc, + pub config: Arc, pub cluster: Cluster, pub metastore: Arc, pub control_plane_service: Option, @@ -152,7 +152,7 @@ async fn balance_channel_for_service( } pub async fn serve_quickwit( - config: QuickwitConfig, + config: NodeConfig, runtimes_config: RuntimesConfig, storage_resolver: StorageResolver, metastore_resolver: MetastoreResolver, diff --git a/quickwit/quickwit-serve/src/node_info_handler.rs b/quickwit/quickwit-serve/src/node_info_handler.rs index db58fea3dab..34775adaf0f 100644 --- a/quickwit/quickwit-serve/src/node_info_handler.rs +++ b/quickwit/quickwit-serve/src/node_info_handler.rs @@ -19,7 +19,7 @@ use std::sync::Arc; -use quickwit_config::QuickwitConfig; +use quickwit_config::NodeConfig; use serde_json::json; use warp::{Filter, Rejection}; @@ -32,7 +32,7 @@ pub struct NodeInfoApi; pub fn node_info_handler( build_info: &'static BuildInfo, runtime_info: &'static RuntimeInfo, - config: Arc, + config: Arc, ) -> impl Filter + Clone { node_version_handler(build_info, runtime_info).or(node_config_handler(config)) } @@ -61,7 +61,7 @@ async fn get_version( #[utoipa::path(get, tag = "Node Info", path = "/config")] fn node_config_handler( - config: Arc, + config: Arc, ) -> impl Filter + Clone { warp::path("config") .and(warp::path::end()) @@ -69,7 +69,7 @@ fn node_config_handler( .then(get_config) } -async fn get_config(config: Arc) -> impl warp::Reply { +async fn get_config(config: Arc) -> impl warp::Reply { // We must redact sensitive information such as credentials. let mut config = (*config).clone(); config.redact(); @@ -89,7 +89,7 @@ mod tests { async fn test_rest_node_info() { let build_info = BuildInfo::get(); let runtime_info = RuntimeInfo::get(); - let mut config = QuickwitConfig::for_test(); + let mut config = NodeConfig::for_test(); config.metastore_uri = Uri::for_test("postgresql://username:password@db"); let handler = node_info_handler(build_info, runtime_info, Arc::new(config.clone())) .recover(recover_fn);