From 80b37b275f7a2e51abbc273f60edde6862354cb8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Ignacio=20Gonz=C3=A1lez?= Date: Tue, 17 Sep 2024 08:54:28 -0300 Subject: [PATCH] feat(zk_toolbox): Add zk_supervisor config-writer command (#2866) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## What ❔ Add zk_supervisor override configs command ## Why ❔ In some cases we want to override some default configs. Right now it is done manually and we need proper automation for regular use/ci. --- .github/workflows/ci-core-reusable.yml | 4 +- .../{mainnet/general.yaml => mainnet.yaml} | 5 +- .../overrides/only_real_proofs.yaml | 3 + .../{testnet/general.yaml => testnet.yaml} | 3 +- .../overrides/tests/loadtest-new.yaml | 7 + .../overrides/tests/loadtest-old.yaml | 7 + etc/env/file_based/overrides/validium.yaml | 6 + zk_toolbox/Cargo.lock | 1 + zk_toolbox/crates/common/src/lib.rs | 1 + zk_toolbox/crates/common/src/yaml.rs | 475 ++++++++++++++++++ zk_toolbox/crates/config/Cargo.toml | 1 + zk_toolbox/crates/config/src/general.rs | 12 +- .../src/commands/chain/genesis.rs | 44 +- .../zk_inception/src/commands/update.rs | 332 +----------- zk_toolbox/crates/zk_inception/src/consts.rs | 4 + .../crates/zk_inception/src/messages.rs | 1 - .../src/commands/config_writer.rs | 35 ++ .../crates/zk_supervisor/src/commands/mod.rs | 1 + zk_toolbox/crates/zk_supervisor/src/main.rs | 16 +- .../crates/zk_supervisor/src/messages.rs | 10 + 20 files changed, 602 insertions(+), 366 deletions(-) rename etc/env/file_based/overrides/{mainnet/general.yaml => mainnet.yaml} (92%) create mode 100644 etc/env/file_based/overrides/only_real_proofs.yaml rename etc/env/file_based/overrides/{testnet/general.yaml => testnet.yaml} (95%) create mode 100644 etc/env/file_based/overrides/tests/loadtest-new.yaml create mode 100644 etc/env/file_based/overrides/tests/loadtest-old.yaml create mode 100644 etc/env/file_based/overrides/validium.yaml create mode 100644 zk_toolbox/crates/common/src/yaml.rs create mode 100644 zk_toolbox/crates/zk_supervisor/src/commands/config_writer.rs diff --git a/.github/workflows/ci-core-reusable.yml b/.github/workflows/ci-core-reusable.yml index 898085a3678..cac585a00a8 100644 --- a/.github/workflows/ci-core-reusable.yml +++ b/.github/workflows/ci-core-reusable.yml @@ -126,9 +126,7 @@ jobs: # `sleep 60` because we need to wait until server added all the tokens - name: Run server run: | - ci_run sed -i -e 's/mode: FULL/mode: LIGHTWEIGHT/' chains/legacy/configs/general.yaml - ci_run sed -i -e 's/state_keeper_fast_vm_mode:.*/state_keeper_fast_vm_mode: ${{ matrix.vm_mode }}/' chains/legacy/configs/general.yaml - ci_run sed -i -e 's/delay_interval:.*/delay_interval: 50/' chains/legacy/configs/general.yaml + ci_run zk_supervisor config-writer --path ${{ matrix.vm_mode == 'NEW' && 'etc/env/file_based/overrides/tests/loadtest-new.yaml' || 'etc/env/file_based/overrides/tests/loadtest-old.yaml' }} --chain legacy ci_run zk_inception server --uring --chain=legacy --components api,tree,eth,state_keeper,housekeeper,commitment_generator,vm_runner_protective_reads &>server.log & ci_run sleep 60 diff --git a/etc/env/file_based/overrides/mainnet/general.yaml b/etc/env/file_based/overrides/mainnet.yaml similarity index 92% rename from etc/env/file_based/overrides/mainnet/general.yaml rename to etc/env/file_based/overrides/mainnet.yaml index 7abe8eb5472..0600abf694c 100644 --- a/etc/env/file_based/overrides/mainnet/general.yaml +++ b/etc/env/file_based/overrides/mainnet.yaml @@ -10,12 +10,13 @@ eth: aggregated_block_prove_deadline: 300 aggregated_block_execute_deadline: 300 timestamp_criteria_max_allowed_lag: 104000 # 29h + wait_confirmations: null gas_adjuster: pricing_formula_parameter_a: 1.06 internal_l1_pricing_multiplier: 1 internal_pubdata_pricing_multiplier: 1.50 poll_period: 60 + watcher: + confirmations_for_eth_event: null observability: log_directives: zksync=info,zksync_state_keeper=debug,zksync_core=debug,zksync_server=debug,zksync_contract_verifier=debug,zksync_dal=debug,zksync_state=debug,zksync_utils=debug,zksync_eth_sender=debug,loadnext=debug,dev_ticker=info,vm=info,block_sizes_test=info,setup_key_generator_and_server=info,zksync_queued_job_processor=debug,slot_index_consistency_checker=debug,zksync_health_check=debug,zksync_consensus_bft=debug,zksync_consensus_network=debug,zksync_consensus_storage=debug,zksync_consensus_executor=debug, - -# remove eth_sender_wait_confirmations, eth_watcher_confirmations_for_eth_event variables diff --git a/etc/env/file_based/overrides/only_real_proofs.yaml b/etc/env/file_based/overrides/only_real_proofs.yaml new file mode 100644 index 00000000000..52747467511 --- /dev/null +++ b/etc/env/file_based/overrides/only_real_proofs.yaml @@ -0,0 +1,3 @@ +eth: + sender: + proof_sending_mode: ONLY_REAL_PROOFS diff --git a/etc/env/file_based/overrides/testnet/general.yaml b/etc/env/file_based/overrides/testnet.yaml similarity index 95% rename from etc/env/file_based/overrides/testnet/general.yaml rename to etc/env/file_based/overrides/testnet.yaml index 43a62f3f0dd..e4da1ac96e2 100644 --- a/etc/env/file_based/overrides/testnet/general.yaml +++ b/etc/env/file_based/overrides/testnet.yaml @@ -10,6 +10,7 @@ eth: aggregated_block_prove_deadline: 300 aggregated_block_execute_deadline: 300 timestamp_criteria_max_allowed_lag: 104000 # 29h + wait_confirmations: null gas_adjuster: pricing_formula_parameter_a: 1.1 internal_l1_pricing_multiplier: 1 @@ -18,5 +19,3 @@ eth: confirmations_for_eth_event: 10 observability: log_directives: zksync=info,zksync_state_keeper=debug,zksync_core=debug,zksync_server=debug,zksync_contract_verifier=debug,zksync_dal=debug,zksync_state=debug,zksync_utils=debug,zksync_eth_sender=debug,loadnext=debug,dev_ticker=info,vm=info,block_sizes_test=info,setup_key_generator_and_server=info,zksync_queued_job_processor=debug,slot_index_consistency_checker=debug,zksync_health_check=debug,zksync_consensus_bft=debug,zksync_consensus_network=debug,zksync_consensus_storage=debug,zksync_consensus_executor=debug, - -# remove eth_sender_wait_confirmations variable diff --git a/etc/env/file_based/overrides/tests/loadtest-new.yaml b/etc/env/file_based/overrides/tests/loadtest-new.yaml new file mode 100644 index 00000000000..2167f7347e0 --- /dev/null +++ b/etc/env/file_based/overrides/tests/loadtest-new.yaml @@ -0,0 +1,7 @@ +db: + merkle_tree: + mode: LIGHTWEIGHT +experimental_vm: + state_keeper_fast_vm_mode: NEW +mempool: + delay_interval: 50 diff --git a/etc/env/file_based/overrides/tests/loadtest-old.yaml b/etc/env/file_based/overrides/tests/loadtest-old.yaml new file mode 100644 index 00000000000..a2d66d1cf4a --- /dev/null +++ b/etc/env/file_based/overrides/tests/loadtest-old.yaml @@ -0,0 +1,7 @@ +db: + merkle_tree: + mode: LIGHTWEIGHT +experimental_vm: + state_keeper_fast_vm_mode: OLD +mempool: + delay_interval: 50 diff --git a/etc/env/file_based/overrides/validium.yaml b/etc/env/file_based/overrides/validium.yaml new file mode 100644 index 00000000000..1af02dd9589 --- /dev/null +++ b/etc/env/file_based/overrides/validium.yaml @@ -0,0 +1,6 @@ +eth: + sender: + pubdata_sending_mode: CUSTOM +state_keeper: + pubdata_overhead_part: 0 + compute_overhead_part: 1 diff --git a/zk_toolbox/Cargo.lock b/zk_toolbox/Cargo.lock index 02da0311991..fd524865d56 100644 --- a/zk_toolbox/Cargo.lock +++ b/zk_toolbox/Cargo.lock @@ -725,6 +725,7 @@ dependencies = [ "rand 0.8.5", "serde", "serde_json", + "serde_yaml", "strum", "thiserror", "types", diff --git a/zk_toolbox/crates/common/src/lib.rs b/zk_toolbox/crates/common/src/lib.rs index b4495d555ec..c23ef920226 100644 --- a/zk_toolbox/crates/common/src/lib.rs +++ b/zk_toolbox/crates/common/src/lib.rs @@ -14,6 +14,7 @@ pub mod git; pub mod server; pub mod version; pub mod wallets; +pub mod yaml; pub use prerequisites::{ check_general_prerequisites, check_prerequisites, GCLOUD_PREREQUISITE, GPU_PREREQUISITES, diff --git a/zk_toolbox/crates/common/src/yaml.rs b/zk_toolbox/crates/common/src/yaml.rs new file mode 100644 index 00000000000..83b59ad6764 --- /dev/null +++ b/zk_toolbox/crates/common/src/yaml.rs @@ -0,0 +1,475 @@ +use anyhow::Context; + +use crate::logger; + +pub(super) const MSG_INVALID_KEY_TYPE_ERR: &str = "Invalid key type"; + +/// Holds the differences between two YAML configurations. +#[derive(Default)] +pub struct ConfigDiff { + /// Fields that have different values between the two configurations + /// This contains the new values + pub differing_values: serde_yaml::Mapping, + + /// Fields that are present in the new configuration but not in the old one. + pub new_fields: serde_yaml::Mapping, +} + +impl ConfigDiff { + pub fn print(&self, msg: &str, is_warning: bool) { + if self.new_fields.is_empty() { + return; + } + + if is_warning { + logger::warn(msg); + logger::warn(logger::object_to_string(&self.new_fields)); + } else { + logger::info(msg); + logger::info(logger::object_to_string(&self.new_fields)); + } + } +} + +fn merge_yaml_internal( + a: &mut serde_yaml::Value, + b: serde_yaml::Value, + current_key: String, + diff: &mut ConfigDiff, + override_values: bool, +) -> anyhow::Result<()> { + match (a, b) { + (serde_yaml::Value::Mapping(a), serde_yaml::Value::Mapping(b)) => { + for (key, value) in b { + let k = key.as_str().context(MSG_INVALID_KEY_TYPE_ERR)?.to_string(); + let current_key = if current_key.is_empty() { + k.clone() + } else { + format!("{}.{}", current_key, k) + }; + + if a.contains_key(&key) { + let a_value = a.get_mut(&key).unwrap(); + if value.is_null() && override_values { + a.remove(&key); + diff.differing_values + .insert(current_key.into(), serde_yaml::Value::Null); + } else { + merge_yaml_internal(a_value, value, current_key, diff, override_values)?; + } + } else if !value.is_null() { + a.insert(key.clone(), value.clone()); + diff.new_fields.insert(current_key.into(), value); + } else if override_values { + diff.differing_values + .insert(current_key.into(), serde_yaml::Value::Null); + } + } + } + (a, b) => { + if a != &b { + diff.differing_values.insert(current_key.into(), b.clone()); + if override_values { + *a = b; + } + } + } + } + Ok(()) +} + +pub fn merge_yaml( + a: &mut serde_yaml::Value, + b: serde_yaml::Value, + override_values: bool, +) -> anyhow::Result { + let mut diff = ConfigDiff::default(); + merge_yaml_internal(a, b, "".into(), &mut diff, override_values)?; + Ok(diff) +} + +#[cfg(test)] +mod tests { + #[test] + fn test_merge_yaml_both_are_equal_returns_no_diff() { + let mut a = serde_yaml::from_str( + r#" + key1: value1 + key2: value2 + key3: + key4: value4 + "#, + ) + .unwrap(); + let b: serde_yaml::Value = serde_yaml::from_str( + r#" + key1: value1 + key2: value2 + key3: + key4: value4 + "#, + ) + .unwrap(); + let expected: serde_yaml::Value = serde_yaml::from_str( + r#" + key1: value1 + key2: value2 + key3: + key4: value4 + "#, + ) + .unwrap(); + let diff = super::merge_yaml(&mut a, b, false).unwrap(); + assert!(diff.differing_values.is_empty()); + assert!(diff.new_fields.is_empty()); + assert_eq!(a, expected); + } + + #[test] + fn test_merge_yaml_b_has_extra_field_returns_diff() { + let mut a = serde_yaml::from_str( + r#" + key1: value1 + key2: value2 + key3: + key4: value4 + "#, + ) + .unwrap(); + let b: serde_yaml::Value = serde_yaml::from_str( + r#" + key1: value1 + key2: value2 + key3: + key4: value4 + key5: value5 + "#, + ) + .unwrap(); + + let expected: serde_yaml::Value = serde_yaml::from_str( + r#" + key1: value1 + key2: value2 + key3: + key4: value4 + key5: value5 + "#, + ) + .unwrap(); + + let diff = super::merge_yaml(&mut a, b.clone(), false).unwrap(); + assert!(diff.differing_values.is_empty()); + assert_eq!(diff.new_fields.len(), 1); + assert_eq!( + diff.new_fields.get::("key5".into()).unwrap(), + b.clone().get("key5").unwrap() + ); + assert_eq!(a, expected); + } + + #[test] + fn test_merge_yaml_a_has_extra_field_no_diff() { + let mut a = serde_yaml::from_str( + r#" + key1: value1 + key2: value2 + key3: + key4: value4 + key5: value5 + "#, + ) + .unwrap(); + let b: serde_yaml::Value = serde_yaml::from_str( + r#" + key1: value1 + key2: value2 + key3: + key4: value4 + "#, + ) + .unwrap(); + + let expected: serde_yaml::Value = serde_yaml::from_str( + r#" + key1: value1 + key2: value2 + key3: + key4: value4 + key5: value5 + "#, + ) + .unwrap(); + + let diff = super::merge_yaml(&mut a, b, false).unwrap(); + assert!(diff.differing_values.is_empty()); + assert!(diff.new_fields.is_empty()); + assert_eq!(a, expected); + } + + #[test] + fn test_merge_yaml_a_has_extra_field_and_b_has_extra_field_returns_diff() { + let mut a = serde_yaml::from_str( + r#" + key1: value1 + key2: value2 + key3: + key4: value4 + key5: value5 + "#, + ) + .unwrap(); + let b: serde_yaml::Value = serde_yaml::from_str( + r#" + key1: value1 + key2: value2 + key3: + key4: value4 + key6: value6 + "#, + ) + .unwrap(); + + let expected: serde_yaml::Value = serde_yaml::from_str( + r#" + key1: value1 + key2: value2 + key3: + key4: value4 + key5: value5 + key6: value6 + "#, + ) + .unwrap(); + + let diff = super::merge_yaml(&mut a, b.clone(), false).unwrap(); + assert_eq!(diff.differing_values.len(), 0); + assert_eq!(diff.new_fields.len(), 1); + assert_eq!( + diff.new_fields.get::("key6".into()).unwrap(), + b.clone().get("key6").unwrap() + ); + assert_eq!(a, expected); + } + + #[test] + fn test_merge_yaml_a_has_different_value_returns_diff() { + let mut a = serde_yaml::from_str( + r#" + key1: value1 + key2: value2 + key3: + key4: value4 + "#, + ) + .unwrap(); + let b: serde_yaml::Value = serde_yaml::from_str( + r#" + key1: value1 + key2: value2 + key3: + key4: value5 + "#, + ) + .unwrap(); + + let expected: serde_yaml::Value = serde_yaml::from_str( + r#" + key1: value1 + key2: value2 + key3: + key4: value4 + "#, + ) + .unwrap(); + + let diff = super::merge_yaml(&mut a, b.clone(), false).unwrap(); + assert_eq!(diff.differing_values.len(), 1); + assert_eq!( + diff.differing_values + .get::("key3.key4".into()) + .unwrap(), + b.get("key3").unwrap().get("key4").unwrap() + ); + assert_eq!(a, expected); + } + + #[test] + fn test_merge_yaml_a_has_different_value_and_b_has_extra_field_returns_diff() { + let mut a = serde_yaml::from_str( + r#" + key1: value1 + key2: value2 + key3: + key4: value4 + "#, + ) + .unwrap(); + let b: serde_yaml::Value = serde_yaml::from_str( + r#" + key1: value1 + key2: value2 + key3: + key4: value5 + key5: value5 + "#, + ) + .unwrap(); + + let expected: serde_yaml::Value = serde_yaml::from_str( + r#" + key1: value1 + key2: value2 + key3: + key4: value4 + key5: value5 + "#, + ) + .unwrap(); + + let diff = super::merge_yaml(&mut a, b.clone(), false).unwrap(); + assert_eq!(diff.differing_values.len(), 1); + assert_eq!( + diff.differing_values + .get::("key3.key4".into()) + .unwrap(), + b.get("key3").unwrap().get("key4").unwrap() + ); + assert_eq!(diff.new_fields.len(), 1); + assert_eq!( + diff.new_fields.get::("key5".into()).unwrap(), + b.get("key5").unwrap() + ); + assert_eq!(a, expected); + } + + #[test] + fn test_merge_yaml_override_values() { + let mut a = serde_yaml::from_str( + r#" + key1: value1 + key2: value2 + key3: + key4: value4 + "#, + ) + .unwrap(); + let b: serde_yaml::Value = serde_yaml::from_str( + r#" + key1: value1 + key2: value2 + key3: + key4: value5 + "#, + ) + .unwrap(); + + let expected: serde_yaml::Value = serde_yaml::from_str( + r#" + key1: value1 + key2: value2 + key3: + key4: value5 + "#, + ) + .unwrap(); + + let diff = super::merge_yaml(&mut a, b.clone(), true).unwrap(); + assert_eq!(diff.differing_values.len(), 1); + assert_eq!( + diff.differing_values + .get::("key3.key4".into()) + .unwrap(), + b.get("key3").unwrap().get("key4").unwrap() + ); + assert_eq!(a, expected); + } + + #[test] + fn test_merge_yaml_override_values_with_extra_field() { + let mut a = serde_yaml::from_str( + r#" + key1: value1 + key2: value2 + key3: + key4: value4 + "#, + ) + .unwrap(); + let b: serde_yaml::Value = serde_yaml::from_str( + r#" + key1: value1 + key2: value2 + key3: + key4: value5 + key5: value5 + "#, + ) + .unwrap(); + + let expected: serde_yaml::Value = serde_yaml::from_str( + r#" + key1: value1 + key2: value2 + key3: + key4: value5 + key5: value5 + "#, + ) + .unwrap(); + + let diff = super::merge_yaml(&mut a, b.clone(), true).unwrap(); + assert_eq!(diff.differing_values.len(), 1); + assert_eq!( + diff.differing_values + .get::("key3.key4".into()) + .unwrap(), + b.get("key3").unwrap().get("key4").unwrap() + ); + assert_eq!(diff.new_fields.len(), 1); + assert_eq!( + diff.new_fields.get::("key5".into()).unwrap(), + b.get("key5").unwrap() + ); + assert_eq!(a, expected); + } + + #[test] + fn test_override_values_with_null() { + let mut a = serde_yaml::from_str( + r#" + key1: value1 + key2: value2 + key3: + key4: value4 + "#, + ) + .unwrap(); + let b: serde_yaml::Value = serde_yaml::from_str( + r#" + key1: value1 + key2: value2 + key3: null + "#, + ) + .unwrap(); + + let expected: serde_yaml::Value = serde_yaml::from_str( + r#" + key1: value1 + key2: value2 + "#, + ) + .unwrap(); + + let diff = super::merge_yaml(&mut a, b.clone(), true).unwrap(); + assert_eq!(diff.differing_values.len(), 1); + assert_eq!( + diff.differing_values + .get::("key3".into()) + .unwrap(), + b.get("key3").unwrap() + ); + assert_eq!(a, expected); + } +} diff --git a/zk_toolbox/crates/config/Cargo.toml b/zk_toolbox/crates/config/Cargo.toml index 5f1419c7ce9..9320beffef2 100644 --- a/zk_toolbox/crates/config/Cargo.toml +++ b/zk_toolbox/crates/config/Cargo.toml @@ -18,6 +18,7 @@ ethers.workspace = true rand.workspace = true serde.workspace = true serde_json.workspace = true +serde_yaml.workspace = true strum.workspace = true thiserror.workspace = true types.workspace = true diff --git a/zk_toolbox/crates/config/src/general.rs b/zk_toolbox/crates/config/src/general.rs index 6498beb0f53..87eb3a7eb19 100644 --- a/zk_toolbox/crates/config/src/general.rs +++ b/zk_toolbox/crates/config/src/general.rs @@ -1,6 +1,7 @@ use std::path::{Path, PathBuf}; use anyhow::Context; +use common::yaml::merge_yaml; use url::Url; use xshell::Shell; pub use zksync_config::configs::GeneralConfig; @@ -10,7 +11,7 @@ use zksync_protobuf_config::{decode_yaml_repr, encode_yaml_repr}; use crate::{ consts::GENERAL_FILE, traits::{ConfigWithL2RpcUrl, FileConfigWithDefaultName, ReadConfig, SaveConfig}, - DEFAULT_CONSENSUS_PORT, + ChainConfig, DEFAULT_CONSENSUS_PORT, }; pub struct RocksDbs { @@ -174,6 +175,15 @@ pub fn update_ports(config: &mut GeneralConfig, ports_config: &PortsConfig) -> a Ok(()) } +pub fn override_config(shell: &Shell, path: PathBuf, chain: &ChainConfig) -> anyhow::Result<()> { + let chain_config_path = chain.path_to_general_config(); + let override_config = serde_yaml::from_str(&shell.read_file(path)?)?; + let mut chain_config = serde_yaml::from_str(&shell.read_file(chain_config_path.clone())?)?; + merge_yaml(&mut chain_config, override_config, true)?; + shell.write_file(chain_config_path, serde_yaml::to_string(&chain_config)?)?; + Ok(()) +} + fn update_port_in_url(http_url: &mut String, port: u16) -> anyhow::Result<()> { let mut http_url_url = Url::parse(http_url)?; if let Err(()) = http_url_url.set_port(Some(port)) { diff --git a/zk_toolbox/crates/zk_inception/src/commands/chain/genesis.rs b/zk_toolbox/crates/zk_inception/src/commands/chain/genesis.rs index bfa3f94916b..187af41489d 100644 --- a/zk_toolbox/crates/zk_inception/src/commands/chain/genesis.rs +++ b/zk_toolbox/crates/zk_inception/src/commands/chain/genesis.rs @@ -9,7 +9,7 @@ use common::{ spinner::Spinner, }; use config::{ - set_databases, set_file_artifacts, set_rocks_db_config, + override_config, set_databases, set_file_artifacts, set_rocks_db_config, traits::{FileConfigWithDefaultName, SaveConfigWithBasePath}, ChainConfig, ContractsConfig, EcosystemConfig, FileArtifacts, GeneralConfig, GenesisConfig, SecretsConfig, WalletsConfig, @@ -17,12 +17,14 @@ use config::{ use types::ProverMode; use xshell::Shell; use zksync_basic_types::commitment::L1BatchCommitmentMode; -use zksync_config::configs::eth_sender::{ProofSendingMode, PubdataSendingMode}; use super::args::genesis::GenesisArgsFinal; use crate::{ commands::chain::args::genesis::GenesisArgs, - consts::{PROVER_MIGRATIONS, SERVER_MIGRATIONS}, + consts::{ + PATH_TO_ONLY_REAL_PROOFS_OVERRIDE_CONFIG, PATH_TO_VALIDIUM_OVERRIDE_CONFIG, + PROVER_MIGRATIONS, SERVER_MIGRATIONS, + }, messages::{ MSG_CHAIN_NOT_INITIALIZED, MSG_FAILED_TO_DROP_PROVER_DATABASE_ERR, MSG_FAILED_TO_DROP_SERVER_DATABASE_ERR, MSG_FAILED_TO_RUN_SERVER_ERR, @@ -55,41 +57,31 @@ pub async fn genesis( ) -> anyhow::Result<()> { shell.create_dir(&config.rocks_db_path)?; + let link_to_code = config.link_to_code.clone(); let rocks_db = recreate_rocksdb_dirs(shell, &config.rocks_db_path, RocksDBDirOption::Main) .context(MSG_RECREATE_ROCKS_DB_ERRROR)?; let mut general = config.get_general_config()?; let file_artifacts = FileArtifacts::new(config.artifacts.clone()); set_rocks_db_config(&mut general, rocks_db)?; set_file_artifacts(&mut general, file_artifacts); + general.save_with_base_path(shell, &config.configs)?; + if config.prover_version != ProverMode::NoProofs { - general - .eth - .as_mut() - .context("eth")? - .sender - .as_mut() - .context("sender")? - .proof_sending_mode = ProofSendingMode::OnlyRealProofs; + override_config( + shell, + link_to_code.join(PATH_TO_ONLY_REAL_PROOFS_OVERRIDE_CONFIG), + config, + )?; } if config.l1_batch_commit_data_generator_mode == L1BatchCommitmentMode::Validium { - general - .eth - .as_mut() - .context("eth")? - .sender - .as_mut() - .context("sender")? - .pubdata_sending_mode = PubdataSendingMode::Custom; - general - .state_keeper_config - .as_mut() - .context("state_keeper_config")? - .pubdata_overhead_part = 0.0; + override_config( + shell, + link_to_code.join(PATH_TO_VALIDIUM_OVERRIDE_CONFIG), + config, + )?; } - general.save_with_base_path(shell, &config.configs)?; - let mut secrets = config.get_secrets_config()?; set_databases(&mut secrets, &args.server_db, &args.prover_db)?; secrets.save_with_base_path(shell, &config.configs)?; diff --git a/zk_toolbox/crates/zk_inception/src/commands/update.rs b/zk_toolbox/crates/zk_inception/src/commands/update.rs index a05ecbe62e0..c140c3a4e9c 100644 --- a/zk_toolbox/crates/zk_inception/src/commands/update.rs +++ b/zk_toolbox/crates/zk_inception/src/commands/update.rs @@ -1,7 +1,11 @@ use std::path::Path; use anyhow::{Context, Ok}; -use common::{git, logger, spinner::Spinner}; +use common::{ + git, logger, + spinner::Spinner, + yaml::{merge_yaml, ConfigDiff}, +}; use config::{ ChainConfig, EcosystemConfig, CONTRACTS_FILE, EN_CONFIG_FILE, ERA_OBSERBAVILITY_DIR, GENERAL_FILE, GENESIS_FILE, SECRETS_FILE, @@ -12,38 +16,11 @@ use super::args::UpdateArgs; use crate::messages::{ msg_diff_contracts_config, msg_diff_genesis_config, msg_diff_secrets, msg_updating_chain, MSG_CHAIN_NOT_FOUND_ERR, MSG_DIFF_EN_CONFIG, MSG_DIFF_EN_GENERAL_CONFIG, - MSG_DIFF_GENERAL_CONFIG, MSG_INVALID_KEY_TYPE_ERR, MSG_PULLING_ZKSYNC_CODE_SPINNER, + MSG_DIFF_GENERAL_CONFIG, MSG_PULLING_ZKSYNC_CODE_SPINNER, MSG_UPDATING_ERA_OBSERVABILITY_SPINNER, MSG_UPDATING_SUBMODULES_SPINNER, MSG_UPDATING_ZKSYNC, MSG_ZKSYNC_UPDATED, }; -/// Holds the differences between two YAML configurations. -#[derive(Default)] -struct ConfigDiff { - /// Fields that have different values between the two configurations - /// This contains the new values - pub differing_values: serde_yaml::Mapping, - - /// Fields that are present in the new configuration but not in the old one. - pub new_fields: serde_yaml::Mapping, -} - -impl ConfigDiff { - fn print(&self, msg: &str, is_warning: bool) { - if self.new_fields.is_empty() { - return; - } - - if is_warning { - logger::warn(msg); - logger::warn(logger::object_to_string(&self.new_fields)); - } else { - logger::info(msg); - logger::info(logger::object_to_string(&self.new_fields)); - } - } -} - pub fn run(shell: &Shell, args: UpdateArgs) -> anyhow::Result<()> { logger::info(MSG_UPDATING_ZKSYNC); let ecosystem = EcosystemConfig::from_file(shell)?; @@ -127,7 +104,7 @@ fn update_config( ) -> anyhow::Result<()> { let original_config = serde_yaml::from_str(&shell.read_file(original_config_path)?)?; let mut chain_config = serde_yaml::from_str(&shell.read_file(chain_config_path)?)?; - let diff = merge_yaml(&mut chain_config, original_config)?; + let diff = merge_yaml(&mut chain_config, original_config, false)?; if save_config { save_updated_config(&shell, chain_config, chain_config_path, diff, msg)?; } else { @@ -202,298 +179,3 @@ fn update_chain( Ok(()) } - -fn merge_yaml_internal( - a: &mut serde_yaml::Value, - b: serde_yaml::Value, - current_key: String, - diff: &mut ConfigDiff, -) -> anyhow::Result<()> { - match (a, b) { - (serde_yaml::Value::Mapping(a), serde_yaml::Value::Mapping(b)) => { - for (key, value) in b { - let k = key.as_str().context(MSG_INVALID_KEY_TYPE_ERR)?.to_string(); - let current_key = if current_key.is_empty() { - k.clone() - } else { - format!("{}.{}", current_key, k) - }; - - if a.contains_key(&key) { - merge_yaml_internal(a.get_mut(&key).unwrap(), value, current_key, diff)?; - } else { - a.insert(key.clone(), value.clone()); - diff.new_fields.insert(current_key.into(), value); - } - } - } - (a, b) => { - if a != &b { - diff.differing_values.insert(current_key.into(), b); - } - } - } - Ok(()) -} - -fn merge_yaml(a: &mut serde_yaml::Value, b: serde_yaml::Value) -> anyhow::Result { - let mut diff = ConfigDiff::default(); - merge_yaml_internal(a, b, "".into(), &mut diff)?; - Ok(diff) -} - -#[cfg(test)] -mod tests { - #[test] - fn test_merge_yaml_both_are_equal_returns_no_diff() { - let mut a = serde_yaml::from_str( - r#" - key1: value1 - key2: value2 - key3: - key4: value4 - "#, - ) - .unwrap(); - let b: serde_yaml::Value = serde_yaml::from_str( - r#" - key1: value1 - key2: value2 - key3: - key4: value4 - "#, - ) - .unwrap(); - let expected: serde_yaml::Value = serde_yaml::from_str( - r#" - key1: value1 - key2: value2 - key3: - key4: value4 - "#, - ) - .unwrap(); - let diff = super::merge_yaml(&mut a, b).unwrap(); - assert!(diff.differing_values.is_empty()); - assert!(diff.new_fields.is_empty()); - assert_eq!(a, expected); - } - - #[test] - fn test_merge_yaml_b_has_extra_field_returns_diff() { - let mut a = serde_yaml::from_str( - r#" - key1: value1 - key2: value2 - key3: - key4: value4 - "#, - ) - .unwrap(); - let b: serde_yaml::Value = serde_yaml::from_str( - r#" - key1: value1 - key2: value2 - key3: - key4: value4 - key5: value5 - "#, - ) - .unwrap(); - - let expected: serde_yaml::Value = serde_yaml::from_str( - r#" - key1: value1 - key2: value2 - key3: - key4: value4 - key5: value5 - "#, - ) - .unwrap(); - - let diff = super::merge_yaml(&mut a, b.clone()).unwrap(); - assert!(diff.differing_values.is_empty()); - assert_eq!(diff.new_fields.len(), 1); - assert_eq!( - diff.new_fields.get::("key5".into()).unwrap(), - b.clone().get("key5").unwrap() - ); - assert_eq!(a, expected); - } - - #[test] - fn test_merge_yaml_a_has_extra_field_no_diff() { - let mut a = serde_yaml::from_str( - r#" - key1: value1 - key2: value2 - key3: - key4: value4 - key5: value5 - "#, - ) - .unwrap(); - let b: serde_yaml::Value = serde_yaml::from_str( - r#" - key1: value1 - key2: value2 - key3: - key4: value4 - "#, - ) - .unwrap(); - - let expected: serde_yaml::Value = serde_yaml::from_str( - r#" - key1: value1 - key2: value2 - key3: - key4: value4 - key5: value5 - "#, - ) - .unwrap(); - - let diff = super::merge_yaml(&mut a, b).unwrap(); - assert!(diff.differing_values.is_empty()); - assert!(diff.new_fields.is_empty()); - assert_eq!(a, expected); - } - - #[test] - fn test_merge_yaml_a_has_extra_field_and_b_has_extra_field_returns_diff() { - let mut a = serde_yaml::from_str( - r#" - key1: value1 - key2: value2 - key3: - key4: value4 - key5: value5 - "#, - ) - .unwrap(); - let b: serde_yaml::Value = serde_yaml::from_str( - r#" - key1: value1 - key2: value2 - key3: - key4: value4 - key6: value6 - "#, - ) - .unwrap(); - - let expected: serde_yaml::Value = serde_yaml::from_str( - r#" - key1: value1 - key2: value2 - key3: - key4: value4 - key5: value5 - key6: value6 - "#, - ) - .unwrap(); - - let diff = super::merge_yaml(&mut a, b.clone()).unwrap(); - assert_eq!(diff.differing_values.len(), 0); - assert_eq!(diff.new_fields.len(), 1); - assert_eq!( - diff.new_fields.get::("key6".into()).unwrap(), - b.clone().get("key6").unwrap() - ); - assert_eq!(a, expected); - } - - #[test] - fn test_merge_yaml_a_has_different_value_returns_diff() { - let mut a = serde_yaml::from_str( - r#" - key1: value1 - key2: value2 - key3: - key4: value4 - "#, - ) - .unwrap(); - let b: serde_yaml::Value = serde_yaml::from_str( - r#" - key1: value1 - key2: value2 - key3: - key4: value5 - "#, - ) - .unwrap(); - - let expected: serde_yaml::Value = serde_yaml::from_str( - r#" - key1: value1 - key2: value2 - key3: - key4: value4 - "#, - ) - .unwrap(); - - let diff = super::merge_yaml(&mut a, b.clone()).unwrap(); - assert_eq!(diff.differing_values.len(), 1); - assert_eq!( - diff.differing_values - .get::("key3.key4".into()) - .unwrap(), - b.get("key3").unwrap().get("key4").unwrap() - ); - assert_eq!(a, expected); - } - - #[test] - fn test_merge_yaml_a_has_different_value_and_b_has_extra_field_returns_diff() { - let mut a = serde_yaml::from_str( - r#" - key1: value1 - key2: value2 - key3: - key4: value4 - "#, - ) - .unwrap(); - let b: serde_yaml::Value = serde_yaml::from_str( - r#" - key1: value1 - key2: value2 - key3: - key4: value5 - key5: value5 - "#, - ) - .unwrap(); - - let expected: serde_yaml::Value = serde_yaml::from_str( - r#" - key1: value1 - key2: value2 - key3: - key4: value4 - key5: value5 - "#, - ) - .unwrap(); - - let diff = super::merge_yaml(&mut a, b.clone()).unwrap(); - assert_eq!(diff.differing_values.len(), 1); - assert_eq!( - diff.differing_values - .get::("key3.key4".into()) - .unwrap(), - b.get("key3").unwrap().get("key4").unwrap() - ); - assert_eq!(diff.new_fields.len(), 1); - assert_eq!( - diff.new_fields.get::("key5".into()).unwrap(), - b.get("key5").unwrap() - ); - assert_eq!(a, expected); - } -} diff --git a/zk_toolbox/crates/zk_inception/src/consts.rs b/zk_toolbox/crates/zk_inception/src/consts.rs index 87315dcd818..9f81847e333 100644 --- a/zk_toolbox/crates/zk_inception/src/consts.rs +++ b/zk_toolbox/crates/zk_inception/src/consts.rs @@ -54,3 +54,7 @@ pub const WITNESS_VECTOR_GENERATOR_BINARY_NAME: &str = "zksync_witness_vector_ge pub const PROVER_BINARY_NAME: &str = "zksync_prover_fri"; pub const COMPRESSOR_BINARY_NAME: &str = "zksync_proof_fri_compressor"; pub const PROVER_JOB_MONITOR_BINARY_NAME: &str = "zksync_prover_job_monitor"; + +pub const PATH_TO_ONLY_REAL_PROOFS_OVERRIDE_CONFIG: &str = + "etc/env/file_based/overrides/only_real_proofs.yaml"; +pub const PATH_TO_VALIDIUM_OVERRIDE_CONFIG: &str = "etc/env/file_based/overrides/validium.yaml"; diff --git a/zk_toolbox/crates/zk_inception/src/messages.rs b/zk_toolbox/crates/zk_inception/src/messages.rs index 5f398f4b8a3..3bbac066dbb 100644 --- a/zk_toolbox/crates/zk_inception/src/messages.rs +++ b/zk_toolbox/crates/zk_inception/src/messages.rs @@ -454,7 +454,6 @@ pub(super) const MSG_DIFF_EN_CONFIG: &str = "Added the following fields to the external node config:"; pub(super) const MSG_DIFF_EN_GENERAL_CONFIG: &str = "Added the following fields to the external node generalconfig:"; -pub(super) const MSG_INVALID_KEY_TYPE_ERR: &str = "Invalid key type"; pub(super) const MSG_UPDATING_ERA_OBSERVABILITY_SPINNER: &str = "Updating era observability..."; pub(super) fn msg_diff_genesis_config(chain: &str) -> String { diff --git a/zk_toolbox/crates/zk_supervisor/src/commands/config_writer.rs b/zk_toolbox/crates/zk_supervisor/src/commands/config_writer.rs new file mode 100644 index 00000000000..3adecb36d06 --- /dev/null +++ b/zk_toolbox/crates/zk_supervisor/src/commands/config_writer.rs @@ -0,0 +1,35 @@ +use anyhow::Context; +use clap::Parser; +use common::{config::global_config, logger, Prompt}; +use config::{override_config, EcosystemConfig}; +use xshell::Shell; + +use crate::messages::{ + msg_overriding_config, MSG_CHAIN_NOT_FOUND_ERR, MSG_OVERRIDE_CONFIG_PATH_HELP, + MSG_OVERRIDE_SUCCESS, MSG_OVERRRIDE_CONFIG_PATH_PROMPT, +}; + +#[derive(Debug, Parser)] +pub struct ConfigWriterArgs { + #[clap(long, short, help = MSG_OVERRIDE_CONFIG_PATH_HELP)] + pub path: Option, +} + +impl ConfigWriterArgs { + pub fn get_config_path(self) -> String { + self.path + .unwrap_or_else(|| Prompt::new(MSG_OVERRRIDE_CONFIG_PATH_PROMPT).ask()) + } +} + +pub fn run(shell: &Shell, args: ConfigWriterArgs) -> anyhow::Result<()> { + let path = args.get_config_path().into(); + let ecosystem = EcosystemConfig::from_file(shell)?; + let chain = ecosystem + .load_chain(global_config().chain_name.clone()) + .context(MSG_CHAIN_NOT_FOUND_ERR)?; + logger::step(msg_overriding_config(chain.name.clone())); + override_config(shell, path, &chain)?; + logger::outro(MSG_OVERRIDE_SUCCESS); + Ok(()) +} diff --git a/zk_toolbox/crates/zk_supervisor/src/commands/mod.rs b/zk_toolbox/crates/zk_supervisor/src/commands/mod.rs index 1f3893e293e..d3cb99f1e34 100644 --- a/zk_toolbox/crates/zk_supervisor/src/commands/mod.rs +++ b/zk_toolbox/crates/zk_supervisor/src/commands/mod.rs @@ -1,4 +1,5 @@ pub mod clean; +pub mod config_writer; pub mod contracts; pub mod database; pub mod fmt; diff --git a/zk_toolbox/crates/zk_supervisor/src/main.rs b/zk_toolbox/crates/zk_supervisor/src/main.rs index 3c34d059656..242affd8a71 100644 --- a/zk_toolbox/crates/zk_supervisor/src/main.rs +++ b/zk_toolbox/crates/zk_supervisor/src/main.rs @@ -1,7 +1,8 @@ use clap::{Parser, Subcommand}; use commands::{ - contracts::ContractsArgs, database::DatabaseCommands, lint::LintArgs, prover::ProverCommands, - send_transactions::args::SendTransactionsArgs, snapshot::SnapshotCommands, test::TestCommands, + config_writer::ConfigWriterArgs, contracts::ContractsArgs, database::DatabaseCommands, + lint::LintArgs, prover::ProverCommands, send_transactions::args::SendTransactionsArgs, + snapshot::SnapshotCommands, test::TestCommands, }; use common::{ check_general_prerequisites, @@ -12,10 +13,10 @@ use common::{ }; use config::EcosystemConfig; use messages::{ - msg_global_chain_does_not_exist, MSG_CONTRACTS_ABOUT, MSG_PROVER_VERSION_ABOUT, - MSG_SEND_TXNS_ABOUT, MSG_SUBCOMMAND_CLEAN, MSG_SUBCOMMAND_DATABASE_ABOUT, - MSG_SUBCOMMAND_FMT_ABOUT, MSG_SUBCOMMAND_LINT_ABOUT, MSG_SUBCOMMAND_SNAPSHOTS_CREATOR_ABOUT, - MSG_SUBCOMMAND_TESTS_ABOUT, + msg_global_chain_does_not_exist, MSG_CONFIG_WRITER_ABOUT, MSG_CONTRACTS_ABOUT, + MSG_PROVER_VERSION_ABOUT, MSG_SEND_TXNS_ABOUT, MSG_SUBCOMMAND_CLEAN, + MSG_SUBCOMMAND_DATABASE_ABOUT, MSG_SUBCOMMAND_FMT_ABOUT, MSG_SUBCOMMAND_LINT_ABOUT, + MSG_SUBCOMMAND_SNAPSHOTS_CREATOR_ABOUT, MSG_SUBCOMMAND_TESTS_ABOUT, }; use xshell::Shell; @@ -59,6 +60,8 @@ enum SupervisorSubcommands { Prover(ProverCommands), #[command(about = MSG_CONTRACTS_ABOUT)] Contracts(ContractsArgs), + #[command(about = MSG_CONFIG_WRITER_ABOUT, alias = "o")] + ConfigWriter(ConfigWriterArgs), #[command(about = MSG_SEND_TXNS_ABOUT)] SendTransactions(SendTransactionsArgs), } @@ -121,6 +124,7 @@ async fn run_subcommand(args: Supervisor, shell: &Shell) -> anyhow::Result<()> { SupervisorSubcommands::Fmt(args) => commands::fmt::run(shell.clone(), args).await?, SupervisorSubcommands::Prover(command) => commands::prover::run(shell, command).await?, SupervisorSubcommands::Contracts(args) => commands::contracts::run(shell, args)?, + SupervisorSubcommands::ConfigWriter(args) => commands::config_writer::run(shell, args)?, SupervisorSubcommands::SendTransactions(args) => { commands::send_transactions::run(shell, args).await? } diff --git a/zk_toolbox/crates/zk_supervisor/src/messages.rs b/zk_toolbox/crates/zk_supervisor/src/messages.rs index 72887e40a2b..5f68630f756 100644 --- a/zk_toolbox/crates/zk_supervisor/src/messages.rs +++ b/zk_toolbox/crates/zk_supervisor/src/messages.rs @@ -14,6 +14,7 @@ pub(super) const MSG_SUBCOMMAND_TESTS_ABOUT: &str = "Run tests"; pub(super) const MSG_SUBCOMMAND_CLEAN: &str = "Clean artifacts"; pub(super) const MSG_SUBCOMMAND_LINT_ABOUT: &str = "Lint code"; pub(super) const MSG_CONTRACTS_ABOUT: &str = "Build contracts"; +pub(super) const MSG_CONFIG_WRITER_ABOUT: &str = "Overwrite general config"; pub(super) const MSG_SUBCOMMAND_FMT_ABOUT: &str = "Format code"; @@ -203,6 +204,15 @@ pub(super) const MSG_WALLETS_TEST_SUCCESS: &str = "Wallets test success"; pub(super) const MSG_LOADTEST_ABOUT: &str = "Run loadtest"; +pub(super) const MSG_OVERRIDE_CONFIG_PATH_HELP: &str = "Path to the config file to override"; +pub(super) const MSG_OVERRRIDE_CONFIG_PATH_PROMPT: &str = + "Provide path to the config file to override"; +pub(super) const MSG_OVERRIDE_SUCCESS: &str = "Config was overridden successfully"; + +pub(super) fn msg_overriding_config(chain: String) -> String { + format!("Overriding general config for chain {}", chain) +} + // Send transactions related messages pub(super) const MSG_SEND_TXNS_ABOUT: &str = "Send transactions from file"; pub(super) const MSG_PROMPT_TRANSACTION_FILE: &str = "Path to transactions file";