diff --git a/compose.yaml b/compose.yaml index f4a619b2..46edd5c3 100644 --- a/compose.yaml +++ b/compose.yaml @@ -7,8 +7,6 @@ services: target: runtime image: consensus-node container_name: consensus-node-1 - environment: - - NODE_ID=node_0 ports: - "3154:3154" networks: @@ -19,8 +17,6 @@ services: node-2: image: consensus-node container_name: consensus-node-2 - environment: - - NODE_ID=node_1 ports: - "3155:3154" networks: diff --git a/docker-entrypoint.sh b/docker-entrypoint.sh index 07a3c781..6b8977c1 100755 --- a/docker-entrypoint.sh +++ b/docker-entrypoint.sh @@ -3,4 +3,4 @@ cd docker_config/${NODE_ID} export RUST_LOG=INFO -../../executor +../../executor $@ diff --git a/node/Cargo.lock b/node/Cargo.lock index 74082815..eae4f239 100644 --- a/node/Cargo.lock +++ b/node/Cargo.lock @@ -117,9 +117,9 @@ dependencies = [ [[package]] name = "anstyle" -version = "1.0.5" +version = "1.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2faccea4cc4ab4a667ce676a30e8ec13922a692c99bb8f5b11f1502c72e04220" +checksum = "8901269c6307e8d93993578286ac0edf7f195079ffff5ebdeea6a59ffb7e36bc" [[package]] name = "anstyle-parse" @@ -466,9 +466,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.4.18" +version = "4.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e578d6ec4194633722ccf9544794b71b1385c3c027efe0c55db226fc880865c" +checksum = "80c21025abd42669a92efc996ef13cfb2c5c627858421ea58d5c3b331a6c134f" dependencies = [ "clap_builder", "clap_derive", @@ -476,21 +476,21 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.4.18" +version = "4.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4df4df40ec50c46000231c914968278b1eb05098cf8f1b3a518a95030e71d1c7" +checksum = "458bf1f341769dfcf849846f65dffdf9146daa56bcd2a47cb4e1de9915567c99" dependencies = [ "anstream", "anstyle", "clap_lex", - "strsim", + "strsim 0.11.0", ] [[package]] name = "clap_derive" -version = "4.4.7" +version = "4.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf9804afaaf59a91e75b022a30fb7229a7901f60c755489cc61c9b423b836442" +checksum = "307bc0538d5f0f83b8248db3087aa92fe504e4691294d0c96c0eabc33f47ba47" dependencies = [ "heck", "proc-macro2", @@ -500,9 +500,9 @@ dependencies = [ [[package]] name = "clap_lex" -version = "0.6.0" +version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "702fc72eb24e5a1e48ce58027a675bc24edd52096d5397d4aea7c6dd9eca0bd1" +checksum = "98cc8fbded0c607b7ba9dd60cd98df59af97e84d24e49c8557331cfc26d301ce" [[package]] name = "colorchoice" @@ -635,9 +635,9 @@ dependencies = [ [[package]] name = "curve25519-dalek" -version = "4.1.1" +version = "4.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e89b8c6a2e4b1f45971ad09761aafb85514a84744b67a95e32c3cc1352d1f65c" +checksum = "0a677b8922c94e01bdbb12126b0bc852f00447528dee1782229af9c720c3f348" dependencies = [ "cfg-if", "cpufeatures", @@ -681,7 +681,7 @@ dependencies = [ "ident_case", "proc-macro2", "quote", - "strsim", + "strsim 0.10.0", "syn 2.0.48", ] @@ -696,12 +696,6 @@ dependencies = [ "syn 2.0.48", ] -[[package]] -name = "data-encoding" -version = "2.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e962a19be5cfc3f3bf6dd8f61eb50107f356ad6270fbb3ed41476571db78be5" - [[package]] name = "der" version = "0.7.8" @@ -782,9 +776,9 @@ dependencies = [ [[package]] name = "ed25519-dalek" -version = "2.1.0" +version = "2.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1f628eaec48bfd21b865dc2950cfa014450c01d2fa2b69a86c2fd5844ec523c0" +checksum = "4a3daa8e81a3963a60642bcc1f90a670680bd4a77535faa384e9d1c79d620871" dependencies = [ "curve25519-dalek", "ed25519", @@ -862,9 +856,9 @@ dependencies = [ [[package]] name = "fiat-crypto" -version = "0.2.5" +version = "0.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "27573eac26f4dd11e2b1916c3fe1baa56407c83c71a773a8ba17ec0bca03b6b7" +checksum = "1676f435fc1dadde4d03e43f5d62b259e1ce5f40bd4ffb21db2b42ebe59c1382" [[package]] name = "fixedbitset" @@ -1072,9 +1066,9 @@ checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8" [[package]] name = "hermit-abi" -version = "0.3.4" +version = "0.3.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5d3d0e0f38255e7fa3cf31335b3a56f05febd18025f4db5ef7a0cfb4f8da651f" +checksum = "d0c62115964e08cb8039170eb33c1d0e2388a256930279edca206fff675f82c3" [[package]] name = "hex" @@ -1185,9 +1179,9 @@ dependencies = [ [[package]] name = "iana-time-zone" -version = "0.1.59" +version = "0.1.60" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b6a67363e2aa4443928ce15e57ebae94fd8949958fd1223c4cfc0cd473ad7539" +checksum = "e7ffbb5a1b541ea2561f8c41c087286cc091e21e556a4f09a8f6cbf17b69b141" dependencies = [ "android_system_properties", "core-foundation-sys", @@ -1266,12 +1260,12 @@ dependencies = [ [[package]] name = "is-terminal" -version = "0.4.10" +version = "0.4.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0bad00257d07be169d870ab665980b06cdb366d792ad690bf2e76876dc503455" +checksum = "fe8f25ce1159c7740ff0b9b2f5cdf4a8428742ba7c112b9f20f22cd5219c7dab" dependencies = [ "hermit-abi", - "rustix", + "libc", "windows-sys 0.52.0", ] @@ -1301,18 +1295,18 @@ checksum = "b1a46d1a171d865aa5f83f92695765caa047a9b4cbae2cbf37dbd613a793fd4c" [[package]] name = "jobserver" -version = "0.1.27" +version = "0.1.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8c37f63953c4c63420ed5fd3d6d398c719489b9f872b9fa683262f8edd363c7d" +checksum = "ab46a6e9526ddef3ae7f787c06f0f2600639ba80ea3eade3d8e670a2230f51d6" dependencies = [ "libc", ] [[package]] name = "js-sys" -version = "0.3.67" +version = "0.3.68" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9a1d36f1235bc969acba30b7f5990b864423a6068a10f7c90ae8f0112e3a59d1" +checksum = "406cda4b368d531c842222cf9d2600a9a4acce8d29423695379c6868a143a9ee" dependencies = [ "wasm-bindgen", ] @@ -1491,7 +1485,6 @@ dependencies = [ "kube-core", "pem", "pin-project", - "rand 0.8.5", "rustls", "rustls-pemfile", "secrecy", @@ -1500,7 +1493,6 @@ dependencies = [ "serde_yaml", "thiserror", "tokio", - "tokio-tungstenite", "tokio-util", "tower", "tower-http", @@ -1755,9 +1747,9 @@ checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" [[package]] name = "miniz_oxide" -version = "0.7.1" +version = "0.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e7810e0be55b428ada41041c41f32c9f1a42817901b4ccf45fa3d4b6561e74c7" +checksum = "9d811f3e15f28568be3407c8e7fdb6514c1cda3cb30683f15b6a1a1dc4ea14a7" dependencies = [ "adler", ] @@ -1818,19 +1810,18 @@ checksum = "51d515d32fb182ee37cda2ccdcb92950d6a3c2893aa280e540671c2cd0f3b1d9" [[package]] name = "num-integer" -version = "0.1.45" +version = "0.1.46" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "225d3389fb3509a24c93f5c29eb6bde2586b98d9f016636dff58d7c6f7569cd9" +checksum = "7969661fd2958a5cb096e56c8e1ad0444ac2bbcd0061bd28660485a44879858f" dependencies = [ - "autocfg", "num-traits", ] [[package]] name = "num-traits" -version = "0.2.17" +version = "0.2.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "39e3200413f237f41ab11ad6d161bc7239c84dcb631773ccd7de3dfe4b5c267c" +checksum = "da0df0e5185db44f69b44f26786fe401b6c293d1907744beaa7fa62b2e5a517a" dependencies = [ "autocfg", ] @@ -1952,9 +1943,9 @@ checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e" [[package]] name = "pest" -version = "2.7.6" +version = "2.7.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1f200d8d83c44a45b21764d1916299752ca035d15ecd46faca3e9a2a2bf6ad06" +checksum = "219c0dcc30b6a27553f9cc242972b67f75b60eb0db71f0b5462f38b058c41546" dependencies = [ "memchr", "thiserror", @@ -1963,9 +1954,9 @@ dependencies = [ [[package]] name = "pest_derive" -version = "2.7.6" +version = "2.7.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bcd6ab1236bbdb3a49027e920e693192ebfe8913f6d60e294de57463a493cfde" +checksum = "22e1288dbd7786462961e69bfd4df7848c1e37e8b74303dbdab82c3a9cdd2809" dependencies = [ "pest", "pest_generator", @@ -1973,9 +1964,9 @@ dependencies = [ [[package]] name = "pest_generator" -version = "2.7.6" +version = "2.7.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2a31940305ffc96863a735bef7c7994a00b325a7138fdbc5bda0f1a0476d3275" +checksum = "1381c29a877c6d34b8c176e734f35d7f7f5b3adaefe940cb4d1bb7af94678e2e" dependencies = [ "pest", "pest_meta", @@ -1986,9 +1977,9 @@ dependencies = [ [[package]] name = "pest_meta" -version = "2.7.6" +version = "2.7.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a7ff62f5259e53b78d1af898941cdcdccfae7385cf7d793a6e55de5d05bb4b7d" +checksum = "d0934d6907f148c22a3acbda520c7eed243ad7487a30f51f6ce52b58b7077a8a" dependencies = [ "once_cell", "pest", @@ -2726,17 +2717,6 @@ dependencies = [ "opaque-debug", ] -[[package]] -name = "sha1" -version = "0.10.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e3bf829a2d51ab4a5ddf1352d8470c140cadc8301b2ae1789db023f01cedd6ba" -dependencies = [ - "cfg-if", - "cpufeatures", - "digest 0.10.7", -] - [[package]] name = "sha2" version = "0.10.8" @@ -2886,6 +2866,12 @@ version = "0.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623" +[[package]] +name = "strsim" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5ee073c9e4cd00e28217186dbe12796d692868f432bf2e97ee73bed0c56dfa01" + [[package]] name = "subtle" version = "2.5.0" @@ -2916,13 +2902,12 @@ dependencies = [ [[package]] name = "tempfile" -version = "3.9.0" +version = "3.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "01ce4141aa927a6d1bd34a041795abd0db1cccba5d5f24b009f694bdf3a1f3fa" +checksum = "a365e8cd18e44762ef95d87f284f4b5cd04107fec2ff3052bd6a3e6069669e67" dependencies = [ "cfg-if", "fastrand", - "redox_syscall", "rustix", "windows-sys 0.52.0", ] @@ -2998,9 +2983,9 @@ dependencies = [ [[package]] name = "time" -version = "0.3.32" +version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fe80ced77cbfb4cb91a94bf72b378b4b6791a0d9b7f09d0be747d1bdff4e68bd" +checksum = "c8248b6521bb14bc45b4067159b9b6ad792e2d6d754d6c41fb50e29fefe38749" dependencies = [ "deranged", "num-conv", @@ -3080,6 +3065,17 @@ dependencies = [ "syn 2.0.48", ] +[[package]] +name = "tokio-retry" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f57eb36ecbe0fc510036adff84824dd3c24bb781e21bfa67b69d556aa85214f" +dependencies = [ + "pin-project", + "rand 0.8.5", + "tokio", +] + [[package]] name = "tokio-rustls" version = "0.24.1" @@ -3101,18 +3097,6 @@ dependencies = [ "tokio", ] -[[package]] -name = "tokio-tungstenite" -version = "0.20.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "212d5dcb2a1ce06d81107c3d0ffa3121fe974b73f068c8282cb1c32328113b6c" -dependencies = [ - "futures-util", - "log", - "tokio", - "tungstenite", -] - [[package]] name = "tokio-util" version = "0.7.10" @@ -3243,9 +3227,9 @@ dependencies = [ [[package]] name = "treediff" -version = "4.0.2" +version = "4.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "52984d277bdf2a751072b5df30ec0377febdb02f7696d64c2d7d54630bac4303" +checksum = "4d127780145176e2b5d16611cc25a900150e86e9fd79d3bde6ff3a37359c9cb5" dependencies = [ "serde_json", ] @@ -3256,25 +3240,6 @@ version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" -[[package]] -name = "tungstenite" -version = "0.20.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9e3dac10fd62eaf6617d3a904ae222845979aec67c615d1c842b4002c7666fb9" -dependencies = [ - "byteorder", - "bytes", - "data-encoding", - "http", - "httparse", - "log", - "rand 0.8.5", - "sha1", - "thiserror", - "url", - "utf-8", -] - [[package]] name = "typenum" version = "1.17.0" @@ -3347,12 +3312,6 @@ dependencies = [ "percent-encoding", ] -[[package]] -name = "utf-8" -version = "0.7.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09cc8ee72d2a9becf2f2febe0205bbed8fc6615b7cb429ad062dc7b7ddd036a9" - [[package]] name = "utf8parse" version = "0.2.1" @@ -3439,9 +3398,9 @@ checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" [[package]] name = "wasm-bindgen" -version = "0.2.90" +version = "0.2.91" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b1223296a201415c7fad14792dbefaace9bd52b62d33453ade1c5b5f07555406" +checksum = "c1e124130aee3fb58c5bdd6b639a0509486b0338acaaae0c84a5124b0f588b7f" dependencies = [ "cfg-if", "wasm-bindgen-macro", @@ -3449,9 +3408,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-backend" -version = "0.2.90" +version = "0.2.91" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fcdc935b63408d58a32f8cc9738a0bffd8f05cc7c002086c6ef20b7312ad9dcd" +checksum = "c9e7e1900c352b609c8488ad12639a311045f40a35491fb69ba8c12f758af70b" dependencies = [ "bumpalo", "log", @@ -3464,9 +3423,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro" -version = "0.2.90" +version = "0.2.91" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3e4c238561b2d428924c49815533a8b9121c664599558a5d9ec51f8a1740a999" +checksum = "b30af9e2d358182b5c7449424f017eba305ed32a7010509ede96cdc4696c46ed" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -3474,9 +3433,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.90" +version = "0.2.91" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bae1abb6806dc1ad9e560ed242107c0f6c84335f1749dd4e8ddb012ebd5e25a7" +checksum = "642f325be6301eb8107a83d12a8ac6c1e1c54345a7ef1a9261962dfefda09e66" dependencies = [ "proc-macro2", "quote", @@ -3487,15 +3446,15 @@ dependencies = [ [[package]] name = "wasm-bindgen-shared" -version = "0.2.90" +version = "0.2.91" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4d91413b1c31d7539ba5ef2451af3f0b833a005eb27a631cec32bc0635a8602b" +checksum = "4f186bd2dcf04330886ce82d6f33dd75a7bfcf69ecf5763b89fcde53b6ac9838" [[package]] name = "web-sys" -version = "0.3.67" +version = "0.3.68" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "58cd2333b6e0be7a39605f0e255892fd7418a682d8da8fe042fe25128794d2ed" +checksum = "96565907687f7aceb35bc5fc03770a8a0471d82e479f25832f54a0e3f4b28446" dependencies = [ "js-sys", "wasm-bindgen", @@ -3914,6 +3873,7 @@ dependencies = [ "serde_json", "tempfile", "tokio", + "tokio-retry", "tower", "tracing", "tracing-subscriber", diff --git a/node/Cargo.toml b/node/Cargo.toml index 98ca4a02..c867def7 100644 --- a/node/Cargo.toml +++ b/node/Cargo.toml @@ -81,9 +81,10 @@ test-casing = "0.1.0" thiserror = "1.0.40" time = "0.3.23" tokio = { version = "1.34.0", features = ["full"] } +tokio-retry = "0.3.0" tracing = { version = "0.1.37", features = ["attributes"] } tracing-subscriber = { version = "0.3.16", features = ["env-filter", "fmt"] } -kube = { version = "0.88.1", features = ["runtime", "derive", "ws"] } +kube = { version = "0.88.1", features = ["runtime", "derive"] } k8s-openapi = { version = "0.21.0", features = ["latest"] } jsonrpsee = { version = "0.21.0", features = ["server", "http-client"] } tower = { version = "0.4.13" } diff --git a/node/tools/Cargo.toml b/node/tools/Cargo.toml index 3546e97c..93e80fa4 100644 --- a/node/tools/Cargo.toml +++ b/node/tools/Cargo.toml @@ -26,6 +26,7 @@ rocksdb.workspace = true serde.workspace = true serde_json.workspace = true tokio.workspace = true +tokio-retry.workspace = true tracing.workspace = true tracing-subscriber.workspace = true vise-exporter.workspace = true diff --git a/node/tools/src/bin/deployer.rs b/node/tools/src/bin/deployer.rs index 5d5056fe..07f4b457 100644 --- a/node/tools/src/bin/deployer.rs +++ b/node/tools/src/bin/deployer.rs @@ -7,7 +7,7 @@ use anyhow::Context; use clap::{Parser, Subcommand}; use rand::Rng; use zksync_consensus_crypto::{Text, TextFmt}; -use zksync_consensus_roles::node::{self, SecretKey}; +use zksync_consensus_roles::node::SecretKey; use zksync_consensus_tools::k8s; use zksync_consensus_tools::AppConfig; use zksync_consensus_tools::NodeAddr; @@ -51,7 +51,7 @@ fn generate_config(nodes: usize) -> anyhow::Result<()> { // Generate the node keys for all the replicas. let rng = &mut rand::thread_rng(); - let node_keys: Vec = (0..nodes).map(|_| rng.gen()).collect(); + let node_keys: Vec = (0..nodes).map(|_| rng.gen()).collect(); let (default_config, validator_keys) = AppConfig::default_for(nodes); let mut cfgs: Vec<_> = (0..nodes).map(|_| default_config.clone()).collect(); @@ -64,7 +64,7 @@ fn generate_config(nodes: usize) -> anyhow::Result<()> { } } - let manifest_path = std::env::var("CARGO_MANIFEST_DIR").unwrap(); + let manifest_path = std::env::var("CARGO_MANIFEST_DIR")?; let root = PathBuf::from(manifest_path).join("k8s_configs"); let _ = fs::remove_dir_all(&root); for (i, cfg) in cfgs.into_iter().enumerate() { @@ -114,29 +114,25 @@ async fn deploy(nodes: usize) -> anyhow::Result<()> { .await?; } - // // Waiting 15 secs to allow the pods to start - // // TODO: should replace with some safer method - // tokio::time::sleep(tokio::time::Duration::from_secs(15)).await; - - // // obtain seed peer(s) IP(s) - // let peer_ips = k8s::get_seed_node_addrs(&client).await; - - // let mut peers = vec![]; - - // for i in 0..seed_nodes { - // let node_id = &format!("node_{i:0>2}"); - // let node_key = read_node_key_from_config(node_id)?; - // let address = peer_ips.get(node_id).context("IP address not found")?; - // peers.push(NodeAddr { - // key: node_key.public(), - // addr: SocketAddr::from_str(&format!("{address}:{NODES_PORT}"))?, - // }); - // } - - // // deploy the rest of nodes - // for i in seed_nodes..nodes { - // k8s::create_deployment(&client, i, false, peers.clone(), NAMESPACE).await?; - // } + // obtain seed peer(s) IP(s) + let peer_ips = k8s::get_seed_node_addrs(&client, seed_nodes).await?; + + let mut peers = vec![]; + + for i in 0..seed_nodes { + let node_id = &format!("node_{i:0>2}"); + let node_key = read_node_key_from_config(node_id)?; + let address = peer_ips.get(node_id).context("IP address not found")?; + peers.push(NodeAddr { + key: node_key.public(), + addr: SocketAddr::from_str(&format!("{address}:{NODES_PORT}"))?, + }); + } + + // deploy the rest of nodes + for i in seed_nodes..nodes { + k8s::create_deployment(&client, i, false, peers.clone(), NAMESPACE).await?; + } Ok(()) } diff --git a/node/tools/src/bin/localnet_config.rs b/node/tools/src/bin/localnet_config.rs index 60a74749..ebe9ba1d 100644 --- a/node/tools/src/bin/localnet_config.rs +++ b/node/tools/src/bin/localnet_config.rs @@ -70,7 +70,7 @@ fn main() -> anyhow::Result<()> { for (i, cfg) in cfgs.into_iter().enumerate() { // Recreate the directory for the node's config. - let root = args.output_dir.join(format!("node_{}", i)); + let root = args.output_dir.join(cfg.public_addr.to_string()); let _ = fs::remove_dir_all(&root); fs::create_dir_all(&root).with_context(|| format!("create_dir_all({:?})", root))?; cfg.write_to_file(&root)?; diff --git a/node/tools/src/config.rs b/node/tools/src/config.rs index 81b18bed..69b603ed 100644 --- a/node/tools/src/config.rs +++ b/node/tools/src/config.rs @@ -1,6 +1,8 @@ //! Node configuration. use crate::{proto, store}; use anyhow::Context as _; +use serde_json::{ser::Formatter, Serializer}; +use std::str::FromStr; use std::{ collections::{HashMap, HashSet}, fs, @@ -11,10 +13,7 @@ use zksync_concurrency::ctx; use zksync_consensus_bft as bft; use zksync_consensus_crypto::{read_optional_text, read_required_text, Text, TextFmt}; use zksync_consensus_executor as executor; -use zksync_consensus_roles::{ - node::{self, PublicKey}, - validator, -}; +use zksync_consensus_roles::{node, validator}; use zksync_consensus_storage::{BlockStore, BlockStoreRunner, PersistentBlockStore}; use zksync_protobuf::{required, serde::Serde, ProtoFmt}; @@ -30,11 +29,19 @@ pub fn decode_json(json: &str) -> anyhow::Result } /// Encodes a generated proto message to json for arbitrary ProtoFmt. -pub fn encode_json(x: &T) -> String { - let mut s = serde_json::Serializer::pretty(vec![]); - T::serialize(x, &mut s).unwrap(); - String::from_utf8(s.into_inner()).unwrap() +pub(crate) fn encode_json(x: &T) -> String { + let s = serde_json::Serializer::pretty(vec![]); + encode_with_serializer(x, s) +} + +pub(crate) fn encode_with_serializer( + x: &T, + mut serializer: Serializer, F>, +) -> String { + T::serialize(x, &mut serializer).unwrap(); + String::from_utf8(serializer.into_inner()).unwrap() } + // pub fn encode_json(x: &T) -> String { // let mut s = serde_json::Serializer::pretty(vec![]); // zksync_protobuf::serde::serialize(x, &mut s).unwrap(); @@ -44,7 +51,7 @@ pub fn encode_json(x: &T) -> String { /// Pair of (public key, ip address) for a gossip network node. #[derive(Debug, Clone)] pub struct NodeAddr { - pub key: PublicKey, + pub key: node::PublicKey, pub addr: SocketAddr, } @@ -78,8 +85,17 @@ pub struct AppConfig { pub max_payload_size: usize, pub gossip_dynamic_inbound_limit: usize, - pub gossip_static_inbound: HashSet, - pub gossip_static_outbound: HashMap, + pub gossip_static_inbound: HashSet, + pub gossip_static_outbound: HashMap, +} + +impl AppConfig { + pub fn check_public_addr(&mut self) -> anyhow::Result<()> { + if let Ok(public_addr) = std::env::var("PUBLIC_ADDR") { + self.public_addr = SocketAddr::from_str(&format!("{public_addr}:{NODES_PORT}"))?; + } + Ok(()) + } } impl ProtoFmt for AppConfig { @@ -266,12 +282,16 @@ impl AppConfig { self } - pub fn add_gossip_static_outbound(&mut self, key: PublicKey, addr: SocketAddr) -> &mut Self { + pub fn add_gossip_static_outbound( + &mut self, + key: node::PublicKey, + addr: SocketAddr, + ) -> &mut Self { self.gossip_static_outbound.insert(key, addr); self } - pub fn add_gossip_static_inbound(&mut self, key: PublicKey) -> &mut Self { + pub fn add_gossip_static_inbound(&mut self, key: node::PublicKey) -> &mut Self { self.gossip_static_inbound.insert(key); self } diff --git a/node/tools/src/k8s.rs b/node/tools/src/k8s.rs index 92e810f5..bd19835f 100644 --- a/node/tools/src/k8s.rs +++ b/node/tools/src/k8s.rs @@ -1,14 +1,18 @@ -use crate::NodeAddr; +use crate::{config, NodeAddr}; +use anyhow::{anyhow, Context}; use k8s_openapi::api::{ apps::v1::Deployment, core::v1::{Namespace, Pod, Service}, }; use kube::{ api::{ListParams, PostParams}, + core::ObjectList, Api, Client, ResourceExt, }; use serde_json::json; use std::collections::HashMap; +use tokio_retry::strategy::FixedInterval; +use tokio_retry::Retry; use tracing::log::info; use zksync_protobuf::serde::Serde; @@ -20,83 +24,46 @@ pub async fn get_client() -> anyhow::Result { /// Creates a namespace in k8s cluster pub async fn create_or_reuse_namespace(client: &Client, name: &str) -> anyhow::Result<()> { let namespaces: Api = Api::all(client.clone()); - let consensus_namespace = namespaces.get_opt(name).await?; - if consensus_namespace.is_none() { - let namespace: Namespace = serde_json::from_value(json!({ - "apiVersion": "v1", - "kind": "Namespace", - "metadata": { - "name": "consensus", - "labels": { - "name": "consensus" - } - } - }))?; - - let namespaces: Api = Api::all(client.clone()); - let post_params = PostParams::default(); - let result = namespaces.create(&post_params, &namespace).await?; - - info!("Namespace: {} ,created", result.metadata.name.unwrap()); - Ok(()) - } else { - info!( - "Namespace: {} ,already exists", - consensus_namespace.unwrap().metadata.name.unwrap() - ); - Ok(()) - } -} - -/// Creates a namespace in k8s cluster -pub async fn create_or_reuse_service( - client: &Client, - name: &str, - node_name: &str, -) -> anyhow::Result<()> { - let services: Api = Api::namespaced(client.clone(), "consensus"); - let example_service = services.get_opt(name).await?; - if example_service.is_none() { - let service: Service = serde_json::from_value(json!({ - "apiVersion": "v1", - "kind": "Service", - "metadata": { - "name": name, - "namespace": "consensus", - "labels": { - "app": node_name - } - }, - "spec": { - "type": "NodePort", - "ports": [ - { - "port": 80, - "targetPort": 3154, - "protocol": "TCP", + match namespaces.get_opt(name).await? { + None => { + let namespace: Namespace = serde_json::from_value(json!({ + "apiVersion": "v1", + "kind": "Namespace", + "metadata": { + "name": "consensus", + "labels": { + "name": "consensus" + } } - ], - "selector": { - "app": node_name - }, - } - }))?; + }))?; - let services: Api = Api::namespaced(client.clone(), "consensus"); - let post_params = PostParams::default(); - let result = services.create(&post_params, &service).await?; + let namespaces: Api = Api::all(client.clone()); + let post_params = PostParams::default(); + let result = namespaces.create(&post_params, &namespace).await?; - info!("Service: {} ,created", result.metadata.name.unwrap()); - Ok(()) - } else { - info!( - "Service: {} ,already exists", - example_service.unwrap().metadata.name.unwrap() - ); - Ok(()) + info!( + "Namespace: {} ,created", + result + .metadata + .name + .context("Name not defined in metadata")? + ); + Ok(()) + } + Some(consensus_namespace) => { + info!( + "Namespace: {} ,already exists", + consensus_namespace + .metadata + .name + .context("Name not defined in metadata")? + ); + Ok(()) + } } } +/// Creates a deployment pub async fn create_deployment( client: &Client, node_number: usize, @@ -138,6 +105,14 @@ pub async fn create_deployment( { "name": "NODE_ID", "value": node_id + }, + { + "name": "PUBLIC_ADDR", + "valueFrom": { + "fieldRef": { + "fieldPath": "status.podIP" + } + } } ], "command": ["./k8s_entrypoint.sh"], @@ -145,8 +120,10 @@ pub async fn create_deployment( "imagePullPolicy": "Never", "ports": [ { - "containerPort": 3154, - "protocol": "TCP" + "containerPort": 3054 + }, + { + "containerPort": 3154 } ], "livenessProbe": { @@ -180,21 +157,58 @@ pub async fn create_deployment( // let pf = pods.portforward(&a.metadata.name.unwrap(), &[3150, 3154]).await; // println!("Portforward: {:?}", pf.is_ok()); - info!("Deployment: {} , created", result.metadata.name.unwrap()); + info!( + "Deployment: {} , created", + result + .metadata + .name + .context("Name not defined in metadata")? + ); Ok(()) } -/// Returns a HashMap with mapping: node_name -> IP address -pub async fn get_seed_node_addrs(client: &Client) -> HashMap { +/// Returns a HashMap with mapping: node_id -> IP address +pub async fn get_seed_node_addrs( + client: &Client, + amount: usize, +) -> anyhow::Result> { let mut seed_nodes = HashMap::new(); let pods: Api = Api::namespaced(client.clone(), "consensus"); - let lp = ListParams::default().labels("seed=true"); - for p in pods.list(&lp).await.unwrap() { + // Will retry 15 times during 15 seconds to allow pods to start and obtain an IP + let retry_strategy = FixedInterval::from_millis(1000).take(15); + let pod_list = Retry::spawn(retry_strategy, || get_seed_pods(&pods, amount)).await?; + + for p in pod_list { let node_id = p.labels()["id"].clone(); - seed_nodes.insert(node_id, p.status.unwrap().pod_ip.unwrap()); + seed_nodes.insert( + node_id, + p.status + .context("Status not present")? + .pod_ip + .context("Pod IP address not present")?, + ); } - seed_nodes + Ok(seed_nodes) +} + +async fn get_seed_pods(pods: &Api, amount: usize) -> anyhow::Result> { + let lp = ListParams::default().labels("seed=true"); + let p = pods.list(&lp).await?; + if p.items.len() == amount && p.iter().all(is_pod_running) { + Ok(p) + } else { + Err(anyhow!("Pods are not ready")) + } +} + +fn is_pod_running(pod: &Pod) -> bool { + if let Some(status) = &pod.status { + if let Some(phase) = &status.phase { + return phase == "Running"; + } + } + false } fn get_cli_args(peers: Vec) -> Vec { @@ -203,20 +217,14 @@ fn get_cli_args(peers: Vec) -> Vec { } else { [ "--add-gossip-static-outbound".to_string(), - encode_json( + config::encode_with_serializer( &peers .iter() .map(|e| Serde(e.clone())) .collect::>>(), + serde_json::Serializer::new(vec![]), ), ] .to_vec() } } - -/// Encodes a generated proto message to json for arbitrary ProtoFmt. -pub fn encode_json(x: &T) -> String { - let mut s = serde_json::Serializer::new(vec![]); - T::serialize(x, &mut s).unwrap(); - String::from_utf8(s.into_inner()).unwrap() -} diff --git a/node/tools/src/main.rs b/node/tools/src/main.rs index ff71e855..428c6c09 100644 --- a/node/tools/src/main.rs +++ b/node/tools/src/main.rs @@ -102,12 +102,15 @@ async fn main() -> anyhow::Result<()> { .load() .context("config_paths().load()")?; + // if `PUBLIC_ADDR` env var is set, use it to override publicAddr in config + configs.app.check_public_addr().context("Public Address")?; + // Add gossipStaticOutbound pairs from cli to config - if let Some(outbound) = args.add_gossip_static_outbound { + if let Some(addrs) = args.add_gossip_static_outbound { configs .app .gossip_static_outbound - .extend(outbound.0.into_iter().map(|e| (e.0.key, e.0.addr))); + .extend(addrs.0.into_iter().map(|e| (e.0.key, e.0.addr))); } let (executor, runner) = configs @@ -121,7 +124,10 @@ async fn main() -> anyhow::Result<()> { } else { rpc_addr.set_port(rpc_addr.port() + 100); } - let rpc_server = RPCServer::new(rpc_addr); + + // cloning configuration to let RPCServer show it + // TODO this should be queried in real time instead, to reflect any possible change in config + let rpc_server = RPCServer::new(rpc_addr, configs.app.clone()); // Initialize the storage. scope::run!(ctx, |ctx, s| async { diff --git a/node/tools/src/rpc/methods/config.rs b/node/tools/src/rpc/methods/config.rs new file mode 100644 index 00000000..0d22ec4d --- /dev/null +++ b/node/tools/src/rpc/methods/config.rs @@ -0,0 +1,47 @@ +//! Peers method for RPC server. +use crate::{config::encode_json, decode_json, AppConfig}; + +use super::RPCMethod; +use jsonrpsee::types::{error::ErrorCode, Params}; +use std::fs::{self}; +use zksync_protobuf::serde::Serde; + +/// Config method for RPC server. +pub(crate) struct ConfigInfo; + +// RPCMethod trait should be more general to allow external parameters like this case +// TODO fix the trait and implement this code in it +impl ConfigInfo { + pub(crate) fn info(config: AppConfig) -> Result { + // This may change in the future since we are assuming that the executor binary is being run inside the config directory. + Ok(serde_json::json!({ + "config": encode_json(&Serde(config)) + })) + } +} + +impl RPCMethod for ConfigInfo { + /// Config response for /config endpoint. + fn callback(_params: Params) -> Result { + // This may change in the future since we are assuming that the executor binary is being run inside the config directory. + let node_config = + fs::read_to_string("config.json").map_err(|_e| ErrorCode::InternalError)?; + let node_config = decode_json::>(&node_config) + .map_err(|_e| ErrorCode::InternalError)? + .0; + let config = encode_json(&Serde(node_config)); + Ok(serde_json::json!({ + "config": config + })) + } + + /// Config method name. + fn method() -> &'static str { + "config" + } + + /// Method path for GET requests. + fn path() -> &'static str { + "/config" + } +} diff --git a/node/tools/src/rpc/methods/mod.rs b/node/tools/src/rpc/methods/mod.rs index 7b5fdc5e..d1f9fa77 100644 --- a/node/tools/src/rpc/methods/mod.rs +++ b/node/tools/src/rpc/methods/mod.rs @@ -10,5 +10,6 @@ pub trait RPCMethod { fn path() -> &'static str; } -pub mod health_check; +pub(crate) mod config; +pub(crate) mod health_check; pub(crate) mod peers; diff --git a/node/tools/src/rpc/server.rs b/node/tools/src/rpc/server.rs index 6d2d5131..3b66dfa0 100644 --- a/node/tools/src/rpc/server.rs +++ b/node/tools/src/rpc/server.rs @@ -1,4 +1,6 @@ -use super::methods::{health_check::HealthCheck, peers::PeersInfo, RPCMethod}; +use crate::AppConfig; + +use super::methods::{config::ConfigInfo, health_check::HealthCheck, peers::PeersInfo, RPCMethod}; use jsonrpsee::server::{middleware::http::ProxyGetRequestLayer, RpcModule, Server}; use std::net::SocketAddr; use zksync_concurrency::{ctx, scope}; @@ -7,11 +9,13 @@ use zksync_concurrency::{ctx, scope}; pub struct RPCServer { /// IP address to bind to. ip_address: SocketAddr, + /// AppConfig + config: AppConfig, } impl RPCServer { - pub fn new(ip_address: SocketAddr) -> Self { - Self { ip_address } + pub fn new(ip_address: SocketAddr, config: AppConfig) -> Self { + Self { ip_address, config } } /// Runs the RPC server. @@ -26,6 +30,10 @@ impl RPCServer { .layer(ProxyGetRequestLayer::new( PeersInfo::path(), PeersInfo::method(), + )?) + .layer(ProxyGetRequestLayer::new( + ConfigInfo::path(), + ConfigInfo::method(), )?); let server = Server::builder() @@ -39,6 +47,12 @@ impl RPCServer { })?; module.register_method(PeersInfo::method(), |params, _| PeersInfo::callback(params))?; + // TODO find a better way to implement this as I had to clone the clone and move it to pass the borrow checker + let config = self.config.clone(); + module.register_method(ConfigInfo::method(), move |_params, _| { + ConfigInfo::info(config.clone()) + })?; + let handle = server.start(module); scope::run!(ctx, |ctx, s| async { s.spawn_bg(async {