From b204fb3411d55046e99b47f26c7ea359de0bf8b9 Mon Sep 17 00:00:00 2001 From: Nacho Avecilla Date: Mon, 5 Feb 2024 17:02:17 +0300 Subject: [PATCH 1/7] Test RPC layer for Standalone Node (#59) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## What ❔ Introduce an RPC layer for the Standalone node, including a test server featuring an endpoint for health-checking the node, and a CLI client for making requests to this server. ## Why ❔ This layer becomes necessary for communication with nodes within a cluster, especially for running tests that involve communication with these nodes. Additionally, it could serve as the initial implementation for a signaling or communication system within the test module, facilitating the determination of test success or failure. --------- Co-authored-by: Bruno França Co-authored-by: Grzegorz Prusak --- Dockerfile | 1 + compose.yaml | 4 + node/Cargo.lock | 351 ++++++++++++++++++--- node/Cargo.toml | 2 + node/deny.toml | 6 +- node/tools/Cargo.toml | 2 + node/tools/src/lib.rs | 2 + node/tools/src/main.rs | 14 +- node/tools/src/rpc/methods/health_check.rs | 23 ++ node/tools/src/rpc/methods/mod.rs | 14 + node/tools/src/rpc/methods/peers.rs | 41 +++ node/tools/src/rpc/mod.rs | 4 + node/tools/src/rpc/server.rs | 55 ++++ 13 files changed, 475 insertions(+), 44 deletions(-) create mode 100644 node/tools/src/rpc/methods/health_check.rs create mode 100644 node/tools/src/rpc/methods/mod.rs create mode 100644 node/tools/src/rpc/methods/peers.rs create mode 100644 node/tools/src/rpc/mod.rs create mode 100644 node/tools/src/rpc/server.rs diff --git a/Dockerfile b/Dockerfile index 3182b42e..bedfa5ae 100644 --- a/Dockerfile +++ b/Dockerfile @@ -20,3 +20,4 @@ RUN chmod +x docker-entrypoint.sh ENTRYPOINT ["./docker-entrypoint.sh"] EXPOSE 3054 +EXPOSE 3051 diff --git a/compose.yaml b/compose.yaml index 19924690..c62357cf 100644 --- a/compose.yaml +++ b/compose.yaml @@ -5,6 +5,8 @@ services: build: . image: consensus-node container_name: consensus-node-1 + ports: + - "3154:3154" networks: node_net: # This allow us to know the ip of the node-1 container to fill the address in the config file @@ -13,6 +15,8 @@ services: node-2: image: consensus-node container_name: consensus-node-2 + ports: + - "3155:3154" networks: node_net: # This allow us to know the ip of the node-2 container to fill the address in the config file diff --git a/node/Cargo.lock b/node/Cargo.lock index fb06aafa..e0ca823a 100644 --- a/node/Cargo.lock +++ b/node/Cargo.lock @@ -83,9 +83,9 @@ dependencies = [ [[package]] name = "anstyle" -version = "1.0.4" +version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7079075b41f533b8c61d2a4d073c4676e1f8b249ff94a393b0595db304e0dd87" +checksum = "2faccea4cc4ab4a667ce676a30e8ec13922a692c99bb8f5b11f1502c72e04220" [[package]] name = "anstyle-parse" @@ -159,6 +159,12 @@ dependencies = [ "rustc-demangle", ] +[[package]] +name = "base64" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e1b586273c5702936fe7b7d6896644d8be71e6314cfe09d3167c95f712589e8" + [[package]] name = "base64" version = "0.21.7" @@ -176,6 +182,9 @@ name = "beef" version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3a8241f3ebb85c056b509d4327ad0358fbbba6ffb340bf388f26350aeda225b1" +dependencies = [ + "serde", +] [[package]] name = "bindgen" @@ -231,7 +240,16 @@ version = "0.10.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "46502ad458c9a52b69d4d4d32775c788b7a1b85e8bc9d482d92250fc0e3f8efe" dependencies = [ - "digest", + "digest 0.10.7", +] + +[[package]] +name = "block-buffer" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4152116fd6e9dadb291ae18fc1ec3575ed6d84c29642d97890f4b4a3417297e4" +dependencies = [ + "generic-array", ] [[package]] @@ -341,9 +359,9 @@ dependencies = [ [[package]] name = "ciborium" -version = "0.2.1" +version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "effd91f6c78e5a4ace8a5d3c0b6bfaec9e2baaef55f3efc00e45fb2e477ee926" +checksum = "42e69ffd6f0917f5c029256a24d0161db17cea3997d185db0d35926308770f0e" dependencies = [ "ciborium-io", "ciborium-ll", @@ -352,15 +370,15 @@ dependencies = [ [[package]] name = "ciborium-io" -version = "0.2.1" +version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cdf919175532b369853f5d5e20b26b43112613fd6fe7aee757e35f7a44642656" +checksum = "05afea1e0a06c9be33d539b876f1ce3692f4afea2cb41f740e7743225ed1c757" [[package]] name = "ciborium-ll" -version = "0.2.1" +version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "defaa24ecc093c77630e6c15e17c51f5e187bf35ee514f4e2d67baaa96dae22b" +checksum = "57663b653d948a338bfb3eeba9bb2fd5fcfaecb9e199e87e1eda4d9e8b240fd9" dependencies = [ "ciborium-io", "half", @@ -515,6 +533,12 @@ version = "0.8.19" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "248e3bacc7dc6baa3b21e405ee045c3047101a49145e7e9eca583ab4c2ca5345" +[[package]] +name = "crunchy" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a81dae078cea95a014a339291cec439d2f232ebe854a9d672b796c6afafa9b7" + [[package]] name = "crypto-common" version = "0.1.6" @@ -544,7 +568,7 @@ dependencies = [ "cfg-if", "cpufeatures", "curve25519-dalek-derive", - "digest", + "digest 0.10.7", "fiat-crypto", "platforms", "rustc_version", @@ -588,13 +612,22 @@ version = "0.1.13" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "56254986775e3233ffa9c4d7d3faaf6d36a2c09d30b20687e9f88bc8bafc16c8" +[[package]] +name = "digest" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3dd60d1080a57a05ab032377049e0591415d2b31afd7028356dbf3cc6dcb066" +dependencies = [ + "generic-array", +] + [[package]] name = "digest" version = "0.10.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" dependencies = [ - "block-buffer", + "block-buffer 0.10.4", "crypto-common", "subtle", ] @@ -719,6 +752,20 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a06f77d526c1a601b7c4cdd98f54b5eaabffc14d5f2f0296febdc7f357c6d3ba" +[[package]] +name = "futures" +version = "0.3.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "645c6916888f6cb6350d2550b80fb63e734897a8498abe35cfb732b6487804b0" +dependencies = [ + "futures-channel", + "futures-core", + "futures-io", + "futures-sink", + "futures-task", + "futures-util", +] + [[package]] name = "futures-channel" version = "0.3.30" @@ -726,6 +773,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "eac8f7d7865dcb88bd4373ab671c8cf4508703796caa2b1985a9ca867b3fcb78" dependencies = [ "futures-core", + "futures-sink", ] [[package]] @@ -734,6 +782,29 @@ version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dfc6580bb841c5a68e9ef15c77ccc837b40a7504914d52e47b8b0e9bbda25a1d" +[[package]] +name = "futures-io" +version = "0.3.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a44623e20b9681a318efdd71c299b6b222ed6f231972bfe2f224ebad6311f0c1" + +[[package]] +name = "futures-macro" +version = "0.3.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87750cf4b7a4c0625b1529e4c543c2182106e4dedc60a2a6455e00d212c489ac" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.48", +] + +[[package]] +name = "futures-sink" +version = "0.3.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9fb8e00e87438d937621c1c6269e53f536c14d3fbd6a042bb24879e57d474fb5" + [[package]] name = "futures-task" version = "0.3.30" @@ -746,10 +817,16 @@ version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3d6401deb83407ab3da39eba7e33987a73c3df0c82b4bb5813ee871c19c41d48" dependencies = [ + "futures-channel", "futures-core", + "futures-io", + "futures-macro", + "futures-sink", "futures-task", + "memchr", "pin-project-lite", "pin-utils", + "slab", ] [[package]] @@ -795,11 +872,34 @@ version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d2fabcfbdc87f4758337ca535fb41a6d701b65693ce38287d856d1674551ec9b" +[[package]] +name = "h2" +version = "0.3.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb2c4422095b67ee78da96fbb51a4cc413b3b25883c7717ff7ca1ab31022c9c9" +dependencies = [ + "bytes", + "fnv", + "futures-core", + "futures-sink", + "futures-util", + "http", + "indexmap", + "slab", + "tokio", + "tokio-util", + "tracing", +] + [[package]] name = "half" -version = "1.8.2" +version = "2.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eabb4a44450da02c90444cf74558da904edde8fb4e9035a9a6a4e15445af0bd7" +checksum = "bc52e53916c08643f1b56ec082790d1e86a32e58dc5268f897f313fbae7b4872" +dependencies = [ + "cfg-if", + "crunchy", +] [[package]] name = "hashbrown" @@ -878,6 +978,7 @@ dependencies = [ "futures-channel", "futures-core", "futures-util", + "h2", "http", "http-body", "httparse", @@ -907,9 +1008,9 @@ dependencies = [ [[package]] name = "indexmap" -version = "2.1.0" +version = "2.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d530e1a18b1cb4c484e6e34556a0d948706958449fca0cab753d649f2bce3d1f" +checksum = "824b2ae422412366ba479e8111fd301f7b5faece8149317bb81925979a53f520" dependencies = [ "equivalent", "hashbrown", @@ -977,6 +1078,77 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "jsonrpsee" +version = "0.21.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9579d0ca9fb30da026bac2f0f7d9576ec93489aeb7cd4971dd5b4617d82c79b2" +dependencies = [ + "jsonrpsee-core", + "jsonrpsee-server", + "jsonrpsee-types", + "tokio", +] + +[[package]] +name = "jsonrpsee-core" +version = "0.21.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "776d009e2f591b78c038e0d053a796f94575d66ca4e77dd84bfc5e81419e436c" +dependencies = [ + "anyhow", + "async-trait", + "beef", + "futures-util", + "hyper", + "jsonrpsee-types", + "parking_lot", + "rand 0.8.5", + "rustc-hash", + "serde", + "serde_json", + "thiserror", + "tokio", + "tracing", +] + +[[package]] +name = "jsonrpsee-server" +version = "0.21.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5cc7c6d1a2c58f6135810284a390d9f823d0f508db74cd914d8237802de80f98" +dependencies = [ + "futures-util", + "http", + "hyper", + "jsonrpsee-core", + "jsonrpsee-types", + "pin-project", + "route-recognizer", + "serde", + "serde_json", + "soketto", + "thiserror", + "tokio", + "tokio-stream", + "tokio-util", + "tower", + "tracing", +] + +[[package]] +name = "jsonrpsee-types" +version = "0.21.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3266dfb045c9174b24c77c2dfe0084914bb23a6b2597d70c9dc6018392e1cd1b" +dependencies = [ + "anyhow", + "beef", + "serde", + "serde_json", + "thiserror", +] + [[package]] name = "keccak" version = "0.1.5" @@ -1000,9 +1172,9 @@ checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55" [[package]] name = "libc" -version = "0.2.152" +version = "0.2.153" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "13e3bf6590cbc649f4d1a3eefc9d5d6eb746f5200ffb04e5e142700b8faa56e7" +checksum = "9c198f91728a82281a64e1f4f9eeb25d82cb32a5de251c6bd1b5154d63a8e7bd" [[package]] name = "libloading" @@ -1032,9 +1204,9 @@ dependencies = [ [[package]] name = "libz-sys" -version = "1.1.14" +version = "1.1.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "295c17e837573c8c821dbaeb3cceb3d745ad082f7572191409e69cbc1b3fd050" +checksum = "037731f5d3aaa87a5675e895b63ddff1a87624bc29f77004ea829809654e48f6" dependencies = [ "cc", "pkg-config", @@ -1226,6 +1398,12 @@ dependencies = [ "num-traits", ] +[[package]] +name = "num-conv" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "51d515d32fb182ee37cda2ccdcb92950d6a3c2893aa280e540671c2cd0f3b1d9" + [[package]] name = "num-integer" version = "0.1.45" @@ -1350,18 +1528,18 @@ dependencies = [ [[package]] name = "pin-project" -version = "1.1.3" +version = "1.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fda4ed1c6c173e3fc7a83629421152e01d7b1f9b7f65fb301e490e8cfc656422" +checksum = "0302c4a0442c456bd56f841aee5c3bfd17967563f6fadc9ceb9f9c23cf3807e0" dependencies = [ "pin-project-internal", ] [[package]] name = "pin-project-internal" -version = "1.1.3" +version = "1.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4359fd9c9171ec6e8c62926d6faaf553a8dc3f64e1507e76da7911b4f6a04405" +checksum = "266c042b60c9c76b8d53061e52b2e0d1116abc57cefc8c5cd671619a56ac3690" dependencies = [ "proc-macro2", "quote", @@ -1568,7 +1746,7 @@ version = "0.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "057237efdb71cf4b3f9396302a3d6599a92fa94063ba537b66130980ea9909f3" dependencies = [ - "base64", + "base64 0.21.7", "logos", "miette", "once_cell", @@ -1745,7 +1923,7 @@ checksum = "b62dbe01f0b06f9d8dc7d49e05a0785f153b00b2c227856282f671e0318c9b15" dependencies = [ "aho-corasick", "memchr", - "regex-automata 0.4.4", + "regex-automata 0.4.5", "regex-syntax 0.8.2", ] @@ -1760,9 +1938,9 @@ dependencies = [ [[package]] name = "regex-automata" -version = "0.4.4" +version = "0.4.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3b7fa1134405e2ec9353fd416b17f8dacd46c473d7d3fd1cf202706a14eb792a" +checksum = "5bb987efffd3c6d0d8f5f89510bb458559eab11e4f869acb20bf845e016259cd" dependencies = [ "aho-corasick", "memchr", @@ -1791,6 +1969,12 @@ dependencies = [ "librocksdb-sys", ] +[[package]] +name = "route-recognizer" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "afab94fb28594581f62d981211a9a4d53cc8130bbcbbb89a0440d9b8e81a7746" + [[package]] name = "rustc-demangle" version = "0.1.23" @@ -1814,9 +1998,9 @@ dependencies = [ [[package]] name = "rustix" -version = "0.38.30" +version = "0.38.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "322394588aaf33c24007e8bb3238ee3e4c5c09c084ab32bc73890b99ff326bca" +checksum = "6ea3e1a662af26cd7a3ba09c0297a31af215563ecf42817c98df621387f4e949" dependencies = [ "bitflags 2.4.2", "errno", @@ -1854,9 +2038,9 @@ checksum = "b97ed7a9823b74f99c7742f5336af7be5ecd3eeafcb1507d1fa93347b1d589b0" [[package]] name = "serde" -version = "1.0.195" +version = "1.0.196" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "63261df402c67811e9ac6def069e4786148c4563f4b50fd4bf30aa370d626b02" +checksum = "870026e60fa08c69f064aa766c10f10b1d62db9ccd4d0abb206472bee0ce3b32" dependencies = [ "serde_derive", ] @@ -1873,9 +2057,9 @@ dependencies = [ [[package]] name = "serde_derive" -version = "1.0.195" +version = "1.0.196" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "46fe8f8603d81ba86327b23a2e9cdf49e1255fb94a4c5f297f6ee0547178ea2c" +checksum = "33c85360c95e7d137454dc81d9a4ed2b8efd8fbe19cee57357b32b9771fccb67" dependencies = [ "proc-macro2", "quote", @@ -1884,15 +2068,28 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.111" +version = "1.0.113" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "176e46fa42316f18edd598015a5166857fc835ec732f5215eac6b7bdbf0a84f4" +checksum = "69801b70b1c3dac963ecb03a364ba0ceda9cf60c71cfe475e99864759c8b8a79" dependencies = [ "itoa", "ryu", "serde", ] +[[package]] +name = "sha-1" +version = "0.9.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "99cd6713db3cf16b6c84e06321e049a9b9f699826e16096d23bbcc44d15d51a6" +dependencies = [ + "block-buffer 0.9.0", + "cfg-if", + "cpufeatures", + "digest 0.9.0", + "opaque-debug", +] + [[package]] name = "sha2" version = "0.10.8" @@ -1901,7 +2098,7 @@ checksum = "793db75ad2bcafc3ffa7c68b215fee268f537982cd901d132f89c6343f3a3dc8" dependencies = [ "cfg-if", "cpufeatures", - "digest", + "digest 0.10.7", ] [[package]] @@ -1910,7 +2107,7 @@ version = "0.10.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "75872d278a8f37ef87fa0ddbda7802605cb18344497949862c0d4dcb291eba60" dependencies = [ - "digest", + "digest 0.10.7", "keccak", ] @@ -1957,6 +2154,15 @@ dependencies = [ "typenum", ] +[[package]] +name = "slab" +version = "0.4.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f92a496fb766b417c996b9c5e57daf2f7ad3b0bebe1ccfca4856390e3d3bb67" +dependencies = [ + "autocfg", +] + [[package]] name = "smallvec" version = "1.13.1" @@ -1965,9 +2171,9 @@ checksum = "e6ecd384b10a64542d77071bd64bd7b231f4ed5940fba55e98c3de13824cf3d7" [[package]] name = "snow" -version = "0.9.4" +version = "0.9.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "58021967fd0a5eeeb23b08df6cc244a4d4a5b4aec1d27c9e02fad1a58b4cd74e" +checksum = "850948bee068e713b8ab860fe1adc4d109676ab4c3b621fd8147f06b261f2f85" dependencies = [ "aes-gcm", "blake2", @@ -1989,6 +2195,22 @@ dependencies = [ "windows-sys 0.48.0", ] +[[package]] +name = "soketto" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41d1c5305e39e09653383c2c7244f2f78b3bcae37cf50c64cb4789c9f5096ec2" +dependencies = [ + "base64 0.13.1", + "bytes", + "futures", + "http", + "httparse", + "log", + "rand 0.8.5", + "sha-1", +] + [[package]] name = "spki" version = "0.7.3" @@ -2113,11 +2335,12 @@ dependencies = [ [[package]] name = "time" -version = "0.3.31" +version = "0.3.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f657ba42c3f86e7680e53c8cd3af8abbe56b5491790b46e22e19c0d57463583e" +checksum = "fe80ced77cbfb4cb91a94bf72b378b4b6791a0d9b7f09d0be747d1bdff4e68bd" dependencies = [ "deranged", + "num-conv", "powerfmt", "serde", "time-core", @@ -2169,6 +2392,49 @@ dependencies = [ "syn 2.0.48", ] +[[package]] +name = "tokio-stream" +version = "0.1.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "397c988d37662c7dda6d2208364a706264bf3d6138b11d436cbac0ad38832842" +dependencies = [ + "futures-core", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "tokio-util" +version = "0.7.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5419f34732d9eb6ee4c3578b7989078579b7f039cbbb9ca2c4da015749371e15" +dependencies = [ + "bytes", + "futures-core", + "futures-io", + "futures-sink", + "pin-project-lite", + "tokio", + "tracing", +] + +[[package]] +name = "tower" +version = "0.4.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8fa9be0de6cf49e536ce1851f987bd21a43b771b09473c3549a6c853db37c1c" +dependencies = [ + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "tower-layer" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c20c8dbed6283a09604c3e69b4b7eeb54e298b8a600d4d5ecb5ad39de609f1d0" + [[package]] name = "tower-service" version = "0.3.2" @@ -2181,6 +2447,7 @@ version = "0.1.40" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c3523ab5a71916ccf420eebdf5521fcef02141234bbc0b8a49f2fdc4544364ef" dependencies = [ + "log", "pin-project-lite", "tracing-attributes", "tracing-core", @@ -2792,6 +3059,7 @@ dependencies = [ "anyhow", "async-trait", "clap", + "jsonrpsee", "prost", "rand 0.8.5", "rocksdb", @@ -2799,6 +3067,7 @@ dependencies = [ "serde_json", "tempfile", "tokio", + "tower", "tracing", "tracing-subscriber", "vise-exporter", diff --git a/node/Cargo.toml b/node/Cargo.toml index bf07a0f6..df0a7b33 100644 --- a/node/Cargo.toml +++ b/node/Cargo.toml @@ -82,6 +82,8 @@ time = "0.3.23" tokio = { version = "1.34.0", features = ["full"] } tracing = { version = "0.1.37", features = ["attributes"] } tracing-subscriber = { version = "0.3.16", features = ["env-filter", "fmt"] } +jsonrpsee = { version = "0.21.0", features = ["server"] } +tower = { version = "0.4.13" } # Note that "bench" profile inherits from "release" profile and # "test" profile inherits from "dev" profile. diff --git a/node/deny.toml b/node/deny.toml index f8535b33..37f6ede5 100644 --- a/node/deny.toml +++ b/node/deny.toml @@ -56,8 +56,10 @@ skip = [ { name = "rand", version = "0.4" }, { name = "syn", version = "1.0" }, - # Old versions required by criterion. - { name = "itertools", version = "0.10.5" }, + # Old versions required by jsonrpsee. + { name = "base64", version = "0.13.1" }, + { name = "block-buffer", version = "0.9.0" }, + { name = "digest", version = "0.10.7" }, ] [sources] diff --git a/node/tools/Cargo.toml b/node/tools/Cargo.toml index 77eb3923..f9b5adb7 100644 --- a/node/tools/Cargo.toml +++ b/node/tools/Cargo.toml @@ -29,6 +29,8 @@ tokio.workspace = true tracing.workspace = true tracing-subscriber.workspace = true vise-exporter.workspace = true +jsonrpsee.workspace = true +tower.workspace = true [dev-dependencies] tempfile.workspace = true diff --git a/node/tools/src/lib.rs b/node/tools/src/lib.rs index 289a729a..f84a00f8 100644 --- a/node/tools/src/lib.rs +++ b/node/tools/src/lib.rs @@ -2,9 +2,11 @@ #![allow(missing_docs)] mod config; mod proto; +mod rpc; mod store; #[cfg(test)] mod tests; pub use config::{decode_json, AppConfig, ConfigPaths, NodeAddr}; +pub use rpc::server::RPCServer; diff --git a/node/tools/src/main.rs b/node/tools/src/main.rs index dfef5fe5..675932ac 100644 --- a/node/tools/src/main.rs +++ b/node/tools/src/main.rs @@ -7,7 +7,7 @@ use tracing::metadata::LevelFilter; use tracing_subscriber::{prelude::*, Registry}; use vise_exporter::MetricsExporter; use zksync_concurrency::{ctx, scope}; -use zksync_consensus_tools::{decode_json, ConfigPaths, NodeAddr}; +use zksync_consensus_tools::{decode_json, ConfigPaths, NodeAddr, RPCServer}; use zksync_consensus_utils::no_copy::NoCopy; use zksync_protobuf::serde::Serde; @@ -38,6 +38,9 @@ struct Args { /// Path to the rocksdb database of the node. #[arg(long, default_value = "./database")] database: PathBuf, + /// Port for the RPC server. + #[arg(long)] + rpc_port: Option, /// IP address and key of the seed peers. #[arg(long)] add_gossip_static_outbound: NodeAddrs, @@ -112,6 +115,14 @@ async fn main() -> anyhow::Result<()> { .await .context("configs.into_executor()")?; + let mut rpc_addr = configs.app.public_addr; + if let Some(port) = args.rpc_port { + rpc_addr.set_port(port); + } else { + rpc_addr.set_port(rpc_addr.port() + 100); + } + let rpc_server = RPCServer::new(rpc_addr); + // Initialize the storage. scope::run!(ctx, |ctx, s| async { if let Some(addr) = configs.app.metrics_server_addr { @@ -127,6 +138,7 @@ async fn main() -> anyhow::Result<()> { } s.spawn_bg(runner.run(ctx)); s.spawn(executor.run(ctx)); + s.spawn(rpc_server.run(ctx)); Ok(()) }) .await diff --git a/node/tools/src/rpc/methods/health_check.rs b/node/tools/src/rpc/methods/health_check.rs new file mode 100644 index 00000000..8d32107a --- /dev/null +++ b/node/tools/src/rpc/methods/health_check.rs @@ -0,0 +1,23 @@ +//! Health check method for RPC server. +use super::RPCMethod; +use jsonrpsee::types::{error::ErrorCode, Params}; + +/// Health check method for RPC server. +pub(crate) struct HealthCheck; + +impl RPCMethod for HealthCheck { + /// Health check response for /health endpoint. + fn callback(_params: Params) -> Result { + Ok(serde_json::json!({"health": true})) + } + + /// Health check method name. + fn method() -> &'static str { + "health_check" + } + + /// Method path for GET requests. + fn path() -> &'static str { + "/health" + } +} diff --git a/node/tools/src/rpc/methods/mod.rs b/node/tools/src/rpc/methods/mod.rs new file mode 100644 index 00000000..2cac6a52 --- /dev/null +++ b/node/tools/src/rpc/methods/mod.rs @@ -0,0 +1,14 @@ +use jsonrpsee::types::{error::ErrorCode, Params}; + +/// Trait to implement for new RPC methods. +pub(crate) trait RPCMethod { + /// Method response logic when called. + fn callback(params: Params) -> Result; + /// Method name. + fn method() -> &'static str; + /// Method path for GET requests. + fn path() -> &'static str; +} + +pub(crate) mod health_check; +pub(crate) mod peers; diff --git a/node/tools/src/rpc/methods/peers.rs b/node/tools/src/rpc/methods/peers.rs new file mode 100644 index 00000000..35f974d4 --- /dev/null +++ b/node/tools/src/rpc/methods/peers.rs @@ -0,0 +1,41 @@ +//! Peers method for RPC server. +use crate::{decode_json, AppConfig}; + +use super::RPCMethod; +use jsonrpsee::types::{error::ErrorCode, Params}; +use std::fs::{self}; +use zksync_consensus_crypto::TextFmt; +use zksync_protobuf::serde::Serde; + +/// Peers method for RPC server. +pub(crate) struct PeersInfo; + +impl RPCMethod for PeersInfo { + /// Peers response for /peers endpoint. + fn callback(_params: Params) -> Result { + // This may change in the future since we are assuming that the executor binary is being run inside the config directory. + let node_config = + fs::read_to_string("config.json").map_err(|_e| ErrorCode::InternalError)?; + let node_config = decode_json::>(&node_config) + .map_err(|_e| ErrorCode::InternalError)? + .0; + let peers: Vec = node_config + .gossip_static_inbound + .iter() + .map(|x| x.encode()) + .collect(); + Ok(serde_json::json!({ + "peers": peers + })) + } + + /// Peers method name. + fn method() -> &'static str { + "peers" + } + + /// Method path for GET requests. + fn path() -> &'static str { + "/peers" + } +} diff --git a/node/tools/src/rpc/mod.rs b/node/tools/src/rpc/mod.rs new file mode 100644 index 00000000..89bd9b90 --- /dev/null +++ b/node/tools/src/rpc/mod.rs @@ -0,0 +1,4 @@ +//! RPC server for testing purposes. +mod methods; +/// Module for the RPC server implementation. +pub(crate) mod server; diff --git a/node/tools/src/rpc/server.rs b/node/tools/src/rpc/server.rs new file mode 100644 index 00000000..6d2d5131 --- /dev/null +++ b/node/tools/src/rpc/server.rs @@ -0,0 +1,55 @@ +use super::methods::{health_check::HealthCheck, peers::PeersInfo, RPCMethod}; +use jsonrpsee::server::{middleware::http::ProxyGetRequestLayer, RpcModule, Server}; +use std::net::SocketAddr; +use zksync_concurrency::{ctx, scope}; + +/// RPC server. +pub struct RPCServer { + /// IP address to bind to. + ip_address: SocketAddr, +} + +impl RPCServer { + pub fn new(ip_address: SocketAddr) -> Self { + Self { ip_address } + } + + /// Runs the RPC server. + pub async fn run(&self, ctx: &ctx::Ctx) -> anyhow::Result<()> { + // Custom tower service to handle the RPC requests + let service_builder = tower::ServiceBuilder::new() + // Proxy `GET /` requests to internal methods. + .layer(ProxyGetRequestLayer::new( + HealthCheck::path(), + HealthCheck::method(), + )?) + .layer(ProxyGetRequestLayer::new( + PeersInfo::path(), + PeersInfo::method(), + )?); + + let server = Server::builder() + .set_http_middleware(service_builder) + .build(self.ip_address) + .await?; + + let mut module = RpcModule::new(()); + module.register_method(HealthCheck::method(), |params, _| { + HealthCheck::callback(params) + })?; + module.register_method(PeersInfo::method(), |params, _| PeersInfo::callback(params))?; + + let handle = server.start(module); + scope::run!(ctx, |ctx, s| async { + s.spawn_bg(async { + ctx.canceled().await; + // Ignore `AlreadyStoppedError`. + let _ = handle.stop(); + Ok(()) + }); + handle.clone().stopped().await; + Ok(()) + }) + .await + } +} From d61288950085d97d87464f6a2b252689fbb9a382 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Esteban=20Dimitroff=20H=C3=B3di?= Date: Wed, 7 Feb 2024 18:43:22 -0300 Subject: [PATCH 2/7] Busy waiting for pods to start to obtain their IPs --- node/Cargo.lock | 12 ++++ node/Cargo.toml | 3 +- node/tools/Cargo.toml | 1 + node/tools/src/bin/deployer.rs | 8 +-- node/tools/src/k8s.rs | 122 +++++++++++++++++++++++---------- 5 files changed, 101 insertions(+), 45 deletions(-) diff --git a/node/Cargo.lock b/node/Cargo.lock index 5f42b1c4..c6c86e46 100644 --- a/node/Cargo.lock +++ b/node/Cargo.lock @@ -3005,6 +3005,17 @@ dependencies = [ "syn 2.0.48", ] +[[package]] +name = "tokio-retry" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f57eb36ecbe0fc510036adff84824dd3c24bb781e21bfa67b69d556aa85214f" +dependencies = [ + "pin-project", + "rand 0.8.5", + "tokio", +] + [[package]] name = "tokio-rustls" version = "0.24.1" @@ -3776,6 +3787,7 @@ dependencies = [ "serde_json", "tempfile", "tokio", + "tokio-retry", "tower", "tracing", "tracing-subscriber", diff --git a/node/Cargo.toml b/node/Cargo.toml index 5791d77b..493a4727 100644 --- a/node/Cargo.toml +++ b/node/Cargo.toml @@ -80,12 +80,13 @@ test-casing = "0.1.0" thiserror = "1.0.40" time = "0.3.23" tokio = { version = "1.34.0", features = ["full"] } +tokio-retry = "0.3.0" tracing = { version = "0.1.37", features = ["attributes"] } tracing-subscriber = { version = "0.3.16", features = ["env-filter", "fmt"] } kube = { version = "0.88.1", features = ["runtime", "derive"] } k8s-openapi = { version = "0.21.0", features = ["latest"] } jsonrpsee = { version = "0.21.0", features = ["server"] } -tower = { version = "0.4.13" } +tower = "0.4.13" # Note that "bench" profile inherits from "release" profile and # "test" profile inherits from "dev" profile. diff --git a/node/tools/Cargo.toml b/node/tools/Cargo.toml index 3546e97c..93e80fa4 100644 --- a/node/tools/Cargo.toml +++ b/node/tools/Cargo.toml @@ -26,6 +26,7 @@ rocksdb.workspace = true serde.workspace = true serde_json.workspace = true tokio.workspace = true +tokio-retry.workspace = true tracing.workspace = true tracing-subscriber.workspace = true vise-exporter.workspace = true diff --git a/node/tools/src/bin/deployer.rs b/node/tools/src/bin/deployer.rs index b9d4d457..16f9234b 100644 --- a/node/tools/src/bin/deployer.rs +++ b/node/tools/src/bin/deployer.rs @@ -64,7 +64,7 @@ fn generate_config(nodes: usize) -> anyhow::Result<()> { } } - let manifest_path = std::env::var("CARGO_MANIFEST_DIR").unwrap(); + let manifest_path = std::env::var("CARGO_MANIFEST_DIR")?; let root = PathBuf::from(manifest_path).join("k8s_configs"); let _ = fs::remove_dir_all(&root); for (i, cfg) in cfgs.into_iter().enumerate() { @@ -108,12 +108,8 @@ async fn deploy(nodes: usize) -> anyhow::Result<()> { .await?; } - // Waiting 15 secs to allow the pods to start - // TODO: should replace with some safer method - tokio::time::sleep(tokio::time::Duration::from_secs(15)).await; - // obtain seed peer(s) IP(s) - let peer_ips = k8s::get_seed_node_addrs(&client).await; + let peer_ips = k8s::get_seed_node_addrs(&client, seed_nodes).await?; let mut peers = vec![]; diff --git a/node/tools/src/k8s.rs b/node/tools/src/k8s.rs index 5af29820..e16bdf43 100644 --- a/node/tools/src/k8s.rs +++ b/node/tools/src/k8s.rs @@ -1,14 +1,18 @@ -use crate::NodeAddr; +use crate::{config, NodeAddr}; +use anyhow::{anyhow, Context}; use k8s_openapi::api::{ apps::v1::Deployment, core::v1::{Namespace, Pod}, }; use kube::{ api::{ListParams, PostParams}, + core::ObjectList, Api, Client, ResourceExt, }; use serde_json::json; use std::collections::HashMap; +use tokio_retry::strategy::FixedInterval; +use tokio_retry::Retry; use tracing::log::info; use zksync_protobuf::serde::Serde; @@ -20,34 +24,46 @@ pub async fn get_client() -> anyhow::Result { /// Creates a namespace in k8s cluster pub async fn create_or_reuse_namespace(client: &Client, name: &str) -> anyhow::Result<()> { let namespaces: Api = Api::all(client.clone()); - let consensus_namespace = namespaces.get_opt(name).await?; - if consensus_namespace.is_none() { - let namespace: Namespace = serde_json::from_value(json!({ - "apiVersion": "v1", - "kind": "Namespace", - "metadata": { - "name": "consensus", - "labels": { - "name": "consensus" + match namespaces.get_opt(name).await? { + None => { + let namespace: Namespace = serde_json::from_value(json!({ + "apiVersion": "v1", + "kind": "Namespace", + "metadata": { + "name": "consensus", + "labels": { + "name": "consensus" + } } - } - }))?; + }))?; - let namespaces: Api = Api::all(client.clone()); - let post_params = PostParams::default(); - let result = namespaces.create(&post_params, &namespace).await?; + let namespaces: Api = Api::all(client.clone()); + let post_params = PostParams::default(); + let result = namespaces.create(&post_params, &namespace).await?; - info!("Namespace: {} ,created", result.metadata.name.unwrap()); - Ok(()) - } else { - info!( - "Namespace: {} ,already exists", - consensus_namespace.unwrap().metadata.name.unwrap() - ); - Ok(()) + info!( + "Namespace: {} ,created", + result + .metadata + .name + .context("Name not defined in metadata")? + ); + Ok(()) + } + Some(consensus_namespace) => { + info!( + "Namespace: {} ,already exists", + consensus_namespace + .metadata + .name + .context("Name not defined in metadata")? + ); + Ok(()) + } } } +/// Creates a deployment pub async fn create_deployment( client: &Client, node_number: usize, @@ -122,21 +138,58 @@ pub async fn create_deployment( let post_params = PostParams::default(); let result = deployments.create(&post_params, &deployment).await?; - info!("Deployment: {} , created", result.metadata.name.unwrap()); + info!( + "Deployment: {} , created", + result + .metadata + .name + .context("Name not defined in metadata")? + ); Ok(()) } -/// Returns a HashMap with mapping: node_name -> IP address -pub async fn get_seed_node_addrs(client: &Client) -> HashMap { +/// Returns a HashMap with mapping: node_id -> IP address +pub async fn get_seed_node_addrs( + client: &Client, + amount: usize, +) -> anyhow::Result> { let mut seed_nodes = HashMap::new(); let pods: Api = Api::namespaced(client.clone(), "consensus"); - let lp = ListParams::default().labels("seed=true"); - for p in pods.list(&lp).await.unwrap() { + // Will retry 15 times during 15 seconds to allow pods to start and obtain an IP + let retry_strategy = FixedInterval::from_millis(1000).take(15); + let pod_list = Retry::spawn(retry_strategy, || get_seed_pods(&pods, amount)).await?; + + for p in pod_list { let node_id = p.labels()["id"].clone(); - seed_nodes.insert(node_id, p.status.unwrap().pod_ip.unwrap()); + seed_nodes.insert( + node_id, + p.status + .context("Status not present")? + .pod_ip + .context("Pod IP address not present")?, + ); } - seed_nodes + Ok(seed_nodes) +} + +async fn get_seed_pods(pods: &Api, amount: usize) -> anyhow::Result> { + let lp = ListParams::default().labels("seed=true"); + let p = pods.list(&lp).await?; + if p.items.len() == amount && p.iter().all(is_pod_running) { + Ok(p) + } else { + Err(anyhow!("Pods are not ready")) + } +} + +fn is_pod_running(pod: &Pod) -> bool { + if let Some(status) = &pod.status { + if let Some(phase) = &status.phase { + return phase == "Running"; + } + } + false } fn get_cli_args(peers: Vec) -> Vec { @@ -145,7 +198,7 @@ fn get_cli_args(peers: Vec) -> Vec { } else { [ "--add-gossip-static-outbound".to_string(), - encode_json( + config::encode_json( &peers .iter() .map(|e| Serde(e.clone())) @@ -155,10 +208,3 @@ fn get_cli_args(peers: Vec) -> Vec { .to_vec() } } - -/// Encodes a generated proto message to json for arbitrary ProtoFmt. -pub fn encode_json(x: &T) -> String { - let mut s = serde_json::Serializer::new(vec![]); - T::serialize(x, &mut s).unwrap(); - String::from_utf8(s.into_inner()).unwrap() -} From 6bd539480b5566d73f20de4ccd72b58e1ac21045 Mon Sep 17 00:00:00 2001 From: IAvecilla Date: Thu, 8 Feb 2024 11:36:36 -0300 Subject: [PATCH 3/7] Revert change to have node ips as directory names for local config --- node/tools/src/bin/localnet_config.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/node/tools/src/bin/localnet_config.rs b/node/tools/src/bin/localnet_config.rs index 60a74749..ebe9ba1d 100644 --- a/node/tools/src/bin/localnet_config.rs +++ b/node/tools/src/bin/localnet_config.rs @@ -70,7 +70,7 @@ fn main() -> anyhow::Result<()> { for (i, cfg) in cfgs.into_iter().enumerate() { // Recreate the directory for the node's config. - let root = args.output_dir.join(format!("node_{}", i)); + let root = args.output_dir.join(cfg.public_addr.to_string()); let _ = fs::remove_dir_all(&root); fs::create_dir_all(&root).with_context(|| format!("create_dir_all({:?})", root))?; cfg.write_to_file(&root)?; From 1d584319b97f7ec55c8763fba841d19b3d078557 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Esteban=20Dimitroff=20H=C3=B3di?= Date: Fri, 9 Feb 2024 15:39:42 -0300 Subject: [PATCH 4/7] Making cli argument optional and forwarding arguments in entrypoint script --- docker-entrypoint.sh | 2 +- node/tools/src/config.rs | 15 ++++++++++++--- node/tools/src/k8s.rs | 10 +++++++--- node/tools/src/main.rs | 14 +++++++------- 4 files changed, 27 insertions(+), 14 deletions(-) diff --git a/docker-entrypoint.sh b/docker-entrypoint.sh index 07a3c781..6b8977c1 100755 --- a/docker-entrypoint.sh +++ b/docker-entrypoint.sh @@ -3,4 +3,4 @@ cd docker_config/${NODE_ID} export RUST_LOG=INFO -../../executor +../../executor $@ diff --git a/node/tools/src/config.rs b/node/tools/src/config.rs index 81b18bed..24e8f4cd 100644 --- a/node/tools/src/config.rs +++ b/node/tools/src/config.rs @@ -1,6 +1,7 @@ //! Node configuration. use crate::{proto, store}; use anyhow::Context as _; +use serde_json::{ser::Formatter, Serializer}; use std::{ collections::{HashMap, HashSet}, fs, @@ -31,10 +32,18 @@ pub fn decode_json(json: &str) -> anyhow::Result /// Encodes a generated proto message to json for arbitrary ProtoFmt. pub fn encode_json(x: &T) -> String { - let mut s = serde_json::Serializer::pretty(vec![]); - T::serialize(x, &mut s).unwrap(); - String::from_utf8(s.into_inner()).unwrap() + let s = serde_json::Serializer::pretty(vec![]); + encode_json_with_serializer(x, s) } + +pub fn encode_json_with_serializer( + x: &T, + mut serializer: Serializer, F>, +) -> String { + T::serialize(x, &mut serializer).unwrap(); + String::from_utf8(serializer.into_inner()).unwrap() +} + // pub fn encode_json(x: &T) -> String { // let mut s = serde_json::Serializer::pretty(vec![]); // zksync_protobuf::serde::serialize(x, &mut s).unwrap(); diff --git a/node/tools/src/k8s.rs b/node/tools/src/k8s.rs index e16bdf43..e03ee98f 100644 --- a/node/tools/src/k8s.rs +++ b/node/tools/src/k8s.rs @@ -113,18 +113,21 @@ pub async fn create_deployment( "ports": [ { "containerPort": 3054 + }, + { + "containerPort": 3154 } ], "livenessProbe": { "httpGet": { "path": "/health", - "port": 3054 + "port": 3154 } }, "readinessProbe": { "httpGet": { "path": "/health", - "port": 3054 + "port": 3154 } } } @@ -198,11 +201,12 @@ fn get_cli_args(peers: Vec) -> Vec { } else { [ "--add-gossip-static-outbound".to_string(), - config::encode_json( + config::encode_json_with_serializer( &peers .iter() .map(|e| Serde(e.clone())) .collect::>>(), + serde_json::Serializer::new(vec![]), ), ] .to_vec() diff --git a/node/tools/src/main.rs b/node/tools/src/main.rs index 675932ac..aad9a4cf 100644 --- a/node/tools/src/main.rs +++ b/node/tools/src/main.rs @@ -43,7 +43,7 @@ struct Args { rpc_port: Option, /// IP address and key of the seed peers. #[arg(long)] - add_gossip_static_outbound: NodeAddrs, + add_gossip_static_outbound: Option, } impl Args { @@ -103,12 +103,12 @@ async fn main() -> anyhow::Result<()> { .context("config_paths().load()")?; // Add gossipStaticOutbound pairs from cli to config - configs.app.gossip_static_outbound.extend( - args.add_gossip_static_outbound - .0 - .into_iter() - .map(|e| (e.0.key, e.0.addr)), - ); + if let Some(addrs) = args.add_gossip_static_outbound { + configs + .app + .gossip_static_outbound + .extend(addrs.0.into_iter().map(|e| (e.0.key, e.0.addr))); + } let (executor, runner) = configs .make_executor(ctx) From c864f51b60f01dd6019d5d61ac5ede4c8f446f74 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Esteban=20Dimitroff=20H=C3=B3di?= Date: Fri, 9 Feb 2024 18:36:48 -0300 Subject: [PATCH 5/7] Fixed compiler warnings --- node/tools/src/bin/deployer.rs | 4 ++-- node/tools/src/config.rs | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/node/tools/src/bin/deployer.rs b/node/tools/src/bin/deployer.rs index 16f9234b..fd1205cc 100644 --- a/node/tools/src/bin/deployer.rs +++ b/node/tools/src/bin/deployer.rs @@ -7,7 +7,7 @@ use anyhow::Context; use clap::{Parser, Subcommand}; use rand::Rng; use zksync_consensus_crypto::{Text, TextFmt}; -use zksync_consensus_roles::node::{self, SecretKey}; +use zksync_consensus_roles::node::SecretKey; use zksync_consensus_tools::k8s; use zksync_consensus_tools::AppConfig; use zksync_consensus_tools::NodeAddr; @@ -51,7 +51,7 @@ fn generate_config(nodes: usize) -> anyhow::Result<()> { // Generate the node keys for all the replicas. let rng = &mut rand::thread_rng(); - let node_keys: Vec = (0..nodes).map(|_| rng.gen()).collect(); + let node_keys: Vec = (0..nodes).map(|_| rng.gen()).collect(); let (default_config, validator_keys) = AppConfig::default_for(nodes); let mut cfgs: Vec<_> = (0..nodes).map(|_| default_config.clone()).collect(); diff --git a/node/tools/src/config.rs b/node/tools/src/config.rs index 24e8f4cd..5697febc 100644 --- a/node/tools/src/config.rs +++ b/node/tools/src/config.rs @@ -31,12 +31,12 @@ pub fn decode_json(json: &str) -> anyhow::Result } /// Encodes a generated proto message to json for arbitrary ProtoFmt. -pub fn encode_json(x: &T) -> String { +pub(crate) fn encode_json(x: &T) -> String { let s = serde_json::Serializer::pretty(vec![]); encode_json_with_serializer(x, s) } -pub fn encode_json_with_serializer( +pub(crate) fn encode_json_with_serializer( x: &T, mut serializer: Serializer, F>, ) -> String { From b5b5ee6ed29231648c3bbaf95bab5e2c51c169a4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Esteban=20Dimitroff=20H=C3=B3di?= Date: Wed, 14 Feb 2024 17:50:43 -0300 Subject: [PATCH 6/7] Consensus node now can obtain it's public address from an ENV VAR. Also added a endpoint to be able to check config from nodes --- node/tools/src/config.rs | 33 ++++++++++++++++++++----------- node/tools/src/k8s.rs | 10 +++++++++- node/tools/src/main.rs | 8 +++++++- node/tools/src/rpc/methods/mod.rs | 1 + node/tools/src/rpc/server.rs | 20 ++++++++++++++++--- 5 files changed, 56 insertions(+), 16 deletions(-) diff --git a/node/tools/src/config.rs b/node/tools/src/config.rs index 5697febc..69b603ed 100644 --- a/node/tools/src/config.rs +++ b/node/tools/src/config.rs @@ -2,6 +2,7 @@ use crate::{proto, store}; use anyhow::Context as _; use serde_json::{ser::Formatter, Serializer}; +use std::str::FromStr; use std::{ collections::{HashMap, HashSet}, fs, @@ -12,10 +13,7 @@ use zksync_concurrency::ctx; use zksync_consensus_bft as bft; use zksync_consensus_crypto::{read_optional_text, read_required_text, Text, TextFmt}; use zksync_consensus_executor as executor; -use zksync_consensus_roles::{ - node::{self, PublicKey}, - validator, -}; +use zksync_consensus_roles::{node, validator}; use zksync_consensus_storage::{BlockStore, BlockStoreRunner, PersistentBlockStore}; use zksync_protobuf::{required, serde::Serde, ProtoFmt}; @@ -33,10 +31,10 @@ pub fn decode_json(json: &str) -> anyhow::Result /// Encodes a generated proto message to json for arbitrary ProtoFmt. pub(crate) fn encode_json(x: &T) -> String { let s = serde_json::Serializer::pretty(vec![]); - encode_json_with_serializer(x, s) + encode_with_serializer(x, s) } -pub(crate) fn encode_json_with_serializer( +pub(crate) fn encode_with_serializer( x: &T, mut serializer: Serializer, F>, ) -> String { @@ -53,7 +51,7 @@ pub(crate) fn encode_json_with_serializer, - pub gossip_static_outbound: HashMap, + pub gossip_static_inbound: HashSet, + pub gossip_static_outbound: HashMap, +} + +impl AppConfig { + pub fn check_public_addr(&mut self) -> anyhow::Result<()> { + if let Ok(public_addr) = std::env::var("PUBLIC_ADDR") { + self.public_addr = SocketAddr::from_str(&format!("{public_addr}:{NODES_PORT}"))?; + } + Ok(()) + } } impl ProtoFmt for AppConfig { @@ -275,12 +282,16 @@ impl AppConfig { self } - pub fn add_gossip_static_outbound(&mut self, key: PublicKey, addr: SocketAddr) -> &mut Self { + pub fn add_gossip_static_outbound( + &mut self, + key: node::PublicKey, + addr: SocketAddr, + ) -> &mut Self { self.gossip_static_outbound.insert(key, addr); self } - pub fn add_gossip_static_inbound(&mut self, key: PublicKey) -> &mut Self { + pub fn add_gossip_static_inbound(&mut self, key: node::PublicKey) -> &mut Self { self.gossip_static_inbound.insert(key); self } diff --git a/node/tools/src/k8s.rs b/node/tools/src/k8s.rs index e03ee98f..c414b593 100644 --- a/node/tools/src/k8s.rs +++ b/node/tools/src/k8s.rs @@ -105,6 +105,14 @@ pub async fn create_deployment( { "name": "NODE_ID", "value": node_id + }, + { + "name": "PUBLIC_ADDR", + "valueFrom": { + "fieldRef": { + "fieldPath": "status.podIP" + } + } } ], "command": ["./k8s_entrypoint.sh"], @@ -201,7 +209,7 @@ fn get_cli_args(peers: Vec) -> Vec { } else { [ "--add-gossip-static-outbound".to_string(), - config::encode_json_with_serializer( + config::encode_with_serializer( &peers .iter() .map(|e| Serde(e.clone())) diff --git a/node/tools/src/main.rs b/node/tools/src/main.rs index aad9a4cf..428c6c09 100644 --- a/node/tools/src/main.rs +++ b/node/tools/src/main.rs @@ -102,6 +102,9 @@ async fn main() -> anyhow::Result<()> { .load() .context("config_paths().load()")?; + // if `PUBLIC_ADDR` env var is set, use it to override publicAddr in config + configs.app.check_public_addr().context("Public Address")?; + // Add gossipStaticOutbound pairs from cli to config if let Some(addrs) = args.add_gossip_static_outbound { configs @@ -121,7 +124,10 @@ async fn main() -> anyhow::Result<()> { } else { rpc_addr.set_port(rpc_addr.port() + 100); } - let rpc_server = RPCServer::new(rpc_addr); + + // cloning configuration to let RPCServer show it + // TODO this should be queried in real time instead, to reflect any possible change in config + let rpc_server = RPCServer::new(rpc_addr, configs.app.clone()); // Initialize the storage. scope::run!(ctx, |ctx, s| async { diff --git a/node/tools/src/rpc/methods/mod.rs b/node/tools/src/rpc/methods/mod.rs index 2cac6a52..ab8f2fee 100644 --- a/node/tools/src/rpc/methods/mod.rs +++ b/node/tools/src/rpc/methods/mod.rs @@ -10,5 +10,6 @@ pub(crate) trait RPCMethod { fn path() -> &'static str; } +pub(crate) mod config; pub(crate) mod health_check; pub(crate) mod peers; diff --git a/node/tools/src/rpc/server.rs b/node/tools/src/rpc/server.rs index 6d2d5131..3b66dfa0 100644 --- a/node/tools/src/rpc/server.rs +++ b/node/tools/src/rpc/server.rs @@ -1,4 +1,6 @@ -use super::methods::{health_check::HealthCheck, peers::PeersInfo, RPCMethod}; +use crate::AppConfig; + +use super::methods::{config::ConfigInfo, health_check::HealthCheck, peers::PeersInfo, RPCMethod}; use jsonrpsee::server::{middleware::http::ProxyGetRequestLayer, RpcModule, Server}; use std::net::SocketAddr; use zksync_concurrency::{ctx, scope}; @@ -7,11 +9,13 @@ use zksync_concurrency::{ctx, scope}; pub struct RPCServer { /// IP address to bind to. ip_address: SocketAddr, + /// AppConfig + config: AppConfig, } impl RPCServer { - pub fn new(ip_address: SocketAddr) -> Self { - Self { ip_address } + pub fn new(ip_address: SocketAddr, config: AppConfig) -> Self { + Self { ip_address, config } } /// Runs the RPC server. @@ -26,6 +30,10 @@ impl RPCServer { .layer(ProxyGetRequestLayer::new( PeersInfo::path(), PeersInfo::method(), + )?) + .layer(ProxyGetRequestLayer::new( + ConfigInfo::path(), + ConfigInfo::method(), )?); let server = Server::builder() @@ -39,6 +47,12 @@ impl RPCServer { })?; module.register_method(PeersInfo::method(), |params, _| PeersInfo::callback(params))?; + // TODO find a better way to implement this as I had to clone the clone and move it to pass the borrow checker + let config = self.config.clone(); + module.register_method(ConfigInfo::method(), move |_params, _| { + ConfigInfo::info(config.clone()) + })?; + let handle = server.start(module); scope::run!(ctx, |ctx, s| async { s.spawn_bg(async { From 702a587a2689a8c908032b4b57b1c08db97caa7b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Esteban=20Dimitroff=20H=C3=B3di?= Date: Thu, 15 Feb 2024 10:42:38 -0300 Subject: [PATCH 7/7] Added config RPC endpoint --- node/tools/src/rpc/methods/config.rs | 47 ++++++++++++++++++++++++++++ 1 file changed, 47 insertions(+) create mode 100644 node/tools/src/rpc/methods/config.rs diff --git a/node/tools/src/rpc/methods/config.rs b/node/tools/src/rpc/methods/config.rs new file mode 100644 index 00000000..0d22ec4d --- /dev/null +++ b/node/tools/src/rpc/methods/config.rs @@ -0,0 +1,47 @@ +//! Peers method for RPC server. +use crate::{config::encode_json, decode_json, AppConfig}; + +use super::RPCMethod; +use jsonrpsee::types::{error::ErrorCode, Params}; +use std::fs::{self}; +use zksync_protobuf::serde::Serde; + +/// Config method for RPC server. +pub(crate) struct ConfigInfo; + +// RPCMethod trait should be more general to allow external parameters like this case +// TODO fix the trait and implement this code in it +impl ConfigInfo { + pub(crate) fn info(config: AppConfig) -> Result { + // This may change in the future since we are assuming that the executor binary is being run inside the config directory. + Ok(serde_json::json!({ + "config": encode_json(&Serde(config)) + })) + } +} + +impl RPCMethod for ConfigInfo { + /// Config response for /config endpoint. + fn callback(_params: Params) -> Result { + // This may change in the future since we are assuming that the executor binary is being run inside the config directory. + let node_config = + fs::read_to_string("config.json").map_err(|_e| ErrorCode::InternalError)?; + let node_config = decode_json::>(&node_config) + .map_err(|_e| ErrorCode::InternalError)? + .0; + let config = encode_json(&Serde(node_config)); + Ok(serde_json::json!({ + "config": config + })) + } + + /// Config method name. + fn method() -> &'static str { + "config" + } + + /// Method path for GET requests. + fn path() -> &'static str { + "/config" + } +}