From efca75f228d61b9abb208aa26c88867fe17a34d1 Mon Sep 17 00:00:00 2001 From: hanabi1224 Date: Thu, 17 Oct 2024 16:12:14 +0800 Subject: [PATCH] feat: add compression support to metrics and RPC servers (#4906) --- Cargo.lock | 138 +++++++++++++++++++++++++++++++-------------- Cargo.toml | 3 +- src/metrics/mod.rs | 2 + src/rpc/mod.rs | 9 +++ 4 files changed, 109 insertions(+), 43 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index c014e7f0e20..968175b7248 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -83,6 +83,21 @@ dependencies = [ "memchr", ] +[[package]] +name = "alloc-no-stdlib" +version = "2.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc7bb162ec39d46ab1ca8c77bf72e890535becd1751bb45f64c597edb4c8c6b3" + +[[package]] +name = "alloc-stdlib" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94fb8275041c72129eb51b7d0322c29b8387a0386127718b096429201a5d6ece" +dependencies = [ + "alloc-no-stdlib", +] + [[package]] name = "allocator-api2" version = "0.2.18" @@ -339,10 +354,12 @@ dependencies = [ [[package]] name = "async-compression" -version = "0.4.15" +version = "0.4.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e26a9844c659a2a293d239c7910b752f8487fe122c6c8bd1659bf85a6507c302" +checksum = "103db485efc3e41214fe4fda9f3dbeae2eb9082f48fd236e6095627a9422066e" dependencies = [ + "brotli", + "flate2", "futures-core", "memchr", "pin-project-lite", @@ -621,7 +638,7 @@ dependencies = [ "http 1.1.0", "http-body 1.0.1", "http-body-util", - "hyper 1.4.1", + "hyper 1.5.0", "hyper-util", "itoa", "matchit", @@ -1006,6 +1023,27 @@ dependencies = [ "cfg_aliases", ] +[[package]] +name = "brotli" +version = "7.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc97b8f16f944bba54f0433f07e30be199b6dc2bd25937444bbad560bcea29bd" +dependencies = [ + "alloc-no-stdlib", + "alloc-stdlib", + "brotli-decompressor", +] + +[[package]] +name = "brotli-decompressor" +version = "4.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a45bd2e4095a8b518033b128020dd4a55aab1c0a381ba4404a472630f4bc362" +dependencies = [ + "alloc-no-stdlib", + "alloc-stdlib", +] + [[package]] name = "bs58" version = "0.5.1" @@ -3316,7 +3354,8 @@ dependencies = [ "tokio-test", "tokio-util", "toml 0.8.19", - "tower 0.5.1", + "tower 0.4.13", + "tower-http", "tracing", "tracing-appender", "tracing-chrome", @@ -4455,9 +4494,9 @@ checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4" [[package]] name = "hyper" -version = "0.14.30" +version = "0.14.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a152ddd61dfaec7273fe8419ab357f33aee0d914c5f4efbf0d96fa749eea5ec9" +checksum = "8c08302e8fa335b151b788c775ff56e7a03ae64ff85c548ee820fecb70356e85" dependencies = [ "bytes", "futures-channel", @@ -4479,9 +4518,9 @@ dependencies = [ [[package]] name = "hyper" -version = "1.4.1" +version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "50dfd22e0e76d0f662d429a5f80fcaf3855009297eab6a0a9f8543834744ba05" +checksum = "bbbff0a806a4728c99295b254c8838933b5b082d75e3cb70c8dab21fdfbcfa9a" dependencies = [ "bytes", "futures-channel", @@ -4506,7 +4545,7 @@ checksum = "08afdbb5c31130e3034af566421053ab03787c640246a446327f550d11bcb333" dependencies = [ "futures-util", "http 1.1.0", - "hyper 1.4.1", + "hyper 1.5.0", "hyper-util", "log", "rustls", @@ -4523,7 +4562,7 @@ version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3203a961e5c83b6f5498933e78b6b263e208c197b63e9c6c53cc82ffd3f63793" dependencies = [ - "hyper 1.4.1", + "hyper 1.5.0", "hyper-util", "pin-project-lite", "tokio", @@ -4541,7 +4580,7 @@ dependencies = [ "futures-util", "http 1.1.0", "http-body 1.0.1", - "hyper 1.4.1", + "hyper 1.5.0", "pin-project-lite", "socket2 0.5.7", "tokio", @@ -4639,7 +4678,7 @@ dependencies = [ "bytes", "futures", "http 0.2.12", - "hyper 0.14.30", + "hyper 0.14.31", "log", "rand", "tokio", @@ -4920,9 +4959,9 @@ dependencies = [ [[package]] name = "jsonrpsee" -version = "0.24.6" +version = "0.24.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "02f01f48e04e0d7da72280ab787c9943695699c9b32b99158ece105e8ad0afea" +checksum = "c5c71d8c1a731cc4227c2f698d377e7848ca12c8a48866fc5e6951c43a4db843" dependencies = [ "jsonrpsee-core", "jsonrpsee-http-client", @@ -4934,9 +4973,9 @@ dependencies = [ [[package]] name = "jsonrpsee-client-transport" -version = "0.24.6" +version = "0.24.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d80eccbd47a7b9f1e67663fd846928e941cb49c65236e297dd11c9ea3c5e3387" +checksum = "548125b159ba1314104f5bb5f38519e03a41862786aa3925cf349aae9cdd546e" dependencies = [ "base64 0.22.1", "futures-util", @@ -4957,9 +4996,9 @@ dependencies = [ [[package]] name = "jsonrpsee-core" -version = "0.24.6" +version = "0.24.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3c2709a32915d816a6e8f625bf72cf74523ebe5d8829f895d6b041b1d3137818" +checksum = "f2882f6f8acb9fdaec7cefc4fd607119a9bd709831df7d7672a1d3b644628280" dependencies = [ "async-trait", "bytes", @@ -4983,14 +5022,14 @@ dependencies = [ [[package]] name = "jsonrpsee-http-client" -version = "0.24.6" +version = "0.24.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cc54db939002b030e794fbfc9d5a925aa2854889c5a2f0352b0bffa54681707e" +checksum = "b3638bc4617f96675973253b3a45006933bde93c2fd8a6170b33c777cc389e5b" dependencies = [ "async-trait", "base64 0.22.1", "http-body 1.0.1", - "hyper 1.4.1", + "hyper 1.5.0", "hyper-rustls", "hyper-util", "jsonrpsee-core", @@ -5008,15 +5047,15 @@ dependencies = [ [[package]] name = "jsonrpsee-server" -version = "0.24.6" +version = "0.24.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e30110d0f2d7866c8cc6c86483bdab2eb9f4d2f0e20db55518b2bca84651ba8e" +checksum = "82ad8ddc14be1d4290cd68046e7d1d37acd408efed6d3ca08aefcc3ad6da069c" dependencies = [ "futures-util", "http 1.1.0", "http-body 1.0.1", "http-body-util", - "hyper 1.4.1", + "hyper 1.5.0", "hyper-util", "jsonrpsee-core", "jsonrpsee-types", @@ -5035,9 +5074,9 @@ dependencies = [ [[package]] name = "jsonrpsee-types" -version = "0.24.6" +version = "0.24.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ca331cd7b3fe95b33432825c2d4c9f5a43963e207fdc01ae67f9fd80ab0930f" +checksum = "a178c60086f24cc35bb82f57c651d0d25d99c4742b4d335de04e97fa1f08a8a1" dependencies = [ "http 1.1.0", "serde", @@ -5047,9 +5086,9 @@ dependencies = [ [[package]] name = "jsonrpsee-ws-client" -version = "0.24.6" +version = "0.24.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "755ca3da1c67671f1fae01cd1a47f41dfb2233a8f19a643e587ab0a663942044" +checksum = "0fe322e0896d0955a3ebdd5bf813571c53fea29edd713bc315b76620b327e86d" dependencies = [ "http 1.1.0", "jsonrpsee-client-transport", @@ -7065,9 +7104,9 @@ dependencies = [ [[package]] name = "proc-macro2" -version = "1.0.87" +version = "1.0.88" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b3e4daa0dcf6feba26f985457cdf104d4b4256fc5a09547140f3631bb076b19a" +checksum = "7c3a7fc5db1e57d5a779a352c8cdb57b29aa4c40cc69c3a68a7fedc815fbf2f9" dependencies = [ "unicode-ident", ] @@ -7585,7 +7624,7 @@ dependencies = [ "http 1.1.0", "http-body 1.0.1", "http-body-util", - "hyper 1.4.1", + "hyper 1.5.0", "hyper-rustls", "hyper-util", "ipnet", @@ -7877,9 +7916,9 @@ dependencies = [ [[package]] name = "rustls" -version = "0.23.14" +version = "0.23.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "415d9944693cb90382053259f89fbb077ea730ad7273047ec63b19bc9b160ba8" +checksum = "5fbb44d7acc4e873d613422379f69f237a1b141928c02f6bc6ccfddddc2d7993" dependencies = [ "log", "once_cell", @@ -7914,9 +7953,9 @@ dependencies = [ [[package]] name = "rustls-pki-types" -version = "1.9.0" +version = "1.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0e696e35370c65c9c541198af4543ccd580cf17fc25d8e05c5a242b202488c55" +checksum = "16f1201b3c9a7ee8039bcadc17b7e605e2945b27eee7631788c1bd2b0643674b" [[package]] name = "rustls-platform-verifier" @@ -9293,7 +9332,7 @@ dependencies = [ "http 1.1.0", "http-body 1.0.1", "http-body-util", - "hyper 1.4.1", + "hyper 1.5.0", "hyper-timeout", "hyper-util", "percent-encoding", @@ -9336,18 +9375,33 @@ checksum = "2873938d487c3cfb9aed7546dc9f2711d867c9f90c46b889989a2cb84eba6b4f" dependencies = [ "futures-core", "futures-util", - "hdrhistogram", - "indexmap 2.6.0", "pin-project-lite", - "slab", "sync_wrapper 0.1.2", "tokio", - "tokio-util", "tower-layer", "tower-service", "tracing", ] +[[package]] +name = "tower-http" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8437150ab6bbc8c5f0f519e3d5ed4aa883a83dd4cdd3d1b21f9482936046cb97" +dependencies = [ + "async-compression", + "bitflags 2.6.0", + "bytes", + "futures-core", + "http 1.1.0", + "http-body 1.0.1", + "pin-project-lite", + "tokio", + "tokio-util", + "tower-layer", + "tower-service", +] + [[package]] name = "tower-layer" version = "0.3.3" @@ -9647,9 +9701,9 @@ checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" [[package]] name = "uuid" -version = "1.10.0" +version = "1.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "81dfa00651efa65069b0b6b651f4aaa31ba9e3c3ce0137aaad053604ee7e0314" +checksum = "f8c5f0a0af699448548ad1a2fbf920fb4bee257eae39953ba95cb84891a0446a" dependencies = [ "getrandom", "serde", diff --git a/Cargo.toml b/Cargo.toml index 164b82ae43b..509bd4fecb0 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -184,7 +184,8 @@ tokio = { version = "1", features = ['full'] } tokio-stream = { version = "0.1", features = ["fs", "io-util"] } tokio-util = { version = "0.7", features = ["compat", "io-util"] } toml = "0.8" -tower = { version = "0.5", features = ["full"] } +tower = { version = "0.4", features = ["util"] } +tower-http = { version = "0.6", features = ["compression-full", "sensitive-headers"] } tracing = "0.1" tracing-appender = "0.2" tracing-subscriber = { version = "0.3", features = ["env-filter", "json"] } diff --git a/src/metrics/mod.rs b/src/metrics/mod.rs index 6b355a5518b..904e333382f 100644 --- a/src/metrics/mod.rs +++ b/src/metrics/mod.rs @@ -18,6 +18,7 @@ use prometheus_client::{ use std::sync::Arc; use std::{path::PathBuf, time::Instant}; use tokio::net::TcpListener; +use tower_http::compression::CompressionLayer; use tracing::warn; static DEFAULT_REGISTRY: Lazy> = @@ -91,6 +92,7 @@ where let app = Router::new() .route("/metrics", get(collect_prometheus_metrics)) .route("/stats/db", get(collect_db_metrics::)) + .layer(CompressionLayer::new()) .with_state(db); // Wait for server to exit diff --git a/src/rpc/mod.rs b/src/rpc/mod.rs index 6b43a10cfac..d0abd4f6740 100644 --- a/src/rpc/mod.rs +++ b/src/rpc/mod.rs @@ -257,6 +257,8 @@ macro_rules! for_each_method { }; } pub(crate) use for_each_method; +use tower_http::compression::CompressionLayer; +use tower_http::sensitive_headers::SetSensitiveRequestHeadersLayer; #[allow(unused)] /// All handler definitions. @@ -485,6 +487,12 @@ where svc_builder, keystore, } = per_conn.clone(); + let http_middleware = tower::ServiceBuilder::new() + .layer(CompressionLayer::new()) + // Mark the `Authorization` request header as sensitive so it doesn't show in logs + .layer(SetSensitiveRequestHeadersLayer::new(std::iter::once( + http::header::AUTHORIZATION, + ))); // NOTE, the rpc middleware must be initialized here to be able to created once per connection // with data from the connection such as the headers in this example let headers = req.headers().clone(); @@ -496,6 +504,7 @@ where .layer(LogLayer::default()) .layer(MetricsLayer::default()); let mut jsonrpsee_svc = svc_builder + .set_http_middleware(http_middleware) .set_rpc_middleware(rpc_middleware) .build(methods, stop_handle);