From e3567ef5f92991e766b429139214091a27d22c4d Mon Sep 17 00:00:00 2001 From: Keyao Shen Date: Fri, 15 Sep 2023 12:13:04 -0700 Subject: [PATCH 01/18] Remove vdemo, rename and move sdemo, extract block impl from demo --- Cargo.lock | 208 +++--- .../examples/web-server-da/multi-validator.rs | 2 +- .../web-server-da/multi-web-server.rs | 2 +- .../examples/web-server-da/orchestrator.rs | 2 +- .../hotshot/examples/web-server-da/types.rs | 2 +- .../examples/web-server-da/validator.rs | 2 +- .../examples/web-server-da/web-server.rs | 2 +- crates/hotshot/src/block_impl.rs | 174 +++++ .../hotshot/src/{demos/sdemo.rs => demo.rs} | 200 +----- crates/hotshot/src/demos.rs | 8 - crates/hotshot/src/demos/vdemo.rs | 607 ------------------ crates/hotshot/src/lib.rs | 5 +- crates/testing/src/node_types.rs | 11 +- crates/testing/tests/da_task.rs | 4 +- crates/testing/tests/network_task.rs | 4 +- testing-macros/src/lib.rs | 2 +- 16 files changed, 312 insertions(+), 923 deletions(-) create mode 100644 crates/hotshot/src/block_impl.rs rename crates/hotshot/src/{demos/sdemo.rs => demo.rs} (52%) delete mode 100644 crates/hotshot/src/demos.rs delete mode 100644 crates/hotshot/src/demos/vdemo.rs diff --git a/Cargo.lock b/Cargo.lock index e577bc0d70..5b26f3cb4d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -178,9 +178,9 @@ dependencies = [ [[package]] name = "anstyle" -version = "1.0.3" +version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b84bf0a05bbb2a83e5eb6fa36bb6e87baa08193c35ff52bbf6b38d8af2890e46" +checksum = "15c4c2c83f81532e5845a733998b6971faca23490340a418e9b72a3ec9de12ea" [[package]] name = "anstyle-parse" @@ -801,7 +801,7 @@ checksum = "16e62a023e7c117e27523144c5d2459f4397fcc3cab0085af8e2224f643a0193" dependencies = [ "proc-macro2", "quote", - "syn 2.0.33", + "syn 2.0.31", ] [[package]] @@ -818,7 +818,7 @@ checksum = "bc00ceb34980c03614e35a3a4e218276a0a824e911d07651cd0d858a51e8c0f0" dependencies = [ "proc-macro2", "quote", - "syn 2.0.33", + "syn 2.0.31", ] [[package]] @@ -854,7 +854,7 @@ version = "0.6.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4057f2c32adbb2fc158e22fb38433c8e9bbf76b75a4732c7c0cbaf695fb65568" dependencies = [ - "bytes 1.5.0", + "bytes 1.4.0", "futures-sink", "futures-util", "memchr", @@ -893,7 +893,7 @@ dependencies = [ "async-trait", "axum-core", "bitflags 1.3.2", - "bytes 1.5.0", + "bytes 1.4.0", "futures-util", "http", "http-body", @@ -919,7 +919,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "759fa577a247914fd3f7f76d62972792636412fbfd634cd452f6a385a74d2d2c" dependencies = [ "async-trait", - "bytes 1.5.0", + "bytes 1.4.0", "futures-util", "http", "http-body", @@ -964,9 +964,9 @@ checksum = "9e1b586273c5702936fe7b7d6896644d8be71e6314cfe09d3167c95f712589e8" [[package]] name = "base64" -version = "0.21.4" +version = "0.21.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ba43ea6f343b788c8764558649e08df62f86c6ef251fdaeb1ffd010a9ae50a2" +checksum = "414dcefbc63d77c526a76b3afcf6fbb9b5e2791c19c3aa2297733208750c6e53" [[package]] name = "base64ct" @@ -1097,9 +1097,9 @@ dependencies = [ [[package]] name = "bumpalo" -version = "3.14.0" +version = "3.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f30e7476521f6f8af1a1c4c0b8cc94f0bee37d91763d0ca2665f299b6cd8aec" +checksum = "a3e2c3daef883ecc1b5d58c15adae93470a91d425f3532ba1695849656af3fc1" [[package]] name = "byte-slice-cast" @@ -1121,9 +1121,9 @@ checksum = "0e4cec68f03f32e44924783795810fa50a7035d8c8ebe78580ad7e6c703fba38" [[package]] name = "bytes" -version = "1.5.0" +version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a2bd12c1caf447e69cd4528f47f94d203fd2582878ecb9e9465484c4148a8223" +checksum = "89b2fd2a0dcf38d7971e2194b6b6eebab45ae01067456a7fd93d5547a61b70be" dependencies = [ "serde", ] @@ -1194,9 +1194,9 @@ dependencies = [ [[package]] name = "chrono" -version = "0.4.31" +version = "0.4.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f2c685bad3eb3d45a01354cedb7d5faa66194d1d58ba6e267a8de788f79db38" +checksum = "95ed24df0632f708f5f6d8082675bef2596f7084dee3dd55f632290bf35bfe0f" dependencies = [ "android-tzdata", "iana-time-zone", @@ -1265,7 +1265,7 @@ dependencies = [ "heck", "proc-macro2", "quote", - "syn 2.0.33", + "syn 2.0.31", ] [[package]] @@ -1572,7 +1572,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "704722d1d929489c8528bb1882805700f1ba20f54325704973e786352320b1ed" dependencies = [ "blake2", - "curve25519-dalek 4.1.0", + "curve25519-dalek 4.0.0", "rand_core 0.6.4", "serdect", ] @@ -1658,9 +1658,9 @@ dependencies = [ [[package]] name = "curve25519-dalek" -version = "4.1.0" +version = "4.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "622178105f911d937a42cdb140730ba4a3ed2becd8ae6ce39c7d28b5d75d4588" +checksum = "f711ade317dd348950a9910f81c5947e3d8907ebd2b83f76203ff1807e6a2bc2" dependencies = [ "cfg-if", "cpufeatures", @@ -1681,7 +1681,7 @@ checksum = "83fdaf97f4804dcebfa5862639bc9ce4121e82140bec2a987ac5140294865b5b" dependencies = [ "proc-macro2", "quote", - "syn 2.0.33", + "syn 2.0.31", ] [[package]] @@ -1749,7 +1749,7 @@ dependencies = [ "proc-macro2", "quote", "strsim", - "syn 2.0.33", + "syn 2.0.31", ] [[package]] @@ -1771,7 +1771,7 @@ checksum = "836a9bbc7ad63342d6d6e7b815ccab164bc77a2d95d84bc3117a8c0d5c98e2d5" dependencies = [ "darling_core 0.20.3", "quote", - "syn 2.0.33", + "syn 2.0.31", ] [[package]] @@ -1865,7 +1865,7 @@ checksum = "53e0efad4403bfc52dc201159c4b842a246a14b98c64b55dfd0f2d89729dfeb8" dependencies = [ "proc-macro2", "quote", - "syn 2.0.33", + "syn 2.0.31", ] [[package]] @@ -1987,7 +1987,7 @@ checksum = "487585f4d0c6655fe74905e2504d8ad6908e4db67f744eb140876906c2f3175d" dependencies = [ "proc-macro2", "quote", - "syn 2.0.33", + "syn 2.0.31", ] [[package]] @@ -2046,7 +2046,7 @@ version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7277392b266383ef8396db7fdeb1e77b6c52fed775f5df15bb24f35b72156980" dependencies = [ - "curve25519-dalek 4.1.0", + "curve25519-dalek 4.0.0", "ed25519", "rand_core 0.6.4", "serde", @@ -2202,9 +2202,9 @@ dependencies = [ [[package]] name = "fiat-crypto" -version = "0.2.1" +version = "0.1.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d0870c84016d4b481be5c9f323c24f65e31e901ae618f0e80f4308fb00de1d2d" +checksum = "e825f6987101665dea6ec934c09ec6d721de7bc1bf92248e1d5810c8cd636b77" [[package]] name = "fixed-hash" @@ -2344,7 +2344,7 @@ checksum = "89ca545a94061b6365f2c7355b4b32bd20df3ff95f02da9329b34ccc3bd6ee72" dependencies = [ "proc-macro2", "quote", - "syn 2.0.33", + "syn 2.0.31", ] [[package]] @@ -2489,7 +2489,7 @@ version = "0.3.21" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "91fc23aa11be92976ef4729127f1a74adf36d8436f7816b185d18df956790833" dependencies = [ - "bytes 1.5.0", + "bytes 1.4.0", "fnv", "futures-core", "futures-sink", @@ -2936,7 +2936,7 @@ version = "0.2.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bd6effc99afb63425aff9b05836f029929e345a6148a14b7ecd5ab67af944482" dependencies = [ - "bytes 1.5.0", + "bytes 1.4.0", "fnv", "itoa 1.0.9", ] @@ -2947,7 +2947,7 @@ version = "0.4.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d5f38f16d184e36f2408a55281cd658ecbd3ca05cce6d6510a176eca393e26d1" dependencies = [ - "bytes 1.5.0", + "bytes 1.4.0", "http", "pin-project-lite 0.2.13", ] @@ -3012,7 +3012,7 @@ version = "0.14.27" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ffb1cfd654a8219eaef89881fdb3bb3b1cdc5fa75ded05d6933b2b382e395468" dependencies = [ - "bytes 1.5.0", + "bytes 1.4.0", "futures-channel", "futures-core", "futures-util", @@ -3228,7 +3228,7 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f97967975f448f1a7ddb12b0bc41069d09ed6a1c161a92687e057325db35d413" dependencies = [ - "bytes 1.5.0", + "bytes 1.4.0", ] [[package]] @@ -3257,7 +3257,7 @@ version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b58db92f96b720de98181bbbe63c831e87005ab460c1bf306eb2622b4707997f" dependencies = [ - "socket2 0.5.4", + "socket2 0.5.3", "widestring", "windows-sys", "winreg", @@ -3455,9 +3455,9 @@ checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" [[package]] name = "libc" -version = "0.2.148" +version = "0.2.147" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9cdc71e17332e86d2e1d38c1f99edcb6288ee11b815fb1a4b049eaa2114d369b" +checksum = "b4668fb0ea861c1df094127ac5f1da3409a82116a4ba74fca2e58ef927159bb3" [[package]] name = "libnghttp2-sys" @@ -3475,7 +3475,7 @@ version = "0.52.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32d07d1502a027366d55afe187621c2d7895dc111a3df13b35fed698049681d7" dependencies = [ - "bytes 1.5.0", + "bytes 1.4.0", "futures", "futures-timer", "getrandom 0.2.10", @@ -3638,9 +3638,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2d157562dba6017193e5285acf6b1054759e83540bfd79f75b69d6ce774c88da" dependencies = [ "asynchronous-codec", - "base64 0.21.4", + "base64 0.21.3", "byteorder", - "bytes 1.5.0", + "bytes 1.4.0", "either", "fnv", "futures", @@ -3715,7 +3715,7 @@ checksum = "fc125f83d8f75322c79e4ade74677d299b34aa5c9d9b5251c03ec28c683cb765" dependencies = [ "arrayvec", "asynchronous-codec", - "bytes 1.5.0", + "bytes 1.4.0", "either", "fnv", "futures", @@ -3752,7 +3752,7 @@ dependencies = [ "log", "rand 0.8.5", "smallvec", - "socket2 0.5.4", + "socket2 0.5.3", "tokio", "trust-dns-proto", "void", @@ -3826,8 +3826,8 @@ version = "0.43.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "71ce70757f2c0d82e9a3ef738fb10ea0723d16cec37f078f719e2c247704c1bb" dependencies = [ - "bytes 1.5.0", - "curve25519-dalek 4.1.0", + "bytes 1.4.0", + "curve25519-dalek 4.0.0", "futures", "libp2p-core", "libp2p-identity", @@ -3870,7 +3870,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "37266c683a757df713f7dcda0cdcb5ad4681355ffa1b37b77c113c176a531195" dependencies = [ "asynchronous-codec", - "bytes 1.5.0", + "bytes 1.4.0", "futures", "libp2p-core", "libp2p-identity", @@ -3900,7 +3900,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4cb763e88f9a043546bfebd3575f340e7dd3d6c1b2cf2629600ec8965360c63a" dependencies = [ "async-std", - "bytes 1.5.0", + "bytes 1.4.0", "futures", "futures-timer", "if-watch", @@ -3912,7 +3912,7 @@ dependencies = [ "quinn", "rand 0.8.5", "rustls", - "socket2 0.5.4", + "socket2 0.5.3", "thiserror", "tokio", ] @@ -3924,7 +3924,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cdb07202cdf103486709fda5d9d10a0297a8ba01c212b1e19b7943c45c1bd7d6" dependencies = [ "asynchronous-codec", - "bytes 1.5.0", + "bytes 1.4.0", "either", "futures", "futures-timer", @@ -4017,7 +4017,7 @@ dependencies = [ "proc-macro-warning", "proc-macro2", "quote", - "syn 2.0.33", + "syn 2.0.31", ] [[package]] @@ -4034,7 +4034,7 @@ dependencies = [ "libp2p-core", "libp2p-identity", "log", - "socket2 0.5.4", + "socket2 0.5.3", "tokio", ] @@ -4418,9 +4418,9 @@ dependencies = [ [[package]] name = "multihash" -version = "0.19.1" +version = "0.19.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "076d548d76a0e2a0d4ab471d0b1c36c577786dfc4471242035d97a12a735c492" +checksum = "2fd59dcc2bbe70baabeac52cd22ae52c55eefe6c38ff11a9439f16a350a939f2" dependencies = [ "core2", "serde", @@ -4433,7 +4433,7 @@ version = "0.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ea0df8e5eec2298a62b326ee4f0d7fe1a6b90a09dfcf9df37b38f947a8c42f19" dependencies = [ - "bytes 1.5.0", + "bytes 1.4.0", "futures", "log", "pin-project", @@ -4542,7 +4542,7 @@ name = "netlink-proto" version = "0.9.2" source = "git+https://github.com/espressosystems/netlink.git#1347bed011eeae7ece9851773906c4a3c80a5d77" dependencies = [ - "bytes 1.5.0", + "bytes 1.4.0", "futures", "log", "netlink-packet-core 0.4.2 (git+https://github.com/espressosystems/netlink.git)", @@ -4556,7 +4556,7 @@ version = "0.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "65b4b14489ab424703c092062176d52ba55485a89c076b4f9db05092b7223aa6" dependencies = [ - "bytes 1.5.0", + "bytes 1.4.0", "futures", "log", "netlink-packet-core 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)", @@ -4571,7 +4571,7 @@ version = "0.8.2" source = "git+https://github.com/espressosystems/netlink.git#1347bed011eeae7ece9851773906c4a3c80a5d77" dependencies = [ "async-io", - "bytes 1.5.0", + "bytes 1.4.0", "futures", "libc", "log", @@ -4584,7 +4584,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6471bf08e7ac0135876a9581bf3217ef0333c191c128d34878079f42ee150411" dependencies = [ "async-io", - "bytes 1.5.0", + "bytes 1.4.0", "futures", "libc", "log", @@ -4735,9 +4735,9 @@ checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" [[package]] name = "openssl-sys" -version = "0.9.93" +version = "0.9.92" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "db4d56a4c0478783083cfafcc42493dd4a981d41669da64b4572a2a089b51b1d" +checksum = "db7e971c2c2bba161b2d2fdf37080177eff520b3bc044787c7f1f5f9e78d869b" dependencies = [ "cc", "libc", @@ -4886,7 +4886,7 @@ dependencies = [ "pest_meta", "proc-macro2", "quote", - "syn 2.0.33", + "syn 2.0.31", ] [[package]] @@ -4917,7 +4917,7 @@ checksum = "4359fd9c9171ec6e8c62926d6faaf553a8dc3f64e1507e76da7911b4f6a04405" dependencies = [ "proc-macro2", "quote", - "syn 2.0.33", + "syn 2.0.31", ] [[package]] @@ -5103,14 +5103,14 @@ checksum = "3d1eaa7fa0aa1929ffdf7eeb6eac234dde6268914a14ad44d23521ab6a9b258e" dependencies = [ "proc-macro2", "quote", - "syn 2.0.33", + "syn 2.0.31", ] [[package]] name = "proc-macro2" -version = "1.0.67" +version = "1.0.66" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3d433d9f1a3e8c1263d9456598b16fec66f4acc9a74dacffd35c7bb09b3a1328" +checksum = "18fb31db3f9bddb2ea821cde30a9f70117e3f119938b5ee630b7403aa6e2ead9" dependencies = [ "unicode-ident", ] @@ -5135,7 +5135,7 @@ checksum = "440f724eba9f6996b75d63681b0a92b06947f1457076d503a4d2e2c8f56442b8" dependencies = [ "proc-macro2", "quote", - "syn 2.0.33", + "syn 2.0.31", ] [[package]] @@ -5144,7 +5144,7 @@ version = "0.11.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0b82eaa1d779e9a4bc1c3217db8ffbeabaae1dca241bf70183242128d48681cd" dependencies = [ - "bytes 1.5.0", + "bytes 1.4.0", "prost-derive", ] @@ -5192,7 +5192,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f8ededb1cd78531627244d51dd0c7139fbe736c7d57af0092a76f0ffb2f56e98" dependencies = [ "asynchronous-codec", - "bytes 1.5.0", + "bytes 1.4.0", "quick-protobuf", "thiserror", "unsigned-varint", @@ -5217,7 +5217,7 @@ checksum = "8cc2c5017e4b43d5995dcea317bc46c1e09404c0a9664d2908f7f02dfe943d75" dependencies = [ "async-io", "async-std", - "bytes 1.5.0", + "bytes 1.4.0", "futures-io", "pin-project-lite 0.2.13", "quinn-proto", @@ -5235,7 +5235,7 @@ version = "0.10.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e13f81c9a9d574310b8351f8666f5a93ac3b0069c45c28ad52c10291389a7cf9" dependencies = [ - "bytes 1.5.0", + "bytes 1.4.0", "rand 0.8.5", "ring", "rustc-hash", @@ -5252,9 +5252,9 @@ version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "055b4e778e8feb9f93c4e439f71dc2156ef13360b432b799e179a8c4cdf0b1d7" dependencies = [ - "bytes 1.5.0", + "bytes 1.4.0", "libc", - "socket2 0.5.4", + "socket2 0.5.3", "tracing", "windows-sys", ] @@ -5483,7 +5483,7 @@ version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bb919243f34364b6bd2fc10ef797edbfa75f33c252e7998527479c6d6b47e1ec" dependencies = [ - "bytes 1.5.0", + "bytes 1.4.0", "rustc-hex", ] @@ -5631,9 +5631,9 @@ dependencies = [ [[package]] name = "rustls-webpki" -version = "0.101.5" +version = "0.101.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "45a27e3b59326c16e23d30aeb7a36a24cc0d29e71d68ff611cdfb4a01d013bed" +checksum = "7d93931baf2d282fff8d3a532bbfd7653f734643161b87e3e01e59a04439bf0d" dependencies = [ "ring", "untrusted", @@ -5740,7 +5740,7 @@ checksum = "4eca7ac642d82aa35b60049a6eccb4be6be75e599bd2e9adb5f875a737654af2" dependencies = [ "proc-macro2", "quote", - "syn 2.0.33", + "syn 2.0.31", ] [[package]] @@ -5817,7 +5817,7 @@ version = "3.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1ca3b16a3d82c4088f343b7480a93550b3eabe1a358569c2dfe38bbcead07237" dependencies = [ - "base64 0.21.4", + "base64 0.21.3", "chrono", "hex", "indexmap 1.9.3", @@ -5837,7 +5837,7 @@ dependencies = [ "darling 0.20.3", "proc-macro2", "quote", - "syn 2.0.33", + "syn 2.0.31", ] [[package]] @@ -5849,7 +5849,7 @@ dependencies = [ "darling 0.20.3", "proc-macro2", "quote", - "syn 2.0.33", + "syn 2.0.31", ] [[package]] @@ -6092,7 +6092,7 @@ dependencies = [ "aes-gcm 0.9.2", "blake2", "chacha20poly1305 0.9.1", - "curve25519-dalek 4.1.0", + "curve25519-dalek 4.0.0", "rand_core 0.6.4", "ring", "rustc_version 0.4.0", @@ -6112,9 +6112,9 @@ dependencies = [ [[package]] name = "socket2" -version = "0.5.4" +version = "0.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4031e820eb552adee9295814c0ced9e5cf38ddf1e8b7d566d6de8e2538ea989e" +checksum = "2538b18701741680e0322a2302176d3253a35388e2e62f172f64f4f16605f877" dependencies = [ "libc", "windows-sys", @@ -6127,7 +6127,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "41d1c5305e39e09653383c2c7244f2f78b3bcae37cf50c64cb4789c9f5096ec2" dependencies = [ "base64 0.13.1", - "bytes 1.5.0", + "bytes 1.4.0", "futures", "httparse", "log", @@ -6274,7 +6274,7 @@ dependencies = [ "proc-macro2", "quote", "rustversion", - "syn 2.0.33", + "syn 2.0.31", ] [[package]] @@ -6404,9 +6404,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.33" +version = "2.0.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9caece70c63bfba29ec2fed841a09851b14a235c60010fa4de58089b6c025668" +checksum = "718fa2415bcb8d8bd775917a1bf12a7931b6dfa890753378538118181e0cb398" dependencies = [ "proc-macro2", "quote", @@ -6548,7 +6548,7 @@ checksum = "49922ecae66cc8a249b77e68d1d0623c1b2c514f0060c27cdc68bd62a1219d35" dependencies = [ "proc-macro2", "quote", - "syn 2.0.33", + "syn 2.0.31", ] [[package]] @@ -6789,14 +6789,14 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "17ed6077ed6cd6c74735e21f37eb16dc3935f96878b1fe961074089cc80893f9" dependencies = [ "backtrace", - "bytes 1.5.0", + "bytes 1.4.0", "libc", "mio", "num_cpus", "parking_lot", "pin-project-lite 0.2.13", "signal-hook-registry", - "socket2 0.5.4", + "socket2 0.5.3", "tokio-macros", "tracing", "windows-sys", @@ -6820,7 +6820,7 @@ checksum = "630bdcf245f78637c13ec01ffae6187cca34625e8c63150d424b59e55af2675e" dependencies = [ "proc-macro2", "quote", - "syn 2.0.33", + "syn 2.0.31", ] [[package]] @@ -6840,7 +6840,7 @@ version = "0.7.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "806fe8c2c87eccc8b3267cbae29ed3ab2d0bd37fca70ab622e46aaa9375ddb7d" dependencies = [ - "bytes 1.5.0", + "bytes 1.4.0", "futures-core", "futures-sink", "pin-project-lite 0.2.13", @@ -6899,8 +6899,8 @@ checksum = "3082666a3a6433f7f511c7192923fa1fe07c69332d3c6a2e6bb040b569199d5a" dependencies = [ "async-trait", "axum", - "base64 0.21.4", - "bytes 1.5.0", + "base64 0.21.3", + "bytes 1.4.0", "futures-core", "futures-util", "h2", @@ -6972,7 +6972,7 @@ checksum = "5f4f31f56159e98206da9efd823404b79b6ef3143b4a7ab76e67b1751b25a4ab" dependencies = [ "proc-macro2", "quote", - "syn 2.0.33", + "syn 2.0.31", ] [[package]] @@ -7119,7 +7119,7 @@ checksum = "5fe8dada8c1a3aeca77d6b51a4f1314e0f4b8e438b7b1b71e3ddaca8080e4093" dependencies = [ "base64 0.13.1", "byteorder", - "bytes 1.5.0", + "bytes 1.4.0", "http", "httparse", "input_buffer", @@ -7139,7 +7139,7 @@ checksum = "983d40747bce878d2fb67d910dcb8bd3eca2b2358540c3cc1b98c027407a3ae3" dependencies = [ "base64 0.13.1", "byteorder", - "bytes 1.5.0", + "bytes 1.4.0", "http", "httparse", "log", @@ -7152,9 +7152,9 @@ dependencies = [ [[package]] name = "typenum" -version = "1.17.0" +version = "1.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "42ff0bf0c66b8238c6f3b578df37d0b7848e55df8577b3f74f92a69acceeb825" +checksum = "497961ef93d974e23eb6f433eb5fe1b7930b659f06d12dec6fc44a8f554c0bba" [[package]] name = "ucd-trie" @@ -7191,9 +7191,9 @@ checksum = "92888ba5573ff080736b3648696b70cafad7d250551175acbaa4e0385b3e1460" [[package]] name = "unicode-ident" -version = "1.0.12" +version = "1.0.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3354b9ac3fae1ff6755cb6db53683adb661634f67557942dea4facebec0fee4b" +checksum = "301abaae475aa91687eb82514b328ab47a211a533026cb25fc3e519b86adfc3c" [[package]] name = "unicode-normalization" @@ -7232,12 +7232,12 @@ dependencies = [ [[package]] name = "unsigned-varint" -version = "0.7.2" +version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6889a77d49f1f013504cec6bf97a2c730394adedaeb1deb5ea08949a50541105" +checksum = "d86a8dc7f45e4c1b0d30e43038c38f274e77af056aa5f74b93c2cf9eb3c1c836" dependencies = [ "asynchronous-codec", - "bytes 1.5.0", + "bytes 1.4.0", ] [[package]] @@ -7380,7 +7380,7 @@ dependencies = [ "once_cell", "proc-macro2", "quote", - "syn 2.0.33", + "syn 2.0.31", "wasm-bindgen-shared", ] @@ -7414,7 +7414,7 @@ checksum = "54681b18a46765f095758388f2d0cf16eb8d4169b639ab575a8f5693af210c7b" dependencies = [ "proc-macro2", "quote", - "syn 2.0.33", + "syn 2.0.31", "wasm-bindgen-backend", "wasm-bindgen-shared", ] @@ -7693,5 +7693,5 @@ checksum = "ce36e65b0d2999d2aafac989fb249189a141aee1f53c612c1f37d72631959f69" dependencies = [ "proc-macro2", "quote", - "syn 2.0.33", + "syn 2.0.31", ] diff --git a/crates/hotshot/examples/web-server-da/multi-validator.rs b/crates/hotshot/examples/web-server-da/multi-validator.rs index 8a0f53c1c9..a16f2951ed 100644 --- a/crates/hotshot/examples/web-server-da/multi-validator.rs +++ b/crates/hotshot/examples/web-server-da/multi-validator.rs @@ -3,7 +3,7 @@ use async_compatibility_layer::{ logging::{setup_backtrace, setup_logging}, }; use clap::Parser; -use hotshot::demos::sdemo::SDemoTypes; +use hotshot::demo::SDemoTypes; use hotshot_orchestrator::client::ValidatorArgs; use std::net::IpAddr; use tracing::instrument; diff --git a/crates/hotshot/examples/web-server-da/multi-web-server.rs b/crates/hotshot/examples/web-server-da/multi-web-server.rs index c41c9c0b03..39a9b7a87e 100644 --- a/crates/hotshot/examples/web-server-da/multi-web-server.rs +++ b/crates/hotshot/examples/web-server-da/multi-web-server.rs @@ -2,7 +2,7 @@ use std::sync::Arc; use async_compatibility_layer::{art::async_spawn, channel::oneshot}; use clap::Parser; -use hotshot::demos::sdemo::SDemoTypes; +use hotshot::demo::SDemoTypes; use tracing::error; #[derive(Parser, Debug)] diff --git a/crates/hotshot/examples/web-server-da/orchestrator.rs b/crates/hotshot/examples/web-server-da/orchestrator.rs index 772d3aa12e..785daeecb1 100644 --- a/crates/hotshot/examples/web-server-da/orchestrator.rs +++ b/crates/hotshot/examples/web-server-da/orchestrator.rs @@ -2,7 +2,7 @@ pub mod types; use async_compatibility_layer::logging::{setup_backtrace, setup_logging}; use clap::Parser; -use hotshot::demos::sdemo::SDemoTypes; +use hotshot::demo::SDemoTypes; use tracing::instrument; use types::ThisMembership; diff --git a/crates/hotshot/examples/web-server-da/types.rs b/crates/hotshot/examples/web-server-da/types.rs index 6827a00fe8..4b4905062e 100644 --- a/crates/hotshot/examples/web-server-da/types.rs +++ b/crates/hotshot/examples/web-server-da/types.rs @@ -1,6 +1,6 @@ use crate::infra_da::WebServerDARun; use hotshot::{ - demos::sdemo::SDemoTypes, + demo::SDemoTypes, traits::{ election::static_committee::GeneralStaticCommittee, implementations::{MemoryStorage, WebCommChannel}, diff --git a/crates/hotshot/examples/web-server-da/validator.rs b/crates/hotshot/examples/web-server-da/validator.rs index ec2415fd65..caf46bdc74 100644 --- a/crates/hotshot/examples/web-server-da/validator.rs +++ b/crates/hotshot/examples/web-server-da/validator.rs @@ -1,6 +1,6 @@ use async_compatibility_layer::logging::{setup_backtrace, setup_logging}; use clap::Parser; -use hotshot::demos::sdemo::SDemoTypes; +use hotshot::demo::SDemoTypes; use tracing::{info, instrument}; use crate::types::{DANetwork, NodeImpl, QuorumNetwork, ThisMembership, ThisRun, ViewSyncNetwork}; diff --git a/crates/hotshot/examples/web-server-da/web-server.rs b/crates/hotshot/examples/web-server-da/web-server.rs index 99d0b12f63..bbf0bbc691 100644 --- a/crates/hotshot/examples/web-server-da/web-server.rs +++ b/crates/hotshot/examples/web-server-da/web-server.rs @@ -1,4 +1,4 @@ -use hotshot::demos::sdemo::SDemoTypes; +use hotshot::demo::SDemoTypes; use std::sync::Arc; use async_compatibility_layer::{ diff --git a/crates/hotshot/src/block_impl.rs b/crates/hotshot/src/block_impl.rs new file mode 100644 index 0000000000..d3e4755129 --- /dev/null +++ b/crates/hotshot/src/block_impl.rs @@ -0,0 +1,174 @@ +//! This module provides an implementation of the `HotShot` suite of traits. +use std::{ + collections::HashSet, + fmt::{Debug, Display}, + ops::Deref, +}; + +use commit::{Commitment, Committable}; +use hotshot_types::traits::{block_contents::Transaction, state::TestableBlock, BlockPayload}; +use serde::{Deserialize, Serialize}; +use snafu::Snafu; + +/// The transaction in a [`VIDBlockPayload`]. +#[derive(PartialEq, Eq, Hash, Serialize, Deserialize, Clone, Debug)] +pub struct VIDTransaction { + /// identifier for the transaction + pub id: u64, + /// padding to add to txn (to make it larger and thereby more realistic) + pub padding: Vec, +} + +impl Deref for VIDTransaction { + type Target = u64; + + fn deref(&self) -> &Self::Target { + &self.id + } +} + +impl Committable for VIDTransaction { + fn commit(&self) -> Commitment { + commit::RawCommitmentBuilder::new("SDemo Txn Comm") + .u64_field("id", self.id) + .finalize() + } + + fn tag() -> String { + "SEQUENCING_DEMO_TXN".to_string() + } +} + +impl Transaction for VIDTransaction {} + +impl VIDTransaction { + /// create a new transaction + #[must_use] + pub fn new(id: u64) -> Self { + Self { + id, + padding: vec![], + } + } +} + +/// The error type for block payload. +#[derive(Snafu, Debug)] +pub enum BlockPayloadError { + /// Previous state commitment does not match + PreviousStateMismatch, + /// Nonce was reused + ReusedTxn, + /// Genesis failure + GenesisFailed, + /// Genesis reencountered after initialization + GenesisAfterStart, + /// no transasctions added to genesis + GenesisCantHaveTransactions, + /// invalid block + InvalidBlock, +} + +/// genesis block +#[derive(PartialEq, Eq, Hash, Serialize, Deserialize, Clone, Debug)] +pub struct GenesisBlockPayload {} + +/// Any block after genesis +#[derive(PartialEq, Eq, Hash, Serialize, Deserialize, Clone, Debug)] +pub struct NormalBlockPayload { + /// [`BlockPayload`] state commitment + pub previous_state: (), + /// [`VIDTransaction`] vector + pub transactions: Vec, +} + +/// The block for the sequencing demo +#[derive(PartialEq, Eq, Hash, Serialize, Deserialize, Clone, Debug)] +pub enum VIDBlockPayload { + /// genesis block payload + Genesis(GenesisBlockPayload), + /// normal block payload + Normal(NormalBlockPayload), +} + +impl Committable for VIDBlockPayload { + fn commit(&self) -> Commitment { + match &self { + VIDBlockPayload::Genesis(_) => { + commit::RawCommitmentBuilder::new("Genesis Comm").finalize() + } + VIDBlockPayload::Normal(block) => { + let mut builder = commit::RawCommitmentBuilder::new("Normal Comm"); + for txn in &block.transactions { + builder = builder.u64_field("transaction", **txn); + } + builder.finalize() + } + } + } + + fn tag() -> String { + "VID_BLOCK_PAYLOAD".to_string() + } +} + +impl Display for VIDBlockPayload { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + VIDBlockPayload::Genesis(_) => { + write!(f, "Genesis BlockPayload") + } + VIDBlockPayload::Normal(block) => { + write!(f, "Normal BlockPayload #txns={}", block.transactions.len()) + } + } + } +} + +impl TestableBlock for VIDBlockPayload { + fn genesis() -> Self { + VIDBlockPayload::Genesis(GenesisBlockPayload {}) + } + + fn txn_count(&self) -> u64 { + match self { + VIDBlockPayload::Genesis(_) => 0, + VIDBlockPayload::Normal(n) => n.transactions.len() as u64, + } + } +} + +impl BlockPayload for VIDBlockPayload { + type Error = BlockPayloadError; + + type Transaction = VIDTransaction; + + fn new() -> Self { + ::genesis() + } + + fn add_transaction_raw( + &self, + tx: &Self::Transaction, + ) -> std::result::Result { + match self { + VIDBlockPayload::Genesis(_) => Err(BlockPayloadError::GenesisCantHaveTransactions), + VIDBlockPayload::Normal(n) => { + let mut new = n.clone(); + new.transactions.push(tx.clone()); + Ok(VIDBlockPayload::Normal(new)) + } + } + } + + fn contained_transactions(&self) -> HashSet> { + match self { + VIDBlockPayload::Genesis(_) => HashSet::new(), + VIDBlockPayload::Normal(n) => n + .transactions + .iter() + .map(commit::Committable::commit) + .collect(), + } + } +} diff --git a/crates/hotshot/src/demos/sdemo.rs b/crates/hotshot/src/demo.rs similarity index 52% rename from crates/hotshot/src/demos/sdemo.rs rename to crates/hotshot/src/demo.rs index 30d12bc579..f5b7361d89 100644 --- a/crates/hotshot/src/demos/sdemo.rs +++ b/crates/hotshot/src/demo.rs @@ -5,14 +5,10 @@ //! //! These implementations are useful in examples and integration testing, but are not suitable for //! production use. -use crate::traits::election::static_committee::{StaticElectionConfig, StaticVoteToken}; -use std::{ - collections::HashSet, - fmt::{Debug, Display}, - marker::PhantomData, - ops::Deref, +use crate::{ + block_impl::{BlockPayloadError, NormalBlockPayload, VIDBlockPayload, VIDTransaction}, + traits::election::static_committee::{StaticElectionConfig, StaticVoteToken}, }; - use commit::{Commitment, Committable}; use derivative::Derivative; use either::Either; @@ -24,101 +20,15 @@ use hotshot_types::{ ViewNumber, }, traits::{ - block_contents::Transaction, election::Membership, node_implementation::NodeType, - state::{ConsensusTime, TestableBlock, TestableState}, + state::{ConsensusTime, TestableState}, BlockPayload, State, }, }; use rand::Rng; use serde::{Deserialize, Serialize}; -use snafu::Snafu; - -/// The transaction for the sequencing demo -#[derive(PartialEq, Eq, Hash, Serialize, Deserialize, Clone, Debug)] -pub struct SDemoTransaction { - /// identifier for the transaction - pub id: u64, - /// padding to add to txn (to make it larger and thereby more realistic) - pub padding: Vec, -} - -impl Deref for SDemoTransaction { - type Target = u64; - - fn deref(&self) -> &Self::Target { - &self.id - } -} - -impl Committable for SDemoTransaction { - fn commit(&self) -> Commitment { - commit::RawCommitmentBuilder::new("SDemo Txn Comm") - .u64_field("id", self.id) - .finalize() - } - - fn tag() -> String { - "SEQUENCING_DEMO_TXN".to_string() - } -} - -impl Transaction for SDemoTransaction {} - -impl SDemoTransaction { - /// create a new transaction - #[must_use] - pub fn new(id: u64) -> Self { - Self { - id, - padding: vec![], - } - } -} - -/// genesis block -#[derive(PartialEq, Eq, Hash, Serialize, Deserialize, Clone, Debug)] -pub struct SDemoGenesisBlock {} - -/// Any block after genesis -#[derive(PartialEq, Eq, Hash, Serialize, Deserialize, Clone, Debug)] -pub struct SDemoNormalBlock { - /// BlockPayload state commitment - pub previous_state: (), - /// Transaction vector - pub transactions: Vec, -} - -/// The block for the sequencing demo -#[derive(PartialEq, Eq, Hash, Serialize, Deserialize, Clone, Debug)] -pub enum SDemoBlock { - /// genesis block - Genesis(SDemoGenesisBlock), - /// normal block - Normal(SDemoNormalBlock), -} - -impl Committable for SDemoBlock { - fn commit(&self) -> Commitment { - match &self { - SDemoBlock::Genesis(_) => { - commit::RawCommitmentBuilder::new("SDemo Genesis Comm").finalize() - } - SDemoBlock::Normal(block) => { - let mut builder = commit::RawCommitmentBuilder::new("SDemo Normal Comm"); - for txn in &block.transactions { - builder = builder.u64_field("transaction", **txn); - } - builder.finalize() - } - } - } - - fn tag() -> String { - "SEQUENCING_DEMO_BLOCK".to_string() - } -} +use std::{fmt::Debug, marker::PhantomData}; /// sequencing demo entry state #[derive(PartialEq, Eq, Hash, Serialize, Deserialize, Clone, Debug)] @@ -155,97 +65,15 @@ impl Default for SDemoState { } } -/// The error type for the sequencing demo -#[derive(Snafu, Debug)] -pub enum SDemoError { - /// Previous state commitment does not match - PreviousStateMismatch, - /// Nonce was reused - ReusedTxn, - /// Genesis failure - GenesisFailed, - /// Genesis reencountered after initialization - GenesisAfterStart, - /// no transasctions added to genesis - GenesisCantHaveTransactions, - /// invalid block - InvalidBlock, -} - -impl Display for SDemoBlock { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - match self { - SDemoBlock::Genesis(_) => { - write!(f, "SDemo Genesis BlockPayload") - } - SDemoBlock::Normal(block) => { - write!( - f, - "SDemo Normal BlockPayload #txns={}", - block.transactions.len() - ) - } - } - } -} - -impl TestableBlock for SDemoBlock { - fn genesis() -> Self { - SDemoBlock::Genesis(SDemoGenesisBlock {}) - } - - fn txn_count(&self) -> u64 { - match self { - SDemoBlock::Genesis(_) => 0, - SDemoBlock::Normal(n) => n.transactions.len() as u64, - } - } -} - -impl BlockPayload for SDemoBlock { - type Error = SDemoError; - - type Transaction = SDemoTransaction; - - fn new() -> Self { - ::genesis() - } - - fn add_transaction_raw( - &self, - tx: &Self::Transaction, - ) -> std::result::Result { - match self { - SDemoBlock::Genesis(_) => Err(SDemoError::GenesisCantHaveTransactions), - SDemoBlock::Normal(n) => { - let mut new = n.clone(); - new.transactions.push(tx.clone()); - Ok(SDemoBlock::Normal(new)) - } - } - } - - fn contained_transactions(&self) -> HashSet> { - match self { - SDemoBlock::Genesis(_) => HashSet::new(), - SDemoBlock::Normal(n) => n - .transactions - .iter() - .map(commit::Committable::commit) - .collect(), - } - } -} - impl State for SDemoState { - type Error = SDemoError; + type Error = BlockPayloadError; - type BlockType = SDemoBlock; + type BlockType = VIDBlockPayload; type Time = ViewNumber; fn next_block(_state: Option) -> Self::BlockType { - SDemoBlock::Normal(SDemoNormalBlock { + VIDBlockPayload::Normal(NormalBlockPayload { previous_state: (), transactions: Vec::new(), }) @@ -253,10 +81,10 @@ impl State for SDemoState { fn validate_block(&self, block: &Self::BlockType, view_number: &Self::Time) -> bool { match block { - SDemoBlock::Genesis(_) => { + VIDBlockPayload::Genesis(_) => { view_number == &ViewNumber::genesis() && view_number == &self.view_number } - SDemoBlock::Normal(_n) => self.view_number < *view_number, + VIDBlockPayload::Normal(_n) => self.view_number < *view_number, } } @@ -266,7 +94,7 @@ impl State for SDemoState { view_number: &Self::Time, ) -> Result { if !self.validate_block(block, view_number) { - return Err(SDemoError::InvalidBlock); + return Err(BlockPayloadError::InvalidBlock); } Ok(SDemoState { @@ -285,7 +113,7 @@ impl TestableState for SDemoState { rng: &mut dyn rand::RngCore, padding: u64, ) -> ::Transaction { - SDemoTransaction { + VIDTransaction { id: rng.gen_range(0..10), padding: vec![0; padding as usize], } @@ -309,10 +137,10 @@ pub struct SDemoTypes; impl NodeType for SDemoTypes { type Time = ViewNumber; - type BlockType = SDemoBlock; + type BlockType = VIDBlockPayload; type SignatureKey = BN254Pub; type VoteTokenType = StaticVoteToken; - type Transaction = SDemoTransaction; + type Transaction = VIDTransaction; type ElectionConfigType = StaticElectionConfig; type StateType = SDemoState; } diff --git a/crates/hotshot/src/demos.rs b/crates/hotshot/src/demos.rs deleted file mode 100644 index 7ddbef89c6..0000000000 --- a/crates/hotshot/src/demos.rs +++ /dev/null @@ -1,8 +0,0 @@ -//! Contains implementations of the `HotShot` traits used in the examples and integration testing. -//! -//! These implementations are not suitable for production use. - -/// this is a demo for sequencing consensus -pub mod sdemo; -/// this is a demo for validating consensus -pub mod vdemo; diff --git a/crates/hotshot/src/demos/vdemo.rs b/crates/hotshot/src/demos/vdemo.rs deleted file mode 100644 index 1d091c60b1..0000000000 --- a/crates/hotshot/src/demos/vdemo.rs +++ /dev/null @@ -1,607 +0,0 @@ -// //! Validating (vanilla) consensus demo -// //! -// //! This module provides an implementation of the `HotShot` suite of traits that implements a -// //! basic demonstration of validating consensus. -// //! -// //! These implementations are useful in examples and integration testing, but are not suitable for -// //! production use. -// -// use crate::traits::{ -// election::static_committee::{StaticElectionConfig, StaticVoteToken}, -// Block, -// }; -// use commit::{Commitment, Committable}; -// use derivative::Derivative; -// -// use hotshot_types::{ -// certificate::{QuorumCertificate, YesNoSignature}, -// constants::genesis_proposer_id, -// data::{random_commitment, LeafType, ValidatingLeaf, ViewNumber}, -// traits::{ -// block_contents::Transaction, -// consensus_type::validating_consensus::ValidatingConsensus, -// election::Membership, -// node_implementation::NodeType, -// signature_key::ed25519::Ed25519Pub, -// state::{ConsensusTime, TestableBlock, TestableState}, -// State, -// }, -// }; -// -// use rand::Rng; -// use serde::{Deserialize, Serialize}; -// use snafu::{ensure, Snafu}; -// use std::{ -// collections::{BTreeMap, HashSet}, -// fmt::{Debug, Display}, -// marker::PhantomData, -// }; -// use tracing::error; -// -// /// The account identifier type used by the demo -// /// -// /// This is a type alias to [`String`] for simplicity. -// pub type Account = String; -// -// /// The account balance type used by the demo -// /// -// /// This is a type alias to [`u64`] for simplicity. -// pub type Balance = u64; -// -// /// Records a reduction in an account balance -// #[derive(PartialEq, Eq, Hash, Serialize, Deserialize, Clone, Debug)] -// pub struct Subtraction { -// /// An account identifier -// pub account: Account, -// /// An account balance -// pub amount: Balance, -// } -// -// /// Records an increase in an account balance -// #[derive(PartialEq, Eq, Hash, Serialize, Deserialize, Clone, Debug)] -// pub struct Addition { -// /// An account identifier -// pub account: Account, -// /// An account balance -// pub amount: Balance, -// } -// -// /// The error type for the validating demo -// #[derive(Snafu, Debug)] -// pub enum VDemoError { -// /// The subtraction and addition amounts for this transaction were not equal -// InconsistentTransaction, -// /// No such input account exists -// NoSuchInputAccount, -// /// No such output account exists -// NoSuchOutputAccount, -// /// Tried to move more money than was in the account -// InsufficentBalance, -// /// Previous state commitment does not match -// PreviousStateMismatch, -// /// Nonce was reused -// ReusedNonce, -// /// Genesis failure -// GenesisFailed, -// /// Genesis reencountered after initialization -// GenesisAfterStart, -// } -// -// /// The transaction for the validating demo -// #[derive(PartialEq, Eq, Hash, Serialize, Deserialize, Clone, Debug)] -// pub struct VDemoTransaction { -// /// An increment to an account balance -// pub add: Addition, -// /// A decrement to an account balance -// pub sub: Subtraction, -// /// The nonce for a transaction, no two transactions can have the same nonce -// pub nonce: u64, -// /// Number of bytes to pad to each transaction -// pub padding: Vec, -// } -// -// impl Transaction for VDemoTransaction {} -// -// impl VDemoTransaction { -// /// Ensures that this transaction is at least consistent with itself -// #[must_use] -// pub fn validate_independence(&self) -> bool { -// // Ensure that we are adding to one account exactly as much as we are subtracting from -// // another -// self.add.amount <= self.sub.amount // TODO why not strict equality? -// } -// } -// -// /// The state for the validating demo -// /// NOTE both fields are btrees because we need -// /// ordered-ing otherwise commitments will not match -// #[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize, Hash)] -// pub struct VDemoState { -// /// Key/value store of accounts and balances -// pub balances: BTreeMap, -// // /// Set of previously seen nonces -// // pub nonces: BTreeSet, -// } -// -// impl Committable for VDemoState { -// fn commit(&self) -> Commitment { -// let mut builder = commit::RawCommitmentBuilder::new("VDemo State Comm"); -// -// for (k, v) in &self.balances { -// builder = builder.u64_field(k, *v); -// } -// builder = builder.constant_str("nonces"); -// -// // for nonce in &self.nonces { -// // builder = builder.u64(*nonce); -// // } -// -// builder.finalize() -// } -// -// fn tag() -> String { -// "VALIDATING_DEMO_STATE".to_string() -// } -// } -// -// /// initializes the first state on genesis -// #[derive(PartialEq, Eq, Hash, Serialize, Deserialize, Clone, Debug)] -// pub struct VDemoGenesisBlock { -// /// initializes the first state -// pub accounts: BTreeMap, -// } -// -// /// Any block after genesis -// #[derive(PartialEq, Eq, Hash, Serialize, Deserialize, Clone, Debug)] -// pub struct VDemoNormalBlock { -// /// Block state commitment -// pub previous_state: Commitment, -// /// Transaction vector -// pub transactions: Vec, -// } -// -// /// The block for the validating demo -// #[derive(PartialEq, Eq, Hash, Serialize, Deserialize, Clone, Debug)] -// pub enum VDemoBlock { -// /// genesis block -// Genesis(VDemoGenesisBlock), -// /// normal block -// Normal(VDemoNormalBlock), -// } -// -// impl Committable for VDemoBlock { -// fn commit(&self) -> Commitment { -// match &self { -// VDemoBlock::Genesis(block) => { -// let mut builder = commit::RawCommitmentBuilder::new("VDemo Genesis Comm") -// .u64_field("account_count", block.accounts.len() as u64); -// for account in &block.accounts { -// builder = builder.u64_field(account.0, *account.1); -// } -// builder.finalize() -// } -// VDemoBlock::Normal(block) => { -// let mut builder = commit::RawCommitmentBuilder::new("VDemo Block Comm") -// .var_size_field("Previous State", block.previous_state.as_ref()); -// -// for txn in &block.transactions { -// builder = builder -// .u64_field(&txn.add.account, txn.add.amount) -// .u64_field(&txn.sub.account, txn.sub.amount) -// .constant_str("nonce") -// .u64_field("nonce", txn.nonce); -// } -// -// builder.finalize() -// } -// } -// } -// -// fn tag() -> String { -// "VALIDATING_DEMO_BLOCK".to_string() -// } -// } -// -// impl Committable for VDemoTransaction { -// fn commit(&self) -> Commitment { -// commit::RawCommitmentBuilder::new("VDemo Txn Comm") -// .u64_field(&self.add.account, self.add.amount) -// .u64_field(&self.sub.account, self.sub.amount) -// .constant_str("nonce") -// .u64_field("nonce", self.nonce) -// .finalize() -// } -// -// fn tag() -> String { -// "VALIDATING_DEMO_TXN".to_string() -// } -// } -// -// impl VDemoBlock { -// /// generate a genesis block with the provided initial accounts and balances -// #[must_use] -// pub fn genesis_from(accounts: BTreeMap) -> Self { -// Self::Genesis(VDemoGenesisBlock { accounts }) -// } -// } -// -// impl State for VDemoState { -// type Error = VDemoError; -// -// type BlockType = VDemoBlock; -// -// type Time = ViewNumber; -// -// #[allow(clippy::panic)] -// fn next_block(state: Option) -> Self::BlockType { -// match state { -// Some(state) => VDemoBlock::Normal(VDemoNormalBlock { -// previous_state: state.commit(), -// transactions: Vec::new(), -// }), -// None => panic!("State is required for the next block"), -// } -// } -// -// // Note: validate_block is actually somewhat redundant, its meant to be a quick and dirty check -// // for clarity, the logic is duplicated with append_to -// fn validate_block(&self, block: &Self::BlockType, _view_number: &Self::Time) -> bool { -// match block { -// VDemoBlock::Genesis(_) => self.balances.is_empty(), // && self.nonces.is_empty(), -// VDemoBlock::Normal(block) => { -// let state = self; -// // A valid block is one in which every transaction is internally consistent, and results in -// // nobody having a negative balance -// // -// // We will check this, in this case, by making a trial copy of our balances map, making -// // trial modifications to it, and then asserting that no balances are negative -// let mut trial_balances = state.balances.clone(); -// for tx in &block.transactions { -// // This is a macro from SNAFU that returns an Err if the condition is not satisfied -// // -// // We first check that the transaction is internally consistent, then apply the change -// // to our trial map -// if !tx.validate_independence() { -// error!("validate_independence failed"); -// return false; -// } -// // Find the input account, and subtract the transfer balance from it, failing if it -// // doesn't exist -// if let Some(input_account) = trial_balances.get_mut(&tx.sub.account) { -// *input_account -= tx.sub.amount; -// } else { -// error!("no such input account"); -// return false; -// } -// // Find the output account, and add the transfer balance to it, failing if it doesn't -// // exist -// if let Some(output_account) = trial_balances.get_mut(&tx.add.account) { -// *output_account += tx.add.amount; -// } else { -// error!("no such output account"); -// return false; -// } -// // // Check to make sure the nonce isn't used -// // if state.nonces.contains(&tx.nonce) { -// // warn!(?state, ?tx, "State nonce is used for transaction"); -// // return false; -// // } -// } -// // This block has now passed all our tests, and thus has not done anything bad, so the block -// // is valid if its previous state hash matches that of the previous state -// let result = block.previous_state == state.commit(); -// if !result { -// error!( -// "hash failure. previous_block: {:?} hash_state: {:?}", -// block.previous_state, -// state.commit() -// ); -// } -// result -// } -// } -// } -// -// fn append( -// &self, -// block: &Self::BlockType, -// _view_number: &Self::Time, -// ) -> std::result::Result { -// match block { -// VDemoBlock::Genesis(block) => { -// if self.balances.is_empty() { -// // && self.nonces.is_empty() -// let mut new_state = Self::default(); -// for account in &block.accounts { -// if new_state -// .balances -// .insert(account.0.clone(), *account.1) -// .is_some() -// { -// error!("Adding the same account twice during application of genesis block!"); -// return Err(VDemoError::GenesisFailed); -// } -// } -// Ok(new_state) -// } else { -// Err(VDemoError::GenesisAfterStart) -// } -// } -// VDemoBlock::Normal(block) => { -// let state = self; -// // A valid block is one in which every transaction is internally consistent, and results in -// // nobody having a negative balance -// // -// // We will check this, in this case, by making a trial copy of our balances map, making -// // trial modifications to it, and then asserting that no balances are negative -// let mut trial_balances = state.balances.clone(); -// for tx in &block.transactions { -// // This is a macro from SNAFU that returns an Err if the condition is not satisfied -// // -// // We first check that the transaction is internally consistent, then apply the change -// // to our trial map -// ensure!(tx.validate_independence(), InconsistentTransactionSnafu); -// // Find the input account, and subtract the transfer balance from it, failing if it -// // doesn't exist -// if let Some(input_account) = trial_balances.get_mut(&tx.sub.account) { -// *input_account -= tx.sub.amount; -// } else { -// return Err(VDemoError::NoSuchInputAccount); -// } -// // Find the output account, and add the transfer balance to it, failing if it doesn't -// // exist -// if let Some(output_account) = trial_balances.get_mut(&tx.add.account) { -// *output_account += tx.add.amount; -// } else { -// return Err(VDemoError::NoSuchOutputAccount); -// } -// // // Check for nonce reuse -// // if state.nonces.contains(&tx.nonce) { -// // return Err(VDemoError::ReusedNonce); -// // } -// } -// // Make sure our previous state commitment matches the provided state -// if block.previous_state == state.commit() { -// // This block has now passed all our tests, and thus has not done anything bad -// // Add the nonces from this block -// // let mut nonces = state.nonces.clone(); -// // for tx in &block.transactions { -// // nonces.insert(tx.nonce); -// // } -// Ok(VDemoState { -// balances: trial_balances, -// // nonces, -// }) -// } else { -// Err(VDemoError::PreviousStateMismatch) -// } -// } -// } -// } -// -// fn on_commit(&self) { -// // Does nothing in this implementation -// } -// } -// -// impl Display for VDemoBlock { -// fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { -// match self { -// VDemoBlock::Genesis(block) => { -// write!(f, "VDemo Genesis Block: {block:#?}") -// } -// VDemoBlock::Normal(block) => { -// write!(f, "VDemo Normal Block #txns={}", block.transactions.len()) -// } -// } -// } -// } -// -// impl TestableState for VDemoState { -// fn create_random_transaction( -// state: Option<&Self>, -// rng: &mut dyn rand::RngCore, -// padding: u64, -// ) -> ::Transaction { -// use rand::seq::IteratorRandom; -// -// let state = state.expect("Missing state"); -// -// let non_zero_balances = state -// .balances -// .iter() -// .filter(|b| *b.1 > 0) -// .collect::>(); -// -// assert!( -// !non_zero_balances.is_empty(), -// "No nonzero balances were available! Unable to generate transaction." -// ); -// -// let input_account = non_zero_balances.iter().choose(rng).unwrap().0; -// let output_account = state.balances.keys().choose(rng).unwrap(); -// let amount = rng.gen_range(0..100); -// -// VDemoTransaction { -// add: Addition { -// account: output_account.to_string(), -// amount, -// }, -// sub: Subtraction { -// account: input_account.to_string(), -// amount, -// }, -// nonce: rng.gen(), -// padding: vec![0; padding as usize], -// } -// } -// } -// -// impl TestableBlock for VDemoBlock { -// fn genesis() -> Self { -// let accounts: BTreeMap = vec![ -// ("Joe", 1_000_000), -// ("Nathan M", 500_000), -// ("John", 400_000), -// ("Nathan Y", 600_000), -// ("Ian", 5_000_000), -// ] -// .into_iter() -// .map(|(x, y)| (x.to_string(), y)) -// .collect(); -// Self::Genesis(VDemoGenesisBlock { accounts }) -// } -// -// fn txn_count(&self) -> u64 { -// if let VDemoBlock::Normal(block) = self { -// block.transactions.len() as u64 -// } else { -// 0 -// } -// } -// } -// -// impl Block for VDemoBlock { -// type Transaction = VDemoTransaction; -// -// type Error = VDemoError; -// -// fn new() -> Self { -// ::genesis() -// } -// -// fn add_transaction_raw( -// &self, -// tx: &Self::Transaction, -// ) -> std::result::Result { -// match self { -// VDemoBlock::Genesis(_) => Err(VDemoError::GenesisAfterStart), -// VDemoBlock::Normal(block) => { -// // first, make sure that the transaction is internally valid -// if tx.validate_independence() { -// // Then check the previous transactions in the block -// if block.transactions.iter().any(|x| x.nonce == tx.nonce) { -// return Err(VDemoError::ReusedNonce); -// } -// let mut new_block = block.clone(); -// // Insert our transaction and return -// new_block.transactions.push(tx.clone()); -// Ok(VDemoBlock::Normal(new_block)) -// } else { -// Err(VDemoError::InconsistentTransaction) -// } -// } -// } -// } -// fn contained_transactions(&self) -> HashSet> { -// match self { -// VDemoBlock::Genesis(_) => HashSet::new(), -// VDemoBlock::Normal(block) => block -// .transactions -// .clone() -// .into_iter() -// .map(|tx| tx.commit()) -// .collect(), -// } -// } -// } -// -// /// Implementation of [`NodeType`] for [`VDemoNode`] -// #[derive( -// Copy, -// Clone, -// Debug, -// Default, -// Hash, -// PartialEq, -// Eq, -// PartialOrd, -// Ord, -// serde::Serialize, -// serde::Deserialize, -// )] -// pub struct VDemoTypes; -// -// impl NodeType for VDemoTypes { -// type ConsensusType = ValidatingConsensus; -// type Time = ViewNumber; -// type BlockType = VDemoBlock; -// type SignatureKey = Ed25519Pub; -// type VoteTokenType = StaticVoteToken; -// type Transaction = VDemoTransaction; -// type ElectionConfigType = StaticElectionConfig; -// type StateType = VDemoState; -// } -// -// /// The node implementation for the validating demo -// #[derive(Derivative)] -// #[derivative(Clone(bound = ""))] -// pub struct VDemoNode(PhantomData) -// where -// MEMBERSHIP: Membership + Debug; -// -// impl VDemoNode -// where -// MEMBERSHIP: Membership + Debug, -// { -// /// Create a new `VDemoNode` -// #[must_use] -// pub fn new() -> Self { -// VDemoNode(PhantomData) -// } -// } -// -// impl Debug for VDemoNode -// where -// MEMBERSHIP: Membership + Debug, -// { -// fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { -// f.debug_struct("VDemoNode") -// .field("_phantom", &"phantom") -// .finish() -// } -// } -// -// impl Default for VDemoNode -// where -// MEMBERSHIP: Membership + Debug, -// { -// fn default() -> Self { -// Self::new() -// } -// } -// -// /// Provides a random [`QuorumCertificate`] -// pub fn random_quorum_certificate>( -// rng: &mut dyn rand::RngCore, -// ) -> QuorumCertificate { -// QuorumCertificate { -// // block_commitment: random_commitment(rng), -// leaf_commitment: random_commitment(rng), -// view_number: TYPES::Time::new(rng.gen()), -// signatures: YesNoSignature::Yes(BTreeMap::default()), -// is_genesis: rng.gen(), -// } -// } -// -// /// Provides a random [`ValidatingLeaf`] -// pub fn random_validating_leaf>( -// deltas: TYPES::BlockType, -// rng: &mut dyn rand::RngCore, -// ) -> ValidatingLeaf { -// let justify_qc = random_quorum_certificate(rng); -// let state = TYPES::StateType::default() -// .append(&deltas, &TYPES::Time::new(42)) -// .unwrap_or_default(); -// ValidatingLeaf { -// view_number: justify_qc.view_number, -// height: rng.next_u64(), -// justify_qc, -// parent_commitment: random_commitment(rng), -// deltas, -// state, -// rejected: Vec::new(), -// timestamp: time::OffsetDateTime::now_utc().unix_timestamp_nanos(), -// proposer_id: genesis_proposer_id(), -// } -// } diff --git a/crates/hotshot/src/lib.rs b/crates/hotshot/src/lib.rs index 065ed8ee80..76a08ba010 100644 --- a/crates/hotshot/src/lib.rs +++ b/crates/hotshot/src/lib.rs @@ -19,12 +19,11 @@ #[cfg(feature = "docs")] pub mod documentation; -/// Data availability support -// pub mod da; +pub mod block_impl; /// Contains structures and functions for committee election pub mod certificate; #[cfg(feature = "demo")] -pub mod demos; +pub mod demo; /// Contains traits consumed by [`HotShot`] pub mod traits; /// Contains types used by the crate diff --git a/crates/testing/src/node_types.rs b/crates/testing/src/node_types.rs index c788006984..9e06d968d0 100644 --- a/crates/testing/src/node_types.rs +++ b/crates/testing/src/node_types.rs @@ -1,8 +1,11 @@ -use hotshot::traits::implementations::CombinedNetworks; +use hotshot::{ + block_impl::{VIDBlockPayload, VIDTransaction}, + traits::implementations::CombinedNetworks, +}; use std::{marker::PhantomData, sync::Arc}; use hotshot::{ - demos::sdemo::{SDemoBlock, SDemoState, SDemoTransaction}, + demo::SDemoState, traits::{ election::static_committee::{StaticCommittee, StaticElectionConfig, StaticVoteToken}, implementations::{ @@ -41,10 +44,10 @@ use serde::{Deserialize, Serialize}; pub struct SequencingTestTypes; impl NodeType for SequencingTestTypes { type Time = ViewNumber; - type BlockType = SDemoBlock; + type BlockType = VIDBlockPayload; type SignatureKey = BN254Pub; type VoteTokenType = StaticVoteToken; - type Transaction = SDemoTransaction; + type Transaction = VIDTransaction; type ElectionConfigType = StaticElectionConfig; type StateType = SDemoState; } diff --git a/crates/testing/tests/da_task.rs b/crates/testing/tests/da_task.rs index 64b70ef40c..f3fb3b0cfb 100644 --- a/crates/testing/tests/da_task.rs +++ b/crates/testing/tests/da_task.rs @@ -21,7 +21,7 @@ use std::collections::HashMap; #[cfg_attr(async_executor_impl = "async-std", async_std::test)] async fn test_da_task() { use hotshot::{ - demos::sdemo::{SDemoBlock, SDemoNormalBlock}, + block_impl::{NormalBlockPayload, VIDBlockPayload}, tasks::add_da_task, }; use hotshot_task_impls::harness::run_harness; @@ -39,7 +39,7 @@ async fn test_da_task() { }; let committee_exchange = api.inner.exchanges.committee_exchange().clone(); let pub_key = *api.public_key(); - let block = SDemoBlock::Normal(SDemoNormalBlock { + let block = VIDBlockPayload::Normal(NormalBlockPayload { previous_state: (), transactions: Vec::new(), }); diff --git a/crates/testing/tests/network_task.rs b/crates/testing/tests/network_task.rs index 2dcd331396..a05cc0bd5e 100644 --- a/crates/testing/tests/network_task.rs +++ b/crates/testing/tests/network_task.rs @@ -21,7 +21,7 @@ use std::collections::HashMap; #[cfg_attr(async_executor_impl = "async-std", async_std::test)] #[ignore] async fn test_network_task() { - use hotshot::demos::sdemo::{SDemoBlock, SDemoNormalBlock}; + use hotshot::block_impl::{NormalBlockPayload, VIDBlockPayload}; use hotshot_task_impls::harness::run_harness; use hotshot_testing::task_helpers::build_system_handle; use hotshot_types::{ @@ -40,7 +40,7 @@ async fn test_network_task() { let committee_exchange = api.inner.exchanges.committee_exchange().clone(); let pub_key = *api.public_key(); let priv_key = api.private_key(); - let block = SDemoBlock::Normal(SDemoNormalBlock { + let block = VIDBlockPayload::Normal(NormalBlockPayload { previous_state: (), transactions: Vec::new(), }); diff --git a/testing-macros/src/lib.rs b/testing-macros/src/lib.rs index 8057569da7..58806d3b9b 100644 --- a/testing-macros/src/lib.rs +++ b/testing-macros/src/lib.rs @@ -616,7 +616,7 @@ pub fn cross_all_types(input: TokenStream) -> TokenStream { slow, } = parse_macro_input!(input as CrossAllTypesSpec); let tokens = quote! { - DemoType: [ /* (SequencingConsensus, hotshot::demos::sdemo::SDemoState), */ (ValidatingConsensus, hotshot::demos::vdemo::VDemoState) ], + DemoType: [ /* (SequencingConsensus, hotshot::demo::SDemoState), */ (ValidatingConsensus, hotshot::demos::vdemo::VDemoState) ], SignatureKey: [ hotshot_types::traits::signature_key::bn254::BN254Pub ], CommChannel: [ hotshot::traits::implementations::Libp2pCommChannel, hotshot::traits::implementations::CentralizedCommChannel ], Time: [ hotshot_types::data::ViewNumber ], From e11b70eae521363d26f71ea9e6cf64d87f66433b Mon Sep 17 00:00:00 2001 From: Keyao Shen Date: Mon, 18 Sep 2023 14:37:24 -0700 Subject: [PATCH 02/18] Combine genesis and normal block, remove previous_state, rename SDemoTypes. --- .../examples/web-server-da/multi-validator.rs | 4 +- .../web-server-da/multi-web-server.rs | 8 +- .../examples/web-server-da/orchestrator.rs | 4 +- .../hotshot/examples/web-server-da/types.rs | 54 ++++++------- .../examples/web-server-da/validator.rs | 4 +- .../examples/web-server-da/web-server.rs | 4 +- crates/hotshot/src/block_impl.rs | 75 ++++--------------- crates/hotshot/src/demo.rs | 30 ++++---- crates/testing/tests/da_task.rs | 10 +-- crates/testing/tests/network_task.rs | 7 +- crates/types/src/traits/block_contents.rs | 2 +- 11 files changed, 70 insertions(+), 132 deletions(-) diff --git a/crates/hotshot/examples/web-server-da/multi-validator.rs b/crates/hotshot/examples/web-server-da/multi-validator.rs index a16f2951ed..3ed46fa979 100644 --- a/crates/hotshot/examples/web-server-da/multi-validator.rs +++ b/crates/hotshot/examples/web-server-da/multi-validator.rs @@ -3,7 +3,7 @@ use async_compatibility_layer::{ logging::{setup_backtrace, setup_logging}, }; use clap::Parser; -use hotshot::demo::SDemoTypes; +use hotshot::demo::DemoTypes; use hotshot_orchestrator::client::ValidatorArgs; use std::net::IpAddr; use tracing::instrument; @@ -49,7 +49,7 @@ async fn main() { for _ in 0..args.num_nodes { let node = async_spawn(async move { infra_da::main_entry_point::< - SDemoTypes, + DemoTypes, ThisMembership, DANetwork, QuorumNetwork, diff --git a/crates/hotshot/examples/web-server-da/multi-web-server.rs b/crates/hotshot/examples/web-server-da/multi-web-server.rs index 39a9b7a87e..f954050ad0 100644 --- a/crates/hotshot/examples/web-server-da/multi-web-server.rs +++ b/crates/hotshot/examples/web-server-da/multi-web-server.rs @@ -2,7 +2,7 @@ use std::sync::Arc; use async_compatibility_layer::{art::async_spawn, channel::oneshot}; use clap::Parser; -use hotshot::demo::SDemoTypes; +use hotshot::demo::DemoTypes; use tracing::error; #[derive(Parser, Debug)] @@ -27,7 +27,7 @@ async fn main() { let cdn_server = async_spawn(async move { if let Err(e) = hotshot_web_server::run_web_server::< - ::SignatureKey, + ::SignatureKey, >(Some(server_shutdown_cdn), args.cdn_port) .await { @@ -37,7 +37,7 @@ async fn main() { }); let da_server = async_spawn(async move { if let Err(e) = hotshot_web_server::run_web_server::< - ::SignatureKey, + ::SignatureKey, >(Some(server_shutdown_da), args.da_port) .await { @@ -47,7 +47,7 @@ async fn main() { }); let vs_server = async_spawn(async move { if let Err(e) = hotshot_web_server::run_web_server::< - ::SignatureKey, + ::SignatureKey, >(Some(server_shutdown_view_sync), args.view_sync_port) .await { diff --git a/crates/hotshot/examples/web-server-da/orchestrator.rs b/crates/hotshot/examples/web-server-da/orchestrator.rs index 785daeecb1..594d004a93 100644 --- a/crates/hotshot/examples/web-server-da/orchestrator.rs +++ b/crates/hotshot/examples/web-server-da/orchestrator.rs @@ -2,7 +2,7 @@ pub mod types; use async_compatibility_layer::logging::{setup_backtrace, setup_logging}; use clap::Parser; -use hotshot::demo::SDemoTypes; +use hotshot::demo::DemoTypes; use tracing::instrument; use types::ThisMembership; @@ -29,7 +29,7 @@ async fn main() { let args = OrchestratorArgs::parse(); run_orchestrator_da::< - SDemoTypes, + DemoTypes, ThisMembership, DANetwork, QuorumNetwork, diff --git a/crates/hotshot/examples/web-server-da/types.rs b/crates/hotshot/examples/web-server-da/types.rs index 4b4905062e..017784b354 100644 --- a/crates/hotshot/examples/web-server-da/types.rs +++ b/crates/hotshot/examples/web-server-da/types.rs @@ -1,6 +1,6 @@ use crate::infra_da::WebServerDARun; use hotshot::{ - demo::SDemoTypes, + demo::DemoTypes, traits::{ election::static_committee::GeneralStaticCommittee, implementations::{MemoryStorage, WebCommChannel}, @@ -22,54 +22,54 @@ use std::fmt::Debug; #[derive(Clone, Debug, Deserialize, Serialize, Hash, PartialEq, Eq)] pub struct NodeImpl {} -pub type ThisLeaf = SequencingLeaf; +pub type ThisLeaf = SequencingLeaf; pub type ThisMembership = - GeneralStaticCommittee::SignatureKey>; -pub type DANetwork = WebCommChannel; -pub type QuorumNetwork = WebCommChannel; -pub type ViewSyncNetwork = WebCommChannel; + GeneralStaticCommittee::SignatureKey>; +pub type DANetwork = WebCommChannel; +pub type QuorumNetwork = WebCommChannel; +pub type ViewSyncNetwork = WebCommChannel; -pub type ThisDAProposal = DAProposal; -pub type ThisDAVote = DAVote; +pub type ThisDAProposal = DAProposal; +pub type ThisDAVote = DAVote; -pub type ThisQuorumProposal = QuorumProposal; -pub type ThisQuorumVote = QuorumVote; +pub type ThisQuorumProposal = QuorumProposal; +pub type ThisQuorumVote = QuorumVote; -pub type ThisViewSyncProposal = ViewSyncCertificate; -pub type ThisViewSyncVote = ViewSyncVote; +pub type ThisViewSyncProposal = ViewSyncCertificate; +pub type ThisViewSyncVote = ViewSyncVote; -impl NodeImplementation for NodeImpl { - type Storage = MemoryStorage; - type Leaf = SequencingLeaf; +impl NodeImplementation for NodeImpl { + type Storage = MemoryStorage; + type Leaf = SequencingLeaf; type Exchanges = SequencingExchanges< - SDemoTypes, - Message, + DemoTypes, + Message, QuorumExchange< - SDemoTypes, + DemoTypes, Self::Leaf, ThisQuorumProposal, ThisMembership, QuorumNetwork, - Message, + Message, >, - CommitteeExchange>, + CommitteeExchange>, ViewSyncExchange< - SDemoTypes, + DemoTypes, ThisViewSyncProposal, ThisMembership, ViewSyncNetwork, - Message, + Message, >, >; - type ConsensusMessage = SequencingMessage; + type ConsensusMessage = SequencingMessage; fn new_channel_maps( - start_view: ::Time, + start_view: ::Time, ) -> ( - ChannelMaps, - Option>, + ChannelMaps, + Option>, ) { (ChannelMaps::new(start_view), None) } } -pub type ThisRun = WebServerDARun; +pub type ThisRun = WebServerDARun; diff --git a/crates/hotshot/examples/web-server-da/validator.rs b/crates/hotshot/examples/web-server-da/validator.rs index caf46bdc74..ab44e02991 100644 --- a/crates/hotshot/examples/web-server-da/validator.rs +++ b/crates/hotshot/examples/web-server-da/validator.rs @@ -1,6 +1,6 @@ use async_compatibility_layer::logging::{setup_backtrace, setup_logging}; use clap::Parser; -use hotshot::demo::SDemoTypes; +use hotshot::demo::DemoTypes; use tracing::{info, instrument}; use crate::types::{DANetwork, NodeImpl, QuorumNetwork, ThisMembership, ThisRun, ViewSyncNetwork}; @@ -29,7 +29,7 @@ async fn main() { args.host, args.port ); infra_da::main_entry_point::< - SDemoTypes, + DemoTypes, ThisMembership, DANetwork, QuorumNetwork, diff --git a/crates/hotshot/examples/web-server-da/web-server.rs b/crates/hotshot/examples/web-server-da/web-server.rs index bbf0bbc691..9c4912c6a3 100644 --- a/crates/hotshot/examples/web-server-da/web-server.rs +++ b/crates/hotshot/examples/web-server-da/web-server.rs @@ -1,4 +1,4 @@ -use hotshot::demo::SDemoTypes; +use hotshot::demo::DemoTypes; use std::sync::Arc; use async_compatibility_layer::{ @@ -23,7 +23,7 @@ async fn main() { let (server_shutdown_sender, server_shutdown) = oneshot(); let _sender = Arc::new(server_shutdown_sender); let _result = hotshot_web_server::run_web_server::< - ::SignatureKey, + ::SignatureKey, >(Some(server_shutdown), args.port) .await; } diff --git a/crates/hotshot/src/block_impl.rs b/crates/hotshot/src/block_impl.rs index d3e4755129..dc7ebf878d 100644 --- a/crates/hotshot/src/block_impl.rs +++ b/crates/hotshot/src/block_impl.rs @@ -63,48 +63,21 @@ pub enum BlockPayloadError { GenesisFailed, /// Genesis reencountered after initialization GenesisAfterStart, - /// no transasctions added to genesis - GenesisCantHaveTransactions, /// invalid block InvalidBlock, } -/// genesis block +/// A [`BlockPayload`] that contains a list of `VIDTransaction`. #[derive(PartialEq, Eq, Hash, Serialize, Deserialize, Clone, Debug)] -pub struct GenesisBlockPayload {} - -/// Any block after genesis -#[derive(PartialEq, Eq, Hash, Serialize, Deserialize, Clone, Debug)] -pub struct NormalBlockPayload { - /// [`BlockPayload`] state commitment - pub previous_state: (), - /// [`VIDTransaction`] vector - pub transactions: Vec, -} - -/// The block for the sequencing demo -#[derive(PartialEq, Eq, Hash, Serialize, Deserialize, Clone, Debug)] -pub enum VIDBlockPayload { - /// genesis block payload - Genesis(GenesisBlockPayload), - /// normal block payload - Normal(NormalBlockPayload), -} +pub struct VIDBlockPayload(pub Vec); impl Committable for VIDBlockPayload { fn commit(&self) -> Commitment { - match &self { - VIDBlockPayload::Genesis(_) => { - commit::RawCommitmentBuilder::new("Genesis Comm").finalize() - } - VIDBlockPayload::Normal(block) => { - let mut builder = commit::RawCommitmentBuilder::new("Normal Comm"); - for txn in &block.transactions { - builder = builder.u64_field("transaction", **txn); - } - builder.finalize() - } + let mut builder = commit::RawCommitmentBuilder::new("Normal Comm"); + for txn in &self.0 { + builder = builder.u64_field("transaction", **txn); } + builder.finalize() } fn tag() -> String { @@ -114,27 +87,17 @@ impl Committable for VIDBlockPayload { impl Display for VIDBlockPayload { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - match self { - VIDBlockPayload::Genesis(_) => { - write!(f, "Genesis BlockPayload") - } - VIDBlockPayload::Normal(block) => { - write!(f, "Normal BlockPayload #txns={}", block.transactions.len()) - } - } + write!(f, "BlockPayload #txns={}", self.0.len()) } } impl TestableBlock for VIDBlockPayload { fn genesis() -> Self { - VIDBlockPayload::Genesis(GenesisBlockPayload {}) + VIDBlockPayload(Vec::new()) } fn txn_count(&self) -> u64 { - match self { - VIDBlockPayload::Genesis(_) => 0, - VIDBlockPayload::Normal(n) => n.transactions.len() as u64, - } + self.0.len() as u64 } } @@ -151,24 +114,12 @@ impl BlockPayload for VIDBlockPayload { &self, tx: &Self::Transaction, ) -> std::result::Result { - match self { - VIDBlockPayload::Genesis(_) => Err(BlockPayloadError::GenesisCantHaveTransactions), - VIDBlockPayload::Normal(n) => { - let mut new = n.clone(); - new.transactions.push(tx.clone()); - Ok(VIDBlockPayload::Normal(new)) - } - } + let mut new = self.0.clone(); + new.push(tx.clone()); + Ok(VIDBlockPayload(new)) } fn contained_transactions(&self) -> HashSet> { - match self { - VIDBlockPayload::Genesis(_) => HashSet::new(), - VIDBlockPayload::Normal(n) => n - .transactions - .iter() - .map(commit::Committable::commit) - .collect(), - } + self.0.iter().map(commit::Committable::commit).collect() } } diff --git a/crates/hotshot/src/demo.rs b/crates/hotshot/src/demo.rs index a4836faebd..eed7f2b785 100644 --- a/crates/hotshot/src/demo.rs +++ b/crates/hotshot/src/demo.rs @@ -6,7 +6,7 @@ //! These implementations are useful in examples and integration testing, but are not suitable for //! production use. use crate::{ - block_impl::{BlockPayloadError, NormalBlockPayload, VIDBlockPayload, VIDTransaction}, + block_impl::{BlockPayloadError, VIDBlockPayload, VIDTransaction}, traits::election::static_committee::{StaticElectionConfig, StaticVoteToken}, }; use commit::{Commitment, Committable}; @@ -73,18 +73,14 @@ impl State for SDemoState { type Time = ViewNumber; fn next_block(_state: Option) -> Self::BlockType { - VIDBlockPayload::Normal(NormalBlockPayload { - previous_state: (), - transactions: Vec::new(), - }) + VIDBlockPayload(Vec::new()) } - fn validate_block(&self, block: &Self::BlockType, view_number: &Self::Time) -> bool { - match block { - VIDBlockPayload::Genesis(_) => { - view_number == &ViewNumber::genesis() && view_number == &self.view_number - } - VIDBlockPayload::Normal(_n) => self.view_number < *view_number, + fn validate_block(&self, _block: &Self::BlockType, view_number: &Self::Time) -> bool { + if view_number == &ViewNumber::genesis() { + &self.view_number == view_number + } else { + self.view_number < *view_number } } @@ -133,9 +129,9 @@ impl TestableState for SDemoState { serde::Serialize, serde::Deserialize, )] -pub struct SDemoTypes; +pub struct DemoTypes; -impl NodeType for SDemoTypes { +impl NodeType for DemoTypes { type Time = ViewNumber; type BlockType = VIDBlockPayload; type SignatureKey = BLSPubKey; @@ -150,11 +146,11 @@ impl NodeType for SDemoTypes { #[derivative(Clone(bound = ""))] pub struct SDemoNode(PhantomData) where - MEMBERSHIP: Membership + std::fmt::Debug; + MEMBERSHIP: Membership + std::fmt::Debug; impl SDemoNode where - MEMBERSHIP: Membership + std::fmt::Debug, + MEMBERSHIP: Membership + std::fmt::Debug, { /// Create a new `SDemoNode` #[must_use] @@ -165,7 +161,7 @@ where impl Debug for SDemoNode where - MEMBERSHIP: Membership + std::fmt::Debug, + MEMBERSHIP: Membership + std::fmt::Debug, { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { f.debug_struct("SDemoNode") @@ -176,7 +172,7 @@ where impl Default for SDemoNode where - MEMBERSHIP: Membership + std::fmt::Debug, + MEMBERSHIP: Membership + std::fmt::Debug, { fn default() -> Self { Self::new() diff --git a/crates/testing/tests/da_task.rs b/crates/testing/tests/da_task.rs index 545442fa20..a9dcbd0c2e 100644 --- a/crates/testing/tests/da_task.rs +++ b/crates/testing/tests/da_task.rs @@ -20,10 +20,7 @@ use std::collections::HashMap; )] #[cfg_attr(async_executor_impl = "async-std", async_std::test)] async fn test_da_task() { - use hotshot::{ - block_impl::{NormalBlockPayload, VIDBlockPayload}, - tasks::add_da_task, - }; + use hotshot::{block_impl::VIDBlockPayload, tasks::add_da_task}; use hotshot_task_impls::harness::run_harness; use hotshot_testing::task_helpers::build_system_handle; use hotshot_types::{message::Proposal, traits::election::CommitteeExchangeType}; @@ -39,10 +36,7 @@ async fn test_da_task() { }; let committee_exchange = api.inner.exchanges.committee_exchange().clone(); let pub_key = *api.public_key(); - let block = VIDBlockPayload::Normal(NormalBlockPayload { - previous_state: (), - transactions: Vec::new(), - }); + let block = VIDBlockPayload(Vec::new()); let block_commitment = block.commit(); let signature = committee_exchange.sign_da_proposal(&block_commitment); let proposal = DAProposal { diff --git a/crates/testing/tests/network_task.rs b/crates/testing/tests/network_task.rs index f1d720946f..8239fc72d4 100644 --- a/crates/testing/tests/network_task.rs +++ b/crates/testing/tests/network_task.rs @@ -21,7 +21,7 @@ use std::collections::HashMap; #[cfg_attr(async_executor_impl = "async-std", async_std::test)] #[ignore] async fn test_network_task() { - use hotshot::block_impl::{NormalBlockPayload, VIDBlockPayload}; + use hotshot::block_impl::VIDBlockPayload; use hotshot_task_impls::harness::run_harness; use hotshot_testing::task_helpers::build_system_handle; use hotshot_types::{ @@ -40,10 +40,7 @@ async fn test_network_task() { let committee_exchange = api.inner.exchanges.committee_exchange().clone(); let pub_key = *api.public_key(); let priv_key = api.private_key(); - let block = VIDBlockPayload::Normal(NormalBlockPayload { - previous_state: (), - transactions: Vec::new(), - }); + let block = VIDBlockPayload(Vec::new()); let block_commitment = block.commit(); let signature = committee_exchange.sign_da_proposal(&block_commitment); let da_proposal = Proposal { diff --git a/crates/types/src/traits/block_contents.rs b/crates/types/src/traits/block_contents.rs index eb4dce2753..a365dd7144 100644 --- a/crates/types/src/traits/block_contents.rs +++ b/crates/types/src/traits/block_contents.rs @@ -134,7 +134,7 @@ pub mod dummy { type Transaction = DummyTransaction; fn new() -> Self { - ::genesis() + Self { nonce: 0 } } fn add_transaction_raw( From 562178af7857844a6d948ee444e9597523b6019b Mon Sep 17 00:00:00 2001 From: Keyao Shen Date: Mon, 18 Sep 2023 15:04:26 -0700 Subject: [PATCH 03/18] Remove id from txn --- crates/hotshot/src/block_impl.rs | 44 ++++++++++++-------------------- crates/hotshot/src/demo.rs | 7 ++--- 2 files changed, 18 insertions(+), 33 deletions(-) diff --git a/crates/hotshot/src/block_impl.rs b/crates/hotshot/src/block_impl.rs index dc7ebf878d..25ae971406 100644 --- a/crates/hotshot/src/block_impl.rs +++ b/crates/hotshot/src/block_impl.rs @@ -2,7 +2,6 @@ use std::{ collections::HashSet, fmt::{Debug, Display}, - ops::Deref, }; use commit::{Commitment, Committable}; @@ -12,30 +11,17 @@ use snafu::Snafu; /// The transaction in a [`VIDBlockPayload`]. #[derive(PartialEq, Eq, Hash, Serialize, Deserialize, Clone, Debug)] -pub struct VIDTransaction { - /// identifier for the transaction - pub id: u64, - /// padding to add to txn (to make it larger and thereby more realistic) - pub padding: Vec, -} - -impl Deref for VIDTransaction { - type Target = u64; - - fn deref(&self) -> &Self::Target { - &self.id - } -} +pub struct VIDTransaction(pub Vec); impl Committable for VIDTransaction { fn commit(&self) -> Commitment { - commit::RawCommitmentBuilder::new("SDemo Txn Comm") - .u64_field("id", self.id) - .finalize() + // TODO: Use use VID block commitment. + // + commit::RawCommitmentBuilder::new("Txn Comm").finalize() } fn tag() -> String { - "SEQUENCING_DEMO_TXN".to_string() + "SEQUENCING_TXN".to_string() } } @@ -44,11 +30,14 @@ impl Transaction for VIDTransaction {} impl VIDTransaction { /// create a new transaction #[must_use] - pub fn new(id: u64) -> Self { - Self { - id, - padding: vec![], - } + pub fn new() -> Self { + Self(Vec::new()) + } +} + +impl Default for VIDTransaction { + fn default() -> Self { + Self::new() } } @@ -73,10 +62,9 @@ pub struct VIDBlockPayload(pub Vec); impl Committable for VIDBlockPayload { fn commit(&self) -> Commitment { - let mut builder = commit::RawCommitmentBuilder::new("Normal Comm"); - for txn in &self.0 { - builder = builder.u64_field("transaction", **txn); - } + // TODO: Use use VID block commitment. + // + let builder = commit::RawCommitmentBuilder::new("BlockPayload Comm"); builder.finalize() } diff --git a/crates/hotshot/src/demo.rs b/crates/hotshot/src/demo.rs index eed7f2b785..feaed2fe6b 100644 --- a/crates/hotshot/src/demo.rs +++ b/crates/hotshot/src/demo.rs @@ -106,13 +106,10 @@ impl State for SDemoState { impl TestableState for SDemoState { fn create_random_transaction( _state: Option<&Self>, - rng: &mut dyn rand::RngCore, + _rng: &mut dyn rand::RngCore, padding: u64, ) -> ::Transaction { - VIDTransaction { - id: rng.gen_range(0..10), - padding: vec![0; padding as usize], - } + VIDTransaction(vec![0; padding as usize]) } } /// Implementation of [`NodeType`] for [`VDemoNode`] From b63b9140749c66aeb9db259b45ecdb0bc6e02bf4 Mon Sep 17 00:00:00 2001 From: Keyao Shen Date: Mon, 18 Sep 2023 17:09:39 -0700 Subject: [PATCH 04/18] Add Keccak hash --- Cargo.lock | 1 + crates/hotshot/Cargo.toml | 1 + crates/hotshot/src/block_impl.rs | 17 +++++++---------- 3 files changed, 9 insertions(+), 10 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 5b26f3cb4d..936c6a2fad 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2654,6 +2654,7 @@ dependencies = [ "rand_chacha 0.3.1", "serde", "serde_json", + "sha3", "snafu", "surf-disco", "time 0.3.28", diff --git a/crates/hotshot/Cargo.toml b/crates/hotshot/Cargo.toml index 1f56a6be66..8f395476b9 100644 --- a/crates/hotshot/Cargo.toml +++ b/crates/hotshot/Cargo.toml @@ -104,6 +104,7 @@ nll = { workspace = true } rand = { workspace = true } rand_chacha = { workspace = true } serde = { workspace = true, features = ["rc"] } +sha3 = "^0.10" snafu = { workspace = true } surf-disco = { workspace = true } time = { workspace = true } diff --git a/crates/hotshot/src/block_impl.rs b/crates/hotshot/src/block_impl.rs index 25ae971406..9991c23a33 100644 --- a/crates/hotshot/src/block_impl.rs +++ b/crates/hotshot/src/block_impl.rs @@ -7,17 +7,20 @@ use std::{ use commit::{Commitment, Committable}; use hotshot_types::traits::{block_contents::Transaction, state::TestableBlock, BlockPayload}; use serde::{Deserialize, Serialize}; +use sha3::{Digest, Keccak256}; use snafu::Snafu; /// The transaction in a [`VIDBlockPayload`]. -#[derive(PartialEq, Eq, Hash, Serialize, Deserialize, Clone, Debug)] +#[derive(Default, PartialEq, Eq, Hash, Serialize, Deserialize, Clone, Debug)] pub struct VIDTransaction(pub Vec); impl Committable for VIDTransaction { fn commit(&self) -> Commitment { - // TODO: Use use VID block commitment. - // - commit::RawCommitmentBuilder::new("Txn Comm").finalize() + let builder = commit::RawCommitmentBuilder::new("Txn Comm"); + let mut hasher = Keccak256::new(); + hasher.update(self.0.clone()); + let generic_array = hasher.finalize(); + builder.generic_byte_array(&generic_array).finalize() } fn tag() -> String { @@ -35,12 +38,6 @@ impl VIDTransaction { } } -impl Default for VIDTransaction { - fn default() -> Self { - Self::new() - } -} - /// The error type for block payload. #[derive(Snafu, Debug)] pub enum BlockPayloadError { From 4ee88fedea01e66506ade0ae259a97b36824bfb6 Mon Sep 17 00:00:00 2001 From: Keyao Shen Date: Mon, 18 Sep 2023 17:18:07 -0700 Subject: [PATCH 05/18] Replace clone with borrow --- crates/hotshot/src/block_impl.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/hotshot/src/block_impl.rs b/crates/hotshot/src/block_impl.rs index 9991c23a33..8960d79158 100644 --- a/crates/hotshot/src/block_impl.rs +++ b/crates/hotshot/src/block_impl.rs @@ -18,7 +18,7 @@ impl Committable for VIDTransaction { fn commit(&self) -> Commitment { let builder = commit::RawCommitmentBuilder::new("Txn Comm"); let mut hasher = Keccak256::new(); - hasher.update(self.0.clone()); + hasher.update(&self.0); let generic_array = hasher.finalize(); builder.generic_byte_array(&generic_array).finalize() } From 295f8471d2883c18f56d61b2fb2c9be6dda8e8d5 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 19 Sep 2023 07:23:01 -0400 Subject: [PATCH 06/18] Bump clap from 4.4.3 to 4.4.4 (#1775) Bumps [clap](https://github.com/clap-rs/clap) from 4.4.3 to 4.4.4. - [Release notes](https://github.com/clap-rs/clap/releases) - [Changelog](https://github.com/clap-rs/clap/blob/master/CHANGELOG.md) - [Commits](https://github.com/clap-rs/clap/compare/v4.4.3...v4.4.4) --- updated-dependencies: - dependency-name: clap dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- Cargo.lock | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index e577bc0d70..344916e52c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1236,9 +1236,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.4.3" +version = "4.4.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "84ed82781cea27b43c9b106a979fe450a13a31aab0500595fb3fc06616de08e6" +checksum = "b1d7b8d5ec32af0fadc644bf1fd509a688c2103b185644bb1e29d164e0703136" dependencies = [ "clap_builder", "clap_derive", @@ -1246,9 +1246,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.4.2" +version = "4.4.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2bb9faaa7c2ef94b2743a21f5a29e6f0010dff4caa69ac8e9d6cf8b6fa74da08" +checksum = "5179bb514e4d7c2051749d8fcefa2ed6d06a9f4e6d69faf3805f5d80b8cf8d56" dependencies = [ "anstream", "anstyle", From c3cb6d5cb9e8e89170c0c53a0b31243b9294d0a4 Mon Sep 17 00:00:00 2001 From: rob-maron <132852777+rob-maron@users.noreply.github.com> Date: Tue, 19 Sep 2023 08:18:21 -0400 Subject: [PATCH 07/18] update pkg-config (#1776) --- flake.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flake.nix b/flake.nix index 2ff009c372..04cbf3c782 100644 --- a/flake.nix +++ b/flake.nix @@ -173,7 +173,7 @@ zlib.out fenix.packages.${system}.rust-analyzer just - pkgconfig + pkg-config openssl.dev openssl.out ] ++ lib.optionals stdenv.isDarwin [ From 8ba608416b8f20c6d6a86d676588844ea1d69c23 Mon Sep 17 00:00:00 2001 From: rob-maron <132852777+rob-maron@users.noreply.github.com> Date: Tue, 19 Sep 2023 09:58:13 -0400 Subject: [PATCH 08/18] Fix libp2p examples (#1735) * fix libp2p examples * fmt * update libp2p run config * libp2p to 10 rounds * fix viewsync by changing DA keys * remove signaturekey comments * update flake * update flake lock * merge main into examples --- crates/hotshot/Cargo.toml | 25 +- crates/hotshot/examples/infra/mod.rs | 1 - crates/hotshot/examples/infra/modDA.rs | 278 ++++++++++++++++-- .../examples/libp2p/multi-validator.rs | 69 +++++ .../hotshot/examples/libp2p/orchestrator.rs | 24 +- crates/hotshot/examples/libp2p/types.rs | 83 +++--- crates/hotshot/examples/libp2p/validator.rs | 30 +- .../default-libp2p-run-config.toml | 9 +- crates/orchestrator/src/lib.rs | 1 - flake.lock | 18 +- 10 files changed, 451 insertions(+), 87 deletions(-) create mode 100644 crates/hotshot/examples/libp2p/multi-validator.rs diff --git a/crates/hotshot/Cargo.toml b/crates/hotshot/Cargo.toml index 1f56a6be66..bb587595d2 100644 --- a/crates/hotshot/Cargo.toml +++ b/crates/hotshot/Cargo.toml @@ -25,16 +25,21 @@ docs = [] doc-images = [] hotshot-testing = [] -# [[example]] -# name = "libp2p-validator" -# required-features = ["demo", "libp2p/rsa"] -# path = "examples/libp2p/validator.rs" -# -# [[example]] -# name = "libp2p-orchestrator" -# required-features = ["demo", "libp2p/rsa"] -# path = "examples/libp2p/orchestrator.rs" -# +[[example]] +name = "libp2p-validator" +required-features = ["demo", "libp2p/rsa"] +path = "examples/libp2p/validator.rs" + +[[example]] +name = "libp2p-multi-validator" +required-features = ["demo", "libp2p/rsa"] +path = "examples/libp2p/multi-validator.rs" + +[[example]] +name = "libp2p-orchestrator" +required-features = ["demo", "libp2p/rsa"] +path = "examples/libp2p/orchestrator.rs" + # [[example]] # name = "web-server-orchestrator" # required-features = ["demo", "libp2p/rsa"] diff --git a/crates/hotshot/examples/infra/mod.rs b/crates/hotshot/examples/infra/mod.rs index 1ffbb04a7f..06d7d9deb3 100644 --- a/crates/hotshot/examples/infra/mod.rs +++ b/crates/hotshot/examples/infra/mod.rs @@ -69,7 +69,6 @@ pub fn load_config_from_file( config } -/// yeesh maybe we should just implement SignatureKey for this... pub fn libp2p_generate_indexed_identity(seed: [u8; 32], index: u64) -> Keypair { let mut hasher = blake3::Hasher::new(); hasher.update(&seed); diff --git a/crates/hotshot/examples/infra/modDA.rs b/crates/hotshot/examples/infra/modDA.rs index 10c6d85cea..ae0f0e40f5 100644 --- a/crates/hotshot/examples/infra/modDA.rs +++ b/crates/hotshot/examples/infra/modDA.rs @@ -1,11 +1,14 @@ use crate::infra::{load_config_from_file, OrchestratorArgs}; use async_compatibility_layer::logging::{setup_backtrace, setup_logging}; +use async_lock::RwLock; use async_trait::async_trait; use futures::StreamExt; use hotshot::{ traits::{ - implementations::{MemoryStorage, WebCommChannel, WebServerNetwork}, + implementations::{ + Libp2pCommChannel, Libp2pNetwork, MemoryStorage, WebCommChannel, WebServerNetwork, + }, NodeImplementation, }, types::{SignatureKey, SystemContextHandle}, @@ -17,6 +20,7 @@ use hotshot_orchestrator::{ config::{NetworkConfig, WebServerConfig}, }; use hotshot_task::task::FilterEvent; +use hotshot_types::HotShotConfig; use hotshot_types::{ certificate::ViewSyncCertificate, data::{QuorumProposal, SequencingLeaf, TestableLeaf}, @@ -33,8 +37,17 @@ use hotshot_types::{ }, state::{ConsensusTime, TestableBlock, TestableState}, }, - HotShotConfig, }; +use libp2p_identity::{ + ed25519::{self, SecretKey}, + Keypair, +}; +use libp2p_networking::{ + network::{MeshParams, NetworkNodeConfigBuilder, NetworkNodeType}, + reexport::Multiaddr, +}; +use std::{collections::BTreeSet, sync::Arc}; +use std::{num::NonZeroUsize, str::FromStr}; // use libp2p::{ // identity::{ // ed25519::{Keypair as EdKeypair, SecretKey}, @@ -43,7 +56,7 @@ use hotshot_types::{ // multiaddr::{self, Protocol}, // Multiaddr, // }; -// use libp2p_identity::PeerId; +use libp2p_identity::PeerId; // use libp2p_networking::network::{MeshParams, NetworkNodeConfigBuilder, NetworkNodeType}; use std::{ //collections::{BTreeSet, VecDeque}, @@ -376,15 +389,6 @@ pub trait RunDA< // WEB SERVER -/// Alias for the [`WebCommChannel`] for sequencing consensus. -type StaticDAComm = WebCommChannel; - -/// Alias for the ['WebCommChannel'] for validating consensus -type StaticQuorumComm = WebCommChannel; - -/// Alias for the ['WebCommChannel'] for view sync consensus -type StaticViewSyncComm = WebCommChannel; - /// Represents a web server-based run pub struct WebServerDARun< TYPES: NodeType, @@ -396,9 +400,9 @@ pub struct WebServerDARun< ::StakeTableEntry, TYPES::ElectionConfigType, >, - quorum_network: StaticQuorumComm, - da_network: StaticDAComm, - view_sync_network: StaticViewSyncComm, + quorum_network: WebCommChannel, + da_network: WebCommChannel, + view_sync_network: WebCommChannel, } #[async_trait] @@ -440,9 +444,9 @@ impl< RunDA< TYPES, MEMBERSHIP, - StaticDAComm, - StaticQuorumComm, - StaticViewSyncComm, + WebCommChannel, + WebCommChannel, + WebCommChannel, NODE, > for WebServerDARun where @@ -530,6 +534,235 @@ where } } +// Libp2p + +/// Represents a libp2p-based run +pub struct Libp2pDARun, MEMBERSHIP: Membership> +{ + config: NetworkConfig< + TYPES::SignatureKey, + ::StakeTableEntry, + TYPES::ElectionConfigType, + >, + quorum_network: Libp2pCommChannel, + da_network: Libp2pCommChannel, + view_sync_network: Libp2pCommChannel, +} + +#[async_trait] +impl< + TYPES: NodeType, + MEMBERSHIP: Membership + Debug, + NODE: NodeImplementation< + TYPES, + Leaf = SequencingLeaf, + Exchanges = SequencingExchanges< + TYPES, + Message, + QuorumExchange< + TYPES, + SequencingLeaf, + QuorumProposal>, + MEMBERSHIP, + Libp2pCommChannel, + Message, + >, + CommitteeExchange< + TYPES, + MEMBERSHIP, + Libp2pCommChannel, + Message, + >, + ViewSyncExchange< + TYPES, + ViewSyncCertificate, + MEMBERSHIP, + Libp2pCommChannel, + Message, + >, + >, + Storage = MemoryStorage>, + ConsensusMessage = SequencingMessage, + >, + > + RunDA< + TYPES, + MEMBERSHIP, + Libp2pCommChannel, + Libp2pCommChannel, + Libp2pCommChannel, + NODE, + > for Libp2pDARun +where + ::StateType: TestableState, + ::BlockType: TestableBlock, + SequencingLeaf: TestableLeaf, + Self: Sync, +{ + async fn initialize_networking( + config: NetworkConfig< + TYPES::SignatureKey, + ::StakeTableEntry, + TYPES::ElectionConfigType, + >, + ) -> Libp2pDARun { + let (pubkey, _privkey) = + <::SignatureKey as SignatureKey>::generated_from_seed_indexed( + config.seed, + config.node_index, + ); + let mut config = config; + let libp2p_config = config + .libp2p_config + .take() + .expect("Configuration is not for a Libp2p network"); + let bs_len = libp2p_config.bootstrap_nodes.len(); + let bootstrap_nodes: Vec<(PeerId, Multiaddr)> = libp2p_config + .bootstrap_nodes + .iter() + .map(|(addr, pair)| { + let kp = Keypair::from_protobuf_encoding(pair).unwrap(); + let peer_id = PeerId::from_public_key(&kp.public()); + let multiaddr = + Multiaddr::from_str(&format!("/ip4/{}/udp/{}/quic-v1", addr.ip(), addr.port())) + .unwrap(); + (peer_id, multiaddr) + }) + .collect(); + let identity = libp2p_generate_indexed_identity(config.seed, config.node_index); + let node_type = if (config.node_index as usize) < bs_len { + NetworkNodeType::Bootstrap + } else { + NetworkNodeType::Regular + }; + let node_index = config.node_index; + let port_index = match libp2p_config.index_ports { + true => node_index, + false => 0, + }; + let bound_addr: Multiaddr = format!( + "/{}/{}/udp/{}/quic-v1", + if libp2p_config.public_ip.is_ipv4() { + "ip4" + } else { + "ip6" + }, + libp2p_config.public_ip, + libp2p_config.base_port as u64 + port_index + ) + .parse() + .unwrap(); + + // generate network + let mut config_builder = NetworkNodeConfigBuilder::default(); + assert!(config.config.total_nodes.get() > 2); + let replicated_nodes = NonZeroUsize::new(config.config.total_nodes.get() - 2).unwrap(); + config_builder.replication_factor(replicated_nodes); + config_builder.identity(identity.clone()); + + config_builder.bound_addr(Some(bound_addr.clone())); + + let to_connect_addrs = bootstrap_nodes + .iter() + .map(|(peer_id, multiaddr)| (Some(*peer_id), multiaddr.clone())) + .collect(); + + config_builder.to_connect_addrs(to_connect_addrs); + + let mesh_params = + // NOTE I'm arbitrarily choosing these. + match node_type { + NetworkNodeType::Bootstrap => MeshParams { + mesh_n_high: libp2p_config.bootstrap_mesh_n_high, + mesh_n_low: libp2p_config.bootstrap_mesh_n_low, + mesh_outbound_min: libp2p_config.bootstrap_mesh_outbound_min, + mesh_n: libp2p_config.bootstrap_mesh_n, + }, + NetworkNodeType::Regular => MeshParams { + mesh_n_high: libp2p_config.mesh_n_high, + mesh_n_low: libp2p_config.mesh_n_low, + mesh_outbound_min: libp2p_config.mesh_outbound_min, + mesh_n: libp2p_config.mesh_n, + }, + NetworkNodeType::Conductor => unreachable!(), + }; + config_builder.mesh_params(Some(mesh_params)); + + let mut all_keys = BTreeSet::new(); + let mut da_keys = BTreeSet::new(); + for i in 0..config.config.total_nodes.get() as u64 { + let privkey = TYPES::SignatureKey::generated_from_seed_indexed([0u8; 32], i).1; + let pubkey = TYPES::SignatureKey::from_private(&privkey); + if i < config.config.da_committee_size as u64 { + da_keys.insert(pubkey.clone()); + } + all_keys.insert(pubkey); + } + + let node_config = config_builder.build().unwrap(); + let underlying_quorum_network = Libp2pNetwork::new( + NoMetrics::boxed(), + node_config, + pubkey.clone(), + Arc::new(RwLock::new( + bootstrap_nodes + .iter() + .map(|(peer_id, addr)| (Some(*peer_id), addr.clone())) + .collect(), + )), + bs_len, + config.node_index as usize, + // NOTE: this introduces an invariant that the keys are assigned using this indexed + // function + all_keys, + da_keys, + ) + .await + .unwrap(); + + underlying_quorum_network.wait_for_ready().await; + + // Create the network + let quorum_network: Libp2pCommChannel = + Libp2pCommChannel::new(underlying_quorum_network.clone().into()); + + let view_sync_network: Libp2pCommChannel = + Libp2pCommChannel::new(underlying_quorum_network.clone().into()); + + let da_network: Libp2pCommChannel = + Libp2pCommChannel::new(underlying_quorum_network.clone().into()); + + Libp2pDARun { + config, + quorum_network, + da_network, + view_sync_network, + } + } + + fn get_da_network(&self) -> Libp2pCommChannel { + self.da_network.clone() + } + + fn get_quorum_network(&self) -> Libp2pCommChannel { + self.quorum_network.clone() + } + + fn get_view_sync_network(&self) -> Libp2pCommChannel { + self.view_sync_network.clone() + } + + fn get_config( + &self, + ) -> NetworkConfig< + TYPES::SignatureKey, + ::StakeTableEntry, + TYPES::ElectionConfigType, + > { + self.config.clone() + } +} + /// Main entry point for validators pub async fn main_entry_point< TYPES: NodeType, @@ -613,3 +846,12 @@ pub async fn main_entry_point< info!("All nodes are ready! Starting HotShot"); run.run_hotshot(hotshot).await; } + +pub fn libp2p_generate_indexed_identity(seed: [u8; 32], index: u64) -> Keypair { + let mut hasher = blake3::Hasher::new(); + hasher.update(&seed); + hasher.update(&index.to_le_bytes()); + let new_seed = *hasher.finalize().as_bytes(); + let sk_bytes = SecretKey::try_from_bytes(new_seed).unwrap(); + >::from(sk_bytes).into() +} diff --git a/crates/hotshot/examples/libp2p/multi-validator.rs b/crates/hotshot/examples/libp2p/multi-validator.rs new file mode 100644 index 0000000000..8a0f53c1c9 --- /dev/null +++ b/crates/hotshot/examples/libp2p/multi-validator.rs @@ -0,0 +1,69 @@ +use async_compatibility_layer::{ + art::async_spawn, + logging::{setup_backtrace, setup_logging}, +}; +use clap::Parser; +use hotshot::demos::sdemo::SDemoTypes; +use hotshot_orchestrator::client::ValidatorArgs; +use std::net::IpAddr; +use tracing::instrument; + +use crate::types::{DANetwork, NodeImpl, QuorumNetwork, ThisMembership, ThisRun, ViewSyncNetwork}; + +pub mod types; + +#[path = "../infra/mod.rs"] +pub mod infra; +#[path = "../infra/modDA.rs"] +pub mod infra_da; + +#[derive(Parser, Debug, Clone)] +struct MultiValidatorArgs { + /// Number of validators to run + pub num_nodes: u16, + /// The address the orchestrator runs on + pub host: IpAddr, + /// The port the orchestrator runs on + pub port: u16, + /// This node's public IP address, for libp2p + /// If no IP address is passed in, it will default to 127.0.0.1 + pub public_ip: Option, +} + +#[cfg_attr( + async_executor_impl = "tokio", + tokio::main(flavor = "multi_thread", worker_threads = 2) +)] +#[cfg_attr(async_executor_impl = "async-std", async_std::main)] +#[instrument] +async fn main() { + setup_logging(); + setup_backtrace(); + let args = MultiValidatorArgs::parse(); + tracing::error!( + "connecting to orchestrator at {:?}:{:?}", + args.host, + args.port + ); + let mut nodes = Vec::new(); + for _ in 0..args.num_nodes { + let node = async_spawn(async move { + infra_da::main_entry_point::< + SDemoTypes, + ThisMembership, + DANetwork, + QuorumNetwork, + ViewSyncNetwork, + NodeImpl, + ThisRun, + >(ValidatorArgs { + host: args.host.to_string(), + port: args.port, + public_ip: args.public_ip, + }) + .await + }); + nodes.push(node); + } + let _result = futures::future::join_all(nodes).await; +} diff --git a/crates/hotshot/examples/libp2p/orchestrator.rs b/crates/hotshot/examples/libp2p/orchestrator.rs index 3bb08103d0..772d3aa12e 100644 --- a/crates/hotshot/examples/libp2p/orchestrator.rs +++ b/crates/hotshot/examples/libp2p/orchestrator.rs @@ -1,15 +1,21 @@ pub mod types; +use async_compatibility_layer::logging::{setup_backtrace, setup_logging}; use clap::Parser; -use hotshot::demos::vdemo::VDemoTypes; +use hotshot::demos::sdemo::SDemoTypes; use tracing::instrument; use types::ThisMembership; -use crate::infra::{run_orchestrator, OrchestratorArgs}; -use crate::types::{NodeImpl, ThisNetwork}; +use crate::{ + infra::OrchestratorArgs, + infra_da::run_orchestrator_da, + types::{DANetwork, NodeImpl, QuorumNetwork, ViewSyncNetwork}, +}; #[path = "../infra/mod.rs"] pub mod infra; +#[path = "../infra/modDA.rs"] +pub mod infra_da; #[cfg_attr( async_executor_impl = "tokio", @@ -18,7 +24,17 @@ pub mod infra; #[cfg_attr(async_executor_impl = "async-std", async_std::main)] #[instrument] async fn main() { + setup_logging(); + setup_backtrace(); let args = OrchestratorArgs::parse(); - run_orchestrator::(args).await; + run_orchestrator_da::< + SDemoTypes, + ThisMembership, + DANetwork, + QuorumNetwork, + ViewSyncNetwork, + NodeImpl, + >(args) + .await; } diff --git a/crates/hotshot/examples/libp2p/types.rs b/crates/hotshot/examples/libp2p/types.rs index 8b3c70e5e1..5289c9d6fe 100644 --- a/crates/hotshot/examples/libp2p/types.rs +++ b/crates/hotshot/examples/libp2p/types.rs @@ -1,60 +1,75 @@ -use crate::infra::Libp2pRun; -use hotshot::traits::implementations::MemoryStorage; +use crate::infra_da::Libp2pDARun; use hotshot::{ - demos::vdemo::VDemoTypes, + demos::sdemo::SDemoTypes, traits::{ - election::static_committee::GeneralStaticCommittee, implementations::Libp2pCommChannel, + election::static_committee::GeneralStaticCommittee, + implementations::{Libp2pCommChannel, MemoryStorage}, }, }; -use hotshot_types::message::{Message, ValidatingMessage}; -use hotshot_types::traits::{ - election::QuorumExchange, - node_implementation::{ChannelMaps, NodeImplementation, ValidatingExchanges}, -}; use hotshot_types::{ - data::{ValidatingLeaf, ValidatingProposal}, - traits::node_implementation::NodeType, - vote::QuorumVote, + certificate::ViewSyncCertificate, + data::{DAProposal, QuorumProposal, SequencingLeaf}, + message::{Message, SequencingMessage}, + traits::{ + election::{CommitteeExchange, QuorumExchange, ViewSyncExchange}, + node_implementation::{ChannelMaps, NodeImplementation, NodeType, SequencingExchanges}, + }, + vote::{DAVote, QuorumVote, ViewSyncVote}, }; use serde::{Deserialize, Serialize}; use std::fmt::Debug; -#[derive(Clone, Debug, Deserialize, Serialize, Hash, Eq, PartialEq)] +#[derive(Clone, Debug, Deserialize, Serialize, Hash, PartialEq, Eq)] pub struct NodeImpl {} -pub type ThisLeaf = ValidatingLeaf; +pub type ThisLeaf = SequencingLeaf; pub type ThisMembership = - GeneralStaticCommittee::SignatureKey>; -pub type ThisNetwork = - Libp2pCommChannel; + GeneralStaticCommittee::SignatureKey>; +pub type DANetwork = Libp2pCommChannel; +pub type QuorumNetwork = Libp2pCommChannel; +pub type ViewSyncNetwork = Libp2pCommChannel; + +pub type ThisDAProposal = DAProposal; +pub type ThisDAVote = DAVote; -pub type ThisProposal = ValidatingProposal; -pub type ThisVote = QuorumVote; +pub type ThisQuorumProposal = QuorumProposal; +pub type ThisQuorumVote = QuorumVote; -impl NodeImplementation for NodeImpl { - type Storage = MemoryStorage; - type Leaf = ValidatingLeaf; - type Exchanges = ValidatingExchanges< - VDemoTypes, - Message, +pub type ThisViewSyncProposal = ViewSyncCertificate; +pub type ThisViewSyncVote = ViewSyncVote; + +impl NodeImplementation for NodeImpl { + type Storage = MemoryStorage; + type Leaf = SequencingLeaf; + type Exchanges = SequencingExchanges< + SDemoTypes, + Message, QuorumExchange< - VDemoTypes, + SDemoTypes, Self::Leaf, - ThisProposal, + ThisQuorumProposal, + ThisMembership, + QuorumNetwork, + Message, + >, + CommitteeExchange>, + ViewSyncExchange< + SDemoTypes, + ThisViewSyncProposal, ThisMembership, - ThisNetwork, - Message, + ViewSyncNetwork, + Message, >, >; - type ConsensusMessage = ValidatingMessage; + type ConsensusMessage = SequencingMessage; fn new_channel_maps( - start_view: ::Time, + start_view: ::Time, ) -> ( - ChannelMaps, - Option>, + ChannelMaps, + Option>, ) { (ChannelMaps::new(start_view), None) } } -pub type ThisRun = Libp2pRun; +pub type ThisRun = Libp2pDARun; diff --git a/crates/hotshot/examples/libp2p/validator.rs b/crates/hotshot/examples/libp2p/validator.rs index 59009c9d8f..ec2415fd65 100644 --- a/crates/hotshot/examples/libp2p/validator.rs +++ b/crates/hotshot/examples/libp2p/validator.rs @@ -1,15 +1,18 @@ -use crate::infra::main_entry_point; +use async_compatibility_layer::logging::{setup_backtrace, setup_logging}; use clap::Parser; -use hotshot::demos::vdemo::VDemoTypes; -use hotshot_orchestrator::client::ValidatorArgs; -use tracing::instrument; +use hotshot::demos::sdemo::SDemoTypes; +use tracing::{info, instrument}; + +use crate::types::{DANetwork, NodeImpl, QuorumNetwork, ThisMembership, ThisRun, ViewSyncNetwork}; -use crate::types::{NodeImpl, ThisMembership, ThisNetwork, ThisRun}; +use hotshot_orchestrator::client::ValidatorArgs; pub mod types; #[path = "../infra/mod.rs"] pub mod infra; +#[path = "../infra/modDA.rs"] +pub mod infra_da; #[cfg_attr( async_executor_impl = "tokio", @@ -18,6 +21,21 @@ pub mod infra; #[cfg_attr(async_executor_impl = "async-std", async_std::main)] #[instrument] async fn main() { + setup_logging(); + setup_backtrace(); let args = ValidatorArgs::parse(); - main_entry_point::(args).await; + info!( + "connecting to orchestrator at {:?}:{:?}", + args.host, args.port + ); + infra_da::main_entry_point::< + SDemoTypes, + ThisMembership, + DANetwork, + QuorumNetwork, + ViewSyncNetwork, + NodeImpl, + ThisRun, + >(args) + .await; } diff --git a/crates/orchestrator/default-libp2p-run-config.toml b/crates/orchestrator/default-libp2p-run-config.toml index 5d43c940fa..5757f4d9f9 100644 --- a/crates/orchestrator/default-libp2p-run-config.toml +++ b/crates/orchestrator/default-libp2p-run-config.toml @@ -1,4 +1,4 @@ -rounds = 100 +rounds = 10 transactions_per_round = 10 node_index = 0 seed = [ @@ -57,17 +57,18 @@ num_txn_per_round = 10 base_port = 9000 [config] -total_nodes = 5 +total_nodes = 10 +committee_nodes = 5 max_transactions = 100 min_transactions = 0 -next_view_timeout = 10000 +next_view_timeout = 30000 timeout_ratio = [ 11, 10, ] round_start_delay = 1 start_delay = 1 -num_bootstrap = 5 +num_bootstrap = 4 [config.propose_min_round_time] secs = 0 diff --git a/crates/orchestrator/src/lib.rs b/crates/orchestrator/src/lib.rs index a7e7ae0c28..93efa501e7 100644 --- a/crates/orchestrator/src/lib.rs +++ b/crates/orchestrator/src/lib.rs @@ -26,7 +26,6 @@ use libp2p::identity::{ Keypair, }; -/// yeesh maybe we should just implement SignatureKey for this... pub fn libp2p_generate_indexed_identity(seed: [u8; 32], index: u64) -> Keypair { let mut hasher = blake3::Hasher::new(); hasher.update(&seed); diff --git a/flake.lock b/flake.lock index ef4fb0daac..ec130ae4cd 100644 --- a/flake.lock +++ b/flake.lock @@ -41,11 +41,11 @@ "rust-analyzer-src": "rust-analyzer-src" }, "locked": { - "lastModified": 1694845188, - "narHash": "sha256-M3Q1PDYcTOcqPRKkvRW+0sOm7dFGW95zK+wFmJYyw2M=", + "lastModified": 1695018106, + "narHash": "sha256-kZd7//cau0fRFqokaC9olL82j60c02yQH3sGAPSAm6Q=", "owner": "nix-community", "repo": "fenix", - "rev": "95d6c30ecac01307772ec4273c649dab40211a9e", + "rev": "529f3af3d0e02a7b474c427464b166c36c531c4e", "type": "github" }, "original": { @@ -72,11 +72,11 @@ }, "nixpkgs": { "locked": { - "lastModified": 1694760568, - "narHash": "sha256-3G07BiXrp2YQKxdcdms22MUx6spc6A++MSePtatCYuI=", + "lastModified": 1694948089, + "narHash": "sha256-d2B282GmQ9o8klc22/Rbbbj6r99EnELQpOQjWMyv0rU=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "46688f8eb5cd6f1298d873d4d2b9cf245e09e88e", + "rev": "5148520bfab61f99fd25fb9ff7bfbb50dad3c9db", "type": "github" }, "original": { @@ -99,11 +99,11 @@ "rust-analyzer-src": { "flake": false, "locked": { - "lastModified": 1694682672, - "narHash": "sha256-AReT7Eis+iRs7ve6L1CeW/obR/QHi2xZpf7m8NX00hU=", + "lastModified": 1694973195, + "narHash": "sha256-+d/GW4MyksvJmNoQLJMpKxApuZTVwaT+yuxxR/Cc/BE=", "owner": "rust-lang", "repo": "rust-analyzer", - "rev": "12e28c35758051dd6bc9cdf419a50dff80fab64d", + "rev": "05666441bafd6010787a4097a6bd44266ad21018", "type": "github" }, "original": { From 3646927d4ae6e823f99604fc8f07c7dcac2c0a59 Mon Sep 17 00:00:00 2001 From: Gus Gutoski Date: Tue, 19 Sep 2023 14:50:19 -0400 Subject: [PATCH 09/18] chore: Remove Committable from VoteData (#1777) * add get_commit method to VoteData * remove Committable from VoteData, replace commit() -> get_commit() * amalgamate impls of VoteData * lint --- crates/types/src/traits/election.rs | 85 ++++++++++------------------- 1 file changed, 29 insertions(+), 56 deletions(-) diff --git a/crates/types/src/traits/election.rs b/crates/types/src/traits/election.rs index 81dfd46d57..6ced8bebbc 100644 --- a/crates/types/src/traits/election.rs +++ b/crates/types/src/traits/election.rs @@ -91,50 +91,21 @@ pub enum VoteData { ViewSyncFinalize(Commitment), } -/// Make different types of `VoteData` committable -impl Committable for VoteData { - fn commit(&self) -> Commitment { +impl VoteData +where + COMMITTABLE: Committable + Serialize + Clone, +{ + /// Return the underlying commitment. + #[must_use] + pub fn get_commit(&self) -> Commitment { + #[allow(clippy::enum_glob_use)] + use VoteData::*; match self { - VoteData::DA(block_commitment) => { - commit::RawCommitmentBuilder::new("DA BlockPayload Commit") - .field("block_commitment", *block_commitment) - .finalize() - } - VoteData::Yes(leaf_commitment) => commit::RawCommitmentBuilder::new("Yes Vote Commit") - .field("leaf_commitment", *leaf_commitment) - .finalize(), - VoteData::No(leaf_commitment) => commit::RawCommitmentBuilder::new("No Vote Commit") - .field("leaf_commitment", *leaf_commitment) - .finalize(), - VoteData::Timeout(view_number_commitment) => { - commit::RawCommitmentBuilder::new("Timeout View Number Commit") - .field("view_number_commitment", *view_number_commitment) - .finalize() - } - VoteData::ViewSyncPreCommit(commitment) => { - commit::RawCommitmentBuilder::new("ViewSyncPreCommit") - .field("commitment", *commitment) - .finalize() - } - VoteData::ViewSyncCommit(commitment) => { - commit::RawCommitmentBuilder::new("ViewSyncCommit") - .field("commitment", *commitment) - .finalize() - } - VoteData::ViewSyncFinalize(commitment) => { - commit::RawCommitmentBuilder::new("ViewSyncFinalize") - .field("commitment", *commitment) - .finalize() - } + DA(c) | Yes(c) | No(c) | Timeout(c) | ViewSyncPreCommit(c) | ViewSyncCommit(c) + | ViewSyncFinalize(c) => *c, } } - fn tag() -> String { - ("VOTE_DATA_COMMIT").to_string() - } -} - -impl VoteData { #[must_use] /// Convert vote data into bytes. /// @@ -367,7 +338,7 @@ pub trait ConsensusExchange: Send + Sync { match qc.signatures() { AssembledSignature::DA(qc) => { - let real_commit = VoteData::DA(leaf_commitment).commit(); + let real_commit = VoteData::DA(leaf_commitment).get_commit(); let real_qc_pp = ::get_public_parameter( self.membership().get_committee_qc_stake_table(), U256::from(self.membership().success_threshold().get()), @@ -375,7 +346,7 @@ pub trait ConsensusExchange: Send + Sync { ::check(&real_qc_pp, real_commit.as_ref(), &qc) } AssembledSignature::Yes(qc) => { - let real_commit = VoteData::Yes(leaf_commitment).commit(); + let real_commit = VoteData::Yes(leaf_commitment).get_commit(); let real_qc_pp = ::get_public_parameter( self.membership().get_committee_qc_stake_table(), U256::from(self.membership().success_threshold().get()), @@ -383,7 +354,7 @@ pub trait ConsensusExchange: Send + Sync { ::check(&real_qc_pp, real_commit.as_ref(), &qc) } AssembledSignature::No(qc) => { - let real_commit = VoteData::No(leaf_commitment).commit(); + let real_commit = VoteData::No(leaf_commitment).get_commit(); let real_qc_pp = ::get_public_parameter( self.membership().get_committee_qc_stake_table(), U256::from(self.membership().success_threshold().get()), @@ -411,7 +382,7 @@ pub trait ConsensusExchange: Send + Sync { let mut is_valid_vote_token = false; let mut is_valid_signature = false; if let Some(key) = ::from_bytes(encoded_key) { - is_valid_signature = key.validate(encoded_signature, data.commit().as_ref()); + is_valid_signature = key.validate(encoded_signature, data.get_commit().as_ref()); let valid_vote_token = self.membership().validate_vote_token(key, vote_token); is_valid_vote_token = match valid_vote_token { Err(_) => { @@ -433,7 +404,7 @@ pub trait ConsensusExchange: Send + Sync { data: &VoteData, vote_token: &Checked, ) -> bool { - let is_valid_signature = key.validate(encoded_signature, data.commit().as_ref()); + let is_valid_signature = key.validate(encoded_signature, data.get_commit().as_ref()); let valid_vote_token = self .membership() .validate_vote_token(key.clone(), vote_token.clone()); @@ -643,7 +614,7 @@ impl< let signature = TYPES::SignatureKey::sign( &self.private_key, VoteData::::DA(block_commitment) - .commit() + .get_commit() .as_ref(), ); (self.public_key.to_bytes(), signature) @@ -688,7 +659,7 @@ impl< let signature = TYPES::SignatureKey::sign( &self.private_key, VoteData::::DA(block_commitment) - .commit() + .get_commit() .as_ref(), ); (self.public_key.to_bytes(), signature) @@ -931,7 +902,7 @@ impl< ) -> (EncodedPublicKey, EncodedSignature) { let signature = TYPES::SignatureKey::sign( &self.private_key, - VoteData::::Yes(leaf_commitment).commit().as_ref(), + VoteData::::Yes(leaf_commitment).get_commit().as_ref(), ); (self.public_key.to_bytes(), signature) } @@ -947,7 +918,7 @@ impl< ) -> (EncodedPublicKey, EncodedSignature) { let signature = TYPES::SignatureKey::sign( &self.private_key, - VoteData::::No(leaf_commitment).commit().as_ref(), + VoteData::::No(leaf_commitment).get_commit().as_ref(), ); (self.public_key.to_bytes(), signature) } @@ -963,7 +934,7 @@ impl< let signature = TYPES::SignatureKey::sign( &self.private_key, VoteData::::Timeout(view_number.commit()) - .commit() + .get_commit() .as_ref(), ); (self.public_key.to_bytes(), signature) @@ -1212,7 +1183,9 @@ impl< ) -> (EncodedPublicKey, EncodedSignature) { let signature = TYPES::SignatureKey::sign( &self.private_key, - VoteData::ViewSyncPreCommit(commitment).commit().as_ref(), + VoteData::ViewSyncPreCommit(commitment) + .get_commit() + .as_ref(), ); (self.public_key.to_bytes(), signature) @@ -1253,7 +1226,7 @@ impl< ) -> (EncodedPublicKey, EncodedSignature) { let signature = TYPES::SignatureKey::sign( &self.private_key, - VoteData::ViewSyncCommit(commitment).commit().as_ref(), + VoteData::ViewSyncCommit(commitment).get_commit().as_ref(), ); (self.public_key.to_bytes(), signature) @@ -1294,7 +1267,7 @@ impl< ) -> (EncodedPublicKey, EncodedSignature) { let signature = TYPES::SignatureKey::sign( &self.private_key, - VoteData::ViewSyncFinalize(commitment).commit().as_ref(), + VoteData::ViewSyncFinalize(commitment).get_commit().as_ref(), ); (self.public_key.to_bytes(), signature) @@ -1325,7 +1298,7 @@ impl< }; match certificate_internal.signatures { AssembledSignature::ViewSyncPreCommit(raw_signatures) => { - let real_commit = VoteData::ViewSyncPreCommit(vote_data.commit()).commit(); + let real_commit = VoteData::ViewSyncPreCommit(vote_data.commit()).get_commit(); let real_qc_pp = ::get_public_parameter( self.membership().get_committee_qc_stake_table(), U256::from(self.membership().failure_threshold().get()), @@ -1337,7 +1310,7 @@ impl< ) } AssembledSignature::ViewSyncCommit(raw_signatures) => { - let real_commit = VoteData::ViewSyncCommit(vote_data.commit()).commit(); + let real_commit = VoteData::ViewSyncCommit(vote_data.commit()).get_commit(); let real_qc_pp = ::get_public_parameter( self.membership().get_committee_qc_stake_table(), U256::from(self.membership().success_threshold().get()), @@ -1349,7 +1322,7 @@ impl< ) } AssembledSignature::ViewSyncFinalize(raw_signatures) => { - let real_commit = VoteData::ViewSyncFinalize(vote_data.commit()).commit(); + let real_commit = VoteData::ViewSyncFinalize(vote_data.commit()).get_commit(); let real_qc_pp = ::get_public_parameter( self.membership().get_committee_qc_stake_table(), U256::from(self.membership().success_threshold().get()), From 028b688cf7ae4d1556478a6b7f6877e84d86e451 Mon Sep 17 00:00:00 2001 From: Gus Gutoski Date: Tue, 19 Sep 2023 16:52:54 -0400 Subject: [PATCH 10/18] chore: remove Committable from VoteData type param (#1779) * remove Committable from VoteData * fix libp2p example build --- .../examples/libp2p/multi-validator.rs | 4 +- .../hotshot/examples/libp2p/orchestrator.rs | 4 +- crates/hotshot/examples/libp2p/types.rs | 54 ++++++------- crates/hotshot/examples/libp2p/validator.rs | 4 +- crates/types/src/certificate.rs | 2 +- crates/types/src/traits/election.rs | 80 +++++++++++-------- crates/types/src/vote.rs | 22 ++--- 7 files changed, 93 insertions(+), 77 deletions(-) diff --git a/crates/hotshot/examples/libp2p/multi-validator.rs b/crates/hotshot/examples/libp2p/multi-validator.rs index 8a0f53c1c9..3ed46fa979 100644 --- a/crates/hotshot/examples/libp2p/multi-validator.rs +++ b/crates/hotshot/examples/libp2p/multi-validator.rs @@ -3,7 +3,7 @@ use async_compatibility_layer::{ logging::{setup_backtrace, setup_logging}, }; use clap::Parser; -use hotshot::demos::sdemo::SDemoTypes; +use hotshot::demo::DemoTypes; use hotshot_orchestrator::client::ValidatorArgs; use std::net::IpAddr; use tracing::instrument; @@ -49,7 +49,7 @@ async fn main() { for _ in 0..args.num_nodes { let node = async_spawn(async move { infra_da::main_entry_point::< - SDemoTypes, + DemoTypes, ThisMembership, DANetwork, QuorumNetwork, diff --git a/crates/hotshot/examples/libp2p/orchestrator.rs b/crates/hotshot/examples/libp2p/orchestrator.rs index 772d3aa12e..594d004a93 100644 --- a/crates/hotshot/examples/libp2p/orchestrator.rs +++ b/crates/hotshot/examples/libp2p/orchestrator.rs @@ -2,7 +2,7 @@ pub mod types; use async_compatibility_layer::logging::{setup_backtrace, setup_logging}; use clap::Parser; -use hotshot::demos::sdemo::SDemoTypes; +use hotshot::demo::DemoTypes; use tracing::instrument; use types::ThisMembership; @@ -29,7 +29,7 @@ async fn main() { let args = OrchestratorArgs::parse(); run_orchestrator_da::< - SDemoTypes, + DemoTypes, ThisMembership, DANetwork, QuorumNetwork, diff --git a/crates/hotshot/examples/libp2p/types.rs b/crates/hotshot/examples/libp2p/types.rs index 5289c9d6fe..79b1ea1419 100644 --- a/crates/hotshot/examples/libp2p/types.rs +++ b/crates/hotshot/examples/libp2p/types.rs @@ -1,6 +1,6 @@ use crate::infra_da::Libp2pDARun; use hotshot::{ - demos::sdemo::SDemoTypes, + demo::DemoTypes, traits::{ election::static_committee::GeneralStaticCommittee, implementations::{Libp2pCommChannel, MemoryStorage}, @@ -22,54 +22,54 @@ use std::fmt::Debug; #[derive(Clone, Debug, Deserialize, Serialize, Hash, PartialEq, Eq)] pub struct NodeImpl {} -pub type ThisLeaf = SequencingLeaf; +pub type ThisLeaf = SequencingLeaf; pub type ThisMembership = - GeneralStaticCommittee::SignatureKey>; -pub type DANetwork = Libp2pCommChannel; -pub type QuorumNetwork = Libp2pCommChannel; -pub type ViewSyncNetwork = Libp2pCommChannel; + GeneralStaticCommittee::SignatureKey>; +pub type DANetwork = Libp2pCommChannel; +pub type QuorumNetwork = Libp2pCommChannel; +pub type ViewSyncNetwork = Libp2pCommChannel; -pub type ThisDAProposal = DAProposal; -pub type ThisDAVote = DAVote; +pub type ThisDAProposal = DAProposal; +pub type ThisDAVote = DAVote; -pub type ThisQuorumProposal = QuorumProposal; -pub type ThisQuorumVote = QuorumVote; +pub type ThisQuorumProposal = QuorumProposal; +pub type ThisQuorumVote = QuorumVote; -pub type ThisViewSyncProposal = ViewSyncCertificate; -pub type ThisViewSyncVote = ViewSyncVote; +pub type ThisViewSyncProposal = ViewSyncCertificate; +pub type ThisViewSyncVote = ViewSyncVote; -impl NodeImplementation for NodeImpl { - type Storage = MemoryStorage; - type Leaf = SequencingLeaf; +impl NodeImplementation for NodeImpl { + type Storage = MemoryStorage; + type Leaf = SequencingLeaf; type Exchanges = SequencingExchanges< - SDemoTypes, - Message, + DemoTypes, + Message, QuorumExchange< - SDemoTypes, + DemoTypes, Self::Leaf, ThisQuorumProposal, ThisMembership, QuorumNetwork, - Message, + Message, >, - CommitteeExchange>, + CommitteeExchange>, ViewSyncExchange< - SDemoTypes, + DemoTypes, ThisViewSyncProposal, ThisMembership, ViewSyncNetwork, - Message, + Message, >, >; - type ConsensusMessage = SequencingMessage; + type ConsensusMessage = SequencingMessage; fn new_channel_maps( - start_view: ::Time, + start_view: ::Time, ) -> ( - ChannelMaps, - Option>, + ChannelMaps, + Option>, ) { (ChannelMaps::new(start_view), None) } } -pub type ThisRun = Libp2pDARun; +pub type ThisRun = Libp2pDARun; diff --git a/crates/hotshot/examples/libp2p/validator.rs b/crates/hotshot/examples/libp2p/validator.rs index ec2415fd65..ab44e02991 100644 --- a/crates/hotshot/examples/libp2p/validator.rs +++ b/crates/hotshot/examples/libp2p/validator.rs @@ -1,6 +1,6 @@ use async_compatibility_layer::logging::{setup_backtrace, setup_logging}; use clap::Parser; -use hotshot::demos::sdemo::SDemoTypes; +use hotshot::demo::DemoTypes; use tracing::{info, instrument}; use crate::types::{DANetwork, NodeImpl, QuorumNetwork, ThisMembership, ThisRun, ViewSyncNetwork}; @@ -29,7 +29,7 @@ async fn main() { args.host, args.port ); infra_da::main_entry_point::< - SDemoTypes, + DemoTypes, ThisMembership, DANetwork, QuorumNetwork, diff --git a/crates/types/src/certificate.rs b/crates/types/src/certificate.rs index 9792026dcf..5309729eb7 100644 --- a/crates/types/src/certificate.rs +++ b/crates/types/src/certificate.rs @@ -146,7 +146,7 @@ pub struct VoteMetaData, /// Data of the vote, yes, no, timeout, or DA - pub data: VoteData, + pub data: VoteData>, /// The votes's token pub vote_token: T, /// View number for the vote diff --git a/crates/types/src/traits/election.rs b/crates/types/src/traits/election.rs index 6ced8bebbc..3d152cc85e 100644 --- a/crates/types/src/traits/election.rs +++ b/crates/types/src/traits/election.rs @@ -74,38 +74,46 @@ pub enum Checked { /// Data to vote on for different types of votes. #[derive(Serialize, Deserialize, Clone, Debug, PartialEq, Eq, Hash)] #[serde(bound(deserialize = ""))] -pub enum VoteData { +pub enum VoteData +where + COMMITMENT: for<'a> Deserialize<'a>, +{ /// Vote to provide availability for a block. - DA(Commitment), + DA(COMMITMENT), /// Vote to append a leaf to the log. - Yes(Commitment), + Yes(COMMITMENT), /// Vote to reject a leaf from the log. - No(Commitment), + No(COMMITMENT), /// Vote to time out and proceed to the next view. - Timeout(Commitment), + Timeout(COMMITMENT), /// Vote to pre-commit the view sync. - ViewSyncPreCommit(Commitment), + ViewSyncPreCommit(COMMITMENT), /// Vote to commit the view sync. - ViewSyncCommit(Commitment), + ViewSyncCommit(COMMITMENT), /// Vote to finalize the view sync. - ViewSyncFinalize(Commitment), + ViewSyncFinalize(COMMITMENT), } -impl VoteData +impl VoteData where - COMMITTABLE: Committable + Serialize + Clone, + COMMITMENT: for<'a> Deserialize<'a> + Clone, { /// Return the underlying commitment. #[must_use] - pub fn get_commit(&self) -> Commitment { + pub fn get_commit(&self) -> COMMITMENT { #[allow(clippy::enum_glob_use)] use VoteData::*; match self { DA(c) | Yes(c) | No(c) | Timeout(c) | ViewSyncPreCommit(c) | ViewSyncCommit(c) - | ViewSyncFinalize(c) => *c, + | ViewSyncFinalize(c) => c.clone(), } } +} +impl VoteData +where + COMMITMENT: Serialize + for<'a> Deserialize<'a>, +{ #[must_use] /// Convert vote data into bytes. /// @@ -269,7 +277,7 @@ pub trait ConsensusExchange: Send + Sync { /// Network used by [`Membership`](Self::Membership) to communicate. type Networking: CommunicationChannel; /// Commitments to items which are the subject of proposals and decisions. - type Commitment: Committable + Serialize + Clone; + type Commitment: Committable + Serialize + for<'a> Deserialize<'a> + Clone; /// Join a [`ConsensusExchange`] with the given identity (`pk` and `sk`). fn create( @@ -322,7 +330,10 @@ pub trait ConsensusExchange: Send + Sync { } /// The contents of a vote on `commit`. - fn vote_data(&self, commit: Commitment) -> VoteData; + fn vote_data( + &self, + commit: Commitment, + ) -> VoteData>; /// Validate a QC. fn is_valid_cert(&self, qc: &Self::Certificate, commit: Commitment) -> bool { @@ -376,7 +387,7 @@ pub trait ConsensusExchange: Send + Sync { &self, encoded_key: &EncodedPublicKey, encoded_signature: &EncodedSignature, - data: VoteData, + data: VoteData>, vote_token: Checked, ) -> bool { let mut is_valid_vote_token = false; @@ -401,7 +412,7 @@ pub trait ConsensusExchange: Send + Sync { &self, key: &TYPES::SignatureKey, encoded_signature: &EncodedSignature, - data: &VoteData, + data: &VoteData>, vote_token: &Checked, ) -> bool { let is_valid_signature = key.validate(encoded_signature, data.get_commit().as_ref()); @@ -439,7 +450,7 @@ pub trait ConsensusExchange: Send + Sync { encoded_key: &EncodedPublicKey, encoded_signature: &EncodedSignature, leaf_commitment: Commitment, - vote_data: VoteData, + vote_data: VoteData>, vote_token: TYPES::VoteTokenType, view_number: TYPES::Time, accumlator: VoteAccumulator, @@ -613,9 +624,7 @@ impl< ) -> (EncodedPublicKey, EncodedSignature) { let signature = TYPES::SignatureKey::sign( &self.private_key, - VoteData::::DA(block_commitment) - .get_commit() - .as_ref(), + VoteData::DA(block_commitment).get_commit().as_ref(), ); (self.public_key.to_bytes(), signature) } @@ -658,9 +667,7 @@ impl< ) -> (EncodedPublicKey, EncodedSignature) { let signature = TYPES::SignatureKey::sign( &self.private_key, - VoteData::::DA(block_commitment) - .get_commit() - .as_ref(), + VoteData::DA(block_commitment).get_commit().as_ref(), ); (self.public_key.to_bytes(), signature) } @@ -710,7 +717,10 @@ impl< .make_vote_token(view_number, &self.private_key) } - fn vote_data(&self, commit: Commitment) -> VoteData { + fn vote_data( + &self, + commit: Commitment, + ) -> VoteData> { VoteData::DA(commit) } @@ -721,7 +731,7 @@ impl< encoded_key: &EncodedPublicKey, encoded_signature: &EncodedSignature, leaf_commitment: Commitment, - vote_data: VoteData, + vote_data: VoteData>, vote_token: TYPES::VoteTokenType, view_number: TYPES::Time, accumlator: VoteAccumulator, @@ -902,7 +912,7 @@ impl< ) -> (EncodedPublicKey, EncodedSignature) { let signature = TYPES::SignatureKey::sign( &self.private_key, - VoteData::::Yes(leaf_commitment).get_commit().as_ref(), + VoteData::Yes(leaf_commitment).get_commit().as_ref(), ); (self.public_key.to_bytes(), signature) } @@ -918,7 +928,7 @@ impl< ) -> (EncodedPublicKey, EncodedSignature) { let signature = TYPES::SignatureKey::sign( &self.private_key, - VoteData::::No(leaf_commitment).get_commit().as_ref(), + VoteData::No(leaf_commitment).get_commit().as_ref(), ); (self.public_key.to_bytes(), signature) } @@ -933,7 +943,7 @@ impl< fn sign_timeout_vote(&self, view_number: TYPES::Time) -> (EncodedPublicKey, EncodedSignature) { let signature = TYPES::SignatureKey::sign( &self.private_key, - VoteData::::Timeout(view_number.commit()) + VoteData::Timeout(view_number.commit()) .get_commit() .as_ref(), ); @@ -1023,7 +1033,10 @@ impl< &self.network } - fn vote_data(&self, commit: Commitment) -> VoteData { + fn vote_data( + &self, + commit: Commitment, + ) -> VoteData> { VoteData::Yes(commit) } @@ -1034,7 +1047,7 @@ impl< encoded_key: &EncodedPublicKey, encoded_signature: &EncodedSignature, leaf_commitment: Commitment, - vote_data: VoteData, + vote_data: VoteData>, vote_token: TYPES::VoteTokenType, view_number: TYPES::Time, accumlator: VoteAccumulator, @@ -1382,7 +1395,10 @@ impl< &self.network } - fn vote_data(&self, _commit: Commitment) -> VoteData { + fn vote_data( + &self, + _commit: Commitment, + ) -> VoteData> { unimplemented!() } @@ -1391,7 +1407,7 @@ impl< encoded_key: &EncodedPublicKey, encoded_signature: &EncodedSignature, leaf_commitment: Commitment>, - vote_data: VoteData, + vote_data: VoteData>, vote_token: TYPES::VoteTokenType, view_number: TYPES::Time, accumlator: VoteAccumulator, TYPES>, diff --git a/crates/types/src/vote.rs b/crates/types/src/vote.rs index d5dda8d825..ef53bc4018 100644 --- a/crates/types/src/vote.rs +++ b/crates/types/src/vote.rs @@ -38,7 +38,7 @@ pub trait VoteType EncodedSignature; /// Get the data this vote was signed over - fn get_data(&self) -> VoteData; + fn get_data(&self) -> VoteData>; /// Get the vote token of this vote fn get_vote_token(&self) -> TYPES::VoteTokenType; } @@ -56,7 +56,7 @@ pub struct DAVote { /// The vote token generated by this replica pub vote_token: TYPES::VoteTokenType, /// The vote data this vote is signed over - pub vote_data: VoteData, + pub vote_data: VoteData>, } /// A positive or negative vote on validating or commitment proposal. @@ -76,7 +76,7 @@ pub struct YesOrNoVote> { /// The vote token generated by this replica pub vote_token: TYPES::VoteTokenType, /// The vote data this vote is signed over - pub vote_data: VoteData, + pub vote_data: VoteData>, } /// A timeout vote. @@ -92,7 +92,7 @@ pub struct TimeoutVote> { /// The vote token generated by this replica pub vote_token: TYPES::VoteTokenType, /// The vote data this vote is signed over - pub vote_data: VoteData, + pub vote_data: VoteData>, } /// The internals of a view sync vote @@ -110,7 +110,7 @@ pub struct ViewSyncVoteInternal { /// The vote token generated by this replica pub vote_token: TYPES::VoteTokenType, /// The vote data this vote is signed over - pub vote_data: VoteData>, + pub vote_data: VoteData>>, } /// The data View Sync votes are signed over @@ -206,7 +206,7 @@ impl VoteType for DAVote { fn get_signature(&self) -> EncodedSignature { self.signature.1.clone() } - fn get_data(&self) -> VoteData { + fn get_data(&self) -> VoteData> { self.vote_data.clone() } fn get_vote_token(&self) -> ::VoteTokenType { @@ -239,7 +239,7 @@ impl> VoteType fn get_signature(&self) -> EncodedSignature { self.signature() } - fn get_data(&self) -> VoteData { + fn get_data(&self) -> VoteData> { match self { QuorumVote::Yes(v) | QuorumVote::No(v) => v.vote_data.clone(), QuorumVote::Timeout(_) => unimplemented!(), @@ -290,7 +290,7 @@ impl VoteType> for ViewSyncVote EncodedSignature { self.signature() } - fn get_data(&self) -> VoteData> { + fn get_data(&self) -> VoteData>> { match self { ViewSyncVote::PreCommit(vote_internal) | ViewSyncVote::Commit(vote_internal) @@ -762,7 +762,7 @@ type VoteMap = HashMap< Commitment, ( u64, - BTreeMap, TOKEN)>, + BTreeMap>, TOKEN)>, ), >; @@ -803,7 +803,7 @@ impl EncodedSignature, Vec<::StakeTableEntry>, usize, - VoteData, + VoteData>, TOKEN, ), ), @@ -824,7 +824,7 @@ where EncodedSignature, Vec<::StakeTableEntry>, usize, - VoteData, + VoteData>, TOKEN, ), ), From 68edb5325a53615e01ca8ab6b08a320d2728ce6a Mon Sep 17 00:00:00 2001 From: rob-maron <132852777+rob-maron@users.noreply.github.com> Date: Wed, 20 Sep 2023 10:32:26 -0400 Subject: [PATCH 11/18] Reduce build times and disk size for CI (#1783) * prune cargos to save on disk and build time * prune github actions * cache test * enable clippy linting --- .github/workflows/build.yml | 38 +++++++-------- .github/workflows/lints.yml | 8 +++- Cargo.lock | 71 ----------------------------- Cargo.toml | 7 ++- crates/hotshot/Cargo.toml | 1 - crates/libp2p-networking/Cargo.toml | 7 +-- crates/orchestrator/Cargo.toml | 1 - crates/task-impls/Cargo.toml | 2 - crates/task/Cargo.toml | 1 - crates/testing/Cargo.toml | 1 - crates/testing/README.md | 2 +- crates/types/Cargo.toml | 5 +- crates/web_server/Cargo.toml | 3 -- testing-macros/Cargo.toml | 1 - 14 files changed, 36 insertions(+), 112 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index c8cbe47c53..e7ac7ecdb4 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -33,6 +33,9 @@ jobs: - uses: Swatinem/rust-cache@v2 name: Enable Rust Caching + with: + shared-key: "" + prefix-key: ${{ matrix.just_variants }} - name: Install dependencies run: | @@ -40,9 +43,6 @@ jobs: sudo apt-get install -y protobuf-compiler cargo install just - - name: Check - run: just ${{ matrix.just_variants }} check - - name: Build all crates in workspace run: just ${{ matrix.just_variants }} build @@ -53,24 +53,24 @@ jobs: env: RUST_BACKTRACE: full - - name: Setup Docker BuildKit (buildx) - if: matrix.just_variants == 'async_std' - uses: docker/setup-buildx-action@v3 + # - name: Setup Docker BuildKit (buildx) + # if: matrix.just_variants == 'async_std' + # uses: docker/setup-buildx-action@v3 - - name: Login to Github Container Repo - uses: docker/login-action@v3 - if: github.event_name != 'pull_request' && matrix.just_variants == 'async_std' - with: - registry: ghcr.io - username: ${{ github.repository_owner }} - password: ${{ secrets.GITHUB_TOKEN }} + # - name: Login to Github Container Repo + # uses: docker/login-action@v3 + # if: github.event_name != 'pull_request' && matrix.just_variants == 'async_std' + # with: + # registry: ghcr.io + # username: ${{ github.repository_owner }} + # password: ${{ secrets.GITHUB_TOKEN }} - - name: Generate docker metadata (web-server) - uses: docker/metadata-action@v5 - if: matrix.just_variants == 'async_std' - id: web-server - with: - images: ghcr.io/espressosystems/hotshot/web-server + # - name: Generate docker metadata (web-server) + # uses: docker/metadata-action@v5 + # if: matrix.just_variants == 'async_std' + # id: web-server + # with: + # images: ghcr.io/espressosystems/hotshot/web-server # - name: Build and push web-server docker # uses: docker/build-push-action@v5 diff --git a/.github/workflows/lints.yml b/.github/workflows/lints.yml index 9ab7bf8c03..61106f6d0c 100644 --- a/.github/workflows/lints.yml +++ b/.github/workflows/lints.yml @@ -13,7 +13,7 @@ on: jobs: lints: runs-on: ubuntu-latest - timeout-minutes: 25 + timeout-minutes: 30 steps: - uses: styfle/cancel-workflow-action@0.11.0 name: Cancel Outdated Builds @@ -27,6 +27,12 @@ jobs: - uses: actions/checkout@v4 name: Checkout Repository + - uses: Swatinem/rust-cache@v2 + name: Enable Rust Caching + with: + shared-key: "" + prefix-key: lint + - name: Format Check run: cargo fmt -- --check diff --git a/Cargo.lock b/Cargo.lock index c23867d746..60b1a74849 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -782,28 +782,6 @@ dependencies = [ "trust-dns-resolver", ] -[[package]] -name = "async-stream" -version = "0.3.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cd56dd203fef61ac097dd65721a419ddccb106b2d2b70ba60a6b529f03961a51" -dependencies = [ - "async-stream-impl", - "futures-core", - "pin-project-lite 0.2.13", -] - -[[package]] -name = "async-stream-impl" -version = "0.3.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "16e62a023e7c117e27523144c5d2459f4397fcc3cab0085af8e2224f643a0193" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.31", -] - [[package]] name = "async-task" version = "4.4.0" @@ -989,12 +967,6 @@ dependencies = [ "serde", ] -[[package]] -name = "bit-vec" -version = "0.6.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "349f9b6a179ed607305526ca489b34ad0a41aed5f7980fa90eb03160b69598fb" - [[package]] name = "bitflags" version = "1.3.2" @@ -1577,12 +1549,6 @@ dependencies = [ "serdect", ] -[[package]] -name = "ct-codecs" -version = "1.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f3b7eb4404b8195a9abb6356f4ac07d8ba267045c8d6d220ac4dc992e6cc75df" - [[package]] name = "ctr" version = "0.6.0" @@ -2030,16 +1996,6 @@ dependencies = [ "signature", ] -[[package]] -name = "ed25519-compact" -version = "2.0.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6a3d382e8464107391c8706b4c14b087808ecb909f6c15c34114bc42e53a9e4c" -dependencies = [ - "ct-codecs", - "getrandom 0.2.10", -] - [[package]] name = "ed25519-dalek" version = "2.0.0" @@ -2649,7 +2605,6 @@ dependencies = [ "libp2p", "libp2p-identity", "libp2p-networking", - "nll", "rand 0.8.5", "rand_chacha 0.3.1", "serde", @@ -2685,7 +2640,6 @@ dependencies = [ "libp2p", "libp2p-core", "libp2p-networking", - "nll", "serde", "serde_json", "snafu", @@ -2777,7 +2731,6 @@ dependencies = [ "atomic_enum", "either", "futures", - "nll", "pin-project", "serde", "snafu", @@ -2792,7 +2745,6 @@ dependencies = [ "async-compatibility-layer", "async-lock", "async-std", - "async-stream", "async-trait", "atomic_enum", "bincode", @@ -2805,7 +2757,6 @@ dependencies = [ "hotshot-types", "hotshot-utils", "jf-primitives", - "nll", "pin-project", "rand_chacha 0.3.1", "serde", @@ -2837,7 +2788,6 @@ dependencies = [ "hotshot-types", "hotshot-utils", "jf-primitives", - "nll", "rand 0.8.5", "rand_chacha 0.3.1", "serde", @@ -2859,7 +2809,6 @@ dependencies = [ "async-std", "async-trait", "bincode", - "bit-vec", "bitvec", "blake3", "commit", @@ -2867,7 +2816,6 @@ dependencies = [ "derivative", "digest 0.10.7", "displaydoc", - "ed25519-compact", "either", "espresso-systems-common 0.4.1", "ethereum-types", @@ -2880,7 +2828,6 @@ dependencies = [ "jf-primitives", "jf-utils", "libp2p-networking", - "nll", "rand 0.8.5", "rand_chacha 0.3.1", "serde", @@ -2917,13 +2864,10 @@ dependencies = [ "hotshot-utils", "jf-primitives", "libp2p-core", - "nll", - "portpicker", "rand 0.8.5", "serde", "serde_json", "snafu", - "surf-disco", "tide", "tide-disco 0.4.1 (git+https://github.com/EspressoSystems/tide-disco.git?tag=v0.4.1)", "tokio", @@ -3788,7 +3732,6 @@ dependencies = [ "async-trait", "bincode", "blake3", - "clap", "color-eyre", "custom_debug", "dashmap", @@ -4627,11 +4570,6 @@ dependencies = [ "libc", ] -[[package]] -name = "nll" -version = "1.0.0" -source = "git+https://github.com/EspressoSystems/nll.git#c84499eb8de11beee5caa9390deff8a8ca3b9514" - [[package]] name = "nohash-hasher" version = "0.2.0" @@ -5028,15 +4966,6 @@ dependencies = [ "universal-hash 0.4.0", ] -[[package]] -name = "portpicker" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "be97d76faf1bfab666e1375477b23fde79eccf0276e9b63b92a39d676a889ba9" -dependencies = [ - "rand 0.8.5", -] - [[package]] name = "ppv-lite86" version = "0.2.17" diff --git a/Cargo.toml b/Cargo.toml index e83b00d2ed..1223493eca 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -58,7 +58,6 @@ jf-utils = { git = "https://github.com/espressosystems/jellyfish", rev = "2fc42a libp2p-identity = "0.2" libp2p-networking = { path = "./crates/libp2p-networking", version = "0.1.0", default-features = false } libp2p-swarm-derive = { version = "=0.33.0" } -nll = { git = "https://github.com/EspressoSystems/nll.git" } rand = "0.8.5" rand_chacha = { version = "0.3.1", default-features = false } serde = { version = "1.0.188", features = ["derive"] } @@ -119,9 +118,15 @@ tokio = { version = "1.32.0", features = [ ## Apply some optimizations to test dependencies in debug/test builds +[profile.dev] +strip = true +debug = 0 + # Generally optimize dependencies a little [profile.dev.package."*"] opt-level = 1 +strip = true +debug = 0 # The default release profile without LTO. [profile.release] diff --git a/crates/hotshot/Cargo.toml b/crates/hotshot/Cargo.toml index def132f6d8..671519ec84 100644 --- a/crates/hotshot/Cargo.toml +++ b/crates/hotshot/Cargo.toml @@ -105,7 +105,6 @@ jf-primitives = { workspace = true } libp2p = { workspace = true } libp2p-identity = { workspace = true } libp2p-networking = { workspace = true } -nll = { workspace = true } rand = { workspace = true } rand_chacha = { workspace = true } serde = { workspace = true, features = ["rc"] } diff --git a/crates/libp2p-networking/Cargo.toml b/crates/libp2p-networking/Cargo.toml index 82b5ed7d87..dfdb02020e 100644 --- a/crates/libp2p-networking/Cargo.toml +++ b/crates/libp2p-networking/Cargo.toml @@ -9,7 +9,7 @@ authors = ["Espresso Systems "] [features] default = ["webui"] -webui = ["tide"] +webui = [] # # this only has effect on linux # lossy_network = [ @@ -47,8 +47,6 @@ snafu = { workspace = true } tide = { version = "0.16", optional = true, default-features = false, features = [ "h1-server", ] } -# TODO do we want this as non optional?? -tokio-stream = "0.1.14" tracing = { workspace = true } void = "1.0.2" dashmap = "5.5.3" @@ -56,6 +54,7 @@ dashmap = "5.5.3" [target.'cfg(all(async_executor_impl = "tokio"))'.dependencies] libp2p = { workspace = true, features = ["tokio"] } tokio = { workspace = true } +tokio-stream = "0.1.14" [target.'cfg(all(async_executor_impl = "async-std"))'.dependencies] libp2p = { workspace = true, features = ["async-std"] } async-std = { workspace = true } @@ -76,5 +75,3 @@ netlink-sys = { git = "https://github.com/espressosystems/netlink.git", version ], optional = true } netlink-packet-generic = { git = "https://github.com/espressosystems/netlink.git", version = "0.2.0", optional = true } -[dev-dependencies] -clap = { version = "4.4", features = ["derive", "env"] } diff --git a/crates/orchestrator/Cargo.toml b/crates/orchestrator/Cargo.toml index f3f57ada3a..4a1cfad69c 100644 --- a/crates/orchestrator/Cargo.toml +++ b/crates/orchestrator/Cargo.toml @@ -16,7 +16,6 @@ blake3 = { workspace = true, features = ["traits-preview"] } hotshot-types = { version = "0.1.0", path = "../types", default-features = false } hotshot-utils = { path = "../utils" } libp2p-networking = { workspace = true } -nll = { workspace = true } tide-disco = { git = "https://github.com/EspressoSystems/tide-disco.git", tag = "v0.4.1" } surf-disco = { workspace = true } tracing = { workspace = true } diff --git a/crates/task-impls/Cargo.toml b/crates/task-impls/Cargo.toml index 5978e10629..08c900b53b 100644 --- a/crates/task-impls/Cargo.toml +++ b/crates/task-impls/Cargo.toml @@ -10,14 +10,12 @@ async-compatibility-layer = { workspace = true } async-trait = { workspace = true } either = { workspace = true } futures = { workspace = true } -nll = { workspace = true } serde = { workspace = true } snafu = { workspace = true } async-lock = { workspace = true } tracing = { workspace = true } atomic_enum = "0.2.0" pin-project = "1.1.3" -async-stream = "0.3.5" hotshot-constants = { path = "../constants", default-features = false } hotshot-types = { path = "../types", default-features = false } hotshot-task = { path = "../task", default-features = false } diff --git a/crates/task/Cargo.toml b/crates/task/Cargo.toml index 720a7aca6a..cb1703831f 100644 --- a/crates/task/Cargo.toml +++ b/crates/task/Cargo.toml @@ -10,7 +10,6 @@ async-compatibility-layer = { workspace = true } async-trait = { workspace = true } either = { workspace = true } futures = { workspace = true } -nll = { workspace = true } serde = { workspace = true } snafu = { workspace = true } async-lock = { workspace = true } diff --git a/crates/testing/Cargo.toml b/crates/testing/Cargo.toml index b361b241d7..935f9e4342 100644 --- a/crates/testing/Cargo.toml +++ b/crates/testing/Cargo.toml @@ -34,7 +34,6 @@ rand = { workspace = true } rand_chacha = { workspace = true } snafu = { workspace = true } tracing = { workspace = true } -nll = { workspace = true } serde = { workspace = true } ethereum-types = { workspace = true } bitvec = { workspace = true } diff --git a/crates/testing/README.md b/crates/testing/README.md index a477450092..59b281f421 100644 --- a/crates/testing/README.md +++ b/crates/testing/README.md @@ -69,4 +69,4 @@ async { }; ``` -See TODO for examples. +See TODO for examples. \ No newline at end of file diff --git a/crates/types/Cargo.toml b/crates/types/Cargo.toml index 74ace15850..742882cce2 100644 --- a/crates/types/Cargo.toml +++ b/crates/types/Cargo.toml @@ -8,7 +8,7 @@ version = "0.1.0" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [features] -demo = ["ed25519-compact"] +demo = [] [dependencies] arbitrary = { version = "1.3", features = ["derive"] } @@ -28,7 +28,6 @@ custom_debug = { workspace = true } derivative = "2.2.0" digest = { workspace = true } displaydoc = { version = "0.2.3", default-features = false } -ed25519-compact = { version = "2.0.4", optional = true } either = { workspace = true, features = ["serde"] } espresso-systems-common = { workspace = true } futures = { workspace = true } @@ -39,7 +38,6 @@ hotshot-utils = { path = "../utils" } hotshot-task = { path = "../task", default-features = false } jf-primitives = { workspace = true, features = ["test-srs"] } jf-utils = { workspace = true } -nll = { workspace = true } libp2p-networking = { workspace = true } rand = { workspace = true } rand_chacha = { workspace = true } @@ -50,7 +48,6 @@ tagged-base64 = { git = "https://github.com/EspressoSystems/tagged-base64", tag time = { workspace = true } tracing = { workspace = true } ethereum-types = { workspace = true } -bit-vec = "0.6.3" typenum = { workspace = true } [dev-dependencies] diff --git a/crates/web_server/Cargo.toml b/crates/web_server/Cargo.toml index 827d5a5258..bc2520065a 100644 --- a/crates/web_server/Cargo.toml +++ b/crates/web_server/Cargo.toml @@ -22,7 +22,6 @@ hotshot-types = { path = "../types", default-features = false } hotshot-utils = { path = "../utils" } jf-primitives = { workspace = true } tide-disco = { git = "https://github.com/EspressoSystems/tide-disco.git", tag = "v0.4.1" } -nll = { workspace = true } tracing = { workspace = true } rand = { workspace = true } serde = { workspace = true } @@ -30,8 +29,6 @@ serde_json = "1.0.96" snafu = { workspace = true } tide = { version = "0.16.0", default-features = false } toml = { workspace = true } -portpicker = "0.1" -surf-disco = { workspace = true } [dev-dependencies] hotshot-types = { path = "../types", default-features = false } diff --git a/testing-macros/Cargo.toml b/testing-macros/Cargo.toml index 5d2d16b485..05efd993c0 100644 --- a/testing-macros/Cargo.toml +++ b/testing-macros/Cargo.toml @@ -22,7 +22,6 @@ hotshot = { path = "../hotshot", default-features = false } hotshot-types = { path = "../types", default-features = false } hotshot-testing = { path = "../testing", default-features = false } jf-primitives = { workspace = true } -nll = { workspace = true } rand = { workspace = true } snafu = { workspace = true } tracing = { workspace = true } From 1f249d513de23694a739ddd29dc8d81fb280e75b Mon Sep 17 00:00:00 2001 From: Gus Gutoski Date: Wed, 20 Sep 2023 12:19:10 -0400 Subject: [PATCH 12/18] chore: update jellyfish (#1785) * update jellyfish dep * add TODO comments --- Cargo.lock | 6 +++--- Cargo.toml | 6 +++--- crates/task-impls/src/da.rs | 8 ++++---- crates/testing/tests/da_task.rs | 6 +++--- crates/testing/tests/network_task.rs | 6 +++--- crates/types/src/data.rs | 6 ++++-- 6 files changed, 20 insertions(+), 18 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 60b1a74849..30f9f438b1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3270,7 +3270,7 @@ checksum = "af150ab688ff2122fcef229be89cb50dd66af9e01a4ff320cc137eecc9bacc38" [[package]] name = "jf-primitives" version = "0.4.0-pre.0" -source = "git+https://github.com/EspressoSystems/jellyfish?rev=2fc42af#2fc42af31c955088f7ff4775cbd405a2f42c9322" +source = "git+https://github.com/EspressoSystems/jellyfish#f6866c5395405f807e62516b5d56d9dc390aeb61" dependencies = [ "anyhow", "ark-bls12-377", @@ -3315,7 +3315,7 @@ dependencies = [ [[package]] name = "jf-relation" version = "0.4.0-pre.0" -source = "git+https://github.com/EspressoSystems/jellyfish?rev=2fc42af#2fc42af31c955088f7ff4775cbd405a2f42c9322" +source = "git+https://github.com/EspressoSystems/jellyfish#f6866c5395405f807e62516b5d56d9dc390aeb61" dependencies = [ "ark-bls12-377", "ark-bls12-381", @@ -3341,7 +3341,7 @@ dependencies = [ [[package]] name = "jf-utils" version = "0.4.0-pre.0" -source = "git+https://github.com/EspressoSystems/jellyfish?rev=2fc42af#2fc42af31c955088f7ff4775cbd405a2f42c9322" +source = "git+https://github.com/EspressoSystems/jellyfish#f6866c5395405f807e62516b5d56d9dc390aeb61" dependencies = [ "ark-ec", "ark-ff", diff --git a/Cargo.toml b/Cargo.toml index 1223493eca..d437aabeeb 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -52,9 +52,9 @@ either = { version = "1.8" } espresso-systems-common = { git = "https://github.com/espressosystems/espresso-systems-common", tag = "0.4.1" } ethereum-types = { version = "0.14.1", features = ["impl-serde"] } futures = "0.3.28" -jf-primitives = { git = "https://github.com/EspressoSystems/jellyfish", rev = "2fc42af" } -jf-relation = { git = "https://github.com/EspressoSystems/jellyfish", rev = "2fc42af" } -jf-utils = { git = "https://github.com/espressosystems/jellyfish", rev = "2fc42af" } +jf-primitives = { git = "https://github.com/EspressoSystems/jellyfish" } +jf-relation = { git = "https://github.com/EspressoSystems/jellyfish" } +jf-utils = { git = "https://github.com/espressosystems/jellyfish" } libp2p-identity = "0.2" libp2p-networking = { path = "./crates/libp2p-networking", version = "0.1.0", default-features = false } libp2p-swarm-derive = { version = "=0.33.0" } diff --git a/crates/task-impls/src/da.rs b/crates/task-impls/src/da.rs index f8711b1d78..ff048ef469 100644 --- a/crates/task-impls/src/da.rs +++ b/crates/task-impls/src/da.rs @@ -671,7 +671,7 @@ where let vid = VidScheme::new(NUM_CHUNKS, NUM_STORAGE_NODES, &srs).unwrap(); let message_bytes = bincode::serialize(&message).unwrap(); - let (shares, common) = vid.dispersal_data(&message_bytes).unwrap(); + let vid_disperse = vid.disperse(&message_bytes).unwrap(); // TODO for now reuse the same block commitment and signature as DA committee // https://github.com/EspressoSystems/jellyfish/issues/369 @@ -680,9 +680,9 @@ where Proposal { data: VidDisperse { view_number: view, - commitment: block.commit(), - shares, - common, + commitment: block.commit(), // TODO GG should be vid_disperse.commit but that's a big change + shares: vid_disperse.shares, + common: vid_disperse.common, }, signature: message.signature, }, diff --git a/crates/testing/tests/da_task.rs b/crates/testing/tests/da_task.rs index a9dcbd0c2e..204038885c 100644 --- a/crates/testing/tests/da_task.rs +++ b/crates/testing/tests/da_task.rs @@ -49,13 +49,13 @@ async fn test_da_task() { }; let vid = vid_init(); let message_bytes = bincode::serialize(&message).unwrap(); - let (shares, common) = vid.dispersal_data(&message_bytes).unwrap(); + let vid_disperse = vid.disperse(&message_bytes).unwrap(); let vid_proposal = Proposal { data: VidDisperse { view_number: message.data.view_number, commitment: block_commitment, - shares, - common, + shares: vid_disperse.shares, + common: vid_disperse.common, }, signature: message.signature.clone(), }; diff --git a/crates/testing/tests/network_task.rs b/crates/testing/tests/network_task.rs index 8239fc72d4..7ad17e2008 100644 --- a/crates/testing/tests/network_task.rs +++ b/crates/testing/tests/network_task.rs @@ -53,15 +53,15 @@ async fn test_network_task() { let quorum_proposal = build_quorum_proposal(&handle, priv_key, 2).await; let vid = vid_init(); let da_proposal_bytes = bincode::serialize(&da_proposal).unwrap(); - let (shares, common) = vid.dispersal_data(&da_proposal_bytes).unwrap(); + let vid_disperse = vid.disperse(&da_proposal_bytes).unwrap(); // TODO for now reuse the same block commitment and signature as DA committee // https://github.com/EspressoSystems/jellyfish/issues/369 let da_vid_disperse = Proposal { data: VidDisperse { view_number: da_proposal.data.view_number, commitment: block_commitment, - shares, - common, + shares: vid_disperse.shares, + common: vid_disperse.common, }, signature: da_proposal.signature.clone(), }; diff --git a/crates/types/src/data.rs b/crates/types/src/data.rs index 8275c8d9b8..080355a195 100644 --- a/crates/types/src/data.rs +++ b/crates/types/src/data.rs @@ -173,11 +173,13 @@ pub struct VidDisperse { /// The view number for which this VID data is intended pub view_number: TYPES::Time, /// Block commitment + /// + /// TODO GG type should be `::Common` but that's a big change. pub commitment: Commitment, /// VID shares dispersed among storage nodes - pub shares: Vec<::StorageShare>, + pub shares: Vec<::Share>, /// VID common data sent to all storage nodes - pub common: ::StorageCommon, + pub common: ::Common, } /// Trusted KZG setup for VID. From a566cb66fdffd877c951b85ace3232bc5c3af8ba Mon Sep 17 00:00:00 2001 From: Brendon Fish Date: Wed, 20 Sep 2023 12:22:17 -0400 Subject: [PATCH 13/18] remove transactions from consensus struct --- crates/hotshot/src/lib.rs | 21 -------------- crates/hotshot/src/tasks/mod.rs | 4 ++- crates/task-impls/src/transactions.rs | 41 +++++++++++++++++---------- crates/types/src/consensus.rs | 14 --------- 4 files changed, 29 insertions(+), 51 deletions(-) diff --git a/crates/hotshot/src/lib.rs b/crates/hotshot/src/lib.rs index b09ea6235d..38c05e19be 100644 --- a/crates/hotshot/src/lib.rs +++ b/crates/hotshot/src/lib.rs @@ -133,11 +133,6 @@ pub struct SystemContextInner> { /// a reference to the metrics that the implementor is using. _metrics: Box, - /// Transactions - /// (this is shared btwn hotshot and `Consensus`) - transactions: - Arc, TYPES::Transaction>>>, - /// The hotstuff implementation consensus: Arc>>, @@ -219,8 +214,6 @@ impl> SystemContext { state_map, cur_view: start_view, last_decided_view: anchored_leaf.get_view_number(), - transactions: Arc::default(), - seen_transactions: HashSet::new(), saved_leaves, saved_blocks, // TODO this is incorrect @@ -231,17 +224,14 @@ impl> SystemContext { invalid_qc: 0, }; let consensus = Arc::new(RwLock::new(consensus)); - let txns = consensus.read().await.get_transactions(); let inner: Arc> = Arc::new(SystemContextInner { id: nonce, channel_maps: I::new_channel_maps(start_view), consensus, - transactions: txns, public_key, private_key, config, - // networking, storage, exchanges: Arc::new(exchanges), event_sender: RwLock::default(), @@ -546,11 +536,6 @@ impl> SystemContext { /// [`HotShot`] implementations that depend on [`TYPES::ConsensusType`]. #[async_trait] pub trait HotShotType> { - /// Get the [`transactions`] field of [`HotShot`]. - fn transactions( - &self, - ) -> &Arc, TYPES::Transaction>>>; - /// Get the [`hotstuff`] field of [`HotShot`]. fn consensus(&self) -> &Arc>>; @@ -684,12 +669,6 @@ where Membership = MEMBERSHIP, > + 'static, { - fn transactions( - &self, - ) -> &Arc, TYPES::Transaction>>> { - &self.inner.transactions - } - fn consensus(&self) -> &Arc>> { &self.inner.consensus } diff --git a/crates/hotshot/src/tasks/mod.rs b/crates/hotshot/src/tasks/mod.rs index 619fdb6d59..a2e68528f4 100644 --- a/crates/hotshot/src/tasks/mod.rs +++ b/crates/hotshot/src/tasks/mod.rs @@ -43,7 +43,7 @@ use hotshot_types::{ vote::{ViewSyncData, VoteType}, }; use serde::Serialize; -use std::{collections::HashMap, marker::PhantomData, sync::Arc, time::Duration}; +use std::{collections::{HashMap, HashSet}, marker::PhantomData, sync::Arc, time::Duration}; /// event for global event stream #[derive(Clone, Debug)] @@ -441,6 +441,8 @@ where registry: registry.clone(), api: c_api.clone(), consensus: handle.hotshot.get_consensus(), + transactions: Arc::default(), + seen_transactions: HashSet::new(), cur_view: TYPES::Time::new(0), committee_exchange: committee_exchange.into(), event_stream: event_stream.clone(), diff --git a/crates/task-impls/src/transactions.rs b/crates/task-impls/src/transactions.rs index 8609708653..b41604376d 100644 --- a/crates/task-impls/src/transactions.rs +++ b/crates/task-impls/src/transactions.rs @@ -1,9 +1,11 @@ use crate::events::SequencingHotShotEvent; +use async_compatibility_layer::async_primitives::subscribable_rwlock::SubscribableRwLock; use async_compatibility_layer::{ art::async_timeout, async_primitives::subscribable_rwlock::ReadView, }; use async_lock::RwLock; use bincode::config::Options; +use commit::Commitment; use commit::Committable; use either::{Either, Left, Right}; use hotshot_task::{ @@ -26,9 +28,16 @@ use hotshot_types::{ }; use hotshot_utils::bincode::bincode_opts; use snafu::Snafu; -use std::{collections::HashSet, sync::Arc, time::Instant}; +use std::{ + collections::{HashMap, HashSet}, + sync::Arc, + time::Instant, +}; use tracing::{debug, error, instrument, warn}; +/// A type alias for `HashMap, T>` +type CommitmentMap = HashMap, T>; + #[derive(Snafu, Debug)] /// Error type for consensus tasks pub struct ConsensusTaskError {} @@ -61,6 +70,12 @@ pub struct TransactionTaskState< /// Reference to consensus. Leader will require a read lock on this. pub consensus: Arc>>>, + /// A list of undecided transactions + pub transactions: Arc>>, + + /// A list of transactions we've seen decided, but didn't receive + pub seen_transactions: HashSet>, + /// the committee exchange pub committee_exchange: Arc>, @@ -97,15 +112,14 @@ where ) -> Option { match event { SequencingHotShotEvent::TransactionsRecv(transactions) => { - let mut consensus = self.consensus.write().await; - consensus - .get_transactions() + let consensus = self.consensus.read().await; + self.transactions .modify(|txns| { for transaction in transactions { let size = bincode_opts().serialized_size(&transaction).unwrap_or(0); // If we didn't already know about this transaction, update our mempool metrics. - if !consensus.seen_transactions.remove(&transaction.commit()) + if !self.seen_transactions.remove(&transaction.commit()) && txns.insert(transaction.commit(), transaction).is_none() { consensus.metrics.outstanding_transactions.update(1); @@ -138,17 +152,16 @@ where Right(_) => {} } } - let mut consensus = self.consensus.write().await; - let txns = consensus.transactions.cloned().await; + let consensus = self.consensus.read().await; + let txns = self.transactions.cloned().await; let _ = included_txns.iter().map(|hash| { if !txns.contains_key(hash) { - consensus.seen_transactions.insert(*hash); + self.seen_transactions.insert(*hash); } }); drop(txns); - consensus - .transactions + self.transactions .modify(|txns| { *txns = txns .drain() @@ -255,12 +268,10 @@ where Either::Right(_commitment) => HashSet::new(), }; - let consensus = self.consensus.read().await; - - let receiver = consensus.transactions.subscribe().await; + let receiver = self.transactions.subscribe().await; loop { - let all_txns = consensus.transactions.cloned().await; + let all_txns = self.transactions.cloned().await; debug!("Size of transactions: {}", all_txns.len()); let unclaimed_txns: Vec<_> = all_txns .iter() @@ -290,7 +301,7 @@ where } break; } - let all_txns = consensus.transactions.cloned().await; + let all_txns = self.transactions.cloned().await; let txns: Vec = all_txns .iter() .filter_map(|(txn_hash, txn)| { diff --git a/crates/types/src/consensus.rs b/crates/types/src/consensus.rs index a29c840056..ce27d7a547 100644 --- a/crates/types/src/consensus.rs +++ b/crates/types/src/consensus.rs @@ -2,8 +2,6 @@ pub use crate::traits::node_implementation::ViewQueue; pub use crate::utils::{View, ViewInner}; -use async_compatibility_layer::async_primitives::subscribable_rwlock::SubscribableRwLock; -use std::collections::HashSet; use crate::utils::Terminator; use crate::{ @@ -41,12 +39,6 @@ pub struct Consensus> { /// last view had a successful decide event pub last_decided_view: TYPES::Time, - /// A list of undecided transactions - pub transactions: Arc>>, - - /// A list of transactions we've seen decided, but didn't receive - pub seen_transactions: HashSet>, - /// Map of leaf hash -> leaf /// - contains undecided leaves /// - includes the MOST RECENT decided leaf @@ -264,12 +256,6 @@ impl> Consensus { self.state_map = self.state_map.split_off(&new_anchor_view); } - /// return a clone of the internal storage of unclaimed transactions - #[must_use] - pub fn get_transactions(&self) -> Arc>> { - self.transactions.clone() - } - /// Gets the last decided state /// # Panics /// if the last decided view's state does not exist in the state map From fb690fb9e36b8453d5a75370849defc30725fb71 Mon Sep 17 00:00:00 2001 From: Brendon Fish Date: Wed, 20 Sep 2023 12:44:33 -0400 Subject: [PATCH 14/18] linting --- crates/hotshot/src/lib.rs | 6 +++--- crates/hotshot/src/tasks/mod.rs | 7 ++++++- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/crates/hotshot/src/lib.rs b/crates/hotshot/src/lib.rs index e543f22901..0f317c9410 100644 --- a/crates/hotshot/src/lib.rs +++ b/crates/hotshot/src/lib.rs @@ -42,12 +42,12 @@ use crate::{ }; use async_compatibility_layer::{ art::{async_spawn, async_spawn_local}, - async_primitives::{broadcast::BroadcastSender, subscribable_rwlock::SubscribableRwLock}, + async_primitives::broadcast::BroadcastSender, channel::UnboundedSender, }; use async_lock::{RwLock, RwLockUpgradableReadGuard, RwLockWriteGuard}; use async_trait::async_trait; -use commit::{Commitment, Committable}; +use commit::Committable; use custom_debug::Debug; use hotshot_task::{ event_stream::{ChannelStream, EventStream}, @@ -83,7 +83,7 @@ use hotshot_types::{ }; use snafu::ResultExt; use std::{ - collections::{BTreeMap, HashMap, HashSet}, + collections::{BTreeMap, HashMap}, marker::PhantomData, num::NonZeroUsize, sync::Arc, diff --git a/crates/hotshot/src/tasks/mod.rs b/crates/hotshot/src/tasks/mod.rs index a2e68528f4..82ea383353 100644 --- a/crates/hotshot/src/tasks/mod.rs +++ b/crates/hotshot/src/tasks/mod.rs @@ -43,7 +43,12 @@ use hotshot_types::{ vote::{ViewSyncData, VoteType}, }; use serde::Serialize; -use std::{collections::{HashMap, HashSet}, marker::PhantomData, sync::Arc, time::Duration}; +use std::{ + collections::{HashMap, HashSet}, + marker::PhantomData, + sync::Arc, + time::Duration, +}; /// event for global event stream #[derive(Clone, Debug)] From 2afb7e544d597088905a70e987733a5f99867126 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 20 Sep 2023 13:58:05 -0400 Subject: [PATCH 15/18] Bump typenum from 1.16.0 to 1.17.0 (#1781) Bumps [typenum](https://github.com/paholg/typenum) from 1.16.0 to 1.17.0. - [Release notes](https://github.com/paholg/typenum/releases) - [Changelog](https://github.com/paholg/typenum/blob/main/CHANGELOG.md) - [Commits](https://github.com/paholg/typenum/compare/v1.16.0...v1.17.0) --- updated-dependencies: - dependency-name: typenum dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- Cargo.lock | 4 ++-- Cargo.toml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 30f9f438b1..e18e2095ed 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -7082,9 +7082,9 @@ dependencies = [ [[package]] name = "typenum" -version = "1.16.0" +version = "1.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "497961ef93d974e23eb6f433eb5fe1b7930b659f06d12dec6fc44a8f554c0bba" +checksum = "42ff0bf0c66b8238c6f3b578df37d0b7848e55df8577b3f74f92a69acceeb825" [[package]] name = "ucd-trie" diff --git a/Cargo.toml b/Cargo.toml index d437aabeeb..c1a4d45a6d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -67,7 +67,7 @@ surf-disco = { git = "https://github.com/EspressoSystems/surf-disco.git", tag = time = "0.3.28" toml = "0.7.8" tracing = "0.1.37" -typenum = "1.16.0" +typenum = "1.17.0" libp2p = { package = "libp2p", version = "0.52.3", features = [ "macros", "autonat", From 6d8f64d1b349e21aa4d627e9c3af46c306b6f333 Mon Sep 17 00:00:00 2001 From: Brendon Fish Date: Wed, 20 Sep 2023 15:10:51 -0400 Subject: [PATCH 16/18] allow more failures in web catchup test --- crates/testing/tests/catchup.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/testing/tests/catchup.rs b/crates/testing/tests/catchup.rs index 9c6bd0bb38..01610f38f2 100644 --- a/crates/testing/tests/catchup.rs +++ b/crates/testing/tests/catchup.rs @@ -164,7 +164,7 @@ async fn test_catchup_one_node() { ..Default::default() }; // only alow for the view which the catchup node hasn't started to fail - metadata.overall_safety_properties.num_failed_views = 1; + metadata.overall_safety_properties.num_failed_views = 5; metadata .gen_launcher::() From 4a29f8f74d750e8b3a02cfda546cae9002fd2f66 Mon Sep 17 00:00:00 2001 From: Gus Gutoski Date: Wed, 20 Sep 2023 17:49:32 -0400 Subject: [PATCH 17/18] chore: remove Committable from 3 more types (#1789) * VoteMap type param Commitment -> COMMITMENT * VoteAccumulator type param commitment instead of Committable * VoteType remove Committable from type param (this was a big one) --- crates/hotshot/src/tasks/mod.rs | 6 +-- crates/types/src/traits/election.rs | 4 +- crates/types/src/vote.rs | 58 ++++++++++++++++------------- 3 files changed, 37 insertions(+), 31 deletions(-) diff --git a/crates/hotshot/src/tasks/mod.rs b/crates/hotshot/src/tasks/mod.rs index 619fdb6d59..3f342b68f7 100644 --- a/crates/hotshot/src/tasks/mod.rs +++ b/crates/hotshot/src/tasks/mod.rs @@ -5,7 +5,7 @@ use crate::{ QuorumCertificate, SequencingQuorumEx, }; use async_compatibility_layer::art::async_sleep; -use commit::Committable; +use commit::{Commitment, Committable}; use futures::FutureExt; use hotshot_task::{ boxed_sync, @@ -66,7 +66,7 @@ pub async fn add_network_message_task< >, COMMITTABLE: Committable + Serialize + Clone, PROPOSAL: ProposalType, - VOTE: VoteType, + VOTE: VoteType>, MEMBERSHIP: Membership, EXCHANGE: ConsensusExchange< TYPES, @@ -176,7 +176,7 @@ pub async fn add_network_event_task< >, COMMITTABLE: Committable + Serialize + Clone, PROPOSAL: ProposalType, - VOTE: VoteType, + VOTE: VoteType>, MEMBERSHIP: Membership, EXCHANGE: ConsensusExchange< TYPES, diff --git a/crates/types/src/traits/election.rs b/crates/types/src/traits/election.rs index 3d152cc85e..fb1ff72a58 100644 --- a/crates/types/src/traits/election.rs +++ b/crates/types/src/traits/election.rs @@ -165,7 +165,7 @@ where TOKEN: VoteToken, { /// `VoteType` that is used in this certificate - type Vote: VoteType; + type Vote: VoteType>; /// `Accumulator` type to accumulate votes. type VoteAccumulator: Accumulator2; @@ -267,7 +267,7 @@ pub trait ConsensusExchange: Send + Sync { type Proposal: ProposalType; /// A vote on a [`Proposal`](Self::Proposal). // TODO ED Make this equal Certificate vote (if possible?) - type Vote: VoteType; + type Vote: VoteType>; /// A [`SignedCertificate`] attesting to a decision taken by the committee. type Certificate: SignedCertificate + Hash diff --git a/crates/types/src/vote.rs b/crates/types/src/vote.rs index ef53bc4018..c470925b54 100644 --- a/crates/types/src/vote.rs +++ b/crates/types/src/vote.rs @@ -28,7 +28,7 @@ use std::{ use tracing::error; /// The vote sent by consensus messages. -pub trait VoteType: +pub trait VoteType Deserialize<'a> + Serialize + Clone>: Debug + Clone + 'static + Serialize + for<'a> Deserialize<'a> + Send + Sync + PartialEq { /// Get the view this vote was cast for @@ -38,7 +38,7 @@ pub trait VoteType EncodedSignature; /// Get the data this vote was signed over - fn get_data(&self) -> VoteData>; + fn get_data(&self) -> VoteData; /// Get the vote token of this vote fn get_vote_token(&self) -> TYPES::VoteTokenType; } @@ -196,7 +196,7 @@ pub enum QuorumVote> { Timeout(TimeoutVote), } -impl VoteType for DAVote { +impl VoteType> for DAVote { fn get_view(&self) -> TYPES::Time { self.current_view } @@ -223,7 +223,7 @@ impl DAVote { } } -impl> VoteType +impl> VoteType> for QuorumVote { fn get_view(&self) -> TYPES::Time { @@ -275,7 +275,7 @@ impl> QuorumVote } } -impl VoteType> for ViewSyncVote { +impl VoteType>> for ViewSyncVote { fn get_view(&self) -> TYPES::Time { match self { ViewSyncVote::PreCommit(v) | ViewSyncVote::Commit(v) | ViewSyncVote::Finalize(v) => { @@ -320,7 +320,7 @@ pub trait Accumulator: Sized { pub trait Accumulator2< TYPES: NodeType, COMMITTABLE: Committable + Serialize + Clone, - VOTE: VoteType, + VOTE: VoteType>, >: Sized { /// Append 1 vote to the accumulator. If the threshold is not reached, return @@ -338,10 +338,10 @@ pub trait Accumulator2< pub struct DAVoteAccumulator< TYPES: NodeType, COMMITTABLE: Committable + Serialize + Clone, - VOTE: VoteType, + VOTE: VoteType>, > { /// Map of all da signatures accumlated so far - pub da_vote_outcomes: VoteMap, + pub da_vote_outcomes: VoteMap, TYPES::VoteTokenType>, /// A quorum's worth of stake, generally 2f + 1 pub success_threshold: NonZeroU64, /// A list of valid signatures for certificate aggregation @@ -355,7 +355,7 @@ pub struct DAVoteAccumulator< impl< TYPES: NodeType, COMMITTABLE: Committable + Serialize + Clone, - VOTE: VoteType, + VOTE: VoteType>, > Accumulator2 for DAVoteAccumulator { fn append( @@ -429,14 +429,14 @@ impl< pub struct QuorumVoteAccumulator< TYPES: NodeType, COMMITTABLE: Committable + Serialize + Clone, - VOTE: VoteType, + VOTE: VoteType>, > { /// Map of all signatures accumlated so far - pub total_vote_outcomes: VoteMap, + pub total_vote_outcomes: VoteMap, TYPES::VoteTokenType>, /// Map of all yes signatures accumlated so far - pub yes_vote_outcomes: VoteMap, + pub yes_vote_outcomes: VoteMap, TYPES::VoteTokenType>, /// Map of all no signatures accumlated so far - pub no_vote_outcomes: VoteMap, + pub no_vote_outcomes: VoteMap, TYPES::VoteTokenType>, /// A quorum's worth of stake, generally 2f + 1 pub success_threshold: NonZeroU64, @@ -453,7 +453,7 @@ pub struct QuorumVoteAccumulator< impl< TYPES: NodeType, COMMITTABLE: Committable + Serialize + Clone, - VOTE: VoteType, + VOTE: VoteType>, > Accumulator2 for QuorumVoteAccumulator { fn append( @@ -560,14 +560,14 @@ impl< pub struct ViewSyncVoteAccumulator< TYPES: NodeType, COMMITTABLE: Committable + Serialize + Clone, - VOTE: VoteType, + VOTE: VoteType>, > { /// Map of all pre_commit signatures accumlated so far - pub pre_commit_vote_outcomes: VoteMap, + pub pre_commit_vote_outcomes: VoteMap, TYPES::VoteTokenType>, /// Map of all ommit signatures accumlated so far - pub commit_vote_outcomes: VoteMap, + pub commit_vote_outcomes: VoteMap, TYPES::VoteTokenType>, /// Map of all finalize signatures accumlated so far - pub finalize_vote_outcomes: VoteMap, + pub finalize_vote_outcomes: VoteMap, TYPES::VoteTokenType>, /// A quorum's worth of stake, generally 2f + 1 pub success_threshold: NonZeroU64, @@ -584,7 +584,7 @@ pub struct ViewSyncVoteAccumulator< impl< TYPES: NodeType, COMMITTABLE: Committable + Serialize + Clone, - VOTE: VoteType, + VOTE: VoteType>, > Accumulator2 for ViewSyncVoteAccumulator { #[allow(clippy::too_many_lines)] @@ -734,7 +734,7 @@ impl< pub struct AccumulatorPlaceholder< TYPES: NodeType, COMMITTABLE: Committable + Serialize + Clone, - VOTE: VoteType, + VOTE: VoteType>, > { /// Phantom data to make compiler happy pub phantom: PhantomData<(TYPES, VOTE, COMMITTABLE)>, @@ -743,7 +743,7 @@ pub struct AccumulatorPlaceholder< impl< TYPES: NodeType, COMMITTABLE: Committable + Serialize + Clone, - VOTE: VoteType, + VOTE: VoteType>, > Accumulator2 for AccumulatorPlaceholder { fn append( @@ -758,17 +758,23 @@ impl< /// Mapping of commitments to vote tokens by key. // TODO ED Remove this whole token generic -type VoteMap = HashMap< - Commitment, +type VoteMap = HashMap< + COMMITMENT, ( u64, - BTreeMap>, TOKEN)>, + BTreeMap, TOKEN)>, ), >; /// Describe the process of collecting signatures on block or leaf commitment, to form a DAC or QC, /// respectively. -pub struct VoteAccumulator { +/// +/// TODO GG used only in election.rs; move this to there and make it private? +pub struct VoteAccumulator< + TOKEN, + COMMITMENT: Serialize + for<'a> Deserialize<'a> + Clone, + TYPES: NodeType, +> { /// Map of all signatures accumlated so far pub total_vote_outcomes: VoteMap, /// Map of all da signatures accumlated so far @@ -809,7 +815,7 @@ impl ), ), AssembledSignature, - > for VoteAccumulator + > for VoteAccumulator, TYPES> where TOKEN: Clone + VoteToken, { From 6f85c106ef7a2c8789c4fe14bc869ed8ab102add Mon Sep 17 00:00:00 2001 From: Gus Gutoski Date: Thu, 21 Sep 2023 13:45:05 -0400 Subject: [PATCH 18/18] chore: Remove Committable from more types (#1792) * remove Committable from Accumulator2 * remove Committable from AccumulatorPlaceholder * remove Committable from DAVoteAccumulator type param * remove Committable from QuorumVoteAccumulator type param * remove Committable from ViewSyncVoteAccumulator type param * remove Committable from SignedCertificate type param (that's a big one) --- crates/task-impls/src/consensus.rs | 4 +- crates/task-impls/src/da.rs | 4 +- crates/task-impls/src/view_sync.rs | 4 +- crates/types/src/certificate.rs | 14 ++++--- crates/types/src/traits/election.rs | 20 +++++----- crates/types/src/vote.rs | 61 +++++++++++++++-------------- 6 files changed, 55 insertions(+), 52 deletions(-) diff --git a/crates/task-impls/src/consensus.rs b/crates/task-impls/src/consensus.rs index dccf7919c3..ef4694a282 100644 --- a/crates/task-impls/src/consensus.rs +++ b/crates/task-impls/src/consensus.rs @@ -4,7 +4,7 @@ use async_lock::{RwLock, RwLockUpgradableReadGuard}; #[cfg(async_executor_impl = "async-std")] use async_std::task::JoinHandle; use bitvec::prelude::*; -use commit::Committable; +use commit::{Commitment, Committable}; use core::time::Duration; use either::{Either, Left, Right}; use futures::FutureExt; @@ -151,7 +151,7 @@ pub struct VoteCollectionTaskState< TYPES, TYPES::Time, TYPES::VoteTokenType, - SequencingLeaf, + Commitment>, >>::VoteAccumulator, QuorumCertificate>, >, diff --git a/crates/task-impls/src/da.rs b/crates/task-impls/src/da.rs index ff048ef469..9b1657324f 100644 --- a/crates/task-impls/src/da.rs +++ b/crates/task-impls/src/da.rs @@ -3,7 +3,7 @@ use async_compatibility_layer::art::async_spawn; use async_lock::RwLock; use bitvec::prelude::*; -use commit::Committable; +use commit::{Commitment, Committable}; use either::{Either, Left, Right}; use futures::FutureExt; use hotshot_task::{ @@ -103,7 +103,7 @@ pub struct DAVoteCollectionTaskState< TYPES, TYPES::Time, TYPES::VoteTokenType, - TYPES::BlockType, + Commitment, >>::VoteAccumulator, DACertificate, >, diff --git a/crates/task-impls/src/view_sync.rs b/crates/task-impls/src/view_sync.rs index f92752f487..eaac060975 100644 --- a/crates/task-impls/src/view_sync.rs +++ b/crates/task-impls/src/view_sync.rs @@ -1,7 +1,7 @@ #![allow(clippy::module_name_repetitions)] use crate::events::SequencingHotShotEvent; use async_compatibility_layer::art::{async_sleep, async_spawn}; -use commit::Committable; +use commit::{Commitment, Committable}; use either::Either::{self, Left, Right}; use futures::FutureExt; use hotshot_task::{ @@ -228,7 +228,7 @@ pub struct ViewSyncRelayTaskState< TYPES, TYPES::Time, TYPES::VoteTokenType, - ViewSyncData, + Commitment>, >>::VoteAccumulator, ViewSyncCertificate, >, diff --git a/crates/types/src/certificate.rs b/crates/types/src/certificate.rs index 5309729eb7..7018490e09 100644 --- a/crates/types/src/certificate.rs +++ b/crates/types/src/certificate.rs @@ -157,11 +157,11 @@ pub struct VoteMetaData> - SignedCertificate + SignedCertificate> for QuorumCertificate { type Vote = QuorumVote; - type VoteAccumulator = QuorumVoteAccumulator; + type VoteAccumulator = QuorumVoteAccumulator, Self::Vote>; fn from_signatures_and_commitment( signatures: AssembledSignature, @@ -234,11 +234,12 @@ impl> Committable } } -impl SignedCertificate +impl + SignedCertificate> for DACertificate { type Vote = DAVote; - type VoteAccumulator = DAVoteAccumulator; + type VoteAccumulator = DAVoteAccumulator, Self::Vote>; fn from_signatures_and_commitment( signatures: AssembledSignature, @@ -323,11 +324,12 @@ impl Committable for ViewSyncCertificate { } impl - SignedCertificate> + SignedCertificate>> for ViewSyncCertificate { type Vote = ViewSyncVote; - type VoteAccumulator = ViewSyncVoteAccumulator, Self::Vote>; + type VoteAccumulator = + ViewSyncVoteAccumulator>, Self::Vote>; /// Build a QC from the threshold signature and commitment fn from_signatures_and_commitment( signatures: AssembledSignature, diff --git a/crates/types/src/traits/election.rs b/crates/types/src/traits/election.rs index fb1ff72a58..7ccde773af 100644 --- a/crates/types/src/traits/election.rs +++ b/crates/types/src/traits/election.rs @@ -158,17 +158,17 @@ pub trait ElectionConfig: } /// A certificate of some property which has been signed by a quroum of nodes. -pub trait SignedCertificate +pub trait SignedCertificate where Self: Send + Sync + Clone + Serialize + for<'a> Deserialize<'a>, - COMMITTABLE: Committable + Serialize + Clone, + COMMITMENT: for<'a> Deserialize<'a> + Serialize + Clone, TOKEN: VoteToken, { /// `VoteType` that is used in this certificate - type Vote: VoteType>; + type Vote: VoteType; /// `Accumulator` type to accumulate votes. - type VoteAccumulator: Accumulator2; + type VoteAccumulator: Accumulator2; /// Build a QC from the threshold signature and commitment // TODO ED Rename this function and rework this function parameters @@ -189,10 +189,10 @@ where // TODO ED Make an issue for this /// Get the leaf commitment. - fn leaf_commitment(&self) -> Commitment; + fn leaf_commitment(&self) -> COMMITMENT; /// Set the leaf commitment. - fn set_leaf_commitment(&mut self, commitment: Commitment); + fn set_leaf_commitment(&mut self, commitment: COMMITMENT); /// Get whether the certificate is for the genesis block. fn is_genesis(&self) -> bool; @@ -269,7 +269,7 @@ pub trait ConsensusExchange: Send + Sync { // TODO ED Make this equal Certificate vote (if possible?) type Vote: VoteType>; /// A [`SignedCertificate`] attesting to a decision taken by the committee. - type Certificate: SignedCertificate + type Certificate: SignedCertificate> + Hash + Eq; /// The committee eligible to make decisions. @@ -468,13 +468,13 @@ pub trait ConsensusExchange: Send + Sync { TYPES, TYPES::Time, TYPES::VoteTokenType, - Self::Commitment, + Commitment, >>::VoteAccumulator, vote: &<>::Certificate as SignedCertificate< TYPES, TYPES::Time, TYPES::VoteTokenType, - Self::Commitment, + Commitment, >>::Vote, _commit: &Commitment, ) -> Either< @@ -482,7 +482,7 @@ pub trait ConsensusExchange: Send + Sync { TYPES, TYPES::Time, TYPES::VoteTokenType, - Self::Commitment, + Commitment, >>::VoteAccumulator, Self::Certificate, > { diff --git a/crates/types/src/vote.rs b/crates/types/src/vote.rs index c470925b54..c27225e4af 100644 --- a/crates/types/src/vote.rs +++ b/crates/types/src/vote.rs @@ -22,6 +22,7 @@ use serde::{Deserialize, Serialize}; use std::{ collections::{BTreeMap, HashMap}, fmt::Debug, + hash::Hash, marker::PhantomData, num::NonZeroU64, }; @@ -319,8 +320,8 @@ pub trait Accumulator: Sized { /// Accumulator trait used to accumulate votes into an `AssembledSignature` pub trait Accumulator2< TYPES: NodeType, - COMMITTABLE: Committable + Serialize + Clone, - VOTE: VoteType>, + COMMITMENT: for<'a> Deserialize<'a> + Serialize + Clone, + VOTE: VoteType, >: Sized { /// Append 1 vote to the accumulator. If the threshold is not reached, return @@ -337,11 +338,11 @@ pub trait Accumulator2< /// Accumulates DA votes pub struct DAVoteAccumulator< TYPES: NodeType, - COMMITTABLE: Committable + Serialize + Clone, - VOTE: VoteType>, + COMMITMENT: for<'a> Deserialize<'a> + Serialize + Clone, + VOTE: VoteType, > { /// Map of all da signatures accumlated so far - pub da_vote_outcomes: VoteMap, TYPES::VoteTokenType>, + pub da_vote_outcomes: VoteMap, /// A quorum's worth of stake, generally 2f + 1 pub success_threshold: NonZeroU64, /// A list of valid signatures for certificate aggregation @@ -354,9 +355,9 @@ pub struct DAVoteAccumulator< impl< TYPES: NodeType, - COMMITTABLE: Committable + Serialize + Clone, - VOTE: VoteType>, - > Accumulator2 for DAVoteAccumulator + COMMITMENT: for<'a> Deserialize<'a> + Serialize + Clone + Copy + PartialEq + Eq + Hash, + VOTE: VoteType, + > Accumulator2 for DAVoteAccumulator { fn append( mut self, @@ -428,15 +429,15 @@ impl< /// Accumulate quorum votes pub struct QuorumVoteAccumulator< TYPES: NodeType, - COMMITTABLE: Committable + Serialize + Clone, - VOTE: VoteType>, + COMMITMENT: for<'a> Deserialize<'a> + Serialize + Clone, + VOTE: VoteType, > { /// Map of all signatures accumlated so far - pub total_vote_outcomes: VoteMap, TYPES::VoteTokenType>, + pub total_vote_outcomes: VoteMap, /// Map of all yes signatures accumlated so far - pub yes_vote_outcomes: VoteMap, TYPES::VoteTokenType>, + pub yes_vote_outcomes: VoteMap, /// Map of all no signatures accumlated so far - pub no_vote_outcomes: VoteMap, TYPES::VoteTokenType>, + pub no_vote_outcomes: VoteMap, /// A quorum's worth of stake, generally 2f + 1 pub success_threshold: NonZeroU64, @@ -452,9 +453,9 @@ pub struct QuorumVoteAccumulator< impl< TYPES: NodeType, - COMMITTABLE: Committable + Serialize + Clone, - VOTE: VoteType>, - > Accumulator2 for QuorumVoteAccumulator + COMMITMENT: for<'a> Deserialize<'a> + Serialize + Clone + Copy + PartialEq + Eq + Hash, + VOTE: VoteType, + > Accumulator2 for QuorumVoteAccumulator { fn append( mut self, @@ -559,15 +560,15 @@ impl< /// Accumulates view sync votes pub struct ViewSyncVoteAccumulator< TYPES: NodeType, - COMMITTABLE: Committable + Serialize + Clone, - VOTE: VoteType>, + COMMITMENT: for<'a> Deserialize<'a> + Serialize + Clone, + VOTE: VoteType, > { /// Map of all pre_commit signatures accumlated so far - pub pre_commit_vote_outcomes: VoteMap, TYPES::VoteTokenType>, + pub pre_commit_vote_outcomes: VoteMap, /// Map of all ommit signatures accumlated so far - pub commit_vote_outcomes: VoteMap, TYPES::VoteTokenType>, + pub commit_vote_outcomes: VoteMap, /// Map of all finalize signatures accumlated so far - pub finalize_vote_outcomes: VoteMap, TYPES::VoteTokenType>, + pub finalize_vote_outcomes: VoteMap, /// A quorum's worth of stake, generally 2f + 1 pub success_threshold: NonZeroU64, @@ -583,9 +584,9 @@ pub struct ViewSyncVoteAccumulator< impl< TYPES: NodeType, - COMMITTABLE: Committable + Serialize + Clone, - VOTE: VoteType>, - > Accumulator2 for ViewSyncVoteAccumulator + COMMITMENT: for<'a> Deserialize<'a> + Serialize + Clone + Copy + PartialEq + Eq + Hash, + VOTE: VoteType, + > Accumulator2 for ViewSyncVoteAccumulator { #[allow(clippy::too_many_lines)] fn append( @@ -733,18 +734,18 @@ impl< /// Placeholder accumulator; will be replaced by accumulator for each certificate type pub struct AccumulatorPlaceholder< TYPES: NodeType, - COMMITTABLE: Committable + Serialize + Clone, - VOTE: VoteType>, + COMMITMENT: for<'a> Deserialize<'a> + Serialize + Clone, + VOTE: VoteType, > { /// Phantom data to make compiler happy - pub phantom: PhantomData<(TYPES, VOTE, COMMITTABLE)>, + pub phantom: PhantomData<(TYPES, VOTE, COMMITMENT)>, } impl< TYPES: NodeType, - COMMITTABLE: Committable + Serialize + Clone, - VOTE: VoteType>, - > Accumulator2 for AccumulatorPlaceholder + COMMITMENT: for<'a> Deserialize<'a> + Serialize + Clone, + VOTE: VoteType, + > Accumulator2 for AccumulatorPlaceholder { fn append( self,