diff --git a/CMakeModules/EosioTester.cmake.in b/CMakeModules/EosioTester.cmake.in index 09ce12f59e..a4600d6f63 100644 --- a/CMakeModules/EosioTester.cmake.in +++ b/CMakeModules/EosioTester.cmake.in @@ -99,6 +99,7 @@ target_link_libraries(EosioChain INTERFACE Boost::interprocess Boost::asio Boost::beast + Boost::crc Boost::signals2 Boost::iostreams "-lz" # Needed by Boost iostreams diff --git a/CMakeModules/EosioTesterBuild.cmake.in b/CMakeModules/EosioTesterBuild.cmake.in index 60371ebe87..db7965dccb 100644 --- a/CMakeModules/EosioTesterBuild.cmake.in +++ b/CMakeModules/EosioTesterBuild.cmake.in @@ -96,6 +96,7 @@ target_link_libraries(EosioChain INTERFACE Boost::interprocess Boost::asio Boost::beast + Boost::crc Boost::signals2 Boost::iostreams "-lz" # Needed by Boost iostreams diff --git a/libraries/chain/controller.cpp b/libraries/chain/controller.cpp index 2cb8e927b2..a0e3724d50 100644 --- a/libraries/chain/controller.cpp +++ b/libraries/chain/controller.cpp @@ -6001,6 +6001,10 @@ bool controller::is_node_finalizer_key(const bls_public_key& key) const { return my->my_finalizers.contains(key); } +const my_finalizers_t& controller::get_node_finalizers() const { + return my->my_finalizers; +} + /// Protocol feature activation handlers: template<> diff --git a/libraries/chain/include/eosio/chain/controller.hpp b/libraries/chain/include/eosio/chain/controller.hpp index 94de0b7f83..599800e753 100644 --- a/libraries/chain/include/eosio/chain/controller.hpp +++ b/libraries/chain/include/eosio/chain/controller.hpp @@ -9,6 +9,7 @@ #include #include #include +#include #include @@ -22,6 +23,10 @@ namespace boost::asio { class thread_pool; } +namespace savanna_cluster { + class node_t; +} + namespace eosio::vm { class wasm_allocator; } namespace eosio::chain { @@ -452,9 +457,13 @@ namespace eosio::chain { // is the bls key a registered finalizer key of this node, thread safe bool is_node_finalizer_key(const bls_public_key& key) const; + private: + const my_finalizers_t& get_node_finalizers() const; // used for tests (purpose is inspecting fsi). + friend class apply_context; friend class transaction_context; + friend class savanna_cluster::node_t; chainbase::database& mutable_db()const; diff --git a/libraries/chain/include/eosio/chain/finality/finalizer.hpp b/libraries/chain/include/eosio/chain/finality/finalizer.hpp index 1ca4c90a2d..e43139dca3 100644 --- a/libraries/chain/include/eosio/chain/finality/finalizer.hpp +++ b/libraries/chain/include/eosio/chain/finality/finalizer.hpp @@ -181,7 +181,7 @@ namespace eosio::chain { fsi_map load_finalizer_safety_info(); // for testing purposes only, not thread safe - const fsi_t& get_fsi(const bls_public_key& k) { return finalizers[k].fsi; } + const fsi_t& get_fsi(const bls_public_key& k) const { return finalizers.at(k).fsi; } void set_fsi(const bls_public_key& k, const fsi_t& fsi) { finalizers[k].fsi = fsi; } private: diff --git a/unittests/finalizer_tests.cpp b/unittests/finalizer_tests.cpp index e78c66a81f..9ed35fb4ee 100644 --- a/unittests/finalizer_tests.cpp +++ b/unittests/finalizer_tests.cpp @@ -162,8 +162,7 @@ BOOST_AUTO_TEST_CASE( corrupt_finalizer_safety_file ) try { finalizer_safety_exception); // make sure the safety info for our finalizer that we saved above is restored correctly - BOOST_CHECK_NE(fset.get_fsi(k.pubkey), fsi); - BOOST_CHECK_EQUAL(fset.get_fsi(k.pubkey), fsi_t()); + BOOST_CHECK(!fset.contains(k.pubkey)); } } FC_LOG_AND_RETHROW() diff --git a/unittests/savanna_cluster.cpp b/unittests/savanna_cluster.cpp index 279ba7f585..0db47d75c9 100644 --- a/unittests/savanna_cluster.cpp +++ b/unittests/savanna_cluster.cpp @@ -4,12 +4,14 @@ namespace savanna_cluster { node_t::node_t(size_t node_idx, cluster_t& cluster, setup_policy policy /* = setup_policy::none */) : tester(policy) - , node_idx(node_idx) { + , _node_idx(node_idx) + , _last_vote({}, false) +{ // since we are creating forks, finalizers may be locked on another fork and unable to vote. do_check_for_votes(false); - voted_block_cb = [&, node_idx](const eosio::chain::vote_signal_params& v) { + _voted_block_cb = [&, node_idx](const eosio::chain::vote_signal_params& v) { // no mutex needed because controller is set in tester (via `disable_async_voting(true)`) // to vote (and emit the `voted_block` signal) synchronously. // -------------------------------------------------------------------------------------- @@ -17,15 +19,25 @@ node_t::node_t(size_t node_idx, cluster_t& cluster, setup_policy policy /* = set if (status == vote_result_t::success) { vote_message_ptr vote_msg = std::get<2>(v); - last_vote = vote_t(vote_msg); - if (propagate_votes) - cluster.dispatch_vote_to_peers(node_idx, skip_self_t::yes, std::get<2>(v)); + _last_vote = vote_t(vote_msg->block_id, vote_msg->strong); + + if (_propagate_votes) { + if (_vote_delay) + _delayed_votes.push_back(std::move(vote_msg)); + while (_delayed_votes.size() > _vote_delay) { + vote_message_ptr vote = _delayed_votes.front(); + _delayed_votes.erase(_delayed_votes.cbegin()); + cluster.dispatch_vote_to_peers(node_idx, skip_self_t::yes, vote); + } + if (!_vote_delay) + cluster.dispatch_vote_to_peers(node_idx, skip_self_t::yes, vote_msg); + } } }; // called on `commit_block`, for both blocks received from `push_block` and produced blocks - accepted_block_cb = [&, node_idx](const eosio::chain::block_signal_params& p) { - if (!pushing_a_block) { + _accepted_block_cb = [&, node_idx](const eosio::chain::block_signal_params& p) { + if (!_pushing_a_block) { // we want to propagate only blocks we produce, not the ones we receive from the network auto& b = std::get<0>(p); cluster.push_block_to_peers(node_idx, skip_self_t::yes, b); @@ -33,9 +45,9 @@ node_t::node_t(size_t node_idx, cluster_t& cluster, setup_policy policy /* = set }; auto node_initialization_fn = [&, node_idx]() { - [[maybe_unused]] auto _a = control->voted_block().connect(voted_block_cb); - [[maybe_unused]] auto _b = control->accepted_block().connect(accepted_block_cb); - tester::set_node_finalizers(node_finalizers); + [[maybe_unused]] auto _a = control->voted_block().connect(_voted_block_cb); + [[maybe_unused]] auto _b = control->accepted_block().connect(_accepted_block_cb); + tester::set_node_finalizers(_node_finalizers); cluster.get_new_blocks_from_peers(node_idx); }; diff --git a/unittests/savanna_cluster.hpp b/unittests/savanna_cluster.hpp index 7c5ac49676..20e2c448c2 100644 --- a/unittests/savanna_cluster.hpp +++ b/unittests/savanna_cluster.hpp @@ -1,6 +1,7 @@ #pragma once #include +#include #include #include @@ -22,6 +23,7 @@ namespace savanna_cluster { using block_header = eosio::chain::block_header; using tester = eosio::testing::tester; using setup_policy = eosio::testing::setup_policy; + using fsi_t = eosio::chain::finalizer_safety_information; class cluster_t; @@ -51,34 +53,68 @@ namespace savanna_cluster { vector privkeys; }; + // two classes for comparisons in BOOST_REQUIRE_EQUAL + // -------------------------------------------------- + struct vote_t { + friend std::ostream& operator<<(std::ostream& s, const vote_t& v) { + s << "vote_t(" << v.id.str().substr(8, 16) << ", " << (v.strong ? "strong" : "weak") << ")"; + return s; + } + bool operator==(const vote_t&) const = default; + + block_id_type id; + bool strong; + }; + + struct strong_vote : public vote_t { + explicit strong_vote(const signed_block_ptr& p) : vote_t(p->calculate_id(), true) {} + }; + struct weak_vote : public vote_t { + explicit weak_vote(const signed_block_ptr& p) : vote_t(p->calculate_id(), false) {} + }; + + + struct qc_s { + explicit qc_s(uint32_t block_num, bool strong) : block_num(block_num), strong(strong) {} + explicit qc_s(const std::optional& qc) : block_num(qc->block_num), strong(qc->is_strong()) {} + + friend std::ostream& operator<<(std::ostream& s, const qc_s& v) { + s << "qc_s(" << v.block_num << ", " << (v.strong ? "strong" : "weak") << ")"; + return s; + } + bool operator==(const qc_s&) const = default; + + uint32_t block_num; // claimed block + bool strong; + }; + + struct strong_qc : public qc_s { + explicit strong_qc(const signed_block_ptr& p) : qc_s(p->block_num(), true) {} + }; + struct weak_qc : public qc_s { + explicit weak_qc(const signed_block_ptr& p) : qc_s(p->block_num(), false) {} + }; + + struct fsi_expect { + const signed_block_ptr& last_vote; + const signed_block_ptr& lock; + block_timestamp_type other_branch_latest_time; + }; + // ---------------------------------------------------------------------------- class node_t : public tester { private: - size_t node_idx; - bool pushing_a_block{false}; - - std::function accepted_block_cb; - std::function voted_block_cb; - - public: - struct vote_t { - vote_t() : strong(false) {} - explicit vote_t(const vote_message_ptr& p) : id(p->block_id), strong(p->strong) {} - explicit vote_t(const signed_block_ptr& p, bool strong) : id(p->calculate_id()), strong(strong) {} - - friend std::ostream& operator<<(std::ostream& s, const vote_t& v) { - s << "vote_t(" << v.id.str().substr(8, 16) << ", " << (v.strong ? "strong" : "weak") << ")"; - return s; - } - bool operator==(const vote_t&) const = default; + size_t _node_idx; + bool _pushing_a_block{false}; + bool _propagate_votes{true}; // if false, votes are dropped + vote_t _last_vote; + std::vector _node_finalizers; - block_id_type id; - bool strong; - }; + size_t _vote_delay{0}; // delay vote propagation by this much + std::vector _delayed_votes; - bool propagate_votes{true}; - vote_t last_vote; - std::vector node_finalizers; + std::function _accepted_block_cb; + std::function _voted_block_cb; public: node_t(size_t node_idx, cluster_t& cluster, setup_policy policy = setup_policy::none); @@ -87,9 +123,18 @@ namespace savanna_cluster { node_t(node_t&&) = default; + bool& propagate_votes() { return _propagate_votes; } + + size_t& vote_delay() { return _vote_delay; } + + const vote_t& last_vote() const { return _last_vote; } + void set_node_finalizers(std::span names) { - node_finalizers = std::vector{ names.begin(), names.end() }; - tester::set_node_finalizers(node_finalizers); + _node_finalizers = std::vector{ names.begin(), names.end() }; + if (control) { + // node is "open", se we can update the tester immediately + tester::set_node_finalizers(_node_finalizers); + } } void transition_to_savanna(std::span finalizer_policy_names) { @@ -167,8 +212,8 @@ namespace savanna_cluster { void push_block(const signed_block_ptr& b) { if (is_open() && !fetch_block_by_id(b->calculate_id())) { - assert(!pushing_a_block); - fc::scoped_set_value set_pushing_a_block(pushing_a_block, true); + assert(!_pushing_a_block); + fc::scoped_set_value set_pushing_a_block(_pushing_a_block, true); tester::push_block(b); } } @@ -189,19 +234,19 @@ namespace savanna_cluster { } std::string snapshot() const { - dlog("node ${i} - taking snapshot", ("i", node_idx)); + dlog("node ${i} - taking snapshot", ("i", _node_idx)); auto writer = buffered_snapshot_suite::get_writer(); control->write_snapshot(writer); return buffered_snapshot_suite::finalize(writer); } void open_from_snapshot(const std::string& snapshot) { - dlog("node ${i} - restoring from snapshot", ("i", node_idx)); + dlog("node ${i} - restoring from snapshot", ("i", _node_idx)); open(buffered_snapshot_suite::get_reader(snapshot)); } std::vector save_fsi() const { - dlog("node ${i} - saving fsi", ("i", node_idx)); + dlog("node ${i} - saving fsi", ("i", _node_idx)); auto finalizer_path = get_fsi_path(); std::ifstream file(finalizer_path.generic_string(), std::ios::binary | std::ios::ate); std::streamsize size = file.tellg(); @@ -214,7 +259,7 @@ namespace savanna_cluster { } void overwrite_fsi(const std::vector& fsi) const { - dlog("node ${i} - overwriting fsi", ("i", node_idx)); + dlog("node ${i} - overwriting fsi", ("i", _node_idx)); auto finalizer_path = get_fsi_path(); std::ofstream file(finalizer_path.generic_string(), std::ios::binary); assert(!fsi.empty()); @@ -222,13 +267,13 @@ namespace savanna_cluster { } void remove_fsi() { - dlog("node ${i} - removing fsi", ("i", node_idx)); + dlog("node ${i} - removing fsi", ("i", _node_idx)); remove_all(get_fsi_path()); } void remove_state() { auto state_path = cfg.state_dir; - dlog("node ${i} - removing state data from: ${state_path}", ("i", node_idx)("${state_path}", state_path)); + dlog("node ${i} - removing state data from: ${state_path}", ("i", _node_idx)("${state_path}", state_path)); remove_all(state_path); fs::create_directories(state_path); } @@ -246,12 +291,26 @@ namespace savanna_cluster { for (auto const& dir_entry : std::filesystem::directory_iterator{path}) { auto path = dir_entry.path(); if (path.filename().generic_string() != "reversible") { - dlog("node ${i} - removing : ${path}", ("i", node_idx)("${path}", path)); + dlog("node ${i} - removing : ${path}", ("i", _node_idx)("${path}", path)); remove_all(path); } } } + const fsi_t& get_fsi(size_t idx = 0) const { + assert(control); + assert(idx < _node_finalizers.size()); + auto [privkey, pubkey, pop] = get_bls_key(_node_finalizers[idx]); + return control->get_node_finalizers().get_fsi(pubkey); + } + + void check_fsi(const fsi_expect& expected) { + const fsi_t& fsi = get_fsi(); + BOOST_REQUIRE_EQUAL(fsi.last_vote.block_id, expected.last_vote->calculate_id()); + BOOST_REQUIRE_EQUAL(fsi.lock.block_id, expected.lock->calculate_id()); + BOOST_REQUIRE_EQUAL(fsi.other_branch_latest_time, expected.other_branch_latest_time); + } + private: // always removes reversible data (`blocks/reversible`) // optionally remove the blocks log as well by deleting the whole `blocks` directory @@ -259,7 +318,7 @@ namespace savanna_cluster { void remove_blocks(bool rm_blocks_log) { auto reversible_path = cfg.blocks_dir / config::reversible_blocks_dir_name; auto& path = rm_blocks_log ? cfg.blocks_dir : reversible_path; - dlog("node ${i} - removing : ${path}", ("i", node_idx)("${path}", path)); + dlog("node ${i} - removing : ${path}", ("i", _node_idx)("${path}", path)); remove_all(path); fs::create_directories(reversible_path); } @@ -471,22 +530,6 @@ namespace savanna_cluster { size_t num_nodes() const { return _num_nodes; } - // Class for comparisons in BOOST_REQUIRE_EQUAL - // -------------------------------------------- - struct qc_s { - explicit qc_s(const signed_block_ptr& p, bool strong) : block_num(p->block_num()), strong(strong) {} - explicit qc_s(const std::optional& qc) : block_num(qc->block_num), strong(qc->is_strong()) {} - - friend std::ostream& operator<<(std::ostream& s, const qc_s& v) { - s << "qc_s(" << v.block_num << ", " << (v.strong ? "strong" : "weak") << ")"; - return s; - } - bool operator==(const qc_s&) const = default; - - uint32_t block_num; // claimed block - bool strong; - }; - static qc_claim_t qc_claim(const signed_block_ptr& b) { return b->extract_header_extension().qc_claim; } diff --git a/unittests/savanna_finalizer_policy_tests.cpp b/unittests/savanna_finalizer_policy_tests.cpp index 4af240c9c2..1cd034bfa5 100644 --- a/unittests/savanna_finalizer_policy_tests.cpp +++ b/unittests/savanna_finalizer_policy_tests.cpp @@ -38,7 +38,7 @@ BOOST_FIXTURE_TEST_CASE(policy_change, savanna_cluster::cluster_t) try { // -------------------------------------------------------------------------------------------------- B.close(); // update `B.node_finalizers` with the new key so that B can vote both on the active and pending policy - B.node_finalizers = std::vector{ _fin_keys[1], _fin_keys[num_nodes()] }; // see node_t::node_t + B.set_node_finalizers(std::vector{ _fin_keys[1], _fin_keys[num_nodes()] }); // see node_t::node_t B.open(); // quick sanity check @@ -144,8 +144,8 @@ BOOST_FIXTURE_TEST_CASE(policy_change_reduce_threshold_replace_all_keys, savanna // ----------------------------------------------------------------------------------------------- A.close(); B.close(); - A.node_finalizers = std::vector{ _fin_keys[0], _fin_keys[num_nodes()] }; - B.node_finalizers = std::vector{ _fin_keys[1], _fin_keys[num_nodes() + 1] }; + A.set_node_finalizers(std::vector{ _fin_keys[0], _fin_keys[num_nodes()] }); + B.set_node_finalizers(std::vector{ _fin_keys[1], _fin_keys[num_nodes() + 1] }); A.open(); B.open(); @@ -206,9 +206,9 @@ BOOST_FIXTURE_TEST_CASE(policy_change_restart_from_snapshot, savanna_cluster::cl A.close(); B.close(); C.close(); - A.node_finalizers = std::vector{ _fin_keys[0], _fin_keys[num_nodes()] }; - B.node_finalizers = std::vector{ _fin_keys[1], _fin_keys[num_nodes() + 1] }; - C.node_finalizers = std::vector{ _fin_keys[2], _fin_keys[num_nodes() + 2] }; + A.set_node_finalizers(std::vector{ _fin_keys[0], _fin_keys[num_nodes()] }); + B.set_node_finalizers(std::vector{ _fin_keys[1], _fin_keys[num_nodes() + 1] }); + C.set_node_finalizers(std::vector{ _fin_keys[2], _fin_keys[num_nodes() + 2] }); A.open(); B.open(); C.open(); diff --git a/unittests/savanna_misc_tests.cpp b/unittests/savanna_misc_tests.cpp index 11285975bb..e77b9c4b68 100644 --- a/unittests/savanna_misc_tests.cpp +++ b/unittests/savanna_misc_tests.cpp @@ -60,8 +60,9 @@ D produces +--------------------| b2 | */ BOOST_FIXTURE_TEST_CASE(weak_masking_issue, savanna_cluster::cluster_t) try { + using namespace savanna_cluster; auto& A=_nodes[0]; auto& B=_nodes[1]; auto& C=_nodes[2]; auto& D=_nodes[3]; - using vote_t = savanna_cluster::node_t::vote_t; + //_debug_mode = true; auto b0 = A.produce_blocks(2); // receives strong votes from all finalizers @@ -87,41 +88,41 @@ BOOST_FIXTURE_TEST_CASE(weak_masking_issue, savanna_cluster::cluster_t) try { // on top of it push_block(1, b2); // push block to B and C, should receive weak votes - BOOST_REQUIRE_EQUAL(B.last_vote, vote_t(b2, false)); - BOOST_REQUIRE_EQUAL(C.last_vote, vote_t(b2, false)); - BOOST_REQUIRE_EQUAL(A.last_vote, vote_t(b1, true));// A should not have seen b2, and therefore not voted on it + BOOST_REQUIRE_EQUAL(B.last_vote(), weak_vote(b2)); + BOOST_REQUIRE_EQUAL(C.last_vote(), weak_vote(b2)); + BOOST_REQUIRE_EQUAL(A.last_vote(), strong_vote(b1));// A should not have seen b2, and therefore not voted on it - BOOST_REQUIRE_EQUAL(qc_s(qc(b2)), qc_s(b0, true)); // b2 should include a strong qc on b0 + BOOST_REQUIRE_EQUAL(qc_s(qc(b2)), strong_qc(b0)); // b2 should include a strong qc on b0 - set_partition(partition); // restore our original partition {A, B, C} and {D} + set_partition(partition); // restore our original partition {A, B, C} and {D} signed_block_ptr b3; { - fc::scoped_set_value tmp(B.propagate_votes, false); // temporarily prevent B from broadcasting its votes) + fc::scoped_set_value tmp(B.propagate_votes(), false);// temporarily prevent B from broadcasting its votes) // so A won't receive them and form a QC on b3 b3 = A.produce_block(_block_interval_us * 2); // A will see its own strong vote on b3, and C's weak vote // (not a quorum) // because B doesn't propagate and D is partitioned away print("b3", b3); - BOOST_REQUIRE_EQUAL(A.last_vote, vote_t(b3, true)); // A didn't vote on b2 so it can vote strong - BOOST_REQUIRE_EQUAL(B.last_vote, vote_t(b3, false)); // but B and C have to vote weak. - BOOST_REQUIRE_EQUAL(C.last_vote, vote_t(b3, false)); // C did vote, but we turned vote propagation off so + BOOST_REQUIRE_EQUAL(A.last_vote(), strong_vote(b3)); // A didn't vote on b2 so it can vote strong + BOOST_REQUIRE_EQUAL(B.last_vote(), weak_vote(b3)); // but B and C have to vote weak. + BOOST_REQUIRE_EQUAL(C.last_vote(), weak_vote(b3)); // C did vote, but we turned vote propagation off so // A will never see C's vote - BOOST_REQUIRE_EQUAL(qc_s(qc(b3)), qc_s(b1, true)); // b3 should include a strong qc on b1 + BOOST_REQUIRE_EQUAL(qc_s(qc(b3)), strong_qc(b1)); // b3 should include a strong qc on b1 } BOOST_REQUIRE_EQUAL(A.lib_number, b0->block_num()); - // Now B broadcasts its votes again, so - auto b4 = A.produce_block(); // b4 should receive 3 weak votes from A, B and C - // and should include a strong QC claim on b1 (repeated) - // since we don't have enough votes to form a QC on b3 + // Now B broadcasts its votes again, so + auto b4 = A.produce_block(); // b4 should receive 3 weak votes from A, B and C + // and should include a strong QC claim on b1 (repeated) + // since we don't have enough votes to form a QC on b3 print("b4", b4); - BOOST_REQUIRE_EQUAL(A.last_vote, vote_t(b4, true)); - BOOST_REQUIRE_EQUAL(B.last_vote, vote_t(b4, false)); - BOOST_REQUIRE_EQUAL(C.last_vote, vote_t(b4, false)); + BOOST_REQUIRE_EQUAL(A.last_vote(), strong_vote(b4)); + BOOST_REQUIRE_EQUAL(B.last_vote(), weak_vote(b4)); + BOOST_REQUIRE_EQUAL(C.last_vote(), weak_vote(b4)); BOOST_REQUIRE_EQUAL(qc_claim(b3), qc_claim(b4)); // A didn't form a QC on b3, so b4 should repeat b3's claim BOOST_REQUIRE(!qc(b4)); // b4 should not have a QC extension (no new QC formed on b3) @@ -132,21 +133,195 @@ BOOST_FIXTURE_TEST_CASE(weak_masking_issue, savanna_cluster::cluster_t) try { // weak QC on b4, which itself had a strong QC on b1. // Upon receiving a strong QC on b5, b4 will be final print("b5", b5); - BOOST_REQUIRE_EQUAL(A.last_vote, vote_t(b5, true)); - BOOST_REQUIRE_EQUAL(B.last_vote, vote_t(b5, true)); - BOOST_REQUIRE_EQUAL(qc_s(qc(b5)), qc_s(b4, false)); // b5 should include a weak qc on b4 + BOOST_REQUIRE_EQUAL(A.last_vote(), strong_vote(b5)); + BOOST_REQUIRE_EQUAL(B.last_vote(), strong_vote(b5)); + BOOST_REQUIRE_EQUAL(qc_s(qc(b5)), weak_qc(b4)); // b5 should include a weak qc on b4 BOOST_REQUIRE_EQUAL(A.lib_number, b0->block_num()); auto b6 = A.produce_block(); // should include a strong QC on b5, b1 should be final print("b6", b6); - BOOST_REQUIRE_EQUAL(qc_s(qc(b6)), qc_s(b5, true)); // b6 should include a strong qc on b5 + BOOST_REQUIRE_EQUAL(qc_s(qc(b6)), strong_qc(b5)); // b6 should include a strong qc on b5 - BOOST_REQUIRE_EQUAL(A.last_vote, vote_t(b6, true)); - BOOST_REQUIRE_EQUAL(B.last_vote, vote_t(b6, true)); + BOOST_REQUIRE_EQUAL(A.last_vote(), strong_vote(b6)); + BOOST_REQUIRE_EQUAL(B.last_vote(), strong_vote(b6)); BOOST_REQUIRE_EQUAL(A.lib_number, b4->block_num()); } FC_LOG_AND_RETHROW() + +// ----------------------------------------------------------------------------------------------------- +// see https://github.com/AntelopeIO/spring/issues/621 explaining the issue that this test demonstrates. +// +// The fix in https://github.com/AntelopeIO/spring/issues/534 for the weak masking issue respected a more +// conservative version of rule 2. This solved the safety concerns due to the weak masking issue, but it +// was unnecessarily restrictive with respect to liveness. +// +// As a consequence if this liveness issue, finalizers may be stuck voting weak if the QC is not formed +// quickly enough. +// +// This testcase fails prior to https://github.com/AntelopeIO/spring/issues/621 being fixed. +// ----------------------------------------------------------------------------------------------------- +/* ----------------------------------------------------------------------------------------------------- + testcase + -------- +Time: t1 t2 t3 t4 t5 t6 t7 t8 +Blocks: + B0 <--- B1 <--- B2 <-|- B3 + | + \--------- B4 <--- B5 <--- B6 <--- B7 <--- B8 +QC claim: + Strong Strong Strong Strong Strong Weak Weak Strong + B0 B1 B2 B2 B2 B4 B5 B6 + +Vote: Strong Strong Strong Weak Weak Strong Strong Strong + + + +In the above example, things are moving along normally until time t4 when a microfork occurs. +Instead of building block B4 off of block B3, the producer builds block B4 off of block B2. +And then going forward, for some reason, it takes slightly longer for votes to propagate that a +QC on a block cannot be formed in time to be included in the very next block; instead the QC goes +in the block after. + +The finalizer of interest is voting on all of the blocks as they come. For this example, it is +sufficient to only have one finalizer. The first time the finalizer is forced to vote weak is on +block B4. As the other blocks continue to build on that new branch, it votes on them appropriately +and the producer collects the vote and forms a QC as soon as it can, which always remains one block +late. The finalizer should begin voting strong again starting with block B6. However, prior to the +changes described in this issue, the finalizer would remain stuck voting weak indefinitely. + +The expected state of the fsi record for the finalizer after each vote is provided below. It also +records what the new LIB should be after processing the block. In addition to checking that the blocks +have the claims as required above and the LIB as noted below, the test should also check that the fsi +record after each vote is as expected below. + +Finalizer fsi after voting strong on block B2 (LIB B0): +last_vote: B2 +lock: B1 +other_branch_latest_time: empty + +Finalizer fsi after voting strong on block B3 (LIB B1): +last_vote: B3 +lock: B2 +other_branch_latest_time: empty + +Finalizer fsi after voting weak on block B4 (LIB B1): +last_vote: B4 +lock: B2 +other_branch_latest_time: t3 + +Finalizer fsi after voting weak on block B5 (LIB B1): +last_vote: B5 +lock: B2 +other_branch_latest_time: t3 + +Finalizer fsi after voting strong on block B6 (LIB B1): +last_vote: B6 +lock: B4 +other_branch_latest_time: empty + +Finalizer fsi after voting strong on block B7 (LIB B1): +last_vote: B7 +lock: B5 +other_branch_latest_time: empty + +Finalizer fsi after voting strong on block B8 (LIB B4): +last_vote: B8 +lock: B6 +other_branch_latest_time: empty +--------------------------------------------------------------------------------------------------------- */ +BOOST_FIXTURE_TEST_CASE(gh_534_liveness_issue, savanna_cluster::cluster_t) try { + using namespace savanna_cluster; + auto& A=_nodes[0]; auto& B=_nodes[1]; auto& C=_nodes[2]; auto& D=_nodes[3]; + + //_debug_mode = true; + auto b0 = A.produce_block(); // receives strong votes from all finalizers + auto b1 = A.produce_block(); // receives strong votes from all finalizers + auto b2 = A.produce_block(); // receives strong votes from all finalizers + print("b1", b1); + print("b2", b2); + BOOST_REQUIRE_EQUAL(A.lib_number, b0->block_num()); + + // partition D out. D will be used to produce blocks on an alternative fork. + // We will have 3 finalizers voting which is enough to reach QCs + // ------------------------------------------------------------------------- + const std::vector partition {3}; + set_partition(partition); + + auto b3 = D.produce_block(); // produce a block on D + print("b3", b3); + + const std::vector tmp_partition {0}; // we temporarily separate A (before pushing b3) + set_partition(tmp_partition); // because we don't want A to see the block produced by D (b3) + // otherwise it will switch forks and build its next block (b4) + // on top of it + + push_block(1, b3); // push block to B and C, should receive strong votes + BOOST_REQUIRE_EQUAL(A.last_vote(), strong_vote(b2)); + BOOST_REQUIRE_EQUAL(B.last_vote(), strong_vote(b3)); + BOOST_REQUIRE_EQUAL(C.last_vote(), strong_vote(b3)); + BOOST_REQUIRE_EQUAL(D.last_vote(), strong_vote(b3)); + BOOST_REQUIRE_EQUAL(qc_s(qc(b3)), strong_qc(b2)); // b3 should include a strong qc on b2 + BOOST_REQUIRE_EQUAL(B.lib_number, b1->block_num()); // don't use A.lib_number as A is partitioned by itself + // so it didn't see b3 and its enclosed QC. + B.check_fsi({.last_vote = b3, .lock = b2, .other_branch_latest_time = {}}); + + set_partition(partition); // restore our original partition {A, B, C} and {D} + + // from now on, to reproduce the scenario where votes are delayed, so the QC we receive don't + // claim the parent block, but an ancestor, we need to artificially delay propagating the votes. + // --------------------------------------------------------------------------------------------- + + fc::scoped_set_value tmp(B.vote_delay(), 1); // delaying just B's votes should be enough to delay QCs + + auto b4 = A.produce_block(_block_interval_us * 2); // b4 skips a slot. receives weak votes from {B, C}. + print("b4", b4); + BOOST_REQUIRE_EQUAL(A.last_vote(), strong_vote(b4)); // A votes strong because it didn't see (and vote on) B3 + BOOST_REQUIRE_EQUAL(B.last_vote(), weak_vote(b4)); // B's last vote even if it wasn't propagated + BOOST_REQUIRE_EQUAL(C.last_vote(), weak_vote(b4)); + BOOST_REQUIRE_EQUAL(qc_s(qc(b4)), strong_qc(b2)); // b4 should include a strong qc on b2 + BOOST_REQUIRE_EQUAL(A.lib_number, b1->block_num()); + B.check_fsi(fsi_expect{.last_vote = b4, .lock = b2, .other_branch_latest_time = b3->timestamp }); + + auto b5 = A.produce_block(); // receives weak votes from {B, C}. + print("b5", b5); + BOOST_REQUIRE_EQUAL(A.last_vote(), strong_vote(b5)); // A votes strong because it didn't see (and vote on) B3 + BOOST_REQUIRE_EQUAL(B.last_vote(), weak_vote(b5)); + BOOST_REQUIRE_EQUAL(C.last_vote(), weak_vote(b5)); + BOOST_REQUIRE(!qc(b5)); // Because B's vote was delayed, b5 should not have a QC + BOOST_REQUIRE_EQUAL(A.lib_number, b1->block_num()); + B.check_fsi(fsi_expect{.last_vote = b5, .lock = b2, .other_branch_latest_time = b3->timestamp }); + + auto b6 = A.produce_block(); // receives strong votes from {A, B, C}. + print("b6", b6); + BOOST_REQUIRE_EQUAL(A.last_vote(), strong_vote(b6)); // A votes strong because it didn't see (and vote on) B3 + BOOST_REQUIRE_EQUAL(B.last_vote(), strong_vote(b6)); // with issue #627 fix, should start voting strong again + BOOST_REQUIRE_EQUAL(C.last_vote(), strong_vote(b6)); // with issue #627 fix, should start voting strong again + BOOST_REQUIRE_EQUAL(qc_s(qc(b6)), weak_qc(b4)); // Because B's vote was delayed, b6 has a weak QC on b4 + BOOST_REQUIRE_EQUAL(A.lib_number, b1->block_num()); + B.check_fsi(fsi_expect{.last_vote = b6, .lock = b4, .other_branch_latest_time = {}}); + + auto b7 = A.produce_block(); // receives strong votes from {A, B, C}. + print("b7", b7); + BOOST_REQUIRE_EQUAL(A.last_vote(), strong_vote(b7)); + BOOST_REQUIRE_EQUAL(B.last_vote(), strong_vote(b7)); + BOOST_REQUIRE_EQUAL(C.last_vote(), strong_vote(b7)); + BOOST_REQUIRE_EQUAL(qc_s(qc(b7)), weak_qc(b5)); // Because B's vote was delayed, b7 has a weak QC on b5 + BOOST_REQUIRE_EQUAL(A.lib_number, b1->block_num()); + B.check_fsi(fsi_expect{.last_vote = b7, .lock = b5, .other_branch_latest_time = {}}); + + auto b8 = A.produce_block(); // receives strong votes from {A, B, C}. + print("b8", b8); + BOOST_REQUIRE_EQUAL(A.last_vote(), strong_vote(b8)); + BOOST_REQUIRE_EQUAL(B.last_vote(), strong_vote(b8)); + BOOST_REQUIRE_EQUAL(C.last_vote(), strong_vote(b8)); + BOOST_REQUIRE_EQUAL(qc_s(qc(b8)), strong_qc(b6)); // Because of the strong votes on b6, b8 has a strong QC on b6 + BOOST_REQUIRE_EQUAL(A.lib_number, b4->block_num()); + B.check_fsi(fsi_expect{.last_vote = b8, .lock = b6, .other_branch_latest_time = {}}); + +} FC_LOG_AND_RETHROW() + + BOOST_AUTO_TEST_SUITE_END()