From 4b6928ad7f757391fe9d89e6f9c20c0cad97c14c Mon Sep 17 00:00:00 2001 From: fcecin Date: Mon, 25 Sep 2023 10:58:21 -0300 Subject: [PATCH 1/5] Implement message propagation - propagate proposals if they are added by the node to its proposal store - non-leaders propagate votes of known proposals that haven't been propagated by them yet - added structure to track seen votes for each proposal (GC'd together with proposals by height) - propagate new view messages that have a High QC that's newer than what we had locally - enabled basic propagation test - added star topology propagation test - added ring topology propagation test - documented that new block messages are not propagated - removed topology restriction for new block messages at test_pacemaker - removed hop count from hotstuff_test_handler::dispatch(); will dispatch until network goes quiet Merging both multi-indexes declared in the qc_chain is possible (makes proposal height gc 2x faster) but I'd leave that for later to minimize merge conflicts. --- .../include/eosio/hotstuff/qc_chain.hpp | 31 ++ libraries/hotstuff/qc_chain.cpp | 57 ++- libraries/hotstuff/test/test_hotstuff.cpp | 341 ++++++++++++++++-- libraries/hotstuff/test/test_pacemaker.cpp | 3 +- 4 files changed, 400 insertions(+), 32 deletions(-) diff --git a/libraries/hotstuff/include/eosio/hotstuff/qc_chain.hpp b/libraries/hotstuff/include/eosio/hotstuff/qc_chain.hpp index 6169b1efa5..eeb23efa4d 100644 --- a/libraries/hotstuff/include/eosio/hotstuff/qc_chain.hpp +++ b/libraries/hotstuff/include/eosio/hotstuff/qc_chain.hpp @@ -88,6 +88,12 @@ namespace eosio::hotstuff { bool quorum_met = false; // not serialized across network }; + struct seen_votes { + fc::sha256 proposal_id; // id of proposal being voted on + uint64_t height; // height of the proposal (for GC) + std::set finalizers; // finalizers that have voted on the proposal + }; + // Concurrency note: qc_chain is a single-threaded and lock-free decision engine. // All thread synchronization, if any, is external. class qc_chain { @@ -113,6 +119,10 @@ namespace eosio::hotstuff { void on_hs_vote_msg(const uint32_t connection_id, const hs_vote_message& msg); void on_hs_proposal_msg(const uint32_t connection_id, const hs_proposal_message& msg); void on_hs_new_view_msg(const uint32_t connection_id, const hs_new_view_message& msg); + + // NOTE: The hotstuff New Block message is not ever propagated (multi-hop) by this method. + // Unit tests do not use network topology emulation for this message. + // The live network does not actually dispatch this message to the wire; this is a local callback. void on_hs_new_block_msg(const uint32_t connection_id, const hs_new_block_message& msg); private: @@ -246,6 +256,27 @@ namespace eosio::hotstuff { proposal_store_type _proposal_store; //internal proposals store #endif + + // Possible optimization: merge _proposal_store and _seen_votes_store. + // Store a struct { set seen_votes; hs_proposal_message p; } in the (now single) multi-index. + struct by_seen_votes_proposal_id{}; + struct by_seen_votes_proposal_height{}; + typedef multi_index_container< + seen_votes, + indexed_by< + hashed_unique< + tag, + BOOST_MULTI_INDEX_MEMBER(seen_votes,fc::sha256,proposal_id) + >, + ordered_non_unique< + tag, + BOOST_MULTI_INDEX_MEMBER(seen_votes,uint64_t,height) + > + > + > seen_votes_store_type; + + // given a height, store a map of proposal IDs at that height and the seen votes for it + seen_votes_store_type _seen_votes_store; }; } /// eosio::hotstuff diff --git a/libraries/hotstuff/qc_chain.cpp b/libraries/hotstuff/qc_chain.cpp index 99018fb36b..27a4eef6fb 100644 --- a/libraries/hotstuff/qc_chain.cpp +++ b/libraries/hotstuff/qc_chain.cpp @@ -418,6 +418,10 @@ namespace eosio::hotstuff { //update internal state update(proposal); + //propagate this proposal since it was new to us + if (! am_i_leader()) + send_hs_proposal_msg(connection_id, proposal); + for (auto &msg : msgs) { send_hs_vote_msg( std::nullopt, msg ); } @@ -437,23 +441,45 @@ namespace eosio::hotstuff { bool am_leader = am_i_leader(); - if (!am_leader) - return; - fc_tlog(_logger, " === Process vote from ${finalizer} : current bitset ${value}" , - ("finalizer", vote.finalizer)("value", _current_qc.get_active_finalizers_string())); - // only leader need to take action on votes - if (vote.proposal_id != _current_qc.get_proposal_id()) { - send_hs_message_warning(connection_id, hs_message_warning::discarded); // example; to be tuned to actual need - return; - } + if (am_leader) { + if (vote.proposal_id != _current_qc.get_proposal_id()) { + send_hs_message_warning(connection_id, hs_message_warning::discarded); // example; to be tuned to actual need + return; + } + } const hs_proposal_message *p = get_proposal( vote.proposal_id ); if (p == nullptr) { - fc_elog(_logger, " *** ${id} couldn't find proposal, vote : ${vote}", ("id",_id)("vote", vote)); + if (am_leader) + fc_elog(_logger, " *** ${id} couldn't find proposal, vote : ${vote}", ("id",_id)("vote", vote)); send_hs_message_warning(connection_id, hs_message_warning::discarded); // example; to be tuned to actual need return; } + // if not leader, check message propagation and quit + if (! am_leader) { + seen_votes_store_type::nth_index<0>::type::iterator itr = _seen_votes_store.get().find( p->proposal_id ); + bool propagate = false; + if (itr == _seen_votes_store.get().end()) { + seen_votes sv = { p->proposal_id, p->get_height(), { vote.finalizer } }; + _seen_votes_store.insert(sv); + propagate = true; + } else { + _seen_votes_store.get().modify(itr, [&](seen_votes& sv) { + if (sv.finalizers.count(vote.finalizer) == 0) { + sv.finalizers.insert(vote.finalizer); + propagate = true; + } + }); + } + if (propagate) + send_hs_vote_msg(connection_id, vote); + return; + } + + fc_tlog(_logger, " === Process vote from ${finalizer} : current bitset ${value}" , + ("finalizer", vote.finalizer)("value", _current_qc.get_active_finalizers_string())); + bool quorum_met = _current_qc.is_quorum_met(); //check if quorum already met // If quorum is already met, we don't need to do anything else. Otherwise, we aggregate the signature. @@ -526,6 +552,11 @@ namespace eosio::hotstuff { auto increment_version = fc::make_scoped_exit([this]() { ++_state_version; }); if (!update_high_qc(quorum_certificate{msg.high_qc})) { increment_version.cancel(); + } else { + // Always propagate a view that's newer than ours. + // If it's not newer, then we have already propagated ours. + // If the recipient doesn't think ours is newer, it has already propagated its own, and so on. + send_hs_new_view_msg(connection_id, msg); } } @@ -988,6 +1019,12 @@ namespace eosio::hotstuff { void qc_chain::gc_proposals(uint64_t cutoff){ //fc_tlog(_logger, " === garbage collection on old data"); + auto sv_end_itr = _seen_votes_store.get().upper_bound(cutoff); + while (_seen_votes_store.get().begin() != sv_end_itr){ + auto itr = _seen_votes_store.get().begin(); + _seen_votes_store.get().erase(itr); + } + #ifdef QC_CHAIN_SIMPLE_PROPOSAL_STORE ps_height_iterator psh_it = _proposal_stores_by_height.begin(); while (psh_it != _proposal_stores_by_height.end()) { diff --git a/libraries/hotstuff/test/test_hotstuff.cpp b/libraries/hotstuff/test/test_hotstuff.cpp index d128cbf0a0..85e51f4dcb 100644 --- a/libraries/hotstuff/test/test_hotstuff.cpp +++ b/libraries/hotstuff/test/test_hotstuff.cpp @@ -129,13 +129,16 @@ class hotstuff_test_handler { std::cout << "\n"; } - void dispatch(test_pacemaker& tpm, int hops, test_pacemaker::hotstuff_message_index msg_type, const std::string& memo = "") { - for (int i=0;isecond->get_state(fs_bpa); BOOST_CHECK_EQUAL(fs_bpa.b_leaf.str(), std::string("a252070cd26d3b231ab2443b9ba97f57fc72e50cca04a020952e45bc7e2d27a8")); @@ -1067,9 +1070,9 @@ BOOST_AUTO_TEST_CASE(hotstuff_7) try { BOOST_CHECK_EQUAL(fs_bpa.b_lock.str(), std::string("0000000000000000000000000000000000000000000000000000000000000000")); BOOST_CHECK_EQUAL(fs_bpa.b_exec.str(), std::string("0000000000000000000000000000000000000000000000000000000000000000")); - ht.dispatch(tpm, 2, test_pacemaker::hs_vote); //send votes on proposal (prepareQC on first block) + ht.dispatch(tpm, test_pacemaker::hs_vote); //send votes on proposal (prepareQC on first block) - ht.dispatch(tpm, 2, test_pacemaker::hs_proposal); //send proposal to replicas (precommit on first block) + ht.dispatch(tpm, test_pacemaker::hs_proposal); //send proposal to replicas (precommit on first block) qcc_bpa->second->get_state(fs_bpa); BOOST_CHECK_EQUAL(fs_bpa.b_leaf.str(), std::string("4b43fb144a8b5e874777f61f3b37d7a8b06c33fbc48db464ce0e8788ff4edb4f")); @@ -1077,9 +1080,9 @@ BOOST_AUTO_TEST_CASE(hotstuff_7) try { BOOST_CHECK_EQUAL(fs_bpa.b_lock.str(), std::string("0000000000000000000000000000000000000000000000000000000000000000")); BOOST_CHECK_EQUAL(fs_bpa.b_exec.str(), std::string("0000000000000000000000000000000000000000000000000000000000000000")); - ht.dispatch(tpm, 2, test_pacemaker::hs_vote); //propagating votes on new proposal (precommitQC on first block) + ht.dispatch(tpm, test_pacemaker::hs_vote); //propagating votes on new proposal (precommitQC on first block) - ht.dispatch(tpm, 2, test_pacemaker::hs_proposal); //send proposal to replicas (commit on first block) + ht.dispatch(tpm, test_pacemaker::hs_proposal); //send proposal to replicas (commit on first block) qcc_bpa->second->get_state(fs_bpa); BOOST_CHECK_EQUAL(fs_bpa.b_leaf.str(), std::string("aedf8bb1ee70bd6e743268f7fe0f8171418aa43a68bb9c6e7329ffa856896c09")); @@ -1089,9 +1092,9 @@ BOOST_AUTO_TEST_CASE(hotstuff_7) try { tpm.set_next_leader("bpb"_n); //leader is set to rotate on next block - ht.dispatch(tpm, 2, test_pacemaker::hs_vote); //propagating votes on new proposal (commitQC on first block) + ht.dispatch(tpm, test_pacemaker::hs_vote); //propagating votes on new proposal (commitQC on first block) - ht.dispatch(tpm, 2, test_pacemaker::hs_proposal); //send proposal to replicas (decide on first block) + ht.dispatch(tpm, test_pacemaker::hs_proposal); //send proposal to replicas (decide on first block) qcc_bpa->second->get_state(fs_bpa); BOOST_CHECK_EQUAL(fs_bpa.b_leaf.str(), std::string("487e5fcbf2c515618941291ae3b6dcebb68942983d8ac3f61c4bdd9901dadbe7")); @@ -1099,7 +1102,7 @@ BOOST_AUTO_TEST_CASE(hotstuff_7) try { BOOST_CHECK_EQUAL(fs_bpa.b_lock.str(), std::string("4b43fb144a8b5e874777f61f3b37d7a8b06c33fbc48db464ce0e8788ff4edb4f")); BOOST_CHECK_EQUAL(fs_bpa.b_exec.str(), std::string("a252070cd26d3b231ab2443b9ba97f57fc72e50cca04a020952e45bc7e2d27a8")); - ht.dispatch(tpm, 2, test_pacemaker::hs_vote); //propagating votes on new proposal (decide on first block) + ht.dispatch(tpm, test_pacemaker::hs_vote); //propagating votes on new proposal (decide on first block) tpm.set_proposer("bpb"_n); //leader has rotated tpm.set_leader("bpb"_n); @@ -1108,7 +1111,7 @@ BOOST_AUTO_TEST_CASE(hotstuff_7) try { tpm.beat(); //produce second block and associated proposal - ht.dispatch(tpm, 2, test_pacemaker::hs_proposal); //send proposal to replicas (prepare on second block) + ht.dispatch(tpm, test_pacemaker::hs_proposal); //send proposal to replicas (prepare on second block) qcc_bpb->second->get_state(fs_bpb); BOOST_CHECK_EQUAL(fs_bpb.b_leaf.str(), std::string("1511035fdcbabdc5e272a3ac19356536252884ed77077cf871ae5029a7502279")); @@ -1116,9 +1119,9 @@ BOOST_AUTO_TEST_CASE(hotstuff_7) try { BOOST_CHECK_EQUAL(fs_bpb.b_lock.str(), std::string("4b43fb144a8b5e874777f61f3b37d7a8b06c33fbc48db464ce0e8788ff4edb4f")); BOOST_CHECK_EQUAL(fs_bpb.b_exec.str(), std::string("a252070cd26d3b231ab2443b9ba97f57fc72e50cca04a020952e45bc7e2d27a8")); - ht.dispatch(tpm, 2, test_pacemaker::hs_vote); //send votes on proposal (prepareQC on second block) + ht.dispatch(tpm, test_pacemaker::hs_vote); //send votes on proposal (prepareQC on second block) - ht.dispatch(tpm, 2, test_pacemaker::hs_proposal); //send proposal to replicas (precommit on second block) + ht.dispatch(tpm, test_pacemaker::hs_proposal); //send proposal to replicas (precommit on second block) qcc_bpb->second->get_state(fs_bpb); BOOST_CHECK_EQUAL(fs_bpb.b_leaf.str(), std::string("6462add7d157da87931c859cb689f722003a20f30c0f1408d11b872020903b85")); @@ -1126,9 +1129,9 @@ BOOST_AUTO_TEST_CASE(hotstuff_7) try { BOOST_CHECK_EQUAL(fs_bpb.b_lock.str(), std::string("aedf8bb1ee70bd6e743268f7fe0f8171418aa43a68bb9c6e7329ffa856896c09")); BOOST_CHECK_EQUAL(fs_bpb.b_exec.str(), std::string("4b43fb144a8b5e874777f61f3b37d7a8b06c33fbc48db464ce0e8788ff4edb4f")); - ht.dispatch(tpm, 2, test_pacemaker::hs_vote); //propagating votes on new proposal (precommitQC on second block) + ht.dispatch(tpm, test_pacemaker::hs_vote); //propagating votes on new proposal (precommitQC on second block) - ht.dispatch(tpm, 2, test_pacemaker::hs_proposal); //send proposal to replicas (commit on second block) + ht.dispatch(tpm, test_pacemaker::hs_proposal); //send proposal to replicas (commit on second block) qcc_bpb->second->get_state(fs_bpb); BOOST_CHECK_EQUAL(fs_bpb.b_leaf.str(), std::string("fd77164bf3898a6a8f27ccff440d17ef6870e75c368fcc93b969066cec70939c")); @@ -1136,9 +1139,9 @@ BOOST_AUTO_TEST_CASE(hotstuff_7) try { BOOST_CHECK_EQUAL(fs_bpb.b_lock.str(), std::string("1511035fdcbabdc5e272a3ac19356536252884ed77077cf871ae5029a7502279")); BOOST_CHECK_EQUAL(fs_bpb.b_exec.str(), std::string("aedf8bb1ee70bd6e743268f7fe0f8171418aa43a68bb9c6e7329ffa856896c09")); - ht.dispatch(tpm, 2, test_pacemaker::hs_vote); //propagating votes on new proposal (commitQC on second block) + ht.dispatch(tpm, test_pacemaker::hs_vote); //propagating votes on new proposal (commitQC on second block) - ht.dispatch(tpm, 2, test_pacemaker::hs_proposal); //send proposal to replicas (decide on second block) + ht.dispatch(tpm, test_pacemaker::hs_proposal); //send proposal to replicas (decide on second block) qcc_bpb->second->get_state(fs_bpb); BOOST_CHECK_EQUAL(fs_bpb.b_leaf.str(), std::string("89f468a127dbadd81b59076067238e3e9c313782d7d83141b16d9da4f2c2b078")); @@ -1301,5 +1304,301 @@ BOOST_AUTO_TEST_CASE(hotstuff_8) try { } FC_LOG_AND_RETHROW(); +BOOST_AUTO_TEST_CASE(hotstuff_9) try { + + //test leader rotation with a star toplogy (message propagation test) + + test_pacemaker tpm; + for (size_t i=0; isecond->get_state(fs_bpa); + auto qcc_bpb = std::find_if(ht._qc_chains.begin(), ht._qc_chains.end(), [&](const auto& q){ return q.first == "bpb"_n; }); + finalizer_state fs_bpb; + qcc_bpb->second->get_state(fs_bpb); + auto qcc_bpc = std::find_if(ht._qc_chains.begin(), ht._qc_chains.end(), [&](const auto& q){ return q.first == "bpc"_n; }); + finalizer_state fs_bpc; + qcc_bpc->second->get_state(fs_bpc); + + tpm.set_current_block_id(ids[0]); //first block + + tpm.beat(); //produce first block and associated proposal + + ht.dispatch(tpm, test_pacemaker::hs_proposal); //send proposal to replicas (prepare on first block) + + qcc_bpa->second->get_state(fs_bpa); + BOOST_CHECK_EQUAL(fs_bpa.b_leaf.str(), std::string("a252070cd26d3b231ab2443b9ba97f57fc72e50cca04a020952e45bc7e2d27a8")); + BOOST_CHECK_EQUAL(fs_bpa.high_qc.proposal_id.str(), std::string("0000000000000000000000000000000000000000000000000000000000000000")); + BOOST_CHECK_EQUAL(fs_bpa.b_lock.str(), std::string("0000000000000000000000000000000000000000000000000000000000000000")); + BOOST_CHECK_EQUAL(fs_bpa.b_exec.str(), std::string("0000000000000000000000000000000000000000000000000000000000000000")); + + ht.dispatch(tpm, test_pacemaker::hs_vote); //send votes on proposal (prepareQC on first block) + + ht.dispatch(tpm, test_pacemaker::hs_proposal); //send proposal to replicas (precommit on first block) + + qcc_bpa->second->get_state(fs_bpa); + BOOST_CHECK_EQUAL(fs_bpa.b_leaf.str(), std::string("4b43fb144a8b5e874777f61f3b37d7a8b06c33fbc48db464ce0e8788ff4edb4f")); + BOOST_CHECK_EQUAL(fs_bpa.high_qc.proposal_id.str(), std::string("a252070cd26d3b231ab2443b9ba97f57fc72e50cca04a020952e45bc7e2d27a8")); + BOOST_CHECK_EQUAL(fs_bpa.b_lock.str(), std::string("0000000000000000000000000000000000000000000000000000000000000000")); + BOOST_CHECK_EQUAL(fs_bpa.b_exec.str(), std::string("0000000000000000000000000000000000000000000000000000000000000000")); + + ht.dispatch(tpm, test_pacemaker::hs_vote); //propagating votes on new proposal (precommitQC on first block) + + ht.dispatch(tpm, test_pacemaker::hs_proposal); //send proposal to replicas (commit on first block) + + qcc_bpa->second->get_state(fs_bpa); + BOOST_CHECK_EQUAL(fs_bpa.b_leaf.str(), std::string("aedf8bb1ee70bd6e743268f7fe0f8171418aa43a68bb9c6e7329ffa856896c09")); //4b4 + BOOST_CHECK_EQUAL(fs_bpa.high_qc.proposal_id.str(), std::string("4b43fb144a8b5e874777f61f3b37d7a8b06c33fbc48db464ce0e8788ff4edb4f"));//a250 + BOOST_CHECK_EQUAL(fs_bpa.b_lock.str(), std::string("a252070cd26d3b231ab2443b9ba97f57fc72e50cca04a020952e45bc7e2d27a8"));//00 + BOOST_CHECK_EQUAL(fs_bpa.b_exec.str(), std::string("0000000000000000000000000000000000000000000000000000000000000000")); + + tpm.set_next_leader("bpb"_n); //leader is set to rotate on next block + + ht.dispatch(tpm, test_pacemaker::hs_vote); //propagating votes on new proposal (commitQC on first block) + + ht.dispatch(tpm, test_pacemaker::hs_new_view); + ht.dispatch(tpm, test_pacemaker::hs_proposal); //send proposal to replicas (decide on first block) + + qcc_bpa->second->get_state(fs_bpa); + BOOST_CHECK_EQUAL(fs_bpa.b_leaf.str(), std::string("487e5fcbf2c515618941291ae3b6dcebb68942983d8ac3f61c4bdd9901dadbe7")); + BOOST_CHECK_EQUAL(fs_bpa.high_qc.proposal_id.str(), std::string("aedf8bb1ee70bd6e743268f7fe0f8171418aa43a68bb9c6e7329ffa856896c09")); + BOOST_CHECK_EQUAL(fs_bpa.b_lock.str(), std::string("4b43fb144a8b5e874777f61f3b37d7a8b06c33fbc48db464ce0e8788ff4edb4f")); + BOOST_CHECK_EQUAL(fs_bpa.b_exec.str(), std::string("a252070cd26d3b231ab2443b9ba97f57fc72e50cca04a020952e45bc7e2d27a8")); + + ht.dispatch(tpm, test_pacemaker::hs_new_view); + ht.dispatch(tpm, test_pacemaker::hs_vote); //propagating votes on new proposal (decide on first block) + + tpm.set_proposer("bpb"_n); //leader has rotated + tpm.set_leader("bpb"_n); + + tpm.set_current_block_id(ids[1]); //second block + + tpm.beat(); //produce second block and associated proposal + + ht.dispatch(tpm, test_pacemaker::hs_proposal); //send proposal to replicas (prepare on second block) + + qcc_bpb->second->get_state(fs_bpb); + BOOST_CHECK_EQUAL(fs_bpb.b_leaf.str(), std::string("1511035fdcbabdc5e272a3ac19356536252884ed77077cf871ae5029a7502279")); + BOOST_CHECK_EQUAL(fs_bpb.high_qc.proposal_id.str(), std::string("aedf8bb1ee70bd6e743268f7fe0f8171418aa43a68bb9c6e7329ffa856896c09")); + BOOST_CHECK_EQUAL(fs_bpb.b_lock.str(), std::string("4b43fb144a8b5e874777f61f3b37d7a8b06c33fbc48db464ce0e8788ff4edb4f")); + BOOST_CHECK_EQUAL(fs_bpb.b_exec.str(), std::string("a252070cd26d3b231ab2443b9ba97f57fc72e50cca04a020952e45bc7e2d27a8")); + + ht.dispatch(tpm, test_pacemaker::hs_vote); //send votes on proposal (prepareQC on second block) + + ht.dispatch(tpm, test_pacemaker::hs_proposal); //send proposal to replicas (precommit on second block) + + qcc_bpb->second->get_state(fs_bpb); + BOOST_CHECK_EQUAL(fs_bpb.b_leaf.str(), std::string("6462add7d157da87931c859cb689f722003a20f30c0f1408d11b872020903b85")); + BOOST_CHECK_EQUAL(fs_bpb.high_qc.proposal_id.str(), std::string("1511035fdcbabdc5e272a3ac19356536252884ed77077cf871ae5029a7502279")); + BOOST_CHECK_EQUAL(fs_bpb.b_lock.str(), std::string("aedf8bb1ee70bd6e743268f7fe0f8171418aa43a68bb9c6e7329ffa856896c09")); + BOOST_CHECK_EQUAL(fs_bpb.b_exec.str(), std::string("4b43fb144a8b5e874777f61f3b37d7a8b06c33fbc48db464ce0e8788ff4edb4f")); + + ht.dispatch(tpm, test_pacemaker::hs_vote); //propagating votes on new proposal (precommitQC on second block) + + ht.dispatch(tpm, test_pacemaker::hs_proposal); //send proposal to replicas (commit on second block) + + qcc_bpb->second->get_state(fs_bpb); + BOOST_CHECK_EQUAL(fs_bpb.b_leaf.str(), std::string("fd77164bf3898a6a8f27ccff440d17ef6870e75c368fcc93b969066cec70939c")); + BOOST_CHECK_EQUAL(fs_bpb.high_qc.proposal_id.str(), std::string("6462add7d157da87931c859cb689f722003a20f30c0f1408d11b872020903b85")); + BOOST_CHECK_EQUAL(fs_bpb.b_lock.str(), std::string("1511035fdcbabdc5e272a3ac19356536252884ed77077cf871ae5029a7502279")); + BOOST_CHECK_EQUAL(fs_bpb.b_exec.str(), std::string("aedf8bb1ee70bd6e743268f7fe0f8171418aa43a68bb9c6e7329ffa856896c09")); + + ht.dispatch(tpm, test_pacemaker::hs_vote); //propagating votes on new proposal (commitQC on second block) + + ht.dispatch(tpm, test_pacemaker::hs_proposal); //send proposal to replicas (decide on second block) + + qcc_bpb->second->get_state(fs_bpb); + BOOST_CHECK_EQUAL(fs_bpb.b_leaf.str(), std::string("89f468a127dbadd81b59076067238e3e9c313782d7d83141b16d9da4f2c2b078")); + BOOST_CHECK_EQUAL(fs_bpb.high_qc.proposal_id.str(), std::string("fd77164bf3898a6a8f27ccff440d17ef6870e75c368fcc93b969066cec70939c")); + BOOST_CHECK_EQUAL(fs_bpb.b_lock.str(), std::string("6462add7d157da87931c859cb689f722003a20f30c0f1408d11b872020903b85")); + BOOST_CHECK_EQUAL(fs_bpb.b_exec.str(), std::string("1511035fdcbabdc5e272a3ac19356536252884ed77077cf871ae5029a7502279")); + + //check bpa as well + qcc_bpa->second->get_state(fs_bpa); + BOOST_CHECK_EQUAL(fs_bpa.high_qc.proposal_id.str(), std::string("fd77164bf3898a6a8f27ccff440d17ef6870e75c368fcc93b969066cec70939c")); + BOOST_CHECK_EQUAL(fs_bpa.b_lock.str(), std::string("6462add7d157da87931c859cb689f722003a20f30c0f1408d11b872020903b85")); + BOOST_CHECK_EQUAL(fs_bpa.b_exec.str(), std::string("1511035fdcbabdc5e272a3ac19356536252884ed77077cf871ae5029a7502279")); + + //check bpc as well + qcc_bpc->second->get_state(fs_bpc); + BOOST_CHECK_EQUAL(fs_bpc.high_qc.proposal_id.str(), std::string("fd77164bf3898a6a8f27ccff440d17ef6870e75c368fcc93b969066cec70939c")); + BOOST_CHECK_EQUAL(fs_bpc.b_lock.str(), std::string("6462add7d157da87931c859cb689f722003a20f30c0f1408d11b872020903b85")); + BOOST_CHECK_EQUAL(fs_bpc.b_exec.str(), std::string("1511035fdcbabdc5e272a3ac19356536252884ed77077cf871ae5029a7502279")); + + BOOST_CHECK_EQUAL(fs_bpa.b_finality_violation.str(), std::string("0000000000000000000000000000000000000000000000000000000000000000")); + +} FC_LOG_AND_RETHROW(); + +BOOST_AUTO_TEST_CASE(hotstuff_10) try { + + //test leader rotation with a ring topology (message propagation test) + + test_pacemaker tpm; + + // zigzag to separate bpa, bpb and bpc. + // cut connections 11,1 *and* 10,0 to see the test fail. + // turning the ring into a line by cutting just one connection is not enough to fail the test. + tpm.connect( { unique_replicas[ 0], unique_replicas[11] } ); + tpm.connect( { unique_replicas[11], unique_replicas[ 1] } ); //cut this to fail (1 of 2) + tpm.connect( { unique_replicas[ 1], unique_replicas[12] } ); + tpm.connect( { unique_replicas[12], unique_replicas[ 2] } ); + tpm.connect( { unique_replicas[ 2], unique_replicas[13] } ); + tpm.connect( { unique_replicas[13], unique_replicas[ 3] } ); + tpm.connect( { unique_replicas[ 3], unique_replicas[14] } ); + tpm.connect( { unique_replicas[14], unique_replicas[ 4] } ); + tpm.connect( { unique_replicas[ 4], unique_replicas[15] } ); + tpm.connect( { unique_replicas[15], unique_replicas[ 5] } ); + tpm.connect( { unique_replicas[ 5], unique_replicas[16] } ); + tpm.connect( { unique_replicas[16], unique_replicas[ 6] } ); + tpm.connect( { unique_replicas[ 6], unique_replicas[17] } ); + tpm.connect( { unique_replicas[17], unique_replicas[ 7] } ); + tpm.connect( { unique_replicas[ 7], unique_replicas[18] } ); + tpm.connect( { unique_replicas[18], unique_replicas[ 8] } ); + tpm.connect( { unique_replicas[ 8], unique_replicas[19] } ); + tpm.connect( { unique_replicas[19], unique_replicas[ 9] } ); + tpm.connect( { unique_replicas[ 9], unique_replicas[20] } ); + tpm.connect( { unique_replicas[20], unique_replicas[10] } ); + tpm.connect( { unique_replicas[10], unique_replicas[ 0] } ); //cut this to fail (2 of 2) + + hotstuff_test_handler ht; + + ht.initialize_qc_chains(tpm, unique_replicas); + + tpm.set_proposer("bpa"_n); + tpm.set_leader("bpa"_n); + tpm.set_next_leader("bpa"_n); + tpm.set_finalizers(unique_replicas); + + auto qcc_bpa = std::find_if(ht._qc_chains.begin(), ht._qc_chains.end(), [&](const auto& q){ return q.first == "bpa"_n; }); + finalizer_state fs_bpa; + qcc_bpa->second->get_state(fs_bpa); + auto qcc_bpb = std::find_if(ht._qc_chains.begin(), ht._qc_chains.end(), [&](const auto& q){ return q.first == "bpb"_n; }); + finalizer_state fs_bpb; + qcc_bpb->second->get_state(fs_bpb); + auto qcc_bpc = std::find_if(ht._qc_chains.begin(), ht._qc_chains.end(), [&](const auto& q){ return q.first == "bpc"_n; }); + finalizer_state fs_bpc; + qcc_bpc->second->get_state(fs_bpc); + + tpm.set_current_block_id(ids[0]); //first block + + tpm.beat(); //produce first block and associated proposal + + ht.dispatch(tpm, test_pacemaker::hs_proposal); //send proposal to replicas (prepare on first block) + + qcc_bpa->second->get_state(fs_bpa); + BOOST_CHECK_EQUAL(fs_bpa.b_leaf.str(), std::string("a252070cd26d3b231ab2443b9ba97f57fc72e50cca04a020952e45bc7e2d27a8")); + BOOST_CHECK_EQUAL(fs_bpa.high_qc.proposal_id.str(), std::string("0000000000000000000000000000000000000000000000000000000000000000")); + BOOST_CHECK_EQUAL(fs_bpa.b_lock.str(), std::string("0000000000000000000000000000000000000000000000000000000000000000")); + BOOST_CHECK_EQUAL(fs_bpa.b_exec.str(), std::string("0000000000000000000000000000000000000000000000000000000000000000")); + + ht.dispatch(tpm, test_pacemaker::hs_vote); //send votes on proposal (prepareQC on first block) + + ht.dispatch(tpm, test_pacemaker::hs_proposal); //send proposal to replicas (precommit on first block) + + qcc_bpa->second->get_state(fs_bpa); + BOOST_CHECK_EQUAL(fs_bpa.b_leaf.str(), std::string("4b43fb144a8b5e874777f61f3b37d7a8b06c33fbc48db464ce0e8788ff4edb4f")); + BOOST_CHECK_EQUAL(fs_bpa.high_qc.proposal_id.str(), std::string("a252070cd26d3b231ab2443b9ba97f57fc72e50cca04a020952e45bc7e2d27a8")); + BOOST_CHECK_EQUAL(fs_bpa.b_lock.str(), std::string("0000000000000000000000000000000000000000000000000000000000000000")); + BOOST_CHECK_EQUAL(fs_bpa.b_exec.str(), std::string("0000000000000000000000000000000000000000000000000000000000000000")); + + ht.dispatch(tpm, test_pacemaker::hs_vote); //propagating votes on new proposal (precommitQC on first block) + + ht.dispatch(tpm, test_pacemaker::hs_proposal); //send proposal to replicas (commit on first block) + + qcc_bpa->second->get_state(fs_bpa); + BOOST_CHECK_EQUAL(fs_bpa.b_leaf.str(), std::string("aedf8bb1ee70bd6e743268f7fe0f8171418aa43a68bb9c6e7329ffa856896c09")); + BOOST_CHECK_EQUAL(fs_bpa.high_qc.proposal_id.str(), std::string("4b43fb144a8b5e874777f61f3b37d7a8b06c33fbc48db464ce0e8788ff4edb4f")); + BOOST_CHECK_EQUAL(fs_bpa.b_lock.str(), std::string("a252070cd26d3b231ab2443b9ba97f57fc72e50cca04a020952e45bc7e2d27a8")); + BOOST_CHECK_EQUAL(fs_bpa.b_exec.str(), std::string("0000000000000000000000000000000000000000000000000000000000000000")); + + tpm.set_next_leader("bpb"_n); //leader is set to rotate on next block + + ht.dispatch(tpm, test_pacemaker::hs_vote); //propagating votes on new proposal (commitQC on first block) + + ht.dispatch(tpm, test_pacemaker::hs_new_view); + ht.dispatch(tpm, test_pacemaker::hs_proposal); //send proposal to replicas (decide on first block) + + qcc_bpa->second->get_state(fs_bpa); + BOOST_CHECK_EQUAL(fs_bpa.b_leaf.str(), std::string("487e5fcbf2c515618941291ae3b6dcebb68942983d8ac3f61c4bdd9901dadbe7")); + BOOST_CHECK_EQUAL(fs_bpa.high_qc.proposal_id.str(), std::string("aedf8bb1ee70bd6e743268f7fe0f8171418aa43a68bb9c6e7329ffa856896c09")); + BOOST_CHECK_EQUAL(fs_bpa.b_lock.str(), std::string("4b43fb144a8b5e874777f61f3b37d7a8b06c33fbc48db464ce0e8788ff4edb4f")); + BOOST_CHECK_EQUAL(fs_bpa.b_exec.str(), std::string("a252070cd26d3b231ab2443b9ba97f57fc72e50cca04a020952e45bc7e2d27a8")); + + ht.dispatch(tpm, test_pacemaker::hs_new_view); + ht.dispatch(tpm, test_pacemaker::hs_vote); //propagating votes on new proposal (decide on first block) + + tpm.set_proposer("bpb"_n); //leader has rotated + tpm.set_leader("bpb"_n); + + tpm.set_current_block_id(ids[1]); //second block + + tpm.beat(); //produce second block and associated proposal + + ht.dispatch(tpm, test_pacemaker::hs_proposal); //send proposal to replicas (prepare on second block) + + qcc_bpb->second->get_state(fs_bpb); + BOOST_CHECK_EQUAL(fs_bpb.b_leaf.str(), std::string("1511035fdcbabdc5e272a3ac19356536252884ed77077cf871ae5029a7502279")); + BOOST_CHECK_EQUAL(fs_bpb.high_qc.proposal_id.str(), std::string("aedf8bb1ee70bd6e743268f7fe0f8171418aa43a68bb9c6e7329ffa856896c09")); + BOOST_CHECK_EQUAL(fs_bpb.b_lock.str(), std::string("4b43fb144a8b5e874777f61f3b37d7a8b06c33fbc48db464ce0e8788ff4edb4f")); + BOOST_CHECK_EQUAL(fs_bpb.b_exec.str(), std::string("a252070cd26d3b231ab2443b9ba97f57fc72e50cca04a020952e45bc7e2d27a8")); + + ht.dispatch(tpm, test_pacemaker::hs_vote); //send votes on proposal (prepareQC on second block) + + ht.dispatch(tpm, test_pacemaker::hs_proposal); //send proposal to replicas (precommit on second block) + + qcc_bpb->second->get_state(fs_bpb); + BOOST_CHECK_EQUAL(fs_bpb.b_leaf.str(), std::string("6462add7d157da87931c859cb689f722003a20f30c0f1408d11b872020903b85")); + BOOST_CHECK_EQUAL(fs_bpb.high_qc.proposal_id.str(), std::string("1511035fdcbabdc5e272a3ac19356536252884ed77077cf871ae5029a7502279")); + BOOST_CHECK_EQUAL(fs_bpb.b_lock.str(), std::string("aedf8bb1ee70bd6e743268f7fe0f8171418aa43a68bb9c6e7329ffa856896c09")); + BOOST_CHECK_EQUAL(fs_bpb.b_exec.str(), std::string("4b43fb144a8b5e874777f61f3b37d7a8b06c33fbc48db464ce0e8788ff4edb4f")); + + ht.dispatch(tpm, test_pacemaker::hs_vote); //propagating votes on new proposal (precommitQC on second block) + + ht.dispatch(tpm, test_pacemaker::hs_proposal); //send proposal to replicas (commit on second block) + + qcc_bpb->second->get_state(fs_bpb); + BOOST_CHECK_EQUAL(fs_bpb.b_leaf.str(), std::string("fd77164bf3898a6a8f27ccff440d17ef6870e75c368fcc93b969066cec70939c")); + BOOST_CHECK_EQUAL(fs_bpb.high_qc.proposal_id.str(), std::string("6462add7d157da87931c859cb689f722003a20f30c0f1408d11b872020903b85")); + BOOST_CHECK_EQUAL(fs_bpb.b_lock.str(), std::string("1511035fdcbabdc5e272a3ac19356536252884ed77077cf871ae5029a7502279")); + BOOST_CHECK_EQUAL(fs_bpb.b_exec.str(), std::string("aedf8bb1ee70bd6e743268f7fe0f8171418aa43a68bb9c6e7329ffa856896c09")); + + ht.dispatch(tpm, test_pacemaker::hs_vote); //propagating votes on new proposal (commitQC on second block) + + ht.dispatch(tpm, test_pacemaker::hs_proposal); //send proposal to replicas (decide on second block) + + qcc_bpb->second->get_state(fs_bpb); + BOOST_CHECK_EQUAL(fs_bpb.b_leaf.str(), std::string("89f468a127dbadd81b59076067238e3e9c313782d7d83141b16d9da4f2c2b078")); + BOOST_CHECK_EQUAL(fs_bpb.high_qc.proposal_id.str(), std::string("fd77164bf3898a6a8f27ccff440d17ef6870e75c368fcc93b969066cec70939c")); + BOOST_CHECK_EQUAL(fs_bpb.b_lock.str(), std::string("6462add7d157da87931c859cb689f722003a20f30c0f1408d11b872020903b85")); + BOOST_CHECK_EQUAL(fs_bpb.b_exec.str(), std::string("1511035fdcbabdc5e272a3ac19356536252884ed77077cf871ae5029a7502279")); + + //check bpa as well + qcc_bpa->second->get_state(fs_bpa); + BOOST_CHECK_EQUAL(fs_bpa.high_qc.proposal_id.str(), std::string("fd77164bf3898a6a8f27ccff440d17ef6870e75c368fcc93b969066cec70939c")); + BOOST_CHECK_EQUAL(fs_bpa.b_lock.str(), std::string("6462add7d157da87931c859cb689f722003a20f30c0f1408d11b872020903b85")); + BOOST_CHECK_EQUAL(fs_bpa.b_exec.str(), std::string("1511035fdcbabdc5e272a3ac19356536252884ed77077cf871ae5029a7502279")); + + //check bpc as well + qcc_bpc->second->get_state(fs_bpc); + BOOST_CHECK_EQUAL(fs_bpc.high_qc.proposal_id.str(), std::string("fd77164bf3898a6a8f27ccff440d17ef6870e75c368fcc93b969066cec70939c")); + BOOST_CHECK_EQUAL(fs_bpc.b_lock.str(), std::string("6462add7d157da87931c859cb689f722003a20f30c0f1408d11b872020903b85")); + BOOST_CHECK_EQUAL(fs_bpc.b_exec.str(), std::string("1511035fdcbabdc5e272a3ac19356536252884ed77077cf871ae5029a7502279")); + + BOOST_CHECK_EQUAL(fs_bpa.b_finality_violation.str(), std::string("0000000000000000000000000000000000000000000000000000000000000000")); + +} FC_LOG_AND_RETHROW(); + + BOOST_AUTO_TEST_SUITE_END() diff --git a/libraries/hotstuff/test/test_pacemaker.cpp b/libraries/hotstuff/test/test_pacemaker.cpp index f285dc899f..0491a67639 100644 --- a/libraries/hotstuff/test/test_pacemaker.cpp +++ b/libraries/hotstuff/test/test_pacemaker.cpp @@ -232,7 +232,8 @@ namespace eosio::hotstuff { void test_pacemaker::on_hs_new_block_msg(const hs_new_block_message& msg, name id) { for (const auto& [qcc_name, qcc_ptr] : _qcc_store) { - if (qcc_ptr->get_id_i() != id && is_qc_chain_active(qcc_name) && is_connected(id, qcc_ptr->get_id_i())) + // New Block msg is not propagated by qc_chain, so it has to go to everyone (no is_connected() check) + if (qcc_ptr->get_id_i() != id && is_qc_chain_active(qcc_name)) qcc_ptr->on_hs_new_block_msg(0, msg); } } From 577a7cd4f9b1c5dd70c02f53fc9bfbd1338d3f26 Mon Sep 17 00:00:00 2001 From: fcecin Date: Mon, 25 Sep 2023 14:03:56 -0300 Subject: [PATCH 2/5] Fix new-view message infinite propagation loop in some testcases --- libraries/hotstuff/qc_chain.cpp | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/libraries/hotstuff/qc_chain.cpp b/libraries/hotstuff/qc_chain.cpp index 27a4eef6fb..500f36715f 100644 --- a/libraries/hotstuff/qc_chain.cpp +++ b/libraries/hotstuff/qc_chain.cpp @@ -729,7 +729,12 @@ namespace eosio::hotstuff { _b_leaf = _high_qc.get_proposal_id(); fc_tlog(_logger, " === ${id} _b_leaf updated (update_high_qc) : ${proposal_id}", ("proposal_id", _high_qc.get_proposal_id())("id", _id)); - return true; + + // avoid looping message propagation when receiving a new-view message with a high_qc.get_proposal_id().empty(). + // not sure if high_qc.get_proposal_id().empty() + _high_qc.get_proposal_id().empty() is something that actually ever happens in the real world. + // not sure if high_qc.get_proposal_id().empty() should be tested and always rejected (return false + no _high_qc / _b_leaf update). + // if this returns false, we won't update the get_finality_status information, but I don't think we care about that at all. + return !high_qc.get_proposal_id().empty(); } else { const hs_proposal_message *old_high_qc_prop = get_proposal( _high_qc.get_proposal_id() ); const hs_proposal_message *new_high_qc_prop = get_proposal( high_qc.get_proposal_id() ); From 33b23e6304ff8685d7c4a95eece04bc050b255d6 Mon Sep 17 00:00:00 2001 From: fcecin Date: Tue, 3 Oct 2023 15:47:36 -0300 Subject: [PATCH 3/5] Delete commented-out code --- libraries/hotstuff/qc_chain.cpp | 9 --------- 1 file changed, 9 deletions(-) diff --git a/libraries/hotstuff/qc_chain.cpp b/libraries/hotstuff/qc_chain.cpp index 2359d4c0dd..59ef5d09f1 100644 --- a/libraries/hotstuff/qc_chain.cpp +++ b/libraries/hotstuff/qc_chain.cpp @@ -405,7 +405,6 @@ namespace eosio::hotstuff { bool am_leader = am_i_leader(); -//<<<<<<< HEAD if (am_leader) { if (vote.proposal_id != _current_qc.get_proposal_id()) { send_hs_message_warning(connection_id, hs_message_warning::discarded); // example; to be tuned to actual need @@ -417,14 +416,6 @@ namespace eosio::hotstuff { if (p == nullptr) { if (am_leader) fc_elog(_logger, " *** ${id} couldn't find proposal, vote : ${vote}", ("id",_id)("vote", vote)); -//======= -// if (!am_leader) -// return; -// fc_tlog(_logger, " === Process vote from ${finalizer_key} : current bitset ${value}" , -// ("finalizer_key", vote.finalizer_key)("value", _current_qc.get_active_finalizers_string())); -// // only leader need to take action on votes -// if (vote.proposal_id != _current_qc.get_proposal_id()) { -//>>>>>>> origin/hotstuff_integration send_hs_message_warning(connection_id, hs_message_warning::discarded); // example; to be tuned to actual need return; } From fd61028a67148b447faf891571679008b69802fa Mon Sep 17 00:00:00 2001 From: fcecin Date: Wed, 18 Oct 2023 17:00:16 -0300 Subject: [PATCH 4/5] Improve seen-votes GC --- libraries/hotstuff/qc_chain.cpp | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/libraries/hotstuff/qc_chain.cpp b/libraries/hotstuff/qc_chain.cpp index 59ef5d09f1..17e94e6485 100644 --- a/libraries/hotstuff/qc_chain.cpp +++ b/libraries/hotstuff/qc_chain.cpp @@ -988,11 +988,8 @@ namespace eosio::hotstuff { void qc_chain::gc_proposals(uint64_t cutoff){ //fc_tlog(_logger, " === garbage collection on old data"); - auto sv_end_itr = _seen_votes_store.get().upper_bound(cutoff); - while (_seen_votes_store.get().begin() != sv_end_itr){ - auto itr = _seen_votes_store.get().begin(); - _seen_votes_store.get().erase(itr); - } + auto& seen_votes_index = _seen_votes_store.get(); + seen_votes_index.erase(seen_votes_index.begin(), seen_votes_index.upper_bound(cutoff)); #ifdef QC_CHAIN_SIMPLE_PROPOSAL_STORE ps_height_iterator psh_it = _proposal_stores_by_height.begin(); From 481b936cf9de29e3a3ea59baad4503f239aaeb15 Mon Sep 17 00:00:00 2001 From: fcecin Date: Fri, 10 Nov 2023 13:36:04 -0300 Subject: [PATCH 5/5] Fix potential bug with proposal propagation --- libraries/hotstuff/qc_chain.cpp | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/libraries/hotstuff/qc_chain.cpp b/libraries/hotstuff/qc_chain.cpp index 17e94e6485..92eda18f96 100644 --- a/libraries/hotstuff/qc_chain.cpp +++ b/libraries/hotstuff/qc_chain.cpp @@ -383,8 +383,7 @@ namespace eosio::hotstuff { update(proposal); //propagate this proposal since it was new to us - if (! am_i_leader()) - send_hs_proposal_msg(connection_id, proposal); + send_hs_proposal_msg(connection_id, proposal); for (auto &msg : msgs) { send_hs_vote_msg( std::nullopt, msg );