From e3583307fdc38602d0d2af978b04848aaeb38ffa Mon Sep 17 00:00:00 2001 From: Huang-Ming Huang Date: Wed, 12 Apr 2023 15:16:55 -0500 Subject: [PATCH 01/58] use python_SITELIB --- scripts/CMakeLists.txt | 4 ++-- scripts/postinst | 10 ++++------ scripts/prerm | 10 +++------- 3 files changed, 9 insertions(+), 15 deletions(-) diff --git a/scripts/CMakeLists.txt b/scripts/CMakeLists.txt index a625aa111d..3fb99184e0 100644 --- a/scripts/CMakeLists.txt +++ b/scripts/CMakeLists.txt @@ -4,6 +4,6 @@ configure_file(eosio-tn_down.sh eosio-tn_down.sh COPYONLY) configure_file(eosio-tn_roll.sh eosio-tn_roll.sh COPYONLY) configure_file(eosio-tn_up.sh eosio-tn_up.sh COPYONLY) configure_file(abi_is_json.py abi_is_json.py COPYONLY) -configure_file(postinst .) -configure_file(prerm .) +configure_file(postinst . @ONLY) +configure_file(prerm . @ONLY) configure_file(install_testharness_symlinks.cmake .) diff --git a/scripts/postinst b/scripts/postinst index f31450bf78..3e4cd9d250 100644 --- a/scripts/postinst +++ b/scripts/postinst @@ -1,13 +1,11 @@ #!/bin/sh set -e +Python_SITELIB=$(python3 -c "from distutils import sysconfig;print(sysconfig.get_python_lib(plat_specific=False,standard_lib=False))") +ln -sf @CMAKE_INSTALL_FULL_DATAROOTDIR@/leap_testing/tests/TestHarness $Python_SITELIB/TestHarness -if [ ! -L ${CMAKE_INSTALL_FULL_LIBDIR}/python3/dist-packages/TestHarness ]; then - mkdir -p ${CMAKE_INSTALL_FULL_LIBDIR}/python3/dist-packages - ln -s ../../../share/leap_testing/tests/TestHarness ${CMAKE_INSTALL_FULL_LIBDIR}/python3/dist-packages/TestHarness -fi # leap_testing is part of the package so should already exist by the time postinst runs -if [ ! -L ${CMAKE_INSTALL_FULL_DATAROOTDIR}/leap_testing/bin ]; then - ln -s ../../bin ${CMAKE_INSTALL_FULL_DATAROOTDIR}/leap_testing/bin +if [ ! -L @CMAKE_INSTALL_FULL_DATAROOTDIR@/leap_testing/bin ]; then + ln -s ../../bin @CMAKE_INSTALL_FULL_DATAROOTDIR@/leap_testing/bin fi \ No newline at end of file diff --git a/scripts/prerm b/scripts/prerm index b6b65101ab..146c43271e 100644 --- a/scripts/prerm +++ b/scripts/prerm @@ -1,10 +1,6 @@ #!/bin/sh # Cleanup symbolic links created during postinst -rm -f ${CMAKE_INSTALL_FULL_LIBDIR}/python3/dist-packages/TestHarness -rm -f ${CMAKE_INSTALL_FULL_DATAROOTDIR}/leap_testing/bin - -# Attempt to clean up directories that may have been created during postinst -# Also may have already existed, so it is acceptable to leave them in place if the rmdir fails -rmdir --ignore-fail-on-non-empty ${CMAKE_INSTALL_FULL_LIBDIR}/python3/dist-packages -rmdir --ignore-fail-on-non-empty ${CMAKE_INSTALL_FULL_LIBDIR}/python3 +Python_SITELIB=$(python3 -c "from distutils import sysconfig;print(sysconfig.get_python_lib(plat_specific=False,standard_lib=False))") +rm -f $Python_SITELIB/TestHarness +rm -f @CMAKE_INSTALL_FULL_DATAROOTDIR@/leap_testing/bin From 9c856538a7eac54d7c63a48e238472c5e7d62218 Mon Sep 17 00:00:00 2001 From: 766C6164 Date: Thu, 13 Apr 2023 18:21:27 -0400 Subject: [PATCH 02/58] Moved snapshot code out of producer plugin --- libraries/chain/CMakeLists.txt | 1 + .../include/eosio/chain/pending_snapshot.hpp | 63 +++++ .../include/eosio/chain}/snapshot_db_json.hpp | 42 ++-- .../eosio/chain/snapshot_scheduler.hpp | 145 +++++++++++ libraries/chain/snapshot_scheduler.cpp | 236 ++++++++++++++++++ .../producer_api_plugin.cpp | 6 +- plugins/producer_plugin/CMakeLists.txt | 1 - .../producer_plugin/pending_snapshot.hpp | 42 ---- .../eosio/producer_plugin/producer_plugin.hpp | 46 +--- .../producer_plugin/snapshot_scheduler.hpp | 213 ---------------- plugins/producer_plugin/pending_snapshot.cpp | 28 --- plugins/producer_plugin/producer_plugin.cpp | 133 +--------- .../test/test_snapshot_scheduler.cpp | 28 +-- tests/test_snapshot_information.cpp | 13 +- 14 files changed, 509 insertions(+), 488 deletions(-) create mode 100644 libraries/chain/include/eosio/chain/pending_snapshot.hpp rename {plugins/producer_plugin/include/eosio/producer_plugin => libraries/chain/include/eosio/chain}/snapshot_db_json.hpp (66%) create mode 100644 libraries/chain/include/eosio/chain/snapshot_scheduler.hpp create mode 100644 libraries/chain/snapshot_scheduler.cpp delete mode 100644 plugins/producer_plugin/include/eosio/producer_plugin/pending_snapshot.hpp delete mode 100644 plugins/producer_plugin/include/eosio/producer_plugin/snapshot_scheduler.hpp delete mode 100644 plugins/producer_plugin/pending_snapshot.cpp diff --git a/libraries/chain/CMakeLists.txt b/libraries/chain/CMakeLists.txt index 6de5d10e3d..7796205658 100644 --- a/libraries/chain/CMakeLists.txt +++ b/libraries/chain/CMakeLists.txt @@ -107,6 +107,7 @@ add_library( eosio_chain abi_serializer.cpp asset.cpp snapshot.cpp + snapshot_scheduler.cpp deep_mind.cpp ${CHAIN_EOSVMOC_SOURCES} diff --git a/libraries/chain/include/eosio/chain/pending_snapshot.hpp b/libraries/chain/include/eosio/chain/pending_snapshot.hpp new file mode 100644 index 0000000000..fa7af9589f --- /dev/null +++ b/libraries/chain/include/eosio/chain/pending_snapshot.hpp @@ -0,0 +1,63 @@ +#pragma once + +#include +#include + +#include + +namespace eosio { +namespace chain { + +namespace bfs = boost::filesystem; + +template +class pending_snapshot { +public: + using next_t = eosio::chain::next_function; + + pending_snapshot(const chain::block_id_type& block_id, next_t& next, std::string pending_path, std::string final_path) + : block_id(block_id), next(next), pending_path(pending_path), final_path(final_path) {} + + uint32_t get_height() const { + return chain::block_header::num_from_id(block_id); + } + + static bfs::path get_final_path(const chain::block_id_type& block_id, const bfs::path& snapshots_dir) { + return snapshots_dir / fc::format_string("snapshot-${id}.bin", fc::mutable_variant_object()("id", block_id)); + } + + static bfs::path get_pending_path(const chain::block_id_type& block_id, const bfs::path& snapshots_dir) { + return snapshots_dir / fc::format_string(".pending-snapshot-${id}.bin", fc::mutable_variant_object()("id", block_id)); + } + + static bfs::path get_temp_path(const chain::block_id_type& block_id, const bfs::path& snapshots_dir) { + return snapshots_dir / fc::format_string(".incomplete-snapshot-${id}.bin", fc::mutable_variant_object()("id", block_id)); + } + + T finalize(const chain::controller& chain) const { + auto block_ptr = chain.fetch_block_by_id(block_id); + auto in_chain = (bool) block_ptr; + boost::system::error_code ec; + + if(!in_chain) { + bfs::remove(bfs::path(pending_path), ec); + EOS_THROW(chain::snapshot_finalization_exception, + "Snapshotted block was forked out of the chain. ID: ${block_id}", + ("block_id", block_id)); + } + + bfs::rename(bfs::path(pending_path), bfs::path(final_path), ec); + EOS_ASSERT(!ec, chain::snapshot_finalization_exception, + "Unable to finalize valid snapshot of block number ${bn}: [code: ${ec}] ${message}", + ("bn", get_height())("ec", ec.value())("message", ec.message())); + + return {block_id, block_ptr->block_num(), block_ptr->timestamp, chain::chain_snapshot_header::current_version, final_path}; + } + + chain::block_id_type block_id; + std::string pending_path; + std::string final_path; + next_t next; +}; +}// namespace chain +}// namespace eosio diff --git a/plugins/producer_plugin/include/eosio/producer_plugin/snapshot_db_json.hpp b/libraries/chain/include/eosio/chain/snapshot_db_json.hpp similarity index 66% rename from plugins/producer_plugin/include/eosio/producer_plugin/snapshot_db_json.hpp rename to libraries/chain/include/eosio/chain/snapshot_db_json.hpp index db9e84837a..3d50b829ba 100644 --- a/plugins/producer_plugin/include/eosio/producer_plugin/snapshot_db_json.hpp +++ b/libraries/chain/include/eosio/chain/snapshot_db_json.hpp @@ -3,19 +3,20 @@ #include #include -#include - #include #include #include namespace eosio { +namespace chain { + +namespace bfs = boost::filesystem; /// this class designed to serialize/deserialize snapshot schedule to a filesystem so it can be restored after restart class snapshot_db_json { public: - snapshot_db_json() = default; - ~snapshot_db_json() = default; + snapshot_db_json() = default; + ~snapshot_db_json() = default; void set_path(bfs::path path) { db_path = std::move(path); @@ -25,17 +26,18 @@ class snapshot_db_json { return db_path / "snapshot-schedule.json"; } - const snapshot_db_json& operator>>(std::vector& sr) { + template + const snapshot_db_json& operator>>(std::vector& sr) { boost::property_tree::ptree root; try { std::ifstream file(get_json_path().string()); - file.exceptions(std::istream::failbit|std::istream::eofbit); + file.exceptions(std::istream::failbit | std::istream::eofbit); boost::property_tree::read_json(file, root); - + // parse ptree for(boost::property_tree::ptree::value_type& req: root.get_child("snapshot_requests")) { - producer_plugin::snapshot_schedule_information ssi; + F ssi; ssi.snapshot_request_id = req.second.get("snapshot_request_id"); ssi.snapshot_description = req.second.get("snapshot_description"); ssi.block_spacing = req.second.get("block_spacing"); @@ -43,17 +45,17 @@ class snapshot_db_json { ssi.end_block_num = req.second.get("end_block_num"); sr.push_back(ssi); } - } - catch (std::ifstream::failure & e) { - elog( "unable to restore snapshots schedule from filesystem ${jsonpath}, details: ${details}", - ("jsonpath", get_json_path().string()) ("details",e.what()) ); - appbase::app().quit(); + } catch(std::ifstream::failure& e) { + elog("unable to restore snapshots schedule from filesystem ${jsonpath}, details: ${details}", + ("jsonpath", get_json_path().string())("details", e.what())); + // appbase::app().quit(); } return *this; } - const snapshot_db_json& operator<<(std::vector& sr) const { + template + const snapshot_db_json& operator<<(std::vector& sr) const { boost::property_tree::ptree root; boost::property_tree::ptree node_srs; @@ -71,13 +73,12 @@ class snapshot_db_json { try { std::ofstream file(get_json_path().string()); - file.exceptions(std::istream::failbit|std::istream::eofbit); + file.exceptions(std::istream::failbit | std::istream::eofbit); boost::property_tree::write_json(file, root); - } - catch (std::ofstream::failure & e) { - elog( "unable to store snapshots schedule to filesystem to ${jsonpath}, details: ${details}", - ("jsonpath", get_json_path().string()) ("details", e.what()) ); - appbase::app().quit(); + } catch(std::ofstream::failure& e) { + elog("unable to store snapshots schedule to filesystem to ${jsonpath}, details: ${details}", + ("jsonpath", get_json_path().string())("details", e.what())); + // appbase::app().quit(); } return *this; @@ -87,4 +88,5 @@ class snapshot_db_json { bfs::path db_path; }; +}// namespace chain }// namespace eosio diff --git a/libraries/chain/include/eosio/chain/snapshot_scheduler.hpp b/libraries/chain/include/eosio/chain/snapshot_scheduler.hpp new file mode 100644 index 0000000000..dea1b1e67d --- /dev/null +++ b/libraries/chain/include/eosio/chain/snapshot_scheduler.hpp @@ -0,0 +1,145 @@ +#pragma once + +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include + + +namespace eosio { +namespace chain { + +namespace bmi = boost::multi_index; +namespace bfs = boost::filesystem; + +class snapshot_scheduler { +public: + struct snapshot_information { + chain::block_id_type head_block_id; + uint32_t head_block_num; + fc::time_point head_block_time; + uint32_t version; + std::string snapshot_name; + }; + + struct snapshot_request_information { + uint32_t block_spacing = 0; + uint32_t start_block_num = 0; + uint32_t end_block_num = 0; + std::string snapshot_description = ""; + }; + + struct snapshot_request_id_information { + uint32_t snapshot_request_id = 0; + }; + + struct snapshot_schedule_result : public snapshot_request_id_information, public snapshot_request_information { + }; + + struct snapshot_schedule_information : public snapshot_request_id_information, public snapshot_request_information { + std::vector pending_snapshots; + }; + + struct get_snapshot_requests_result { + std::vector snapshot_requests; + }; + + template + using next_function = eosio::chain::next_function; + + struct by_height; + + using pending_snapshot_index = bmi::multi_index_container< + pending_snapshot, + indexed_by< + bmi::hashed_unique, BOOST_MULTI_INDEX_MEMBER(pending_snapshot, block_id_type, block_id)>, + bmi::ordered_non_unique, BOOST_MULTI_INDEX_CONST_MEM_FUN(pending_snapshot, uint32_t, get_height)>>>; + + +private: + struct by_snapshot_id; + struct by_snapshot_value; + struct as_vector; + + using snapshot_requests = bmi::multi_index_container< + snapshot_scheduler::snapshot_schedule_information, + indexed_by< + bmi::hashed_unique, BOOST_MULTI_INDEX_MEMBER(snapshot_scheduler::snapshot_request_id_information, uint32_t, snapshot_request_id)>, + bmi::random_access>, + bmi::ordered_unique, + composite_key>>>; + snapshot_requests _snapshot_requests; + snapshot_db_json _snapshot_db; + pending_snapshot_index _pending_snapshot_index; + chain::controller* _chain = NULL; + + uint32_t _snapshot_id = 0; + uint32_t _inflight_sid = 0; + + // path to write the snapshots to + bfs::path _snapshots_dir; + + void x_serialize() { + auto& vec = _snapshot_requests.get(); + std::vector sr(vec.begin(), vec.end()); + _snapshot_db << sr; + } + +public: + snapshot_scheduler() = default; + + // snapshot scheduler listener + void on_start_block(uint32_t height); + + // to promote pending snapshots + void on_irreversible_block(const signed_block_ptr& lib); + + // snapshot scheduler handlers + snapshot_scheduler::snapshot_schedule_result schedule_snapshot(const snapshot_scheduler::snapshot_request_information& sri); + snapshot_scheduler::snapshot_schedule_result unschedule_snapshot(uint32_t sri); + snapshot_scheduler::get_snapshot_requests_result get_snapshot_requests(); + + // initialize with storage + void set_db_path(bfs::path db_path); + + // set snapshot path + void set_snapshots_path(bfs::path sn_path); + + // set controller + void set_controller(chain::controller* chain) { + _chain = chain; + } + + // add pending snapshot info to inflight snapshot request + void add_pending_snapshot_info(const snapshot_scheduler::snapshot_information& si); + + // execute snapshot + void execute_snapshot(uint32_t srid); + + // former producer_plugin snapshot fn + void create_snapshot(snapshot_scheduler::next_function next); +}; +}// namespace chain +}// namespace eosio + +FC_REFLECT(eosio::chain::snapshot_scheduler::snapshot_information, (head_block_id)(head_block_num)(head_block_time)(version)(snapshot_name)) +FC_REFLECT(eosio::chain::snapshot_scheduler::snapshot_request_information, (block_spacing)(start_block_num)(end_block_num)(snapshot_description)) +FC_REFLECT(eosio::chain::snapshot_scheduler::snapshot_request_id_information, (snapshot_request_id)) +FC_REFLECT(eosio::chain::snapshot_scheduler::get_snapshot_requests_result, (snapshot_requests)) +FC_REFLECT_DERIVED(eosio::chain::snapshot_scheduler::snapshot_schedule_information, (eosio::chain::snapshot_scheduler::snapshot_request_id_information)(eosio::chain::snapshot_scheduler::snapshot_request_information), (pending_snapshots)) +FC_REFLECT_DERIVED(eosio::chain::snapshot_scheduler::snapshot_schedule_result, (eosio::chain::snapshot_scheduler::snapshot_request_id_information)(eosio::chain::snapshot_scheduler::snapshot_request_information), ) diff --git a/libraries/chain/snapshot_scheduler.cpp b/libraries/chain/snapshot_scheduler.cpp new file mode 100644 index 0000000000..74ffb1f5de --- /dev/null +++ b/libraries/chain/snapshot_scheduler.cpp @@ -0,0 +1,236 @@ +#include +#include +#include +#include +#include + +namespace eosio { +namespace chain { + +// snapshot_scheduler_listener +void snapshot_scheduler::on_start_block(uint32_t height) { + bool serialize_needed = false; + bool snapshot_executed = false; + + auto execute_snapshot_with_log = [this, height, &snapshot_executed](const auto& req) { + // one snapshot per height + if(!snapshot_executed) { + dlog("snapshot scheduler creating a snapshot from the request [start_block_num:${start_block_num}, end_block_num=${end_block_num}, block_spacing=${block_spacing}], height=${height}", + ("start_block_num", req.start_block_num)("end_block_num", req.end_block_num)("block_spacing", req.block_spacing)("height", height)); + + execute_snapshot(req.snapshot_request_id); + snapshot_executed = true; + } + }; + + std::vector unschedule_snapshot_request_ids; + for(const auto& req: _snapshot_requests.get<0>()) { + // -1 since its called from start block + bool recurring_snapshot = req.block_spacing && (height >= req.start_block_num + 1) && (!((height - req.start_block_num - 1) % req.block_spacing)); + bool onetime_snapshot = (!req.block_spacing) && (height == req.start_block_num + 1); + + // assume "asap" for snapshot with missed/zero start, it can have spacing + if(!req.start_block_num) { + // update start_block_num with current height only if this is recurring + // if non recurring, will be executed and unscheduled + if(req.block_spacing && height) { + auto& snapshot_by_id = _snapshot_requests.get(); + auto it = snapshot_by_id.find(req.snapshot_request_id); + _snapshot_requests.modify(it, [&height](auto& p) { p.start_block_num = height - 1; }); + serialize_needed = true; + } + execute_snapshot_with_log(req); + } else if(recurring_snapshot || onetime_snapshot) { + execute_snapshot_with_log(req); + } + + // cleanup - remove expired (or invalid) request + if((!req.start_block_num && !req.block_spacing) || + (!req.block_spacing && height >= (req.start_block_num + 1)) || + (req.end_block_num > 0 && height >= (req.end_block_num + 1))) { + unschedule_snapshot_request_ids.push_back(req.snapshot_request_id); + } + } + + for(const auto& i: unschedule_snapshot_request_ids) { + unschedule_snapshot(i); + } + + // store db to filesystem + if(serialize_needed) x_serialize(); +} + +void snapshot_scheduler::on_irreversible_block(const signed_block_ptr& lib) { + auto& snapshots_by_height = _pending_snapshot_index.get(); + uint32_t lib_height = lib->block_num(); + + while(!snapshots_by_height.empty() && snapshots_by_height.begin()->get_height() <= lib_height) { + const auto& pending = snapshots_by_height.begin(); + auto next = pending->next; + + try { + next(pending->finalize(*_chain)); + } + CATCH_AND_CALL(next); + + snapshots_by_height.erase(snapshots_by_height.begin()); + } +} + +snapshot_scheduler::snapshot_schedule_result snapshot_scheduler::schedule_snapshot(const snapshot_scheduler::snapshot_request_information& sri) { + auto& snapshot_by_value = _snapshot_requests.get(); + auto existing = snapshot_by_value.find(std::make_tuple(sri.block_spacing, sri.start_block_num, sri.end_block_num)); + EOS_ASSERT(existing == snapshot_by_value.end(), chain::duplicate_snapshot_request, "Duplicate snapshot request"); + + if(sri.end_block_num > 0) { + // if "end" is specified, it should be greater then start + EOS_ASSERT(sri.start_block_num <= sri.end_block_num, chain::invalid_snapshot_request, "End block number should be greater or equal to start block number"); + // if also block_spacing specified, check it + if(sri.block_spacing > 0) { + EOS_ASSERT(sri.start_block_num + sri.block_spacing <= sri.end_block_num, chain::invalid_snapshot_request, "Block spacing exceeds defined by start and end range"); + } + } + + _snapshot_requests.emplace(snapshot_scheduler::snapshot_schedule_information{{_snapshot_id++}, {sri.block_spacing, sri.start_block_num, sri.end_block_num, sri.snapshot_description}, {}}); + x_serialize(); + + // returning snapshot_schedule_result + return snapshot_scheduler::snapshot_schedule_result{{_snapshot_id - 1}, {sri.block_spacing, sri.start_block_num, sri.end_block_num, sri.snapshot_description}}; +} + +snapshot_scheduler::snapshot_schedule_result snapshot_scheduler::unschedule_snapshot(uint32_t sri) { + auto& snapshot_by_id = _snapshot_requests.get(); + auto existing = snapshot_by_id.find(sri); + EOS_ASSERT(existing != snapshot_by_id.end(), chain::snapshot_request_not_found, "Snapshot request not found"); + + snapshot_scheduler::snapshot_schedule_result result{{existing->snapshot_request_id}, {existing->block_spacing, existing->start_block_num, existing->end_block_num, existing->snapshot_description}}; + _snapshot_requests.erase(existing); + x_serialize(); + + // returning snapshot_schedule_result + return result; +} + +snapshot_scheduler::get_snapshot_requests_result snapshot_scheduler::get_snapshot_requests() { + snapshot_scheduler::get_snapshot_requests_result result; + auto& asvector = _snapshot_requests.get(); + result.snapshot_requests.reserve(asvector.size()); + result.snapshot_requests.insert(result.snapshot_requests.begin(), asvector.begin(), asvector.end()); + return result; +} + +void snapshot_scheduler::set_db_path(bfs::path db_path) { + _snapshot_db.set_path(std::move(db_path)); + // init from db + if(fc::exists(_snapshot_db.get_json_path())) { + std::vector sr; + _snapshot_db >> sr; + // if db read succeeded, clear/load + _snapshot_requests.get().clear(); + _snapshot_requests.insert(sr.begin(), sr.end()); + } +} + +void snapshot_scheduler::set_snapshots_path(bfs::path sn_path) { + _snapshots_dir = std::move(sn_path); +} + +void snapshot_scheduler::add_pending_snapshot_info(const snapshot_scheduler::snapshot_information& si) { + auto& snapshot_by_id = _snapshot_requests.get(); + auto snapshot_req = snapshot_by_id.find(_inflight_sid); + if(snapshot_req != snapshot_by_id.end()) { + _snapshot_requests.modify(snapshot_req, [&si](auto& p) { + p.pending_snapshots.emplace_back(si); + }); + } +} + +void snapshot_scheduler::execute_snapshot(uint32_t srid) { + _inflight_sid = srid; + auto next = [srid, this](const chain::next_function_variant& result) { + if(std::holds_alternative(result)) { + try { + std::get(result)->dynamic_rethrow_exception(); + } catch(const fc::exception& e) { + elog("snapshot creation error: ${details}", ("details", e.to_detail_string())); + //appbase::app().quit(); + } catch(const std::exception& e) { + elog("snapshot creation error: ${details}", ("details", e.what())); + //appbase::app().quit(); + } + } else { + // success, snapshot finalized + auto snapshot_info = std::get(result); + auto& snapshot_by_id = _snapshot_requests.get(); + auto snapshot_req = snapshot_by_id.find(srid); + + if(snapshot_req != snapshot_by_id.end()) { + _snapshot_requests.modify(snapshot_req, [&](auto& p) { + auto& pending = p.pending_snapshots; + auto it = std::remove_if(pending.begin(), pending.end(), [&snapshot_info](const snapshot_scheduler::snapshot_information& s) { return s.head_block_num <= snapshot_info.head_block_num; }); + pending.erase(it, pending.end()); + }); + } + } + }; + create_snapshot(next); +} + +void snapshot_scheduler::create_snapshot(snapshot_scheduler::next_function next) { + auto head_id = _chain->head_block_id(); + const auto head_block_num = _chain->head_block_num(); + const auto head_block_time = _chain->head_block_time(); + const auto& snapshot_path = pending_snapshot::get_final_path(head_id, _snapshots_dir); + const auto& temp_path = pending_snapshot::get_temp_path(head_id, _snapshots_dir); + + // maintain legacy exception if the snapshot exists + if(fc::is_regular_file(snapshot_path)) { + auto ex = snapshot_exists_exception(FC_LOG_MESSAGE(error, "snapshot named ${name} already exists", ("name", _snapshots_dir.generic_string()))); + next(ex.dynamic_copy_exception()); + return; + } + + auto write_snapshot = [&](const bfs::path& p) -> void { + /* + auto reschedule = fc::make_scoped_exit([this](){ + my->schedule_production_loop(); + }); + + if (_chain.is_building_block()) { + // abort the pending block + my->abort_block(); + } else { + reschedule.cancel(); + } +*/ + bfs::create_directory(p.parent_path()); + + // create the snapshot + auto snap_out = std::ofstream(p.generic_string(), (std::ios::out | std::ios::binary)); + auto writer = std::make_shared(snap_out); + _chain->write_snapshot(writer); + writer->finalize(); + snap_out.flush(); + snap_out.close(); + }; + + // If in irreversible mode, create snapshot and return path to snapshot immediately. + if(_chain->get_read_mode() == db_read_mode::IRREVERSIBLE) { + try { + write_snapshot(temp_path); + + boost::system::error_code ec; + bfs::rename(temp_path, snapshot_path, ec); + EOS_ASSERT(!ec, snapshot_finalization_exception, + "Unable to finalize valid snapshot of block number ${bn}: [code: ${ec}] ${message}", + ("bn", head_block_num)("ec", ec.value())("message", ec.message())); + + next(snapshot_information{head_id, head_block_num, head_block_time, chain_snapshot_header::current_version, snapshot_path.generic_string()}); + } + CATCH_AND_CALL(next); + return; + } +} + +}// namespace chain +}// namespace eosio diff --git a/plugins/producer_api_plugin/producer_api_plugin.cpp b/plugins/producer_api_plugin/producer_api_plugin.cpp index 185655de4f..9d477ed425 100644 --- a/plugins/producer_api_plugin/producer_api_plugin.cpp +++ b/plugins/producer_api_plugin/producer_api_plugin.cpp @@ -138,12 +138,12 @@ void producer_api_plugin::plugin_startup() { INVOKE_V_R(producer, remove_greylist_accounts, producer_plugin::greylist_params), 201), CALL_WITH_400(producer, producer, set_whitelist_blacklist, INVOKE_V_R(producer, set_whitelist_blacklist, producer_plugin::whitelist_blacklist), 201), - CALL_ASYNC(producer, producer, create_snapshot, producer_plugin::snapshot_information, + CALL_ASYNC(producer, producer, create_snapshot, chain::snapshot_scheduler::snapshot_information, INVOKE_R_V_ASYNC(producer, create_snapshot), 201), CALL_WITH_400(producer, producer, schedule_snapshot, - INVOKE_R_R_II(producer, schedule_snapshot, producer_plugin::snapshot_request_information), 201), + INVOKE_R_R_II(producer, schedule_snapshot, chain::snapshot_scheduler::snapshot_request_information), 201), CALL_WITH_400(producer, producer, unschedule_snapshot, - INVOKE_R_R(producer, unschedule_snapshot, producer_plugin::snapshot_request_id_information), 201), + INVOKE_R_R(producer, unschedule_snapshot, chain::snapshot_scheduler::snapshot_request_id_information), 201), CALL_WITH_400(producer, producer, get_integrity_hash, INVOKE_R_V(producer, get_integrity_hash), 201), CALL_WITH_400(producer, producer, schedule_protocol_feature_activations, diff --git a/plugins/producer_plugin/CMakeLists.txt b/plugins/producer_plugin/CMakeLists.txt index a64518f7c9..1353284054 100644 --- a/plugins/producer_plugin/CMakeLists.txt +++ b/plugins/producer_plugin/CMakeLists.txt @@ -2,7 +2,6 @@ file(GLOB HEADERS "include/eosio/producer_plugin/*.hpp") add_library( producer_plugin producer_plugin.cpp - pending_snapshot.cpp ${HEADERS} ) diff --git a/plugins/producer_plugin/include/eosio/producer_plugin/pending_snapshot.hpp b/plugins/producer_plugin/include/eosio/producer_plugin/pending_snapshot.hpp deleted file mode 100644 index 02814ca824..0000000000 --- a/plugins/producer_plugin/include/eosio/producer_plugin/pending_snapshot.hpp +++ /dev/null @@ -1,42 +0,0 @@ -#pragma once - -#include - -namespace eosio { - -class pending_snapshot { -public: - using next_t = producer_plugin::next_function; - - pending_snapshot(const chain::block_id_type& block_id, next_t& next, std::string pending_path, std::string final_path) - : block_id(block_id) - , next(next) - , pending_path(pending_path) - , final_path(final_path) - {} - - uint32_t get_height() const { - return chain::block_header::num_from_id(block_id); - } - - static bfs::path get_final_path(const chain::block_id_type& block_id, const bfs::path& snapshots_dir) { - return snapshots_dir / fc::format_string("snapshot-${id}.bin", fc::mutable_variant_object()("id", block_id)); - } - - static bfs::path get_pending_path(const chain::block_id_type& block_id, const bfs::path& snapshots_dir) { - return snapshots_dir / fc::format_string(".pending-snapshot-${id}.bin", fc::mutable_variant_object()("id", block_id)); - } - - static bfs::path get_temp_path(const chain::block_id_type& block_id, const bfs::path& snapshots_dir) { - return snapshots_dir / fc::format_string(".incomplete-snapshot-${id}.bin", fc::mutable_variant_object()("id", block_id)); - } - - producer_plugin::snapshot_information finalize( const chain::controller& chain ) const; - - chain::block_id_type block_id; - next_t next; - std::string pending_path; - std::string final_path; -}; - -} // namespace eosio diff --git a/plugins/producer_plugin/include/eosio/producer_plugin/producer_plugin.hpp b/plugins/producer_plugin/include/eosio/producer_plugin/producer_plugin.hpp index 82a9284581..eac8c4418c 100644 --- a/plugins/producer_plugin/include/eosio/producer_plugin/producer_plugin.hpp +++ b/plugins/producer_plugin/include/eosio/producer_plugin/producer_plugin.hpp @@ -2,6 +2,7 @@ #include #include +#include #include #include @@ -73,37 +74,6 @@ class producer_plugin : public appbase::plugin { chain::digest_type integrity_hash; }; - struct snapshot_information { - chain::block_id_type head_block_id; - uint32_t head_block_num; - fc::time_point head_block_time; - uint32_t version; - std::string snapshot_name; - }; - - struct snapshot_request_information { - uint32_t block_spacing = 0; - uint32_t start_block_num = 0; - uint32_t end_block_num = 0; - std::string snapshot_description = ""; - - }; - - struct snapshot_request_id_information { - uint32_t snapshot_request_id = 0; - }; - - struct snapshot_schedule_result : public snapshot_request_id_information, public snapshot_request_information { - }; - - struct snapshot_schedule_information : public snapshot_request_id_information, public snapshot_request_information { - std::vector pending_snapshots; - }; - - struct get_snapshot_requests_result { - std::vector snapshot_requests; - }; - struct scheduled_protocol_feature_activations { std::vector protocol_features_to_activate; }; @@ -160,10 +130,10 @@ class producer_plugin : public appbase::plugin { integrity_hash_information get_integrity_hash() const; - void create_snapshot(next_function next); - snapshot_schedule_result schedule_snapshot(const snapshot_request_information& schedule); - snapshot_schedule_result unschedule_snapshot(const snapshot_request_id_information& schedule); - get_snapshot_requests_result get_snapshot_requests() const; + void create_snapshot(next_function next); + chain::snapshot_scheduler::snapshot_schedule_result schedule_snapshot(const chain::snapshot_scheduler::snapshot_request_information& schedule); + chain::snapshot_scheduler::snapshot_schedule_result unschedule_snapshot(const chain::snapshot_scheduler::snapshot_request_id_information& schedule); + chain::snapshot_scheduler::get_snapshot_requests_result get_snapshot_requests() const; scheduled_protocol_feature_activations get_scheduled_protocol_feature_activations() const; void schedule_protocol_feature_activations(const scheduled_protocol_feature_activations& schedule); @@ -221,12 +191,6 @@ FC_REFLECT(eosio::producer_plugin::runtime_options, (max_transaction_time)(max_i FC_REFLECT(eosio::producer_plugin::greylist_params, (accounts)); FC_REFLECT(eosio::producer_plugin::whitelist_blacklist, (actor_whitelist)(actor_blacklist)(contract_whitelist)(contract_blacklist)(action_blacklist)(key_blacklist) ) FC_REFLECT(eosio::producer_plugin::integrity_hash_information, (head_block_id)(integrity_hash)) -FC_REFLECT(eosio::producer_plugin::snapshot_information, (head_block_id)(head_block_num)(head_block_time)(version)(snapshot_name)) -FC_REFLECT(eosio::producer_plugin::snapshot_request_information, (block_spacing)(start_block_num)(end_block_num)(snapshot_description)) -FC_REFLECT(eosio::producer_plugin::snapshot_request_id_information, (snapshot_request_id)) -FC_REFLECT(eosio::producer_plugin::get_snapshot_requests_result, (snapshot_requests)) -FC_REFLECT_DERIVED(eosio::producer_plugin::snapshot_schedule_information, (eosio::producer_plugin::snapshot_request_id_information)(eosio::producer_plugin::snapshot_request_information), (pending_snapshots)) -FC_REFLECT_DERIVED(eosio::producer_plugin::snapshot_schedule_result, (eosio::producer_plugin::snapshot_request_id_information)(eosio::producer_plugin::snapshot_request_information),) FC_REFLECT(eosio::producer_plugin::scheduled_protocol_feature_activations, (protocol_features_to_activate)) FC_REFLECT(eosio::producer_plugin::get_supported_protocol_features_params, (exclude_disabled)(exclude_unactivatable)) FC_REFLECT(eosio::producer_plugin::get_account_ram_corrections_params, (lower_bound)(upper_bound)(limit)(reverse)) diff --git a/plugins/producer_plugin/include/eosio/producer_plugin/snapshot_scheduler.hpp b/plugins/producer_plugin/include/eosio/producer_plugin/snapshot_scheduler.hpp deleted file mode 100644 index 2dd86b85fe..0000000000 --- a/plugins/producer_plugin/include/eosio/producer_plugin/snapshot_scheduler.hpp +++ /dev/null @@ -1,213 +0,0 @@ -#pragma once - -#include - -#include -#include -#include -#include -#include -#include -#include - -#include -#include -#include -#include -#include - -namespace eosio { - -namespace bmi = boost::multi_index; - -class snapshot_scheduler { -private: - struct by_snapshot_id; - struct by_snapshot_value; - struct as_vector; - - using snapshot_requests = bmi::multi_index_container< - producer_plugin::snapshot_schedule_information, - indexed_by< - bmi::hashed_unique, BOOST_MULTI_INDEX_MEMBER(producer_plugin::snapshot_request_id_information, uint32_t, snapshot_request_id)>, - bmi::random_access>, - bmi::ordered_unique, - composite_key>>>; - snapshot_requests _snapshot_requests; - snapshot_db_json _snapshot_db; - uint32_t _snapshot_id = 0; - uint32_t _inflight_sid = 0; - std::function)> _create_snapshot; - - void x_serialize() { - auto& vec = _snapshot_requests.get(); - std::vector sr(vec.begin(), vec.end()); - _snapshot_db << sr; - } - -public: - snapshot_scheduler() = default; - - // snapshot_scheduler_listener - void on_start_block(uint32_t height) { - bool serialize_needed = false; - bool snapshot_executed = false; - - auto execute_snapshot_with_log = [this, height, &snapshot_executed](const auto & req) { - // one snapshot per height - if (!snapshot_executed) { - dlog("snapshot scheduler creating a snapshot from the request [start_block_num:${start_block_num}, end_block_num=${end_block_num}, block_spacing=${block_spacing}], height=${height}", - ("start_block_num", req.start_block_num) - ("end_block_num", req.end_block_num) - ("block_spacing", req.block_spacing) - ("height", height)); - - execute_snapshot(req.snapshot_request_id); - snapshot_executed = true; - } - }; - - std::vector unschedule_snapshot_request_ids; - for(const auto& req: _snapshot_requests.get<0>()) { - // -1 since its called from start block - bool recurring_snapshot = req.block_spacing && (height >= req.start_block_num + 1) && (!((height - req.start_block_num - 1) % req.block_spacing)); - bool onetime_snapshot = (!req.block_spacing) && (height == req.start_block_num + 1); - - // assume "asap" for snapshot with missed/zero start, it can have spacing - if(!req.start_block_num) { - // update start_block_num with current height only if this is recurring - // if non recurring, will be executed and unscheduled - if (req.block_spacing && height) { - auto& snapshot_by_id = _snapshot_requests.get(); - auto it = snapshot_by_id.find(req.snapshot_request_id); - _snapshot_requests.modify(it, [&height](auto& p) { p.start_block_num = height - 1; }); - serialize_needed = true; - } - execute_snapshot_with_log(req); - } else if(recurring_snapshot || onetime_snapshot) { - execute_snapshot_with_log(req); - } - - // cleanup - remove expired (or invalid) request - if((!req.start_block_num && !req.block_spacing) || - (!req.block_spacing && height >= (req.start_block_num + 1)) || - (req.end_block_num > 0 && height >= (req.end_block_num + 1))) { - unschedule_snapshot_request_ids.push_back(req.snapshot_request_id); - } - } - - for( const auto& i : unschedule_snapshot_request_ids ) { - unschedule_snapshot(i); - } - - // store db to filesystem - if (serialize_needed) x_serialize(); - } - - // snapshot_scheduler_handler - producer_plugin::snapshot_schedule_result schedule_snapshot(const producer_plugin::snapshot_request_information& sri) { - auto& snapshot_by_value = _snapshot_requests.get(); - auto existing = snapshot_by_value.find(std::make_tuple(sri.block_spacing, sri.start_block_num, sri.end_block_num)); - EOS_ASSERT(existing == snapshot_by_value.end(), chain::duplicate_snapshot_request, "Duplicate snapshot request"); - - if(sri.end_block_num > 0) { - // if "end" is specified, it should be greater then start - EOS_ASSERT(sri.start_block_num <= sri.end_block_num, chain::invalid_snapshot_request, "End block number should be greater or equal to start block number"); - // if also block_spacing specified, check it - if(sri.block_spacing > 0) { - EOS_ASSERT(sri.start_block_num + sri.block_spacing <= sri.end_block_num, chain::invalid_snapshot_request, "Block spacing exceeds defined by start and end range"); - } - } - - _snapshot_requests.emplace(producer_plugin::snapshot_schedule_information {{_snapshot_id++}, {sri.block_spacing, sri.start_block_num, sri.end_block_num, sri.snapshot_description},{}}); - x_serialize(); - - // returning snapshot_schedule_result - return producer_plugin::snapshot_schedule_result{{_snapshot_id - 1}, {sri.block_spacing, sri.start_block_num, sri.end_block_num, sri.snapshot_description}}; - } - - producer_plugin::snapshot_schedule_result unschedule_snapshot(uint32_t sri) { - auto& snapshot_by_id = _snapshot_requests.get(); - auto existing = snapshot_by_id.find(sri); - EOS_ASSERT(existing != snapshot_by_id.end(), chain::snapshot_request_not_found, "Snapshot request not found"); - - producer_plugin::snapshot_schedule_result result{{existing->snapshot_request_id}, {existing->block_spacing, existing->start_block_num, existing->end_block_num, existing->snapshot_description}}; - _snapshot_requests.erase(existing); - x_serialize(); - - // returning snapshot_schedule_result - return result; - } - - producer_plugin::get_snapshot_requests_result get_snapshot_requests() { - producer_plugin::get_snapshot_requests_result result; - auto& asvector = _snapshot_requests.get(); - result.snapshot_requests.reserve(asvector.size()); - result.snapshot_requests.insert(result.snapshot_requests.begin(), asvector.begin(), asvector.end()); - return result; - } - - // initialize with storage - void set_db_path(bfs::path db_path) { - _snapshot_db.set_path(std::move(db_path)); - // init from db - if(fc::exists(_snapshot_db.get_json_path())) { - std::vector sr; - _snapshot_db >> sr; - // if db read succeeded, clear/load - _snapshot_requests.get().clear(); - _snapshot_requests.insert(sr.begin(), sr.end()); - } - } - - // add pending snapshot info to inflight snapshot request - void add_pending_snapshot_info(const producer_plugin::snapshot_information & si) { - auto& snapshot_by_id = _snapshot_requests.get(); - auto snapshot_req = snapshot_by_id.find(_inflight_sid); - if (snapshot_req != snapshot_by_id.end()) { - _snapshot_requests.modify(snapshot_req, [&si](auto& p) { - p.pending_snapshots.emplace_back(si); - }); - } - } - - // snapshot executor - void set_create_snapshot_fn(std::function)> fn) { - _create_snapshot = std::move(fn); - } - - void execute_snapshot(uint32_t srid) { - _inflight_sid = srid; - auto next = [srid, this](const chain::next_function_variant& result) { - if(std::holds_alternative(result)) { - try { - std::get(result)->dynamic_rethrow_exception(); - } catch(const fc::exception& e) { - elog( "snapshot creation error: ${details}", ("details",e.to_detail_string()) ); - appbase::app().quit(); - } catch(const std::exception& e) { - elog( "snapshot creation error: ${details}", ("details",e.what()) ); - appbase::app().quit(); - } - } else { - // success, snapshot finalized - auto snapshot_info = std::get(result); - auto& snapshot_by_id = _snapshot_requests.get(); - auto snapshot_req = snapshot_by_id.find(srid); - - if (snapshot_req != snapshot_by_id.end()) { - _snapshot_requests.modify(snapshot_req, [&](auto& p) { - auto & pending = p.pending_snapshots; - auto it = std::remove_if(pending.begin(), pending.end(), [&snapshot_info](const producer_plugin::snapshot_information & s){ return s.head_block_num <= snapshot_info.head_block_num; }); - pending.erase(it, pending.end()); - }); - } - } - }; - _create_snapshot(next); - } -}; -}// namespace eosio diff --git a/plugins/producer_plugin/pending_snapshot.cpp b/plugins/producer_plugin/pending_snapshot.cpp deleted file mode 100644 index cd94291c0d..0000000000 --- a/plugins/producer_plugin/pending_snapshot.cpp +++ /dev/null @@ -1,28 +0,0 @@ -#include -#include - -namespace eosio { - -producer_plugin::snapshot_information pending_snapshot::finalize( const chain::controller& chain ) const { - auto block_ptr = chain.fetch_block_by_id( block_id ); - auto in_chain = (bool)block_ptr; - boost::system::error_code ec; - - if (!in_chain) { - bfs::remove(bfs::path(pending_path), ec); - EOS_THROW(chain::snapshot_finalization_exception, - "Snapshotted block was forked out of the chain. ID: ${block_id}", - ("block_id", block_id)); - } - - bfs::rename(bfs::path(pending_path), bfs::path(final_path), ec); - EOS_ASSERT(!ec, chain::snapshot_finalization_exception, - "Unable to finalize valid snapshot of block number ${bn}: [code: ${ec}] ${message}", - ("bn", get_height()) - ("ec", ec.value()) - ("message", ec.message())); - - return {block_id, block_ptr->block_num(), block_ptr->timestamp, chain::chain_snapshot_header::current_version, final_path}; -} - -} // namespace eosio diff --git a/plugins/producer_plugin/producer_plugin.cpp b/plugins/producer_plugin/producer_plugin.cpp index 7b338a1627..bc53148612 100644 --- a/plugins/producer_plugin/producer_plugin.cpp +++ b/plugins/producer_plugin/producer_plugin.cpp @@ -1,8 +1,7 @@ #include -#include #include -#include #include + #include #include #include @@ -10,6 +9,9 @@ #include #include #include +#include +#include + #include #include @@ -125,17 +127,6 @@ using transaction_id_with_expiry_index = multi_index_container< > >; -struct by_height; - -using pending_snapshot_index = multi_index_container< - pending_snapshot, - indexed_by< - hashed_unique, BOOST_MULTI_INDEX_MEMBER(pending_snapshot, block_id_type, block_id)>, - ordered_non_unique, BOOST_MULTI_INDEX_CONST_MEM_FUN( pending_snapshot, uint32_t, get_height)> - > ->; - - namespace { // track multiple failures on unapplied transactions @@ -390,7 +381,6 @@ class producer_plugin_impl : public std::enable_shared_from_thischain(); EOS_ASSERT(chain.is_write_window(), producer_exception, "write window is expected for on_irreversible_block signal"); _irreversible_block_time = lib->timestamp.to_time_point(); - - // promote any pending snapshots - auto& snapshots_by_height = _pending_snapshot_index.get(); - uint32_t lib_height = lib->block_num(); - - while (!snapshots_by_height.empty() && snapshots_by_height.begin()->get_height() <= lib_height) { - const auto& pending = snapshots_by_height.begin(); - auto next = pending->next; - - try { - next(pending->finalize(chain)); - } CATCH_AND_CALL(next); - - snapshots_by_height.erase(snapshots_by_height.begin()); - } + _snapshot_scheduler.on_irreversible_block(lib); } void update_block_metrics() { @@ -1184,7 +1160,7 @@ void producer_plugin::plugin_initialize(const boost::program_options::variables_ } my->_snapshot_scheduler.set_db_path(my->_snapshots_dir); - my->_snapshot_scheduler.set_create_snapshot_fn([this](producer_plugin::next_function next){create_snapshot(next);}); + my->_snapshot_scheduler.set_snapshots_path(my->_snapshots_dir); } FC_LOG_AND_RETHROW() } using namespace std::chrono_literals; @@ -1214,6 +1190,8 @@ void producer_plugin::plugin_startup() my->_irreversible_block_connection.emplace(chain.irreversible_block.connect( [this]( const auto& bsp ){ my->on_irreversible_block( bsp->block ); } )); my->_block_start_connection.emplace(chain.block_start.connect( [this]( uint32_t bs ){ my->_snapshot_scheduler.on_start_block(bs); } )); + my->_snapshot_scheduler.set_controller(&chain); + const auto lib_num = chain.last_irreversible_block_num(); const auto lib = chain.fetch_block_by_number(lib_num); if (lib) { @@ -1448,106 +1426,21 @@ producer_plugin::integrity_hash_information producer_plugin::get_integrity_hash( return {chain.head_block_id(), chain.calculate_integrity_hash()}; } -void producer_plugin::create_snapshot(producer_plugin::next_function next) { - chain::controller& chain = my->chain_plug->chain(); - - auto head_id = chain.head_block_id(); - const auto head_block_num = chain.head_block_num(); - const auto head_block_time = chain.head_block_time(); - const auto& snapshot_path = pending_snapshot::get_final_path(head_id, my->_snapshots_dir); - const auto& temp_path = pending_snapshot::get_temp_path(head_id, my->_snapshots_dir); - - // maintain legacy exception if the snapshot exists - if( fc::is_regular_file(snapshot_path) ) { - auto ex = snapshot_exists_exception( FC_LOG_MESSAGE( error, "snapshot named ${name} already exists", ("name", snapshot_path.generic_string()) ) ); - next(ex.dynamic_copy_exception()); - return; - } - - auto write_snapshot = [&]( const bfs::path& p ) -> void { - auto reschedule = fc::make_scoped_exit([this](){ - my->schedule_production_loop(); - }); - - if (chain.is_building_block()) { - // abort the pending block - my->abort_block(); - } else { - reschedule.cancel(); - } - - bfs::create_directory( p.parent_path() ); - - // create the snapshot - auto snap_out = std::ofstream(p.generic_string(), (std::ios::out | std::ios::binary)); - auto writer = std::make_shared(snap_out); - chain.write_snapshot(writer); - writer->finalize(); - snap_out.flush(); - snap_out.close(); - }; - - // If in irreversible mode, create snapshot and return path to snapshot immediately. - if( chain.get_read_mode() == db_read_mode::IRREVERSIBLE ) { - try { - write_snapshot( temp_path ); - - boost::system::error_code ec; - bfs::rename(temp_path, snapshot_path, ec); - EOS_ASSERT(!ec, snapshot_finalization_exception, - "Unable to finalize valid snapshot of block number ${bn}: [code: ${ec}] ${message}", - ("bn", head_block_num) - ("ec", ec.value()) - ("message", ec.message())); - - next( producer_plugin::snapshot_information{head_id, head_block_num, head_block_time, chain_snapshot_header::current_version, snapshot_path.generic_string()} ); - } CATCH_AND_CALL (next); - return; - } - - // Otherwise, the result will be returned when the snapshot becomes irreversible. - - // determine if this snapshot is already in-flight - auto& pending_by_id = my->_pending_snapshot_index.get(); - auto existing = pending_by_id.find(head_id); - if( existing != pending_by_id.end() ) { - // if a snapshot at this block is already pending, attach this requests handler to it - pending_by_id.modify(existing, [&next]( auto& entry ){ - entry.next = [prev = entry.next, next](const next_function_variant& res){ - prev(res); - next(res); - }; - }); - } else { - const auto& pending_path = pending_snapshot::get_pending_path(head_id, my->_snapshots_dir); - - try { - write_snapshot( temp_path ); // create a new pending snapshot - - boost::system::error_code ec; - bfs::rename(temp_path, pending_path, ec); - EOS_ASSERT(!ec, snapshot_finalization_exception, - "Unable to promote temp snapshot to pending for block number ${bn}: [code: ${ec}] ${message}", - ("bn", head_block_num) - ("ec", ec.value()) - ("message", ec.message())); - my->_pending_snapshot_index.emplace(head_id, next, pending_path.generic_string(), snapshot_path.generic_string()); - my->_snapshot_scheduler.add_pending_snapshot_info( producer_plugin::snapshot_information{head_id, head_block_num, head_block_time, chain_snapshot_header::current_version, pending_path.generic_string()} ); - } CATCH_AND_CALL (next); - } +void producer_plugin::create_snapshot(producer_plugin::next_function next) { + my->_snapshot_scheduler.create_snapshot(next); } -producer_plugin::snapshot_schedule_result producer_plugin::schedule_snapshot(const snapshot_request_information& sri) +chain::snapshot_scheduler::snapshot_schedule_result producer_plugin::schedule_snapshot(const chain::snapshot_scheduler::snapshot_request_information& sri) { return my->_snapshot_scheduler.schedule_snapshot(sri); } -producer_plugin::snapshot_schedule_result producer_plugin::unschedule_snapshot(const snapshot_request_id_information& sri) +chain::snapshot_scheduler::snapshot_schedule_result producer_plugin::unschedule_snapshot(const chain::snapshot_scheduler::snapshot_request_id_information& sri) { return my->_snapshot_scheduler.unschedule_snapshot(sri.snapshot_request_id); } -producer_plugin::get_snapshot_requests_result producer_plugin::get_snapshot_requests() const +chain::snapshot_scheduler::get_snapshot_requests_result producer_plugin::get_snapshot_requests() const { return my->_snapshot_scheduler.get_snapshot_requests(); } diff --git a/plugins/producer_plugin/test/test_snapshot_scheduler.cpp b/plugins/producer_plugin/test/test_snapshot_scheduler.cpp index 40ae2b9a30..fd0036f06f 100644 --- a/plugins/producer_plugin/test/test_snapshot_scheduler.cpp +++ b/plugins/producer_plugin/test/test_snapshot_scheduler.cpp @@ -3,7 +3,7 @@ #include #include -#include +#include #include namespace { @@ -19,8 +19,8 @@ BOOST_AUTO_TEST_CASE(snapshot_scheduler_test) { { // add/remove test - producer_plugin::snapshot_request_information sri1 = {.block_spacing = 100, .start_block_num = 5000, .end_block_num = 10000, .snapshot_description = "Example of recurring snapshot"}; - producer_plugin::snapshot_request_information sri2 = {.block_spacing = 0, .start_block_num = 5200, .end_block_num = 5200, .snapshot_description = "Example of one-time snapshot"}; + snapshot_scheduler::snapshot_request_information sri1 = {.block_spacing = 100, .start_block_num = 5000, .end_block_num = 10000, .snapshot_description = "Example of recurring snapshot"}; + snapshot_scheduler::snapshot_request_information sri2 = {.block_spacing = 0, .start_block_num = 5200, .end_block_num = 5200, .snapshot_description = "Example of one-time snapshot"}; scheduler.schedule_snapshot(sri1); scheduler.schedule_snapshot(sri2); @@ -31,27 +31,27 @@ BOOST_AUTO_TEST_CASE(snapshot_scheduler_test) { return e.to_detail_string().find("Duplicate snapshot request") != std::string::npos; }); - producer_plugin::snapshot_request_id_information sri_delete_1 = {.snapshot_request_id = 0}; + snapshot_scheduler::snapshot_request_id_information sri_delete_1 = {.snapshot_request_id = 0}; scheduler.unschedule_snapshot(sri_delete_1); BOOST_CHECK_EQUAL(1, scheduler.get_snapshot_requests().snapshot_requests.size()); - producer_plugin::snapshot_request_id_information sri_delete_none = {.snapshot_request_id = 2}; + snapshot_scheduler::snapshot_request_id_information sri_delete_none = {.snapshot_request_id = 2}; BOOST_CHECK_EXCEPTION(scheduler.unschedule_snapshot(sri_delete_none), snapshot_request_not_found, [](const fc::assert_exception& e) { return e.to_detail_string().find("Snapshot request not found") != std::string::npos; }); - producer_plugin::snapshot_request_id_information sri_delete_2 = {.snapshot_request_id = 1}; + snapshot_scheduler::snapshot_request_id_information sri_delete_2 = {.snapshot_request_id = 1}; scheduler.unschedule_snapshot(sri_delete_2); BOOST_CHECK_EQUAL(0, scheduler.get_snapshot_requests().snapshot_requests.size()); - producer_plugin::snapshot_request_information sri_large_spacing = {.block_spacing = 1000, .start_block_num = 5000, .end_block_num = 5010}; + snapshot_scheduler::snapshot_request_information sri_large_spacing = {.block_spacing = 1000, .start_block_num = 5000, .end_block_num = 5010}; BOOST_CHECK_EXCEPTION(scheduler.schedule_snapshot(sri_large_spacing), invalid_snapshot_request, [](const fc::assert_exception& e) { return e.to_detail_string().find("Block spacing exceeds defined by start and end range") != std::string::npos; }); - producer_plugin::snapshot_request_information sri_start_end = {.block_spacing = 1000, .start_block_num = 50000, .end_block_num = 5000}; + snapshot_scheduler::snapshot_request_information sri_start_end = {.block_spacing = 1000, .start_block_num = 50000, .end_block_num = 5000}; BOOST_CHECK_EXCEPTION(scheduler.schedule_snapshot(sri_start_end), invalid_snapshot_request, [](const fc::assert_exception& e) { return e.to_detail_string().find("End block number should be greater or equal to start block number") != std::string::npos; }); @@ -94,7 +94,7 @@ BOOST_AUTO_TEST_CASE(snapshot_scheduler_test) { // catching pending snapshot if (!pp->get_snapshot_requests().snapshot_requests.empty()) { const auto& snapshot_requests = pp->get_snapshot_requests().snapshot_requests; - auto it = find_if(snapshot_requests.begin(), snapshot_requests.end(), [](const producer_plugin::snapshot_schedule_information& obj) {return obj.snapshot_request_id == 0;}); + auto it = find_if(snapshot_requests.begin(), snapshot_requests.end(), [](const snapshot_scheduler::snapshot_schedule_information& obj) {return obj.snapshot_request_id == 0;}); // we should have a pending snapshot for request id = 0 BOOST_REQUIRE(it != snapshot_requests.end()); auto& pending = it->pending_snapshots; @@ -104,9 +104,9 @@ BOOST_AUTO_TEST_CASE(snapshot_scheduler_test) { } }); - producer_plugin::snapshot_request_information sri1 = {.block_spacing = 8, .start_block_num = 1, .end_block_num = 300000, .snapshot_description = "Example of recurring snapshot 1"}; - producer_plugin::snapshot_request_information sri2 = {.block_spacing = 5000, .start_block_num = 100000, .end_block_num = 300000, .snapshot_description = "Example of recurring snapshot 2 that will never happen"}; - producer_plugin::snapshot_request_information sri3 = {.block_spacing = 2, .start_block_num = 0, .end_block_num = 3, .snapshot_description = "Example of recurring snapshot 3 that will expire"}; + snapshot_scheduler::snapshot_request_information sri1 = {.block_spacing = 8, .start_block_num = 1, .end_block_num = 300000, .snapshot_description = "Example of recurring snapshot 1"}; + snapshot_scheduler::snapshot_request_information sri2 = {.block_spacing = 5000, .start_block_num = 100000, .end_block_num = 300000, .snapshot_description = "Example of recurring snapshot 2 that will never happen"}; + snapshot_scheduler::snapshot_request_information sri3 = {.block_spacing = 2, .start_block_num = 0, .end_block_num = 3, .snapshot_description = "Example of recurring snapshot 3 that will expire"}; pp->schedule_snapshot(sri1); pp->schedule_snapshot(sri2); @@ -122,7 +122,7 @@ BOOST_AUTO_TEST_CASE(snapshot_scheduler_test) { // check whether no pending snapshots present for a snapshot with id 0 const auto& snapshot_requests = pp->get_snapshot_requests().snapshot_requests; - auto it = find_if(snapshot_requests.begin(), snapshot_requests.end(),[](const producer_plugin::snapshot_schedule_information& obj) {return obj.snapshot_request_id == 0;}); + auto it = find_if(snapshot_requests.begin(), snapshot_requests.end(),[](const snapshot_scheduler::snapshot_schedule_information& obj) {return obj.snapshot_request_id == 0;}); // snapshot request with id = 0 should be found and should not have any pending snapshots BOOST_REQUIRE(it != snapshot_requests.end()); @@ -134,7 +134,7 @@ BOOST_AUTO_TEST_CASE(snapshot_scheduler_test) { // lets check whether schedule can be read back after restart snapshot_db_json db; - std::vector ssi; + std::vector ssi; db.set_path(temp / "snapshots"); db >> ssi; BOOST_CHECK_EQUAL(2, ssi.size()); diff --git a/tests/test_snapshot_information.cpp b/tests/test_snapshot_information.cpp index cbbb2e3c27..e6cd56db8d 100644 --- a/tests/test_snapshot_information.cpp +++ b/tests/test_snapshot_information.cpp @@ -3,21 +3,22 @@ #include #include "snapshot_suites.hpp" #include -#include +#include #include #include using namespace eosio; +using namespace eosio::chain; using namespace eosio::testing; using namespace boost::system; namespace { - eosio::producer_plugin::snapshot_information test_snap_info; + snapshot_scheduler::snapshot_information test_snap_info; } BOOST_AUTO_TEST_SUITE(producer_snapshot_tests) -using next_t = eosio::producer_plugin::next_function; +using next_t = eosio::producer_plugin::next_function; BOOST_AUTO_TEST_CASE_TEMPLATE(test_snapshot_information, SNAPSHOT_SUITE, snapshot_suites) { tester chain; @@ -53,12 +54,12 @@ BOOST_AUTO_TEST_CASE_TEMPLATE(test_snapshot_information, SNAPSHOT_SUITE, snapsho snap_out.close(); }; - auto final_path = eosio::pending_snapshot::get_final_path(block2->previous, "../snapshots/"); - auto pending_path = eosio::pending_snapshot::get_pending_path(block2->previous, "../snapshots/"); + auto final_path = pending_snapshot::get_final_path(block2->previous, "../snapshots/"); + auto pending_path = pending_snapshot::get_pending_path(block2->previous, "../snapshots/"); write_snapshot( pending_path ); next_t next; - eosio::pending_snapshot pending{ block2->previous, next, pending_path.generic_string(), final_path.generic_string() }; + pending_snapshot pending{ block2->previous, next, pending_path.generic_string(), final_path.generic_string() }; test_snap_info = pending.finalize(*chain.control); BOOST_REQUIRE_EQUAL(test_snap_info.head_block_num, 6); BOOST_REQUIRE_EQUAL(test_snap_info.version, chain_snapshot_header::current_version); From 4bb4d65b749aa78fed3a313103b222b76723b4e9 Mon Sep 17 00:00:00 2001 From: 766C6164 Date: Thu, 13 Apr 2023 21:02:28 -0400 Subject: [PATCH 03/58] Multiple fixes --- .../include/eosio/chain/pending_snapshot.hpp | 2 +- .../eosio/chain/snapshot_scheduler.hpp | 21 ++++-- libraries/chain/snapshot_scheduler.cpp | 65 +++++++++++++------ plugins/producer_plugin/producer_plugin.cpp | 35 +++++++++- 4 files changed, 98 insertions(+), 25 deletions(-) diff --git a/libraries/chain/include/eosio/chain/pending_snapshot.hpp b/libraries/chain/include/eosio/chain/pending_snapshot.hpp index fa7af9589f..89dd8dd038 100644 --- a/libraries/chain/include/eosio/chain/pending_snapshot.hpp +++ b/libraries/chain/include/eosio/chain/pending_snapshot.hpp @@ -55,9 +55,9 @@ class pending_snapshot { } chain::block_id_type block_id; + next_t next; std::string pending_path; std::string final_path; - next_t next; }; }// namespace chain }// namespace eosio diff --git a/libraries/chain/include/eosio/chain/snapshot_scheduler.hpp b/libraries/chain/include/eosio/chain/snapshot_scheduler.hpp index dea1b1e67d..a7c641855d 100644 --- a/libraries/chain/include/eosio/chain/snapshot_scheduler.hpp +++ b/libraries/chain/include/eosio/chain/snapshot_scheduler.hpp @@ -86,7 +86,6 @@ class snapshot_scheduler { snapshot_requests _snapshot_requests; snapshot_db_json _snapshot_db; pending_snapshot_index _pending_snapshot_index; - chain::controller* _chain = NULL; uint32_t _snapshot_id = 0; uint32_t _inflight_sid = 0; @@ -94,11 +93,20 @@ class snapshot_scheduler { // path to write the snapshots to bfs::path _snapshots_dir; + // predicate to snapshot creation + std::function _predicate = [=]() { + }; + + // predicate to snapshot creation + std::function _chain = [=]() -> chain::controller* { + return NULL; + }; + void x_serialize() { auto& vec = _snapshot_requests.get(); std::vector sr(vec.begin(), vec.end()); _snapshot_db << sr; - } + }; public: snapshot_scheduler() = default; @@ -120,9 +128,14 @@ class snapshot_scheduler { // set snapshot path void set_snapshots_path(bfs::path sn_path); + // set predicate + void set_predicate(std::function fun) { + _predicate = std::move(fun); + } + // set controller - void set_controller(chain::controller* chain) { - _chain = chain; + void set_controller(std::function chain) { + _chain = std::move(chain); } // add pending snapshot info to inflight snapshot request diff --git a/libraries/chain/snapshot_scheduler.cpp b/libraries/chain/snapshot_scheduler.cpp index 74ffb1f5de..95c45b6fee 100644 --- a/libraries/chain/snapshot_scheduler.cpp +++ b/libraries/chain/snapshot_scheduler.cpp @@ -63,13 +63,15 @@ void snapshot_scheduler::on_start_block(uint32_t height) { void snapshot_scheduler::on_irreversible_block(const signed_block_ptr& lib) { auto& snapshots_by_height = _pending_snapshot_index.get(); uint32_t lib_height = lib->block_num(); + chain::controller * chain = _chain(); + EOS_ASSERT(chain, snapshot_exception, "Controller is not set"); while(!snapshots_by_height.empty() && snapshots_by_height.begin()->get_height() <= lib_height) { const auto& pending = snapshots_by_height.begin(); auto next = pending->next; try { - next(pending->finalize(*_chain)); + next(pending->finalize(*chain)); } CATCH_AND_CALL(next); @@ -153,10 +155,10 @@ void snapshot_scheduler::execute_snapshot(uint32_t srid) { std::get(result)->dynamic_rethrow_exception(); } catch(const fc::exception& e) { elog("snapshot creation error: ${details}", ("details", e.to_detail_string())); - //appbase::app().quit(); + return; } catch(const std::exception& e) { elog("snapshot creation error: ${details}", ("details", e.what())); - //appbase::app().quit(); + return; } } else { // success, snapshot finalized @@ -177,9 +179,11 @@ void snapshot_scheduler::execute_snapshot(uint32_t srid) { } void snapshot_scheduler::create_snapshot(snapshot_scheduler::next_function next) { - auto head_id = _chain->head_block_id(); - const auto head_block_num = _chain->head_block_num(); - const auto head_block_time = _chain->head_block_time(); + chain::controller * chain = _chain(); + EOS_ASSERT(chain, snapshot_exception, "Controller is not set"); + auto head_id = chain->head_block_id(); + const auto head_block_num = chain->head_block_num(); + const auto head_block_time = chain->head_block_time(); const auto& snapshot_path = pending_snapshot::get_final_path(head_id, _snapshots_dir); const auto& temp_path = pending_snapshot::get_temp_path(head_id, _snapshots_dir); @@ -191,32 +195,24 @@ void snapshot_scheduler::create_snapshot(snapshot_scheduler::next_function void { - /* - auto reschedule = fc::make_scoped_exit([this](){ - my->schedule_production_loop(); - }); + _predicate(); - if (_chain.is_building_block()) { - // abort the pending block - my->abort_block(); - } else { - reschedule.cancel(); - } -*/ bfs::create_directory(p.parent_path()); // create the snapshot auto snap_out = std::ofstream(p.generic_string(), (std::ios::out | std::ios::binary)); auto writer = std::make_shared(snap_out); - _chain->write_snapshot(writer); + chain->write_snapshot(writer); writer->finalize(); snap_out.flush(); snap_out.close(); }; // If in irreversible mode, create snapshot and return path to snapshot immediately. - if(_chain->get_read_mode() == db_read_mode::IRREVERSIBLE) { + if(chain->get_read_mode() == db_read_mode::IRREVERSIBLE) { try { + EOS_ASSERT(false, snapshot_exception, "POOP"); + write_snapshot(temp_path); boost::system::error_code ec; @@ -230,6 +226,37 @@ void snapshot_scheduler::create_snapshot(snapshot_scheduler::next_function(); + auto existing = pending_by_id.find(head_id); + if( existing != pending_by_id.end() ) { + // if a snapshot at this block is already pending, attach this requests handler to it + pending_by_id.modify(existing, [&next]( auto& entry ){ + entry.next = [prev = entry.next, next](const next_function_variant& res){ + prev(res); + next(res); + }; + }); + } else { + const auto& pending_path = pending_snapshot::get_pending_path(head_id, _snapshots_dir); + + try { + write_snapshot( temp_path ); // create a new pending snapshot + + boost::system::error_code ec; + bfs::rename(temp_path, pending_path, ec); + EOS_ASSERT(!ec, snapshot_finalization_exception, + "Unable to promote temp snapshot to pending for block number ${bn}: [code: ${ec}] ${message}", + ("bn", head_block_num) + ("ec", ec.value()) + ("message", ec.message())); + _pending_snapshot_index.emplace(head_id, next, pending_path.generic_string(), snapshot_path.generic_string()); + add_pending_snapshot_info( snapshot_scheduler::snapshot_information{head_id, head_block_num, head_block_time, chain_snapshot_header::current_version, pending_path.generic_string()} ); + } CATCH_AND_CALL (next); + } } }// namespace chain diff --git a/plugins/producer_plugin/producer_plugin.cpp b/plugins/producer_plugin/producer_plugin.cpp index bc53148612..11ebc1bc11 100644 --- a/plugins/producer_plugin/producer_plugin.cpp +++ b/plugins/producer_plugin/producer_plugin.cpp @@ -1190,8 +1190,27 @@ void producer_plugin::plugin_startup() my->_irreversible_block_connection.emplace(chain.irreversible_block.connect( [this]( const auto& bsp ){ my->on_irreversible_block( bsp->block ); } )); my->_block_start_connection.emplace(chain.block_start.connect( [this]( uint32_t bs ){ my->_snapshot_scheduler.on_start_block(bs); } )); - my->_snapshot_scheduler.set_controller(&chain); + // controller hook + my->_snapshot_scheduler.set_controller([this] { + return &(my->chain_plug->chain()); + }); + + // to be executed right before taking snapshot + my->_snapshot_scheduler.set_predicate([this] { + auto reschedule = fc::make_scoped_exit([this](){ + my->schedule_production_loop(); + }); + + chain::controller& chain = my->chain_plug->chain(); + if (chain.is_building_block()) { + // abort the pending block + my->abort_block(); + } else { + reschedule.cancel(); + } + }); + const auto lib_num = chain.last_irreversible_block_num(); const auto lib = chain.fetch_block_by_number(lib_num); if (lib) { @@ -1427,6 +1446,20 @@ producer_plugin::integrity_hash_information producer_plugin::get_integrity_hash( } void producer_plugin::create_snapshot(producer_plugin::next_function next) { + /* + auto predicate = [] { + auto reschedule = fc::make_scoped_exit([this](){ + my->schedule_production_loop(); + }); + + if (_chain.is_building_block()) { + // abort the pending block + my->abort_block(); + } else { + reschedule.cancel(); + } + }; +*/ my->_snapshot_scheduler.create_snapshot(next); } From 16c437514ea9b5a260c22d0661b85a7e9762a666 Mon Sep 17 00:00:00 2001 From: Huang-Ming Huang Date: Fri, 14 Apr 2023 04:09:24 -0500 Subject: [PATCH 04/58] update Python_SITELIB --- scripts/postinst | 2 +- scripts/prerm | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/postinst b/scripts/postinst index 3e4cd9d250..b8b54fa14f 100644 --- a/scripts/postinst +++ b/scripts/postinst @@ -1,7 +1,7 @@ #!/bin/sh set -e -Python_SITELIB=$(python3 -c "from distutils import sysconfig;print(sysconfig.get_python_lib(plat_specific=False,standard_lib=False))") +Python_SITELIB=$(python3 -c "import sysconfig; print(sysconfig.get_path('purelib'))") ln -sf @CMAKE_INSTALL_FULL_DATAROOTDIR@/leap_testing/tests/TestHarness $Python_SITELIB/TestHarness diff --git a/scripts/prerm b/scripts/prerm index 146c43271e..9884fd352b 100644 --- a/scripts/prerm +++ b/scripts/prerm @@ -1,6 +1,6 @@ #!/bin/sh # Cleanup symbolic links created during postinst -Python_SITELIB=$(python3 -c "from distutils import sysconfig;print(sysconfig.get_python_lib(plat_specific=False,standard_lib=False))") +Python_SITELIB=$(python3 -c "import sysconfig; print(sysconfig.get_path('purelib'))") rm -f $Python_SITELIB/TestHarness rm -f @CMAKE_INSTALL_FULL_DATAROOTDIR@/leap_testing/bin From fdeb7bee942dde507d3d9c9748a081be71e04646 Mon Sep 17 00:00:00 2001 From: 766C6164 Date: Fri, 14 Apr 2023 11:41:35 -0400 Subject: [PATCH 05/58] Cleanup --- .../eosio/chain/snapshot_scheduler.hpp | 27 ++---------- libraries/chain/snapshot_scheduler.cpp | 38 ++++++---------- plugins/producer_plugin/producer_plugin.cpp | 44 +++++-------------- 3 files changed, 30 insertions(+), 79 deletions(-) diff --git a/libraries/chain/include/eosio/chain/snapshot_scheduler.hpp b/libraries/chain/include/eosio/chain/snapshot_scheduler.hpp index a7c641855d..1c913f42f7 100644 --- a/libraries/chain/include/eosio/chain/snapshot_scheduler.hpp +++ b/libraries/chain/include/eosio/chain/snapshot_scheduler.hpp @@ -93,15 +93,6 @@ class snapshot_scheduler { // path to write the snapshots to bfs::path _snapshots_dir; - // predicate to snapshot creation - std::function _predicate = [=]() { - }; - - // predicate to snapshot creation - std::function _chain = [=]() -> chain::controller* { - return NULL; - }; - void x_serialize() { auto& vec = _snapshot_requests.get(); std::vector sr(vec.begin(), vec.end()); @@ -112,10 +103,10 @@ class snapshot_scheduler { snapshot_scheduler() = default; // snapshot scheduler listener - void on_start_block(uint32_t height); + void on_start_block(uint32_t height, chain::controller& chain); // to promote pending snapshots - void on_irreversible_block(const signed_block_ptr& lib); + void on_irreversible_block(const signed_block_ptr& lib, const chain::controller& chain); // snapshot scheduler handlers snapshot_scheduler::snapshot_schedule_result schedule_snapshot(const snapshot_scheduler::snapshot_request_information& sri); @@ -128,24 +119,14 @@ class snapshot_scheduler { // set snapshot path void set_snapshots_path(bfs::path sn_path); - // set predicate - void set_predicate(std::function fun) { - _predicate = std::move(fun); - } - - // set controller - void set_controller(std::function chain) { - _chain = std::move(chain); - } - // add pending snapshot info to inflight snapshot request void add_pending_snapshot_info(const snapshot_scheduler::snapshot_information& si); // execute snapshot - void execute_snapshot(uint32_t srid); + void execute_snapshot(uint32_t srid, chain::controller& chain); // former producer_plugin snapshot fn - void create_snapshot(snapshot_scheduler::next_function next); + void create_snapshot(snapshot_scheduler::next_function next, chain::controller& chain, std::function predicate); }; }// namespace chain }// namespace eosio diff --git a/libraries/chain/snapshot_scheduler.cpp b/libraries/chain/snapshot_scheduler.cpp index 95c45b6fee..818b784ef1 100644 --- a/libraries/chain/snapshot_scheduler.cpp +++ b/libraries/chain/snapshot_scheduler.cpp @@ -8,17 +8,17 @@ namespace eosio { namespace chain { // snapshot_scheduler_listener -void snapshot_scheduler::on_start_block(uint32_t height) { +void snapshot_scheduler::on_start_block(uint32_t height, chain::controller& chain) { bool serialize_needed = false; bool snapshot_executed = false; - auto execute_snapshot_with_log = [this, height, &snapshot_executed](const auto& req) { + auto execute_snapshot_with_log = [this, height, &snapshot_executed, &chain](const auto& req) { // one snapshot per height if(!snapshot_executed) { dlog("snapshot scheduler creating a snapshot from the request [start_block_num:${start_block_num}, end_block_num=${end_block_num}, block_spacing=${block_spacing}], height=${height}", ("start_block_num", req.start_block_num)("end_block_num", req.end_block_num)("block_spacing", req.block_spacing)("height", height)); - execute_snapshot(req.snapshot_request_id); + execute_snapshot(req.snapshot_request_id, chain); snapshot_executed = true; } }; @@ -60,18 +60,16 @@ void snapshot_scheduler::on_start_block(uint32_t height) { if(serialize_needed) x_serialize(); } -void snapshot_scheduler::on_irreversible_block(const signed_block_ptr& lib) { +void snapshot_scheduler::on_irreversible_block(const signed_block_ptr& lib, const chain::controller& chain) { auto& snapshots_by_height = _pending_snapshot_index.get(); uint32_t lib_height = lib->block_num(); - chain::controller * chain = _chain(); - EOS_ASSERT(chain, snapshot_exception, "Controller is not set"); while(!snapshots_by_height.empty() && snapshots_by_height.begin()->get_height() <= lib_height) { const auto& pending = snapshots_by_height.begin(); auto next = pending->next; try { - next(pending->finalize(*chain)); + next(pending->finalize(chain)); } CATCH_AND_CALL(next); @@ -147,7 +145,7 @@ void snapshot_scheduler::add_pending_snapshot_info(const snapshot_scheduler::sna } } -void snapshot_scheduler::execute_snapshot(uint32_t srid) { +void snapshot_scheduler::execute_snapshot(uint32_t srid, chain::controller& chain) { _inflight_sid = srid; auto next = [srid, this](const chain::next_function_variant& result) { if(std::holds_alternative(result)) { @@ -175,15 +173,13 @@ void snapshot_scheduler::execute_snapshot(uint32_t srid) { } } }; - create_snapshot(next); + create_snapshot(next, chain, {}); } -void snapshot_scheduler::create_snapshot(snapshot_scheduler::next_function next) { - chain::controller * chain = _chain(); - EOS_ASSERT(chain, snapshot_exception, "Controller is not set"); - auto head_id = chain->head_block_id(); - const auto head_block_num = chain->head_block_num(); - const auto head_block_time = chain->head_block_time(); +void snapshot_scheduler::create_snapshot(snapshot_scheduler::next_function next, chain::controller& chain, std::function predicate) { + auto head_id = chain.head_block_id(); + const auto head_block_num = chain.head_block_num(); + const auto head_block_time = chain.head_block_time(); const auto& snapshot_path = pending_snapshot::get_final_path(head_id, _snapshots_dir); const auto& temp_path = pending_snapshot::get_temp_path(head_id, _snapshots_dir); @@ -195,26 +191,20 @@ void snapshot_scheduler::create_snapshot(snapshot_scheduler::next_function void { - _predicate(); - + if (predicate) predicate(); bfs::create_directory(p.parent_path()); - - // create the snapshot auto snap_out = std::ofstream(p.generic_string(), (std::ios::out | std::ios::binary)); auto writer = std::make_shared(snap_out); - chain->write_snapshot(writer); + chain.write_snapshot(writer); writer->finalize(); snap_out.flush(); snap_out.close(); }; // If in irreversible mode, create snapshot and return path to snapshot immediately. - if(chain->get_read_mode() == db_read_mode::IRREVERSIBLE) { + if(chain.get_read_mode() == db_read_mode::IRREVERSIBLE) { try { - EOS_ASSERT(false, snapshot_exception, "POOP"); - write_snapshot(temp_path); - boost::system::error_code ec; bfs::rename(temp_path, snapshot_path, ec); EOS_ASSERT(!ec, snapshot_finalization_exception, diff --git a/plugins/producer_plugin/producer_plugin.cpp b/plugins/producer_plugin/producer_plugin.cpp index 9b49e0e302..fe73a01c92 100644 --- a/plugins/producer_plugin/producer_plugin.cpp +++ b/plugins/producer_plugin/producer_plugin.cpp @@ -507,7 +507,7 @@ class producer_plugin_impl : public std::enable_shared_from_thischain(); EOS_ASSERT(chain.is_write_window(), producer_exception, "write window is expected for on_irreversible_block signal"); _irreversible_block_time = lib->timestamp.to_time_point(); - _snapshot_scheduler.on_irreversible_block(lib); + _snapshot_scheduler.on_irreversible_block(lib, chain); } void update_block_metrics() { @@ -1188,29 +1188,8 @@ void producer_plugin::plugin_startup() my->_accepted_block_connection.emplace(chain.accepted_block.connect( [this]( const auto& bsp ){ my->on_block( bsp ); } )); my->_accepted_block_header_connection.emplace(chain.accepted_block_header.connect( [this]( const auto& bsp ){ my->on_block_header( bsp ); } )); my->_irreversible_block_connection.emplace(chain.irreversible_block.connect( [this]( const auto& bsp ){ my->on_irreversible_block( bsp->block ); } )); - my->_block_start_connection.emplace(chain.block_start.connect( [this]( uint32_t bs ){ my->_snapshot_scheduler.on_start_block(bs); } )); - - // controller hook - my->_snapshot_scheduler.set_controller([this] { - return &(my->chain_plug->chain()); - }); - - // to be executed right before taking snapshot - my->_snapshot_scheduler.set_predicate([this] { - auto reschedule = fc::make_scoped_exit([this](){ - my->schedule_production_loop(); - }); - - chain::controller& chain = my->chain_plug->chain(); - if (chain.is_building_block()) { - // abort the pending block - my->abort_block(); - } else { - reschedule.cancel(); - } - }); - - + my->_block_start_connection.emplace(chain.block_start.connect( [this, &chain]( uint32_t bs ){ my->_snapshot_scheduler.on_start_block(bs, chain); } )); + const auto lib_num = chain.last_irreversible_block_num(); const auto lib = chain.fetch_block_by_number(lib_num); if (lib) { @@ -1446,21 +1425,22 @@ producer_plugin::integrity_hash_information producer_plugin::get_integrity_hash( } void producer_plugin::create_snapshot(producer_plugin::next_function next) { - /* - auto predicate = [] { - auto reschedule = fc::make_scoped_exit([this](){ - my->schedule_production_loop(); - }); + chain::controller& chain = my->chain_plug->chain(); + + auto reschedule = fc::make_scoped_exit([this](){ + my->schedule_production_loop(); + }); - if (_chain.is_building_block()) { + auto predicate = [&]() -> void { + if (chain.is_building_block()) { // abort the pending block my->abort_block(); } else { reschedule.cancel(); } }; -*/ - my->_snapshot_scheduler.create_snapshot(next); + + my->_snapshot_scheduler.create_snapshot(next, chain, predicate); } chain::snapshot_scheduler::snapshot_schedule_result producer_plugin::schedule_snapshot(const chain::snapshot_scheduler::snapshot_request_information& sri) From 720035653057c596cc67cef36880dd8850b08fb1 Mon Sep 17 00:00:00 2001 From: 766C6164 Date: Mon, 17 Apr 2023 11:56:44 -0400 Subject: [PATCH 06/58] Added graceful shutdown on snapshot execution exception --- .../chain/include/eosio/chain/exceptions.hpp | 2 ++ libraries/chain/snapshot_scheduler.cpp | 33 ++++++++++--------- plugins/producer_plugin/producer_plugin.cpp | 11 ++++++- 3 files changed, 29 insertions(+), 17 deletions(-) diff --git a/libraries/chain/include/eosio/chain/exceptions.hpp b/libraries/chain/include/eosio/chain/exceptions.hpp index d1d50edd8a..151021a284 100644 --- a/libraries/chain/include/eosio/chain/exceptions.hpp +++ b/libraries/chain/include/eosio/chain/exceptions.hpp @@ -603,6 +603,8 @@ namespace eosio { namespace chain { 3170014, "Snapshot request not found" ) FC_DECLARE_DERIVED_EXCEPTION( invalid_snapshot_request, producer_exception, 3170015, "Invalid snapshot request" ) + FC_DECLARE_DERIVED_EXCEPTION( snapshot_execution_exception, producer_exception, + 3170016, "Snapshot execution exception" ) FC_DECLARE_DERIVED_EXCEPTION( reversible_blocks_exception, chain_exception, 3180000, "Reversible Blocks exception" ) diff --git a/libraries/chain/snapshot_scheduler.cpp b/libraries/chain/snapshot_scheduler.cpp index 818b784ef1..01dc947f14 100644 --- a/libraries/chain/snapshot_scheduler.cpp +++ b/libraries/chain/snapshot_scheduler.cpp @@ -152,11 +152,13 @@ void snapshot_scheduler::execute_snapshot(uint32_t srid, chain::controller& chai try { std::get(result)->dynamic_rethrow_exception(); } catch(const fc::exception& e) { - elog("snapshot creation error: ${details}", ("details", e.to_detail_string())); - return; + EOS_THROW(snapshot_execution_exception, + "Snapshot creation error: ${details}", + ("details", e.to_detail_string())); } catch(const std::exception& e) { - elog("snapshot creation error: ${details}", ("details", e.what())); - return; + EOS_THROW(snapshot_execution_exception, + "Snapshot creation error: ${details}", + ("details", e.what())); } } else { // success, snapshot finalized @@ -176,7 +178,7 @@ void snapshot_scheduler::execute_snapshot(uint32_t srid, chain::controller& chai create_snapshot(next, chain, {}); } -void snapshot_scheduler::create_snapshot(snapshot_scheduler::next_function next, chain::controller& chain, std::function predicate) { +void snapshot_scheduler::create_snapshot(snapshot_scheduler::next_function next, chain::controller& chain, std::function predicate) { auto head_id = chain.head_block_id(); const auto head_block_num = chain.head_block_num(); const auto head_block_time = chain.head_block_time(); @@ -191,7 +193,7 @@ void snapshot_scheduler::create_snapshot(snapshot_scheduler::next_function void { - if (predicate) predicate(); + if(predicate) predicate(); bfs::create_directory(p.parent_path()); auto snap_out = std::ofstream(p.generic_string(), (std::ios::out | std::ios::binary)); auto writer = std::make_shared(snap_out); @@ -222,10 +224,10 @@ void snapshot_scheduler::create_snapshot(snapshot_scheduler::next_function(); auto existing = pending_by_id.find(head_id); - if( existing != pending_by_id.end() ) { + if(existing != pending_by_id.end()) { // if a snapshot at this block is already pending, attach this requests handler to it - pending_by_id.modify(existing, [&next]( auto& entry ){ - entry.next = [prev = entry.next, next](const next_function_variant& res){ + pending_by_id.modify(existing, [&next](auto& entry) { + entry.next = [prev = entry.next, next](const next_function_variant& res) { prev(res); next(res); }; @@ -234,18 +236,17 @@ void snapshot_scheduler::create_snapshot(snapshot_scheduler::next_function::get_pending_path(head_id, _snapshots_dir); try { - write_snapshot( temp_path ); // create a new pending snapshot + write_snapshot(temp_path);// create a new pending snapshot boost::system::error_code ec; bfs::rename(temp_path, pending_path, ec); EOS_ASSERT(!ec, snapshot_finalization_exception, - "Unable to promote temp snapshot to pending for block number ${bn}: [code: ${ec}] ${message}", - ("bn", head_block_num) - ("ec", ec.value()) - ("message", ec.message())); + "Unable to promote temp snapshot to pending for block number ${bn}: [code: ${ec}] ${message}", + ("bn", head_block_num)("ec", ec.value())("message", ec.message())); _pending_snapshot_index.emplace(head_id, next, pending_path.generic_string(), snapshot_path.generic_string()); - add_pending_snapshot_info( snapshot_scheduler::snapshot_information{head_id, head_block_num, head_block_time, chain_snapshot_header::current_version, pending_path.generic_string()} ); - } CATCH_AND_CALL (next); + add_pending_snapshot_info(snapshot_scheduler::snapshot_information{head_id, head_block_num, head_block_time, chain_snapshot_header::current_version, pending_path.generic_string()}); + } + CATCH_AND_CALL(next); } } diff --git a/plugins/producer_plugin/producer_plugin.cpp b/plugins/producer_plugin/producer_plugin.cpp index fe73a01c92..ca4bb420e9 100644 --- a/plugins/producer_plugin/producer_plugin.cpp +++ b/plugins/producer_plugin/producer_plugin.cpp @@ -1188,7 +1188,16 @@ void producer_plugin::plugin_startup() my->_accepted_block_connection.emplace(chain.accepted_block.connect( [this]( const auto& bsp ){ my->on_block( bsp ); } )); my->_accepted_block_header_connection.emplace(chain.accepted_block_header.connect( [this]( const auto& bsp ){ my->on_block_header( bsp ); } )); my->_irreversible_block_connection.emplace(chain.irreversible_block.connect( [this]( const auto& bsp ){ my->on_irreversible_block( bsp->block ); } )); - my->_block_start_connection.emplace(chain.block_start.connect( [this, &chain]( uint32_t bs ){ my->_snapshot_scheduler.on_start_block(bs, chain); } )); + + my->_block_start_connection.emplace(chain.block_start.connect( [this, &chain]( uint32_t bs ) { + try { + my->_snapshot_scheduler.on_start_block(bs, chain); + } + catch (const snapshot_execution_exception & e) { + fc_elog( _log, "Exception during snapshot execution: ${e}", ("e", e.to_detail_string()) ); + app().quit(); + } + } )); const auto lib_num = chain.last_irreversible_block_num(); const auto lib = chain.fetch_block_by_number(lib_num); From 4b104260ba99a59a7a744cbe2161b3783b2de0b4 Mon Sep 17 00:00:00 2001 From: Huang-Ming Huang Date: Tue, 18 Apr 2023 09:30:00 -0500 Subject: [PATCH 07/58] make sure Python_SITELIB exists --- scripts/postinst | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/postinst b/scripts/postinst index b8b54fa14f..3269e66c78 100644 --- a/scripts/postinst +++ b/scripts/postinst @@ -2,6 +2,7 @@ set -e Python_SITELIB=$(python3 -c "import sysconfig; print(sysconfig.get_path('purelib'))") +mkdir -p $Python_SITELIB ln -sf @CMAKE_INSTALL_FULL_DATAROOTDIR@/leap_testing/tests/TestHarness $Python_SITELIB/TestHarness From f74763b8e6a2f583f81e5a6396992f09c5e04d24 Mon Sep 17 00:00:00 2001 From: Huang-Ming Huang Date: Tue, 18 Apr 2023 13:03:26 -0500 Subject: [PATCH 08/58] use dist-package --- scripts/postinst | 6 +++--- scripts/prerm | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/scripts/postinst b/scripts/postinst index 3269e66c78..cef3eec3d8 100644 --- a/scripts/postinst +++ b/scripts/postinst @@ -1,9 +1,9 @@ #!/bin/sh set -e -Python_SITELIB=$(python3 -c "import sysconfig; print(sysconfig.get_path('purelib'))") -mkdir -p $Python_SITELIB -ln -sf @CMAKE_INSTALL_FULL_DATAROOTDIR@/leap_testing/tests/TestHarness $Python_SITELIB/TestHarness +Python_DISTLIB=$(python3 -c "import sysconfig; print(sysconfig.get_path('purelib'))" | sed 's/site-packages/dist-packages/') +mkdir -p $Python_DISTLIB +ln -sf @CMAKE_INSTALL_FULL_DATAROOTDIR@/leap_testing/tests/TestHarness $Python_DISTLIB/TestHarness # leap_testing is part of the package so should already exist by the time postinst runs diff --git a/scripts/prerm b/scripts/prerm index 9884fd352b..4d36f88924 100644 --- a/scripts/prerm +++ b/scripts/prerm @@ -1,6 +1,6 @@ #!/bin/sh # Cleanup symbolic links created during postinst -Python_SITELIB=$(python3 -c "import sysconfig; print(sysconfig.get_path('purelib'))") -rm -f $Python_SITELIB/TestHarness +Python_DISTLIB=$(python3 -c "import sysconfig; print(sysconfig.get_path('purelib'))" | sed 's/site-packages/dist-packages/') +rm -f $Python_DISTLIB/TestHarness rm -f @CMAKE_INSTALL_FULL_DATAROOTDIR@/leap_testing/bin From 306e9145a3326df7c71aa4371a0a069b7ac4cb1b Mon Sep 17 00:00:00 2001 From: Huang-Ming Huang Date: Tue, 18 Apr 2023 13:03:48 -0500 Subject: [PATCH 09/58] add test --- .github/workflows/build.yaml | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 4d2d77e3a4..ed914a7cbc 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -103,12 +103,21 @@ jobs: zstdcat build.tar.zst | tar x cd build cpack + - name: Install dev package + run: | + apt update + export DEBIAN_FRONTEND=noninteractive + export TZ="America/New_York" + apt install -y ./build/leap_*.deb + apt install -y ./build/leap-dev*.deb + - name: Test using TestHarness + run: | + python3 -c "from TestHarness import Cluster" - name: Upload dev package uses: actions/upload-artifact@v3 with: name: leap-dev-ubuntu20-amd64 path: build/leap-dev*.deb - tests: name: Tests needs: [d, Build] From 94540634be104eb54cccda7e4691ba185d369e17 Mon Sep 17 00:00:00 2001 From: 766C6164 Date: Thu, 20 Apr 2023 15:48:23 -0400 Subject: [PATCH 10/58] Merge --- libraries/chain/snapshot_scheduler.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libraries/chain/snapshot_scheduler.cpp b/libraries/chain/snapshot_scheduler.cpp index 01dc947f14..9495173a0d 100644 --- a/libraries/chain/snapshot_scheduler.cpp +++ b/libraries/chain/snapshot_scheduler.cpp @@ -187,7 +187,7 @@ void snapshot_scheduler::create_snapshot(snapshot_scheduler::next_function Date: Thu, 20 Apr 2023 17:00:19 -0400 Subject: [PATCH 11/58] fixes to merge conflicts --- libraries/chain/include/eosio/chain/pending_snapshot.hpp | 4 ++-- libraries/chain/include/eosio/chain/snapshot_db_json.hpp | 2 +- .../chain/include/eosio/chain/snapshot_scheduler.hpp | 2 +- libraries/chain/snapshot_scheduler.cpp | 8 ++++---- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/libraries/chain/include/eosio/chain/pending_snapshot.hpp b/libraries/chain/include/eosio/chain/pending_snapshot.hpp index 89dd8dd038..667e1cb802 100644 --- a/libraries/chain/include/eosio/chain/pending_snapshot.hpp +++ b/libraries/chain/include/eosio/chain/pending_snapshot.hpp @@ -8,7 +8,7 @@ namespace eosio { namespace chain { -namespace bfs = boost::filesystem; +namespace bfs = std::filesystem; template class pending_snapshot { @@ -37,7 +37,7 @@ class pending_snapshot { T finalize(const chain::controller& chain) const { auto block_ptr = chain.fetch_block_by_id(block_id); auto in_chain = (bool) block_ptr; - boost::system::error_code ec; + std::error_code ec; if(!in_chain) { bfs::remove(bfs::path(pending_path), ec); diff --git a/libraries/chain/include/eosio/chain/snapshot_db_json.hpp b/libraries/chain/include/eosio/chain/snapshot_db_json.hpp index 763e9a13b1..42a4858908 100644 --- a/libraries/chain/include/eosio/chain/snapshot_db_json.hpp +++ b/libraries/chain/include/eosio/chain/snapshot_db_json.hpp @@ -10,7 +10,7 @@ namespace eosio { namespace chain { -namespace bfs = boost::filesystem; +namespace bfs = std::filesystem; /// this class designed to serialize/deserialize snapshot schedule to a filesystem so it can be restored after restart class snapshot_db_json { diff --git a/libraries/chain/include/eosio/chain/snapshot_scheduler.hpp b/libraries/chain/include/eosio/chain/snapshot_scheduler.hpp index 1c913f42f7..c391f885a6 100644 --- a/libraries/chain/include/eosio/chain/snapshot_scheduler.hpp +++ b/libraries/chain/include/eosio/chain/snapshot_scheduler.hpp @@ -22,7 +22,7 @@ namespace eosio { namespace chain { namespace bmi = boost::multi_index; -namespace bfs = boost::filesystem; +namespace bfs = std::filesystem; class snapshot_scheduler { public: diff --git a/libraries/chain/snapshot_scheduler.cpp b/libraries/chain/snapshot_scheduler.cpp index 9495173a0d..05f906a900 100644 --- a/libraries/chain/snapshot_scheduler.cpp +++ b/libraries/chain/snapshot_scheduler.cpp @@ -122,7 +122,7 @@ snapshot_scheduler::get_snapshot_requests_result snapshot_scheduler::get_snapsho void snapshot_scheduler::set_db_path(bfs::path db_path) { _snapshot_db.set_path(std::move(db_path)); // init from db - if(fc::exists(_snapshot_db.get_json_path())) { + if(std::filesystem::exists(_snapshot_db.get_json_path())) { std::vector sr; _snapshot_db >> sr; // if db read succeeded, clear/load @@ -186,7 +186,7 @@ void snapshot_scheduler::create_snapshot(snapshot_scheduler::next_function::get_temp_path(head_id, _snapshots_dir); // maintain legacy exception if the snapshot exists - if(fc::is_regular_file(snapshot_path)) { + if(bfs::is_regular_file(snapshot_path)) { auto ex = snapshot_exists_exception(FC_LOG_MESSAGE(error, "snapshot named ${name} already exists", ("name", _snapshots_dir))); next(ex.dynamic_copy_exception()); return; @@ -207,7 +207,7 @@ void snapshot_scheduler::create_snapshot(snapshot_scheduler::next_function Date: Fri, 21 Apr 2023 14:09:00 -0500 Subject: [PATCH 12/58] Factor out TestHelperConfig and ClusterConfig setup from args into one reusable location/helper function. --- tests/performance_tests/performance_test.py | 32 +--------- .../performance_test_basic.py | 63 +++++++++++-------- 2 files changed, 39 insertions(+), 56 deletions(-) diff --git a/tests/performance_tests/performance_test.py b/tests/performance_tests/performance_test.py index 6695ff684d..07197e1ddf 100755 --- a/tests/performance_tests/performance_test.py +++ b/tests/performance_tests/performance_test.py @@ -538,35 +538,8 @@ def main(): args = PerfTestArgumentsHandler.parseArgs() Utils.Debug = args.v - testHelperConfig = PerformanceTestBasic.TestHelperConfig(killAll=args.clean_run, dontKill=args.leave_running, keepLogs=not args.del_perf_logs, - dumpErrorDetails=args.dump_error_details, delay=args.d, nodesFile=args.nodes_file, - verbose=args.v) - - chainPluginArgs = ChainPluginArgs(signatureCpuBillablePct=args.signature_cpu_billable_pct, - chainThreads=args.chain_threads, databaseMapMode=args.database_map_mode, - wasmRuntime=args.wasm_runtime, contractsConsole=args.contracts_console, - eosVmOcCacheSizeMb=args.eos_vm_oc_cache_size_mb, eosVmOcCompileThreads=args.eos_vm_oc_compile_threads, - blockLogRetainBlocks=args.block_log_retain_blocks, - chainStateDbSizeMb=args.chain_state_db_size_mb, abiSerializerMaxTimeMs=990000) - - producerPluginArgs = ProducerPluginArgs(disableSubjectiveBilling=args.disable_subjective_billing, - cpuEffortPercent=args.cpu_effort_percent, - producerThreads=args.producer_threads, maxTransactionTime=-1) - httpPluginArgs = HttpPluginArgs(httpMaxResponseTimeMs=args.http_max_response_time_ms, httpMaxBytesInFlightMb=args.http_max_bytes_in_flight_mb, - httpThreads=args.http_threads) - netPluginArgs = NetPluginArgs(netThreads=args.net_threads, maxClients=0) - nodeosVers=Utils.getNodeosVersion().split('.')[0] - resourceMonitorPluginArgs = ResourceMonitorPluginArgs(resourceMonitorNotShutdownOnThresholdExceeded=not nodeosVers == "v2") - ENA = PerformanceTestBasic.ClusterConfig.ExtraNodeosArgs - extraNodeosArgs = ENA(chainPluginArgs=chainPluginArgs, httpPluginArgs=httpPluginArgs, producerPluginArgs=producerPluginArgs, netPluginArgs=netPluginArgs, - resourceMonitorPluginArgs=resourceMonitorPluginArgs) - SC = PerformanceTestBasic.ClusterConfig.SpecifiedContract - specifiedContract=SC(contractDir=args.contract_dir, wasmFile=args.wasm_file, abiFile=args.abi_file, account=Account(args.account_name)) - testClusterConfig = PerformanceTestBasic.ClusterConfig(pnodes=args.p, totalNodes=args.n, topo=args.s, genesisPath=args.genesis, - prodsEnableTraceApi=args.prods_enable_trace_api, extraNodeosArgs=extraNodeosArgs, - specifiedContract=specifiedContract, loggingLevel=args.cluster_log_lvl, - nodeosVers=nodeosVers, nonProdsEosVmOcEnable=args.non_prods_eos_vm_oc_enable) - + testHelperConfig = PerformanceTestBasic.setupTestHelperConfig(args) + testClusterConfig = PerformanceTestBasic.setupClusterConfig(args) ptConfig = PerformanceTest.PtConfig(testDurationSec=args.test_iteration_duration_sec, finalDurationSec=args.final_iterations_duration_sec, @@ -586,6 +559,7 @@ def main(): userTrxDataFile=Path(args.user_trx_data_file) if args.user_trx_data_file is not None else None) myTest = PerformanceTest(testHelperConfig=testHelperConfig, clusterConfig=testClusterConfig, ptConfig=ptConfig) + perfRunSuccessful = myTest.runTest() exitCode = 0 if perfRunSuccessful else 1 diff --git a/tests/performance_tests/performance_test_basic.py b/tests/performance_tests/performance_test_basic.py index 7ffb4310f6..cc01b2bc62 100755 --- a/tests/performance_tests/performance_test_basic.py +++ b/tests/performance_tests/performance_test_basic.py @@ -532,6 +532,38 @@ def runTest(self) -> bool: return testSuccessful + def setupTestHelperConfig(args) -> TestHelperConfig: + return PerformanceTestBasic.TestHelperConfig(killAll=args.clean_run, dontKill=args.leave_running, keepLogs=not args.del_perf_logs, + dumpErrorDetails=args.dump_error_details, delay=args.d, nodesFile=args.nodes_file, + verbose=args.v) + + def setupClusterConfig(args) -> ClusterConfig: + + chainPluginArgs = ChainPluginArgs(signatureCpuBillablePct=args.signature_cpu_billable_pct, + chainThreads=args.chain_threads, databaseMapMode=args.database_map_mode, + wasmRuntime=args.wasm_runtime, contractsConsole=args.contracts_console, + eosVmOcCacheSizeMb=args.eos_vm_oc_cache_size_mb, eosVmOcCompileThreads=args.eos_vm_oc_compile_threads, + blockLogRetainBlocks=args.block_log_retain_blocks, + chainStateDbSizeMb=args.chain_state_db_size_mb, abiSerializerMaxTimeMs=990000) + + producerPluginArgs = ProducerPluginArgs(disableSubjectiveBilling=args.disable_subjective_billing, + cpuEffortPercent=args.cpu_effort_percent, + producerThreads=args.producer_threads, maxTransactionTime=-1) + httpPluginArgs = HttpPluginArgs(httpMaxResponseTimeMs=args.http_max_response_time_ms, httpMaxBytesInFlightMb=args.http_max_bytes_in_flight_mb, + httpThreads=args.http_threads) + netPluginArgs = NetPluginArgs(netThreads=args.net_threads, maxClients=0) + nodeosVers=Utils.getNodeosVersion().split('.')[0] + resourceMonitorPluginArgs = ResourceMonitorPluginArgs(resourceMonitorNotShutdownOnThresholdExceeded=not nodeosVers == "v2") + ENA = PerformanceTestBasic.ClusterConfig.ExtraNodeosArgs + extraNodeosArgs = ENA(chainPluginArgs=chainPluginArgs, httpPluginArgs=httpPluginArgs, producerPluginArgs=producerPluginArgs, netPluginArgs=netPluginArgs, + resourceMonitorPluginArgs=resourceMonitorPluginArgs) + SC = PerformanceTestBasic.ClusterConfig.SpecifiedContract + specifiedContract=SC(contractDir=args.contract_dir, wasmFile=args.wasm_file, abiFile=args.abi_file, account=Account(args.account_name)) + return PerformanceTestBasic.ClusterConfig(pnodes=args.p, totalNodes=args.n, topo=args.s, genesisPath=args.genesis, + prodsEnableTraceApi=args.prods_enable_trace_api, extraNodeosArgs=extraNodeosArgs, + specifiedContract=specifiedContract, loggingLevel=args.cluster_log_lvl, + nodeosVers=nodeosVers, nonProdsEosVmOcEnable=args.non_prods_eos_vm_oc_enable) + class PtbArgumentsHandler(object): @staticmethod def createBaseArgumentParser(suppressHelp: bool=False): @@ -619,37 +651,13 @@ def main(): args = PtbArgumentsHandler.parseArgs() Utils.Debug = args.v - testHelperConfig = PerformanceTestBasic.TestHelperConfig(killAll=args.clean_run, dontKill=args.leave_running, keepLogs=not args.del_perf_logs, - dumpErrorDetails=args.dump_error_details, delay=args.d, nodesFile=args.nodes_file, verbose=args.v, unshared=args.unshared) - - chainPluginArgs = ChainPluginArgs(signatureCpuBillablePct=args.signature_cpu_billable_pct, - chainThreads=args.chain_threads, databaseMapMode=args.database_map_mode, - wasmRuntime=args.wasm_runtime, contractsConsole=args.contracts_console, - eosVmOcCacheSizeMb=args.eos_vm_oc_cache_size_mb, eosVmOcCompileThreads=args.eos_vm_oc_compile_threads, - blockLogRetainBlocks=args.block_log_retain_blocks, - chainStateDbSizeMb=args.chain_state_db_size_mb, abiSerializerMaxTimeMs=990000) - - producerPluginArgs = ProducerPluginArgs(disableSubjectiveBilling=args.disable_subjective_billing, - cpuEffortPercent=args.cpu_effort_percent, - producerThreads=args.producer_threads, maxTransactionTime=-1) - httpPluginArgs = HttpPluginArgs(httpMaxResponseTimeMs=args.http_max_response_time_ms, httpMaxBytesInFlightMb=args.http_max_bytes_in_flight_mb, - httpThreads=args.http_threads) - netPluginArgs = NetPluginArgs(netThreads=args.net_threads, maxClients=0) - nodeosVers=Utils.getNodeosVersion().split('.')[0] - resourceMonitorPluginArgs = ResourceMonitorPluginArgs(resourceMonitorNotShutdownOnThresholdExceeded=not nodeosVers == "v2") - ENA = PerformanceTestBasic.ClusterConfig.ExtraNodeosArgs - extraNodeosArgs = ENA(chainPluginArgs=chainPluginArgs, httpPluginArgs=httpPluginArgs, producerPluginArgs=producerPluginArgs, netPluginArgs=netPluginArgs, - resourceMonitorPluginArgs=resourceMonitorPluginArgs) - SC = PerformanceTestBasic.ClusterConfig.SpecifiedContract - specifiedContract=SC(contractDir=args.contract_dir, wasmFile=args.wasm_file, abiFile=args.abi_file, account=Account(args.account_name)) - testClusterConfig = PerformanceTestBasic.ClusterConfig(pnodes=args.p, totalNodes=args.n, topo=args.s, genesisPath=args.genesis, - prodsEnableTraceApi=args.prods_enable_trace_api, extraNodeosArgs=extraNodeosArgs, - specifiedContract=specifiedContract, loggingLevel=args.cluster_log_lvl, - nodeosVers=nodeosVers, nonProdsEosVmOcEnable=args.non_prods_eos_vm_oc_enable) + testHelperConfig = PerformanceTestBasic.setupTestHelperConfig(args) + testClusterConfig = PerformanceTestBasic.setupClusterConfig(args) if args.contracts_console and testClusterConfig.loggingLevel != "debug" and testClusterConfig.loggingLevel != "all": print("Enabling contracts-console will not print anything unless debug level is 'debug' or higher." f" Current debug level is: {testClusterConfig.loggingLevel}") + ptbConfig = PerformanceTestBasic.PtbConfig(targetTps=args.target_tps, testTrxGenDurationSec=args.test_duration_sec, tpsLimitPerGenerator=args.tps_limit_per_generator, @@ -659,6 +667,7 @@ def main(): delPerfLogs=args.del_perf_logs, printMissingTransactions=args.print_missing_transactions, userTrxDataFile=Path(args.user_trx_data_file) if args.user_trx_data_file is not None else None) + myTest = PerformanceTestBasic(testHelperConfig=testHelperConfig, clusterConfig=testClusterConfig, ptbConfig=ptbConfig) testSuccessful = myTest.runTest() From e994655f0d0c10be6775ae80bb1c2f7a7a2ccfa4 Mon Sep 17 00:00:00 2001 From: Peter Oschwald Date: Fri, 21 Apr 2023 14:46:34 -0500 Subject: [PATCH 13/58] Slight reorg of config items in op modes doc showing associated plugin. --- tests/performance_tests/ph_op_modes.md | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/tests/performance_tests/ph_op_modes.md b/tests/performance_tests/ph_op_modes.md index 2cfc348a7d..abd372d1e3 100644 --- a/tests/performance_tests/ph_op_modes.md +++ b/tests/performance_tests/ph_op_modes.md @@ -53,10 +53,11 @@ so as to not adversely affect the producer's ability to process transactions by Additional node included over the base configuration. This additional node will have the http_plugin enabled and will be the recipient of all generated transaction traffic. Configure unlimited: -- `max_bytes_in_flight` +- `chain_plugin` + - `max_bytes_in_flight` + - `max_requests_in_flight` + - `http_max_response_time` - `max_messages_in_flight` -- `max_requests_in_flight` -- `http_max_response_time` ### Performance Measurements From f9ac95cb9bd66251800ee77df3f0c85ec9b7c6a8 Mon Sep 17 00:00:00 2001 From: Peter Oschwald Date: Fri, 21 Apr 2023 14:47:06 -0500 Subject: [PATCH 14/58] Add command line support for http-max-in-flight-requests. --- tests/performance_tests/performance_test_basic.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/performance_tests/performance_test_basic.py b/tests/performance_tests/performance_test_basic.py index cc01b2bc62..ac1f4822d7 100755 --- a/tests/performance_tests/performance_test_basic.py +++ b/tests/performance_tests/performance_test_basic.py @@ -549,8 +549,8 @@ def setupClusterConfig(args) -> ClusterConfig: producerPluginArgs = ProducerPluginArgs(disableSubjectiveBilling=args.disable_subjective_billing, cpuEffortPercent=args.cpu_effort_percent, producerThreads=args.producer_threads, maxTransactionTime=-1) - httpPluginArgs = HttpPluginArgs(httpMaxResponseTimeMs=args.http_max_response_time_ms, httpMaxBytesInFlightMb=args.http_max_bytes_in_flight_mb, - httpThreads=args.http_threads) + httpPluginArgs = HttpPluginArgs(httpMaxBytesInFlightMb=args.http_max_bytes_in_flight_mb, httpMaxInFlightRequests=args.http_max_in_flight_requests, + httpMaxResponseTimeMs=args.http_max_response_time_ms, httpThreads=args.http_threads) netPluginArgs = NetPluginArgs(netThreads=args.net_threads, maxClients=0) nodeosVers=Utils.getNodeosVersion().split('.')[0] resourceMonitorPluginArgs = ResourceMonitorPluginArgs(resourceMonitorNotShutdownOnThresholdExceeded=not nodeosVers == "v2") @@ -595,6 +595,7 @@ def createBaseArgumentParser(suppressHelp: bool=False): ptbBaseParserGroup.add_argument("--disable-subjective-billing", type=bool, help=argparse.SUPPRESS if suppressHelp else "Disable subjective CPU billing for API/P2P transactions", default=True) ptbBaseParserGroup.add_argument("--cpu-effort-percent", type=int, help=argparse.SUPPRESS if suppressHelp else "Percentage of cpu block production time used to produce block. Whole number percentages, e.g. 80 for 80%%", default=100) ptbBaseParserGroup.add_argument("--producer-threads", type=int, help=argparse.SUPPRESS if suppressHelp else "Number of worker threads in producer thread pool", default=2) + ptbBaseParserGroup.add_argument("--http-max-in-flight-requests", type=int, help=argparse.SUPPRESS if suppressHelp else "Maximum number of requests http_plugin should use for processing http requests. 429 error response when exceeded. -1 for unlimited", default=-1) ptbBaseParserGroup.add_argument("--http-max-response-time-ms", type=int, help=argparse.SUPPRESS if suppressHelp else "Maximum time for processing a request, -1 for unlimited", default=-1) ptbBaseParserGroup.add_argument("--http-max-bytes-in-flight-mb", type=int, help=argparse.SUPPRESS if suppressHelp else "Maximum size in megabytes http_plugin should use for processing http requests. -1 for unlimited. 429\ error response when exceeded.", default=-1) From fdc3bf6f387cf24273947dc514111fb4826f433a Mon Sep 17 00:00:00 2001 From: greg7mdp Date: Fri, 21 Apr 2023 15:57:00 -0400 Subject: [PATCH 15/58] Changes for building with c++20. With these changes, plus the ones in the PRs opened in the following submodules: - bn256 - eos-vm - CLI11 - abieos Leap builds with clang-16 in c++20 mode. --- libraries/chain/include/eosio/chain/types.hpp | 8 ++ .../chain/unapplied_transaction_queue.hpp | 5 +- .../eosio/chain/wasm_eosio_injection.hpp | 120 ++++++++-------- libraries/chain/webassembly/crypto.cpp | 8 +- plugins/chain_plugin/account_query_db.cpp | 3 +- .../test_trx_finality_status_processing.cpp | 128 +++++++++--------- plugins/chain_plugin/trx_retry_db.cpp | 5 +- .../file_space_handler.hpp | 2 +- 8 files changed, 149 insertions(+), 130 deletions(-) diff --git a/libraries/chain/include/eosio/chain/types.hpp b/libraries/chain/include/eosio/chain/types.hpp index b0089f3a15..0a71ea9fdf 100644 --- a/libraries/chain/include/eosio/chain/types.hpp +++ b/libraries/chain/include/eosio/chain/types.hpp @@ -87,6 +87,14 @@ namespace eosio::chain { #endif struct void_t{}; + struct noncopyable { + noncopyable() = default; + noncopyable(noncopyable&&) = default; + noncopyable& operator=(noncopyable&&) = default; + noncopyable(const noncopyable&) = delete; + noncopyable& operator=(const noncopyable&) = delete; + }; + using chainbase::allocator; using shared_string = chainbase::shared_string; template diff --git a/libraries/chain/include/eosio/chain/unapplied_transaction_queue.hpp b/libraries/chain/include/eosio/chain/unapplied_transaction_queue.hpp index 7677caa5b5..65775e6a7a 100644 --- a/libraries/chain/include/eosio/chain/unapplied_transaction_queue.hpp +++ b/libraries/chain/include/eosio/chain/unapplied_transaction_queue.hpp @@ -37,11 +37,14 @@ struct unapplied_transaction { const transaction_id_type& id()const { return trx_meta->id(); } fc::time_point_sec expiration()const { return trx_meta->packed_trx()->expiration(); } - +#if __cplusplus >= 202002L + [[no_unique_address]] noncopyable _; +#else unapplied_transaction(const unapplied_transaction&) = delete; unapplied_transaction() = delete; unapplied_transaction& operator=(const unapplied_transaction&) = delete; unapplied_transaction(unapplied_transaction&&) = default; +#endif }; /** diff --git a/libraries/chain/include/eosio/chain/wasm_eosio_injection.hpp b/libraries/chain/include/eosio/chain/wasm_eosio_injection.hpp index 48ee05c2a3..020a4ad202 100644 --- a/libraries/chain/include/eosio/chain/wasm_eosio_injection.hpp +++ b/libraries/chain/include/eosio/chain/wasm_eosio_injection.hpp @@ -168,121 +168,121 @@ namespace eosio { namespace chain { namespace wasm_injections { constexpr const char* inject_which_op( uint16_t opcode ) { switch ( opcode ) { case wasm_ops::f32_add_code: - return u8"_eosio_f32_add"; + return "_eosio_f32_add"; case wasm_ops::f32_sub_code: - return u8"_eosio_f32_sub"; + return "_eosio_f32_sub"; case wasm_ops::f32_mul_code: - return u8"_eosio_f32_mul"; + return "_eosio_f32_mul"; case wasm_ops::f32_div_code: - return u8"_eosio_f32_div"; + return "_eosio_f32_div"; case wasm_ops::f32_min_code: - return u8"_eosio_f32_min"; + return "_eosio_f32_min"; case wasm_ops::f32_max_code: - return u8"_eosio_f32_max"; + return "_eosio_f32_max"; case wasm_ops::f32_copysign_code: - return u8"_eosio_f32_copysign"; + return "_eosio_f32_copysign"; case wasm_ops::f32_abs_code: - return u8"_eosio_f32_abs"; + return "_eosio_f32_abs"; case wasm_ops::f32_neg_code: - return u8"_eosio_f32_neg"; + return "_eosio_f32_neg"; case wasm_ops::f32_sqrt_code: - return u8"_eosio_f32_sqrt"; + return "_eosio_f32_sqrt"; case wasm_ops::f32_ceil_code: - return u8"_eosio_f32_ceil"; + return "_eosio_f32_ceil"; case wasm_ops::f32_floor_code: - return u8"_eosio_f32_floor"; + return "_eosio_f32_floor"; case wasm_ops::f32_trunc_code: - return u8"_eosio_f32_trunc"; + return "_eosio_f32_trunc"; case wasm_ops::f32_nearest_code: - return u8"_eosio_f32_nearest"; + return "_eosio_f32_nearest"; case wasm_ops::f32_eq_code: - return u8"_eosio_f32_eq"; + return "_eosio_f32_eq"; case wasm_ops::f32_ne_code: - return u8"_eosio_f32_ne"; + return "_eosio_f32_ne"; case wasm_ops::f32_lt_code: - return u8"_eosio_f32_lt"; + return "_eosio_f32_lt"; case wasm_ops::f32_le_code: - return u8"_eosio_f32_le"; + return "_eosio_f32_le"; case wasm_ops::f32_gt_code: - return u8"_eosio_f32_gt"; + return "_eosio_f32_gt"; case wasm_ops::f32_ge_code: - return u8"_eosio_f32_ge"; + return "_eosio_f32_ge"; case wasm_ops::f64_add_code: - return u8"_eosio_f64_add"; + return "_eosio_f64_add"; case wasm_ops::f64_sub_code: - return u8"_eosio_f64_sub"; + return "_eosio_f64_sub"; case wasm_ops::f64_mul_code: - return u8"_eosio_f64_mul"; + return "_eosio_f64_mul"; case wasm_ops::f64_div_code: - return u8"_eosio_f64_div"; + return "_eosio_f64_div"; case wasm_ops::f64_min_code: - return u8"_eosio_f64_min"; + return "_eosio_f64_min"; case wasm_ops::f64_max_code: - return u8"_eosio_f64_max"; + return "_eosio_f64_max"; case wasm_ops::f64_copysign_code: - return u8"_eosio_f64_copysign"; + return "_eosio_f64_copysign"; case wasm_ops::f64_abs_code: - return u8"_eosio_f64_abs"; + return "_eosio_f64_abs"; case wasm_ops::f64_neg_code: - return u8"_eosio_f64_neg"; + return "_eosio_f64_neg"; case wasm_ops::f64_sqrt_code: - return u8"_eosio_f64_sqrt"; + return "_eosio_f64_sqrt"; case wasm_ops::f64_ceil_code: - return u8"_eosio_f64_ceil"; + return "_eosio_f64_ceil"; case wasm_ops::f64_floor_code: - return u8"_eosio_f64_floor"; + return "_eosio_f64_floor"; case wasm_ops::f64_trunc_code: - return u8"_eosio_f64_trunc"; + return "_eosio_f64_trunc"; case wasm_ops::f64_nearest_code: - return u8"_eosio_f64_nearest"; + return "_eosio_f64_nearest"; case wasm_ops::f64_eq_code: - return u8"_eosio_f64_eq"; + return "_eosio_f64_eq"; case wasm_ops::f64_ne_code: - return u8"_eosio_f64_ne"; + return "_eosio_f64_ne"; case wasm_ops::f64_lt_code: - return u8"_eosio_f64_lt"; + return "_eosio_f64_lt"; case wasm_ops::f64_le_code: - return u8"_eosio_f64_le"; + return "_eosio_f64_le"; case wasm_ops::f64_gt_code: - return u8"_eosio_f64_gt"; + return "_eosio_f64_gt"; case wasm_ops::f64_ge_code: - return u8"_eosio_f64_ge"; + return "_eosio_f64_ge"; case wasm_ops::f64_promote_f32_code: - return u8"_eosio_f32_promote"; + return "_eosio_f32_promote"; case wasm_ops::f32_demote_f64_code: - return u8"_eosio_f64_demote"; + return "_eosio_f64_demote"; case wasm_ops::i32_trunc_u_f32_code: - return u8"_eosio_f32_trunc_i32u"; + return "_eosio_f32_trunc_i32u"; case wasm_ops::i32_trunc_s_f32_code: - return u8"_eosio_f32_trunc_i32s"; + return "_eosio_f32_trunc_i32s"; case wasm_ops::i32_trunc_u_f64_code: - return u8"_eosio_f64_trunc_i32u"; + return "_eosio_f64_trunc_i32u"; case wasm_ops::i32_trunc_s_f64_code: - return u8"_eosio_f64_trunc_i32s"; + return "_eosio_f64_trunc_i32s"; case wasm_ops::i64_trunc_u_f32_code: - return u8"_eosio_f32_trunc_i64u"; + return "_eosio_f32_trunc_i64u"; case wasm_ops::i64_trunc_s_f32_code: - return u8"_eosio_f32_trunc_i64s"; + return "_eosio_f32_trunc_i64s"; case wasm_ops::i64_trunc_u_f64_code: - return u8"_eosio_f64_trunc_i64u"; + return "_eosio_f64_trunc_i64u"; case wasm_ops::i64_trunc_s_f64_code: - return u8"_eosio_f64_trunc_i64s"; + return "_eosio_f64_trunc_i64s"; case wasm_ops::f32_convert_s_i32_code: - return u8"_eosio_i32_to_f32"; + return "_eosio_i32_to_f32"; case wasm_ops::f32_convert_u_i32_code: - return u8"_eosio_ui32_to_f32"; + return "_eosio_ui32_to_f32"; case wasm_ops::f32_convert_s_i64_code: - return u8"_eosio_i64_f32"; + return "_eosio_i64_f32"; case wasm_ops::f32_convert_u_i64_code: - return u8"_eosio_ui64_to_f32"; + return "_eosio_ui64_to_f32"; case wasm_ops::f64_convert_s_i32_code: - return u8"_eosio_i32_to_f64"; + return "_eosio_i32_to_f64"; case wasm_ops::f64_convert_u_i32_code: - return u8"_eosio_ui32_to_f64"; + return "_eosio_ui32_to_f64"; case wasm_ops::f64_convert_s_i64_code: - return u8"_eosio_i64_to_f64"; + return "_eosio_i64_to_f64"; case wasm_ops::f64_convert_u_i64_code: - return u8"_eosio_ui64_to_f64"; + return "_eosio_ui64_to_f64"; default: FC_THROW_EXCEPTION( eosio::chain::wasm_execution_error, "Error, unknown opcode in injection ${op}", ("op", opcode)); @@ -483,7 +483,7 @@ namespace eosio { namespace chain { namespace wasm_injections { static void init() {} static void accept( wasm_ops::instr* inst, wasm_ops::visitor_arg& arg ) { int32_t idx; - injector_utils::add_import( *(arg.module), u8"_eosio_f32_promote", idx ); + injector_utils::add_import( *(arg.module), "_eosio_f32_promote", idx ); wasm_ops::op_types<>::call_t f32promote; f32promote.field = idx; f32promote.pack(arg.new_code); @@ -496,7 +496,7 @@ namespace eosio { namespace chain { namespace wasm_injections { static void init() {} static void accept( wasm_ops::instr* inst, wasm_ops::visitor_arg& arg ) { int32_t idx; - injector_utils::add_import( *(arg.module), u8"_eosio_f64_demote", idx ); + injector_utils::add_import( *(arg.module), "_eosio_f64_demote", idx ); wasm_ops::op_types<>::call_t f32promote; f32promote.field = idx; f32promote.pack(arg.new_code); diff --git a/libraries/chain/webassembly/crypto.cpp b/libraries/chain/webassembly/crypto.cpp index 43991610f9..095ee4c1d9 100644 --- a/libraries/chain/webassembly/crypto.cpp +++ b/libraries/chain/webassembly/crypto.cpp @@ -118,14 +118,18 @@ namespace eosio { namespace chain { namespace webassembly { int32_t interface::alt_bn128_add(span op1, span op2, span result ) const { if (op1.size() != 64 || op2.size() != 64 || result.size() < 64 || - bn256::g1_add({(const uint8_t*)op1.data(), 64}, {(const uint8_t*)op2.data(), 64}, { (uint8_t*)result.data(), 64}) == -1) + bn256::g1_add(std::span{(const uint8_t*)op1.data(), 64}, + std::span{(const uint8_t*)op2.data(), 64}, + std::span{ (uint8_t*)result.data(), 64}) == -1) return return_code::failure; return return_code::success; } int32_t interface::alt_bn128_mul(span g1_point, span scalar, span result) const { if (g1_point.size() != 64 || scalar.size() != 32 || result.size() < 64 || - bn256::g1_scalar_mul({(const uint8_t*)g1_point.data(), 64}, {(const uint8_t*)scalar.data(), 32}, { (uint8_t*)result.data(), 64}) == -1) + bn256::g1_scalar_mul(std::span{(const uint8_t*)g1_point.data(), 64}, + std::span{(const uint8_t*)scalar.data(), 32}, + std::span{ (uint8_t*)result.data(), 64}) == -1) return return_code::failure; return return_code::success; } diff --git a/plugins/chain_plugin/account_query_db.cpp b/plugins/chain_plugin/account_query_db.cpp index f0206eabcb..0c2442f6da 100644 --- a/plugins/chain_plugin/account_query_db.cpp +++ b/plugins/chain_plugin/account_query_db.cpp @@ -266,7 +266,8 @@ namespace eosio::chain_apis { } else { const auto& po = *itr; - uint32_t last_updated_height = po.last_updated == bsp->header.timestamp ? bsp->block_num : last_updated_time_to_height(po.last_updated); + uint32_t last_updated_height = chain::block_timestamp_type(po.last_updated) == bsp->header.timestamp ? + bsp->block_num : last_updated_time_to_height(po.last_updated); index.modify(index.iterator_to(pi), [&po, last_updated_height](auto& mutable_pi) { mutable_pi.last_updated_height = last_updated_height; diff --git a/plugins/chain_plugin/test/test_trx_finality_status_processing.cpp b/plugins/chain_plugin/test/test_trx_finality_status_processing.cpp index 938c07d304..4dd9a1748a 100644 --- a/plugins/chain_plugin/test/test_trx_finality_status_processing.cpp +++ b/plugins/chain_plugin/test/test_trx_finality_status_processing.cpp @@ -203,28 +203,28 @@ BOOST_AUTO_TEST_CASE(trx_finality_status_logic) { try { op_ts ts = status.get_trx_state(std::get<1>(trx_pairs_20[0])->id()); BOOST_REQUIRE(ts); BOOST_CHECK(ts->block_id == eosio::chain::block_id_type{}); - BOOST_CHECK(ts->block_timestamp == eosio::chain::block_timestamp_type{}); + BOOST_CHECK(block_timestamp_type(ts->block_timestamp) == eosio::chain::block_timestamp_type{}); BOOST_CHECK_EQUAL(std::string(ts->received), pre_block_20_time); BOOST_CHECK_EQUAL(ts->status, "LOCALLY_APPLIED"); ts = status.get_trx_state(std::get<1>(trx_pairs_20[1])->id()); BOOST_REQUIRE(ts); BOOST_CHECK(ts->block_id == eosio::chain::block_id_type{}); - BOOST_CHECK(ts->block_timestamp == eosio::chain::block_timestamp_type{}); + BOOST_CHECK(block_timestamp_type(ts->block_timestamp) == eosio::chain::block_timestamp_type{}); BOOST_CHECK_EQUAL(std::string(ts->received), pre_block_20_time); BOOST_CHECK_EQUAL(ts->status, "LOCALLY_APPLIED"); ts = status.get_trx_state(std::get<1>(trx_pairs_20[2])->id()); BOOST_REQUIRE(ts); BOOST_CHECK(ts->block_id == eosio::chain::block_id_type{}); - BOOST_CHECK(ts->block_timestamp == eosio::chain::block_timestamp_type{}); + BOOST_CHECK(block_timestamp_type(ts->block_timestamp) == eosio::chain::block_timestamp_type{}); BOOST_CHECK_EQUAL(std::string(ts->received), pre_block_20_time); BOOST_CHECK_EQUAL(ts->status, "LOCALLY_APPLIED"); ts = status.get_trx_state(std::get<1>(trx_pairs_20[3])->id()); BOOST_REQUIRE(ts); BOOST_CHECK(ts->block_id == eosio::chain::block_id_type{}); - BOOST_CHECK(ts->block_timestamp == eosio::chain::block_timestamp_type{}); + BOOST_CHECK(block_timestamp_type(ts->block_timestamp) == eosio::chain::block_timestamp_type{}); BOOST_CHECK_EQUAL(std::string(ts->received), pre_block_20_time); BOOST_CHECK_EQUAL(ts->status, "LOCALLY_APPLIED"); @@ -271,14 +271,14 @@ BOOST_AUTO_TEST_CASE(trx_finality_status_logic) { try { ts = status.get_trx_state(std::get<1>(trx_pairs_20[0])->id()); BOOST_REQUIRE(ts); BOOST_CHECK(ts->block_id == bs_20->id); - BOOST_CHECK(ts->block_timestamp == bs_20->block->timestamp); + BOOST_CHECK(block_timestamp_type(ts->block_timestamp) == bs_20->block->timestamp); BOOST_CHECK_EQUAL(std::string(ts->received), pre_block_20_time); BOOST_CHECK_EQUAL(ts->status, "IN_BLOCK"); ts = status.get_trx_state(std::get<1>(trx_pairs_20[1])->id()); BOOST_REQUIRE(ts); BOOST_CHECK(ts->block_id == bs_20->id); - BOOST_CHECK(ts->block_timestamp == bs_20->block->timestamp); + BOOST_CHECK(block_timestamp_type(ts->block_timestamp) == bs_20->block->timestamp); BOOST_CHECK(fc::time_point_sec(ts->expiration) == (std::get<1>(trx_pairs_20[1])->expiration())); BOOST_CHECK_EQUAL(std::string(ts->received), pre_block_20_time); BOOST_CHECK_EQUAL(ts->status, "IN_BLOCK"); @@ -286,28 +286,28 @@ BOOST_AUTO_TEST_CASE(trx_finality_status_logic) { try { ts = status.get_trx_state(std::get<1>(trx_pairs_20[2])->id()); BOOST_REQUIRE(ts); BOOST_CHECK(ts->block_id == bs_20->id); - BOOST_CHECK(ts->block_timestamp == bs_20->block->timestamp); + BOOST_CHECK(block_timestamp_type(ts->block_timestamp) == bs_20->block->timestamp); BOOST_CHECK_EQUAL(std::string(ts->received), block_20_time); BOOST_CHECK_EQUAL(ts->status, "IN_BLOCK"); ts = status.get_trx_state(std::get<1>(trx_pairs_20[3])->id()); BOOST_REQUIRE(ts); BOOST_CHECK(ts->block_id == bs_20->id); - BOOST_CHECK(ts->block_timestamp == bs_20->block->timestamp); + BOOST_CHECK(block_timestamp_type(ts->block_timestamp) == bs_20->block->timestamp); BOOST_CHECK_EQUAL(std::string(ts->received), block_20_time); BOOST_CHECK_EQUAL(ts->status, "IN_BLOCK"); ts = status.get_trx_state(std::get<1>(hold_pairs[0])->id()); BOOST_REQUIRE(ts); BOOST_CHECK(ts->block_id == eosio::chain::block_id_type{}); - BOOST_CHECK(ts->block_timestamp == eosio::chain::block_timestamp_type{}); + BOOST_CHECK(block_timestamp_type(ts->block_timestamp) == eosio::chain::block_timestamp_type{}); BOOST_CHECK_EQUAL(std::string(ts->received), pre_block_20_time); BOOST_CHECK_EQUAL(ts->status, "LOCALLY_APPLIED"); ts = status.get_trx_state(std::get<1>(hold_pairs[1])->id()); BOOST_REQUIRE(ts); BOOST_CHECK(ts->block_id == eosio::chain::block_id_type{}); - BOOST_CHECK(ts->block_timestamp == eosio::chain::block_timestamp_type{}); + BOOST_CHECK(block_timestamp_type(ts->block_timestamp) == eosio::chain::block_timestamp_type{}); BOOST_CHECK_EQUAL(std::string(ts->received), pre_block_20_time); BOOST_CHECK_EQUAL(ts->status, "LOCALLY_APPLIED"); @@ -334,49 +334,49 @@ BOOST_AUTO_TEST_CASE(trx_finality_status_logic) { try { ts = status.get_trx_state(std::get<1>(trx_pairs_20[0])->id()); BOOST_REQUIRE(ts); BOOST_CHECK(ts->block_id == bs_20->id); - BOOST_CHECK(ts->block_timestamp == bs_20->block->timestamp); + BOOST_CHECK(block_timestamp_type(ts->block_timestamp) == bs_20->block->timestamp); BOOST_CHECK_EQUAL(std::string(ts->received), pre_block_20_time); BOOST_CHECK_EQUAL(ts->status, "IN_BLOCK"); ts = status.get_trx_state(std::get<1>(trx_pairs_20[1])->id()); BOOST_REQUIRE(ts); BOOST_CHECK(ts->block_id == bs_20->id); - BOOST_CHECK(ts->block_timestamp == bs_20->block->timestamp); + BOOST_CHECK(block_timestamp_type(ts->block_timestamp) == bs_20->block->timestamp); BOOST_CHECK_EQUAL(std::string(ts->received), pre_block_20_time); BOOST_CHECK_EQUAL(ts->status, "IN_BLOCK"); ts = status.get_trx_state(std::get<1>(trx_pairs_20[2])->id()); BOOST_REQUIRE(ts); BOOST_CHECK(ts->block_id == bs_20->id); - BOOST_CHECK(ts->block_timestamp == bs_20->block->timestamp); + BOOST_CHECK(block_timestamp_type(ts->block_timestamp) == bs_20->block->timestamp); BOOST_CHECK_EQUAL(std::string(ts->received), block_20_time); BOOST_CHECK_EQUAL(ts->status, "IN_BLOCK"); ts = status.get_trx_state(std::get<1>(trx_pairs_20[3])->id()); BOOST_REQUIRE(ts); BOOST_CHECK(ts->block_id == bs_20->id); - BOOST_CHECK(ts->block_timestamp == bs_20->block->timestamp); + BOOST_CHECK(block_timestamp_type(ts->block_timestamp) == bs_20->block->timestamp); BOOST_CHECK_EQUAL(std::string(ts->received), block_20_time); BOOST_CHECK_EQUAL(ts->status, "IN_BLOCK"); ts = status.get_trx_state(std::get<1>(hold_pairs[0])->id()); BOOST_REQUIRE(ts); BOOST_CHECK(ts->block_id == eosio::chain::block_id_type{}); - BOOST_CHECK(ts->block_timestamp == eosio::chain::block_timestamp_type{}); + BOOST_CHECK(block_timestamp_type(ts->block_timestamp) == eosio::chain::block_timestamp_type{}); BOOST_CHECK_EQUAL(std::string(ts->received), pre_block_20_time); BOOST_CHECK_EQUAL(ts->status, "LOCALLY_APPLIED"); ts = status.get_trx_state(std::get<1>(hold_pairs[1])->id()); BOOST_REQUIRE(ts); BOOST_CHECK(ts->block_id == eosio::chain::block_id_type{}); - BOOST_CHECK(ts->block_timestamp == eosio::chain::block_timestamp_type{}); + BOOST_CHECK(block_timestamp_type(ts->block_timestamp) == eosio::chain::block_timestamp_type{}); BOOST_CHECK_EQUAL(std::string(ts->received), pre_block_20_time); BOOST_CHECK_EQUAL(ts->status, "LOCALLY_APPLIED"); ts = status.get_trx_state(std::get<1>(trx_pairs_21[0])->id()); BOOST_REQUIRE(ts); BOOST_CHECK(ts->block_id == bs_21->id); - BOOST_CHECK(ts->block_timestamp == bs_21->block->timestamp); + BOOST_CHECK(block_timestamp_type(ts->block_timestamp) == bs_21->block->timestamp); BOOST_CHECK_EQUAL(std::string(ts->received), block_21_time); BOOST_CHECK_EQUAL(ts->status, "IN_BLOCK"); @@ -404,56 +404,56 @@ BOOST_AUTO_TEST_CASE(trx_finality_status_logic) { try { ts = status.get_trx_state(std::get<1>(trx_pairs_20[0])->id()); BOOST_REQUIRE(ts); BOOST_CHECK(ts->block_id == bs_20->id); - BOOST_CHECK(ts->block_timestamp == bs_20->block->timestamp); + BOOST_CHECK(block_timestamp_type(ts->block_timestamp) == bs_20->block->timestamp); BOOST_CHECK_EQUAL(std::string(ts->received), pre_block_20_time); BOOST_CHECK_EQUAL(ts->status, "IN_BLOCK"); ts = status.get_trx_state(std::get<1>(trx_pairs_20[1])->id()); BOOST_REQUIRE(ts); BOOST_CHECK(ts->block_id == bs_20->id); - BOOST_CHECK(ts->block_timestamp == bs_20->block->timestamp); + BOOST_CHECK(block_timestamp_type(ts->block_timestamp) == bs_20->block->timestamp); BOOST_CHECK_EQUAL(std::string(ts->received), pre_block_20_time); BOOST_CHECK_EQUAL(ts->status, "IN_BLOCK"); ts = status.get_trx_state(std::get<1>(trx_pairs_20[2])->id()); BOOST_REQUIRE(ts); BOOST_CHECK(ts->block_id == bs_20->id); - BOOST_CHECK(ts->block_timestamp == bs_20->block->timestamp); + BOOST_CHECK(block_timestamp_type(ts->block_timestamp) == bs_20->block->timestamp); BOOST_CHECK_EQUAL(std::string(ts->received), block_20_time); BOOST_CHECK_EQUAL(ts->status, "IN_BLOCK"); ts = status.get_trx_state(std::get<1>(trx_pairs_20[3])->id()); BOOST_REQUIRE(ts); BOOST_CHECK(ts->block_id == bs_20->id); - BOOST_CHECK(ts->block_timestamp == bs_20->block->timestamp); + BOOST_CHECK(block_timestamp_type(ts->block_timestamp) == bs_20->block->timestamp); BOOST_CHECK_EQUAL(std::string(ts->received), block_20_time); BOOST_CHECK_EQUAL(ts->status, "IN_BLOCK"); ts = status.get_trx_state(std::get<1>(hold_pairs[0])->id()); BOOST_REQUIRE(ts); BOOST_CHECK(ts->block_id == eosio::chain::block_id_type{}); - BOOST_CHECK(ts->block_timestamp == eosio::chain::block_timestamp_type{}); + BOOST_CHECK(block_timestamp_type(ts->block_timestamp) == eosio::chain::block_timestamp_type{}); BOOST_CHECK_EQUAL(std::string(ts->received), pre_block_20_time); BOOST_CHECK_EQUAL(ts->status, "LOCALLY_APPLIED"); ts = status.get_trx_state(std::get<1>(hold_pairs[1])->id()); BOOST_REQUIRE(ts); BOOST_CHECK(ts->block_id == eosio::chain::block_id_type{}); - BOOST_CHECK(ts->block_timestamp == eosio::chain::block_timestamp_type{}); + BOOST_CHECK(block_timestamp_type(ts->block_timestamp) == eosio::chain::block_timestamp_type{}); BOOST_CHECK_EQUAL(std::string(ts->received), pre_block_20_time); BOOST_CHECK_EQUAL(ts->status, "LOCALLY_APPLIED"); ts = status.get_trx_state(std::get<1>(trx_pairs_21[0])->id()); BOOST_REQUIRE(ts); BOOST_CHECK(ts->block_id == bs_21->id); - BOOST_CHECK(ts->block_timestamp == bs_21->block->timestamp); + BOOST_CHECK(block_timestamp_type(ts->block_timestamp) == bs_21->block->timestamp); BOOST_CHECK_EQUAL(std::string(ts->received), block_21_time); BOOST_CHECK_EQUAL(ts->status, "IN_BLOCK"); ts = status.get_trx_state(std::get<1>(trx_pairs_22[0])->id()); BOOST_REQUIRE(ts); BOOST_CHECK(ts->block_id == bs_22->id); - BOOST_CHECK(ts->block_timestamp == bs_22->block->timestamp); + BOOST_CHECK(block_timestamp_type(ts->block_timestamp) == bs_22->block->timestamp); BOOST_CHECK_EQUAL(std::string(ts->received), block_22_time); BOOST_CHECK_EQUAL(ts->status, "IN_BLOCK"); @@ -482,63 +482,63 @@ BOOST_AUTO_TEST_CASE(trx_finality_status_logic) { try { ts = status.get_trx_state(std::get<1>(trx_pairs_20[0])->id()); BOOST_REQUIRE(ts); BOOST_CHECK(ts->block_id == bs_20->id); - BOOST_CHECK(ts->block_timestamp == bs_20->block->timestamp); + BOOST_CHECK(block_timestamp_type(ts->block_timestamp) == bs_20->block->timestamp); BOOST_CHECK_EQUAL(std::string(ts->received), pre_block_20_time); BOOST_CHECK_EQUAL(ts->status, "IN_BLOCK"); ts = status.get_trx_state(std::get<1>(trx_pairs_20[1])->id()); BOOST_REQUIRE(ts); BOOST_CHECK(ts->block_id == bs_20->id); - BOOST_CHECK(ts->block_timestamp == bs_20->block->timestamp); + BOOST_CHECK(block_timestamp_type(ts->block_timestamp) == bs_20->block->timestamp); BOOST_CHECK_EQUAL(std::string(ts->received), pre_block_20_time); BOOST_CHECK_EQUAL(ts->status, "IN_BLOCK"); ts = status.get_trx_state(std::get<1>(trx_pairs_20[2])->id()); BOOST_REQUIRE(ts); BOOST_CHECK(ts->block_id == bs_20->id); - BOOST_CHECK(ts->block_timestamp == bs_20->block->timestamp); + BOOST_CHECK(block_timestamp_type(ts->block_timestamp) == bs_20->block->timestamp); BOOST_CHECK_EQUAL(std::string(ts->received), block_20_time); BOOST_CHECK_EQUAL(ts->status, "IN_BLOCK"); ts = status.get_trx_state(std::get<1>(trx_pairs_20[3])->id()); BOOST_REQUIRE(ts); BOOST_CHECK(ts->block_id == bs_20->id); - BOOST_CHECK(ts->block_timestamp == bs_20->block->timestamp); + BOOST_CHECK(block_timestamp_type(ts->block_timestamp) == bs_20->block->timestamp); BOOST_CHECK_EQUAL(std::string(ts->received), block_20_time); BOOST_CHECK_EQUAL(ts->status, "IN_BLOCK"); ts = status.get_trx_state(std::get<1>(hold_pairs[0])->id()); BOOST_REQUIRE(ts); BOOST_CHECK(ts->block_id == eosio::chain::block_id_type{}); - BOOST_CHECK(ts->block_timestamp == eosio::chain::block_timestamp_type{}); + BOOST_CHECK(block_timestamp_type(ts->block_timestamp) == eosio::chain::block_timestamp_type{}); BOOST_CHECK_EQUAL(std::string(ts->received), pre_block_20_time); BOOST_CHECK_EQUAL(ts->status, "FAILED"); ts = status.get_trx_state(std::get<1>(hold_pairs[1])->id()); BOOST_REQUIRE(ts); BOOST_CHECK(ts->block_id == eosio::chain::block_id_type{}); - BOOST_CHECK(ts->block_timestamp == eosio::chain::block_timestamp_type{}); + BOOST_CHECK(block_timestamp_type(ts->block_timestamp) == eosio::chain::block_timestamp_type{}); BOOST_CHECK_EQUAL(std::string(ts->received), pre_block_20_time); BOOST_CHECK_EQUAL(ts->status, "FAILED"); ts = status.get_trx_state(std::get<1>(trx_pairs_21[0])->id()); BOOST_REQUIRE(ts); BOOST_CHECK(ts->block_id == bs_21->id); - BOOST_CHECK(ts->block_timestamp == bs_21->block->timestamp); + BOOST_CHECK(block_timestamp_type(ts->block_timestamp) == bs_21->block->timestamp); BOOST_CHECK_EQUAL(std::string(ts->received), block_21_time); BOOST_CHECK_EQUAL(ts->status, "IN_BLOCK"); ts = status.get_trx_state(std::get<1>(trx_pairs_22[0])->id()); BOOST_REQUIRE(ts); BOOST_CHECK(ts->block_id == bs_22->id); - BOOST_CHECK(ts->block_timestamp == bs_22->block->timestamp); + BOOST_CHECK(block_timestamp_type(ts->block_timestamp) == bs_22->block->timestamp); BOOST_CHECK_EQUAL(std::string(ts->received), block_22_time); BOOST_CHECK_EQUAL(ts->status, "FORKED_OUT"); ts = status.get_trx_state(std::get<1>(trx_pairs_22_alt[0])->id()); BOOST_REQUIRE(ts); BOOST_CHECK(ts->block_id == bs_22_alt->id); - BOOST_CHECK(ts->block_timestamp == bs_22_alt->block->timestamp); + BOOST_CHECK(block_timestamp_type(ts->block_timestamp) == bs_22_alt->block->timestamp); BOOST_CHECK_EQUAL(std::string(ts->received), block_22_alt_time); BOOST_CHECK_EQUAL(ts->status, "IN_BLOCK"); @@ -567,49 +567,49 @@ BOOST_AUTO_TEST_CASE(trx_finality_status_logic) { try { ts = status.get_trx_state(std::get<1>(trx_pairs_20[0])->id()); BOOST_REQUIRE(ts); BOOST_CHECK(ts->block_id == bs_20->id); - BOOST_CHECK(ts->block_timestamp == bs_20->block->timestamp); + BOOST_CHECK(block_timestamp_type(ts->block_timestamp) == bs_20->block->timestamp); BOOST_CHECK_EQUAL(std::string(ts->received), pre_block_20_time); BOOST_CHECK_EQUAL(ts->status, "FAILED"); ts = status.get_trx_state(std::get<1>(trx_pairs_20[1])->id()); BOOST_REQUIRE(ts); BOOST_CHECK(ts->block_id == bs_20->id); - BOOST_CHECK(ts->block_timestamp == bs_20->block->timestamp); + BOOST_CHECK(block_timestamp_type(ts->block_timestamp) == bs_20->block->timestamp); BOOST_CHECK_EQUAL(std::string(ts->received), pre_block_20_time); BOOST_CHECK_EQUAL(ts->status, "FAILED"); ts = status.get_trx_state(std::get<1>(trx_pairs_20[2])->id()); BOOST_REQUIRE(ts); BOOST_CHECK(ts->block_id == bs_20->id); - BOOST_CHECK(ts->block_timestamp == bs_20->block->timestamp); + BOOST_CHECK(block_timestamp_type(ts->block_timestamp) == bs_20->block->timestamp); BOOST_CHECK_EQUAL(std::string(ts->received), block_20_time); BOOST_CHECK_EQUAL(ts->status, "FAILED"); ts = status.get_trx_state(std::get<1>(trx_pairs_20[3])->id()); BOOST_REQUIRE(ts); BOOST_CHECK(ts->block_id == bs_20->id); - BOOST_CHECK(ts->block_timestamp == bs_20->block->timestamp); + BOOST_CHECK(block_timestamp_type(ts->block_timestamp) == bs_20->block->timestamp); BOOST_CHECK_EQUAL(std::string(ts->received), block_20_time); BOOST_CHECK_EQUAL(ts->status, "FAILED"); ts = status.get_trx_state(std::get<1>(hold_pairs[0])->id()); BOOST_REQUIRE(ts); BOOST_CHECK(ts->block_id == eosio::chain::block_id_type{}); - BOOST_CHECK(ts->block_timestamp == eosio::chain::block_timestamp_type{}); + BOOST_CHECK(block_timestamp_type(ts->block_timestamp) == eosio::chain::block_timestamp_type{}); BOOST_CHECK_EQUAL(std::string(ts->received), pre_block_20_time); BOOST_CHECK_EQUAL(ts->status, "FAILED"); ts = status.get_trx_state(std::get<1>(hold_pairs[1])->id()); BOOST_REQUIRE(ts); BOOST_CHECK(ts->block_id == eosio::chain::block_id_type{}); - BOOST_CHECK(ts->block_timestamp == eosio::chain::block_timestamp_type{}); + BOOST_CHECK(block_timestamp_type(ts->block_timestamp) == eosio::chain::block_timestamp_type{}); BOOST_CHECK_EQUAL(std::string(ts->received), pre_block_20_time); BOOST_CHECK_EQUAL(ts->status, "FAILED"); ts = status.get_trx_state(std::get<1>(trx_pairs_21[0])->id()); BOOST_REQUIRE(ts); BOOST_CHECK(ts->block_id == bs_21->id); - BOOST_CHECK(ts->block_timestamp == bs_21->block->timestamp); + BOOST_CHECK(block_timestamp_type(ts->block_timestamp) == bs_21->block->timestamp); BOOST_CHECK_EQUAL(std::string(ts->received), block_21_time); BOOST_CHECK_EQUAL(ts->status, "FAILED"); @@ -617,21 +617,21 @@ BOOST_AUTO_TEST_CASE(trx_finality_status_logic) { try { ts = status.get_trx_state(std::get<1>(trx_pairs_22[0])->id()); BOOST_REQUIRE(ts); BOOST_CHECK(ts->block_id == bs_22->id); - BOOST_CHECK(ts->block_timestamp == bs_22->block->timestamp); + BOOST_CHECK(block_timestamp_type(ts->block_timestamp) == bs_22->block->timestamp); BOOST_CHECK_EQUAL(std::string(ts->received), block_22_time); BOOST_CHECK_EQUAL(ts->status, "FAILED"); ts = status.get_trx_state(std::get<1>(trx_pairs_22_alt[0])->id()); BOOST_REQUIRE(ts); BOOST_CHECK(ts->block_id == bs_22_alt->id); - BOOST_CHECK(ts->block_timestamp == bs_22_alt->block->timestamp); + BOOST_CHECK(block_timestamp_type(ts->block_timestamp) == bs_22_alt->block->timestamp); BOOST_CHECK_EQUAL(std::string(ts->received), block_22_alt_time); BOOST_CHECK_EQUAL(ts->status, "FORKED_OUT"); ts = status.get_trx_state(std::get<1>(trx_pairs_19[0])->id()); BOOST_REQUIRE(ts); BOOST_CHECK(ts->block_id == bs_19->id); - BOOST_CHECK(ts->block_timestamp == bs_19->block->timestamp); + BOOST_CHECK(block_timestamp_type(ts->block_timestamp) == bs_19->block->timestamp); BOOST_CHECK_EQUAL(std::string(ts->received), block_19_time); BOOST_CHECK_EQUAL(ts->status, "IN_BLOCK"); @@ -675,49 +675,49 @@ BOOST_AUTO_TEST_CASE(trx_finality_status_logic) { try { ts = status.get_trx_state(std::get<1>(trx_pairs_20[0])->id()); BOOST_REQUIRE(ts); BOOST_CHECK(ts->block_id == bs_19_alt->id); - BOOST_CHECK(ts->block_timestamp == bs_19_alt->block->timestamp); + BOOST_CHECK(block_timestamp_type(ts->block_timestamp) == bs_19_alt->block->timestamp); BOOST_CHECK_EQUAL(std::string(ts->received), pre_block_20_time); BOOST_CHECK_EQUAL(ts->status, "IN_BLOCK"); ts = status.get_trx_state(std::get<1>(trx_pairs_20[1])->id()); BOOST_REQUIRE(ts); BOOST_CHECK(ts->block_id == bs_19_alt->id); - BOOST_CHECK(ts->block_timestamp == bs_19_alt->block->timestamp); + BOOST_CHECK(block_timestamp_type(ts->block_timestamp) == bs_19_alt->block->timestamp); BOOST_CHECK_EQUAL(std::string(ts->received), pre_block_20_time); BOOST_CHECK_EQUAL(ts->status, "IN_BLOCK"); ts = status.get_trx_state(std::get<1>(trx_pairs_20[2])->id()); BOOST_REQUIRE(ts); BOOST_CHECK(ts->block_id == bs_19_alt->id); - BOOST_CHECK(ts->block_timestamp == bs_19_alt->block->timestamp); + BOOST_CHECK(block_timestamp_type(ts->block_timestamp) == bs_19_alt->block->timestamp); BOOST_CHECK_EQUAL(std::string(ts->received), block_20_time); BOOST_CHECK_EQUAL(ts->status, "IN_BLOCK"); ts = status.get_trx_state(std::get<1>(trx_pairs_20[3])->id()); BOOST_REQUIRE(ts); BOOST_CHECK(ts->block_id == bs_19_alt->id); - BOOST_CHECK(ts->block_timestamp == bs_19_alt->block->timestamp); + BOOST_CHECK(block_timestamp_type(ts->block_timestamp) == bs_19_alt->block->timestamp); BOOST_CHECK_EQUAL(std::string(ts->received), block_20_time); BOOST_CHECK_EQUAL(ts->status, "IN_BLOCK"); ts = status.get_trx_state(std::get<1>(hold_pairs[0])->id()); BOOST_REQUIRE(ts); BOOST_CHECK(ts->block_id == bs_19_alt->id); - BOOST_CHECK(ts->block_timestamp == bs_19_alt->block->timestamp); + BOOST_CHECK(block_timestamp_type(ts->block_timestamp) == bs_19_alt->block->timestamp); BOOST_CHECK_EQUAL(std::string(ts->received), pre_block_20_time); BOOST_CHECK_EQUAL(ts->status, "IN_BLOCK"); ts = status.get_trx_state(std::get<1>(hold_pairs[1])->id()); BOOST_REQUIRE(ts); BOOST_CHECK(ts->block_id == eosio::chain::block_id_type{}); - BOOST_CHECK(ts->block_timestamp == eosio::chain::block_timestamp_type{}); + BOOST_CHECK(block_timestamp_type(ts->block_timestamp) == eosio::chain::block_timestamp_type{}); BOOST_CHECK_EQUAL(std::string(ts->received), pre_block_20_time); BOOST_CHECK_EQUAL(ts->status, "LOCALLY_APPLIED"); ts = status.get_trx_state(std::get<1>(trx_pairs_21[0])->id()); BOOST_REQUIRE(ts); BOOST_CHECK(ts->block_id == bs_21->id); - BOOST_CHECK(ts->block_timestamp == bs_21->block->timestamp); + BOOST_CHECK(block_timestamp_type(ts->block_timestamp) == bs_21->block->timestamp); BOOST_CHECK_EQUAL(std::string(ts->received), block_21_time); BOOST_CHECK_EQUAL(ts->status, "FORKED_OUT"); @@ -725,21 +725,21 @@ BOOST_AUTO_TEST_CASE(trx_finality_status_logic) { try { ts = status.get_trx_state(std::get<1>(trx_pairs_22[0])->id()); BOOST_REQUIRE(ts); BOOST_CHECK(ts->block_id == bs_22->id); - BOOST_CHECK(ts->block_timestamp == bs_22->block->timestamp); + BOOST_CHECK(block_timestamp_type(ts->block_timestamp) == bs_22->block->timestamp); BOOST_CHECK_EQUAL(std::string(ts->received), block_22_time); BOOST_CHECK_EQUAL(ts->status, "FORKED_OUT"); ts = status.get_trx_state(std::get<1>(trx_pairs_22_alt[0])->id()); BOOST_REQUIRE(ts); BOOST_CHECK(ts->block_id == bs_22_alt->id); - BOOST_CHECK(ts->block_timestamp == bs_22_alt->block->timestamp); + BOOST_CHECK(block_timestamp_type(ts->block_timestamp) == bs_22_alt->block->timestamp); BOOST_CHECK_EQUAL(std::string(ts->received), block_22_alt_time); BOOST_CHECK_EQUAL(ts->status, "FORKED_OUT"); ts = status.get_trx_state(std::get<1>(trx_pairs_19[0])->id()); BOOST_REQUIRE(ts); BOOST_CHECK(ts->block_id == bs_19_alt->id); - BOOST_CHECK(ts->block_timestamp == bs_19_alt->block->timestamp); + BOOST_CHECK(block_timestamp_type(ts->block_timestamp) == bs_19_alt->block->timestamp); BOOST_CHECK_EQUAL(std::string(ts->received), block_19_time); BOOST_CHECK_EQUAL(ts->status, "IN_BLOCK"); @@ -763,49 +763,49 @@ BOOST_AUTO_TEST_CASE(trx_finality_status_logic) { try { ts = status.get_trx_state(std::get<1>(trx_pairs_20[0])->id()); BOOST_REQUIRE(ts); BOOST_CHECK(ts->block_id == bs_19_alt->id); - BOOST_CHECK(ts->block_timestamp == bs_19_alt->block->timestamp); + BOOST_CHECK(block_timestamp_type(ts->block_timestamp) == bs_19_alt->block->timestamp); BOOST_CHECK_EQUAL(std::string(ts->received), pre_block_20_time); BOOST_CHECK_EQUAL(ts->status, "IRREVERSIBLE"); ts = status.get_trx_state(std::get<1>(trx_pairs_20[1])->id()); BOOST_REQUIRE(ts); BOOST_CHECK(ts->block_id == bs_19_alt->id); - BOOST_CHECK(ts->block_timestamp == bs_19_alt->block->timestamp); + BOOST_CHECK(block_timestamp_type(ts->block_timestamp) == bs_19_alt->block->timestamp); BOOST_CHECK_EQUAL(std::string(ts->received), pre_block_20_time); BOOST_CHECK_EQUAL(ts->status, "IRREVERSIBLE"); ts = status.get_trx_state(std::get<1>(trx_pairs_20[2])->id()); BOOST_REQUIRE(ts); BOOST_CHECK(ts->block_id == bs_19_alt->id); - BOOST_CHECK(ts->block_timestamp == bs_19_alt->block->timestamp); + BOOST_CHECK(block_timestamp_type(ts->block_timestamp) == bs_19_alt->block->timestamp); BOOST_CHECK_EQUAL(std::string(ts->received), block_20_time); BOOST_CHECK_EQUAL(ts->status, "IRREVERSIBLE"); ts = status.get_trx_state(std::get<1>(trx_pairs_20[3])->id()); BOOST_REQUIRE(ts); BOOST_CHECK(ts->block_id == bs_19_alt->id); - BOOST_CHECK(ts->block_timestamp == bs_19_alt->block->timestamp); + BOOST_CHECK(block_timestamp_type(ts->block_timestamp) == bs_19_alt->block->timestamp); BOOST_CHECK_EQUAL(std::string(ts->received), block_20_time); BOOST_CHECK_EQUAL(ts->status, "IRREVERSIBLE"); ts = status.get_trx_state(std::get<1>(hold_pairs[0])->id()); BOOST_REQUIRE(ts); BOOST_CHECK(ts->block_id == bs_19_alt->id); - BOOST_CHECK(ts->block_timestamp == bs_19_alt->block->timestamp); + BOOST_CHECK(block_timestamp_type(ts->block_timestamp) == bs_19_alt->block->timestamp); BOOST_CHECK_EQUAL(std::string(ts->received), pre_block_20_time); BOOST_CHECK_EQUAL(ts->status, "IRREVERSIBLE"); ts = status.get_trx_state(std::get<1>(hold_pairs[1])->id()); BOOST_REQUIRE(ts); BOOST_CHECK(ts->block_id == eosio::chain::block_id_type{}); - BOOST_CHECK(ts->block_timestamp == eosio::chain::block_timestamp_type{}); + BOOST_CHECK(block_timestamp_type(ts->block_timestamp) == eosio::chain::block_timestamp_type{}); BOOST_CHECK_EQUAL(std::string(ts->received), pre_block_20_time); BOOST_CHECK_EQUAL(ts->status, "LOCALLY_APPLIED"); ts = status.get_trx_state(std::get<1>(trx_pairs_21[0])->id()); BOOST_REQUIRE(ts); BOOST_CHECK(ts->block_id == bs_21->id); - BOOST_CHECK(ts->block_timestamp == bs_21->block->timestamp); + BOOST_CHECK(block_timestamp_type(ts->block_timestamp) == bs_21->block->timestamp); BOOST_CHECK_EQUAL(std::string(ts->received), block_21_time); BOOST_CHECK_EQUAL(ts->status, "FORKED_OUT"); @@ -813,21 +813,21 @@ BOOST_AUTO_TEST_CASE(trx_finality_status_logic) { try { ts = status.get_trx_state(std::get<1>(trx_pairs_22[0])->id()); BOOST_REQUIRE(ts); BOOST_CHECK(ts->block_id == bs_22->id); - BOOST_CHECK(ts->block_timestamp == bs_22->block->timestamp); + BOOST_CHECK(block_timestamp_type(ts->block_timestamp) == bs_22->block->timestamp); BOOST_CHECK_EQUAL(std::string(ts->received), block_22_time); BOOST_CHECK_EQUAL(ts->status, "FORKED_OUT"); ts = status.get_trx_state(std::get<1>(trx_pairs_22_alt[0])->id()); BOOST_REQUIRE(ts); BOOST_CHECK(ts->block_id == bs_22_alt->id); - BOOST_CHECK(ts->block_timestamp == bs_22_alt->block->timestamp); + BOOST_CHECK(block_timestamp_type(ts->block_timestamp) == bs_22_alt->block->timestamp); BOOST_CHECK_EQUAL(std::string(ts->received), block_22_alt_time); BOOST_CHECK_EQUAL(ts->status, "FORKED_OUT"); ts = status.get_trx_state(std::get<1>(trx_pairs_19[0])->id()); BOOST_REQUIRE(ts); BOOST_CHECK(ts->block_id == bs_19_alt->id); - BOOST_CHECK(ts->block_timestamp == bs_19_alt->block->timestamp); + BOOST_CHECK(block_timestamp_type(ts->block_timestamp) == bs_19_alt->block->timestamp); BOOST_CHECK_EQUAL(std::string(ts->received), block_19_time); BOOST_CHECK_EQUAL(ts->status, "IRREVERSIBLE"); diff --git a/plugins/chain_plugin/trx_retry_db.cpp b/plugins/chain_plugin/trx_retry_db.cpp index dd617279f0..a9863ce233 100644 --- a/plugins/chain_plugin/trx_retry_db.cpp +++ b/plugins/chain_plugin/trx_retry_db.cpp @@ -52,11 +52,14 @@ struct tracked_transaction { } size_t memory_size()const { return ptrx->get_estimated_size() + trx_trace_v.estimated_size() + sizeof(*this); } - +#if __cplusplus >= 202002L + [[no_unique_address]] noncopyable _; +#else tracked_transaction(const tracked_transaction&) = delete; tracked_transaction() = delete; tracked_transaction& operator=(const tracked_transaction&) = delete; tracked_transaction(tracked_transaction&&) = default; +#endif }; struct by_trx_id; diff --git a/plugins/resource_monitor_plugin/include/eosio/resource_monitor_plugin/file_space_handler.hpp b/plugins/resource_monitor_plugin/include/eosio/resource_monitor_plugin/file_space_handler.hpp index 6a5828c7fe..230edf62fb 100644 --- a/plugins/resource_monitor_plugin/include/eosio/resource_monitor_plugin/file_space_handler.hpp +++ b/plugins/resource_monitor_plugin/include/eosio/resource_monitor_plugin/file_space_handler.hpp @@ -138,7 +138,7 @@ namespace eosio::resource_monitor { timer.expires_from_now( boost::posix_time::seconds( sleep_time_in_secs )); - timer.async_wait([this](auto& ec) { + timer.async_wait([this](const auto& ec) { if ( ec ) { wlog("Exit due to error: ${ec}, message: ${message}", ("ec", ec.value()) From 6d4a3ac00d516e1535b32d5deaf6747e438a803f Mon Sep 17 00:00:00 2001 From: Peter Oschwald Date: Fri, 21 Apr 2023 15:55:04 -0500 Subject: [PATCH 16/58] Remove duplicate of max_requests_in_flight --- tests/performance_tests/ph_op_modes.md | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/performance_tests/ph_op_modes.md b/tests/performance_tests/ph_op_modes.md index abd372d1e3..195cbdfa9d 100644 --- a/tests/performance_tests/ph_op_modes.md +++ b/tests/performance_tests/ph_op_modes.md @@ -57,7 +57,6 @@ Configure unlimited: - `max_bytes_in_flight` - `max_requests_in_flight` - `http_max_response_time` -- `max_messages_in_flight` ### Performance Measurements From b40f23f64f84e46639f3dd8ea17b4e7c30a19dbd Mon Sep 17 00:00:00 2001 From: greg7mdp Date: Mon, 24 Apr 2023 10:00:19 -0400 Subject: [PATCH 17/58] Remove `#ifdef` preserving the `nocopy` constraint as per PR comment --- .../include/eosio/chain/unapplied_transaction_queue.hpp | 8 -------- plugins/chain_plugin/trx_retry_db.cpp | 8 -------- 2 files changed, 16 deletions(-) diff --git a/libraries/chain/include/eosio/chain/unapplied_transaction_queue.hpp b/libraries/chain/include/eosio/chain/unapplied_transaction_queue.hpp index 65775e6a7a..51f07e3cc7 100644 --- a/libraries/chain/include/eosio/chain/unapplied_transaction_queue.hpp +++ b/libraries/chain/include/eosio/chain/unapplied_transaction_queue.hpp @@ -37,14 +37,6 @@ struct unapplied_transaction { const transaction_id_type& id()const { return trx_meta->id(); } fc::time_point_sec expiration()const { return trx_meta->packed_trx()->expiration(); } -#if __cplusplus >= 202002L - [[no_unique_address]] noncopyable _; -#else - unapplied_transaction(const unapplied_transaction&) = delete; - unapplied_transaction() = delete; - unapplied_transaction& operator=(const unapplied_transaction&) = delete; - unapplied_transaction(unapplied_transaction&&) = default; -#endif }; /** diff --git a/plugins/chain_plugin/trx_retry_db.cpp b/plugins/chain_plugin/trx_retry_db.cpp index a9863ce233..5b93cb1943 100644 --- a/plugins/chain_plugin/trx_retry_db.cpp +++ b/plugins/chain_plugin/trx_retry_db.cpp @@ -52,14 +52,6 @@ struct tracked_transaction { } size_t memory_size()const { return ptrx->get_estimated_size() + trx_trace_v.estimated_size() + sizeof(*this); } -#if __cplusplus >= 202002L - [[no_unique_address]] noncopyable _; -#else - tracked_transaction(const tracked_transaction&) = delete; - tracked_transaction() = delete; - tracked_transaction& operator=(const tracked_transaction&) = delete; - tracked_transaction(tracked_transaction&&) = default; -#endif }; struct by_trx_id; From c81ae9a766f95cb53308a15996fcb655d06b0a89 Mon Sep 17 00:00:00 2001 From: Huang-Ming Huang Date: Mon, 24 Apr 2023 09:24:47 -0500 Subject: [PATCH 18/58] revert to use distutils --- .cicd/platforms/ubuntu20.Dockerfile | 27 ++++++++++++++------------- .cicd/platforms/ubuntu22.Dockerfile | 27 ++++++++++++++------------- .github/workflows/build.yaml | 7 ++----- scripts/postinst | 6 +++--- scripts/prerm | 4 ++-- 5 files changed, 35 insertions(+), 36 deletions(-) diff --git a/.cicd/platforms/ubuntu20.Dockerfile b/.cicd/platforms/ubuntu20.Dockerfile index e30a888c61..baccb7c937 100644 --- a/.cicd/platforms/ubuntu20.Dockerfile +++ b/.cicd/platforms/ubuntu20.Dockerfile @@ -1,15 +1,16 @@ FROM ubuntu:focal - +ENV TZ="America/New_York" +ENV DEBIAN_FRONTEND=noninteractive RUN apt-get update && apt-get upgrade -y && \ - DEBIAN_FRONTEND=noninteractive apt-get install -y build-essential \ - cmake \ - git \ - jq \ - libboost-all-dev \ - libcurl4-openssl-dev \ - libgmp-dev \ - libssl-dev \ - llvm-11-dev \ - ninja-build \ - python3-numpy \ - zstd + apt-get install -y build-essential \ + cmake \ + git \ + jq \ + libboost-all-dev \ + libcurl4-openssl-dev \ + libgmp-dev \ + libssl-dev \ + llvm-11-dev \ + ninja-build \ + python3-numpy \ + zstd diff --git a/.cicd/platforms/ubuntu22.Dockerfile b/.cicd/platforms/ubuntu22.Dockerfile index 7e4de4cc5c..1e5a936a4d 100644 --- a/.cicd/platforms/ubuntu22.Dockerfile +++ b/.cicd/platforms/ubuntu22.Dockerfile @@ -1,15 +1,16 @@ FROM ubuntu:jammy - +ENV TZ="America/New_York" +ENV DEBIAN_FRONTEND=noninteractive RUN apt-get update && apt-get upgrade -y && \ - DEBIAN_FRONTEND=noninteractive apt-get install -y build-essential \ - cmake \ - git \ - jq \ - libboost-all-dev \ - libcurl4-openssl-dev \ - libgmp-dev \ - libssl-dev \ - llvm-11-dev \ - ninja-build \ - python3-numpy \ - zstd + apt-get install -y build-essential \ + cmake \ + git \ + jq \ + libboost-all-dev \ + libcurl4-openssl-dev \ + libgmp-dev \ + libssl-dev \ + llvm-11-dev \ + ninja-build \ + python3-numpy \ + zstd diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index c6bae57dbe..e3f60238f7 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -105,11 +105,8 @@ jobs: cpack - name: Install dev package run: | - apt update - export DEBIAN_FRONTEND=noninteractive - export TZ="America/New_York" - apt install -y ./build/leap_*.deb - apt install -y ./build/leap-dev*.deb + apt update && apt upgrade -y + apt install -y ./build/leap_*.deb ./build/leap-dev*.deb - name: Test using TestHarness run: | python3 -c "from TestHarness import Cluster" diff --git a/scripts/postinst b/scripts/postinst index cef3eec3d8..2496f70229 100644 --- a/scripts/postinst +++ b/scripts/postinst @@ -1,9 +1,9 @@ #!/bin/sh set -e -Python_DISTLIB=$(python3 -c "import sysconfig; print(sysconfig.get_path('purelib'))" | sed 's/site-packages/dist-packages/') -mkdir -p $Python_DISTLIB -ln -sf @CMAKE_INSTALL_FULL_DATAROOTDIR@/leap_testing/tests/TestHarness $Python_DISTLIB/TestHarness +Python_SITELIB=$(python3 -c "from distutils import sysconfig;print(sysconfig.get_python_lib(plat_specific=False,standard_lib=False))") +mkdir -p $Python_SITELIB +ln -sf @CMAKE_INSTALL_FULL_DATAROOTDIR@/leap_testing/tests/TestHarness $Python_SITELIB/TestHarness # leap_testing is part of the package so should already exist by the time postinst runs diff --git a/scripts/prerm b/scripts/prerm index 4d36f88924..146c43271e 100644 --- a/scripts/prerm +++ b/scripts/prerm @@ -1,6 +1,6 @@ #!/bin/sh # Cleanup symbolic links created during postinst -Python_DISTLIB=$(python3 -c "import sysconfig; print(sysconfig.get_path('purelib'))" | sed 's/site-packages/dist-packages/') -rm -f $Python_DISTLIB/TestHarness +Python_SITELIB=$(python3 -c "from distutils import sysconfig;print(sysconfig.get_python_lib(plat_specific=False,standard_lib=False))") +rm -f $Python_SITELIB/TestHarness rm -f @CMAKE_INSTALL_FULL_DATAROOTDIR@/leap_testing/bin From 5cbdf9bf1caeda49328c9c2ff20c03414c924623 Mon Sep 17 00:00:00 2001 From: Huang-Ming Huang Date: Mon, 24 Apr 2023 09:46:20 -0500 Subject: [PATCH 19/58] refactor prometheus plugin --- .../include/eosio/chain/plugin_metrics.hpp | 62 ------- plugins/http_plugin/http_plugin.cpp | 8 +- .../eosio/http_plugin/beast_http_session.hpp | 6 +- .../include/eosio/http_plugin/common.hpp | 25 +-- .../include/eosio/http_plugin/http_plugin.hpp | 9 +- .../include/eosio/net_plugin/net_plugin.hpp | 26 +-- plugins/net_plugin/net_plugin.cpp | 36 +++- .../eosio/producer_plugin/producer_plugin.hpp | 60 +++--- plugins/producer_plugin/producer_plugin.cpp | 56 +++--- plugins/prometheus_plugin/metrics.hpp | 171 ++++++++++++++++++ .../prometheus_plugin/prometheus_plugin.cpp | 167 +---------------- 11 files changed, 284 insertions(+), 342 deletions(-) delete mode 100644 plugins/chain_interface/include/eosio/chain/plugin_metrics.hpp create mode 100644 plugins/prometheus_plugin/metrics.hpp diff --git a/plugins/chain_interface/include/eosio/chain/plugin_metrics.hpp b/plugins/chain_interface/include/eosio/chain/plugin_metrics.hpp deleted file mode 100644 index 6079074e42..0000000000 --- a/plugins/chain_interface/include/eosio/chain/plugin_metrics.hpp +++ /dev/null @@ -1,62 +0,0 @@ -#pragma once - -#include - -#include -#include -#include - -namespace eosio::chain::plugin_interface { - - // - // prometheus metrics - // - - enum class metric_type { - gauge = 1, - counter = 2 - }; - - struct runtime_metric { - metric_type type = metric_type::gauge; - std::string family; - std::string label; - int64_t value = 0; - }; - - using metrics_listener = std::function)>; - - struct plugin_metrics { - - virtual ~plugin_metrics() = default; - virtual std::vector metrics()=0; - - bool should_post() { - return (_listener && (fc::time_point::now() > (_last_post + _min_post_interval_us))); - } - - bool post_metrics() { - if (should_post()){ - _listener(metrics()); - _last_post = fc::time_point::now(); - return true; - } - - return false; - } - - void register_listener(metrics_listener listener) { - _listener = std::move(listener); - } - - explicit plugin_metrics(fc::microseconds min_post_interval_us = fc::milliseconds(250)) - : _min_post_interval_us(min_post_interval_us) - , _listener(nullptr) {} - - private: - fc::microseconds _min_post_interval_us; - metrics_listener _listener; - fc::time_point _last_post; - }; - -} diff --git a/plugins/http_plugin/http_plugin.cpp b/plugins/http_plugin/http_plugin.cpp index b5549fb6a7..40fa7ab9f3 100644 --- a/plugins/http_plugin/http_plugin.cpp +++ b/plugins/http_plugin/http_plugin.cpp @@ -474,10 +474,6 @@ namespace eosio { return result; } - void http_plugin::register_metrics_listener(chain::plugin_interface::metrics_listener listener) { - my->plugin_state->metrics.register_listener(std::move(listener)); - } - fc::microseconds http_plugin::get_max_response_time()const { return my->plugin_state->max_response_time; } @@ -486,4 +482,8 @@ namespace eosio { return my->plugin_state->max_body_size; } + void http_plugin::register_update_metrics(std::function&& fun) { + my->plugin_state->update_metrics = std::move(fun); + } + } diff --git a/plugins/http_plugin/include/eosio/http_plugin/beast_http_session.hpp b/plugins/http_plugin/include/eosio/http_plugin/beast_http_session.hpp index eafc7ee059..4c2384ca14 100644 --- a/plugins/http_plugin/include/eosio/http_plugin/beast_http_session.hpp +++ b/plugins/http_plugin/include/eosio/http_plugin/beast_http_session.hpp @@ -177,8 +177,10 @@ class beast_http_session : public detail::abstract_conn { std::string body = req.body(); auto content_type = handler_itr->second.content_type; set_content_type_header(content_type); - handler_itr->second.call_count.value++; - plugin_state_->metrics.post_metrics(); + + if (plugin_state_->update_metrics) + plugin_state_->update_metrics({resource}); + handler_itr->second.fn(derived().shared_from_this(), std::move(resource), std::move(body), diff --git a/plugins/http_plugin/include/eosio/http_plugin/common.hpp b/plugins/http_plugin/include/eosio/http_plugin/common.hpp index fbc4d31918..1974733a98 100644 --- a/plugins/http_plugin/include/eosio/http_plugin/common.hpp +++ b/plugins/http_plugin/include/eosio/http_plugin/common.hpp @@ -1,7 +1,6 @@ #pragma once #include // for thread pool -#include #include #include @@ -70,11 +69,9 @@ using abstract_conn_ptr = std::shared_ptr; */ using internal_url_handler_fn = std::function; struct internal_url_handler { - chain::plugin_interface::runtime_metric call_count; internal_url_handler_fn fn; http_content_type content_type = http_content_type::json; - explicit internal_url_handler(const string& url) - : call_count{chain::plugin_interface::metric_type::gauge, "num_calls", url} {} + explicit internal_url_handler(const string& url){} }; /** * Helper method to calculate the "in flight" size of a fc::variant @@ -110,24 +107,6 @@ static size_t in_flight_sizeof(const std::optional& o) { // key -> priority, url_handler typedef map url_handlers_type; -struct http_plugin_metrics : chain::plugin_interface::plugin_metrics { - url_handlers_type& _handlers; - - vector metrics() final { - vector m; - m.reserve(_handlers.size()); - - for (const auto& p : _handlers) { - m.push_back(p.second.call_count); - } - - return m; - } - - explicit http_plugin_metrics(url_handlers_type& handlers) - : _handlers(handlers) {} -}; - struct http_plugin_state { string access_control_allow_origin; string access_control_allow_headers; @@ -154,8 +133,8 @@ struct http_plugin_state { eosio::chain::named_thread_pool thread_pool; fc::logger& logger; + std::function update_metrics; - http_plugin_metrics metrics{url_handlers}; explicit http_plugin_state(fc::logger& log) : logger(log) {} }; diff --git a/plugins/http_plugin/include/eosio/http_plugin/http_plugin.hpp b/plugins/http_plugin/include/eosio/http_plugin/http_plugin.hpp index 50c47e038b..5de6eb3efa 100644 --- a/plugins/http_plugin/include/eosio/http_plugin/http_plugin.hpp +++ b/plugins/http_plugin/include/eosio/http_plugin/http_plugin.hpp @@ -2,7 +2,6 @@ #include #include -#include #include #include #include @@ -120,10 +119,14 @@ namespace eosio { /// @return the configured http-max-response-time-ms fc::microseconds get_max_response_time()const; - void register_metrics_listener(chain::plugin_interface::metrics_listener listener); - size_t get_max_body_size()const; + struct metrics { + std::string target; + }; + + void register_update_metrics(std::function&& fun); + private: std::shared_ptr my; }; diff --git a/plugins/net_plugin/include/eosio/net_plugin/net_plugin.hpp b/plugins/net_plugin/include/eosio/net_plugin/net_plugin.hpp index 80e2b86035..488b9bcf6a 100644 --- a/plugins/net_plugin/include/eosio/net_plugin/net_plugin.hpp +++ b/plugins/net_plugin/include/eosio/net_plugin/net_plugin.hpp @@ -2,28 +2,11 @@ #include #include -#include #include namespace eosio { using namespace appbase; - struct net_plugin_metrics : chain::plugin_interface::plugin_metrics { - chain::plugin_interface::runtime_metric num_peers{ chain::plugin_interface::metric_type::gauge, "num_peers", "num_peers", 0 }; - chain::plugin_interface::runtime_metric num_clients{ chain::plugin_interface::metric_type::gauge, "num_clients", "num_clients", 0 }; - chain::plugin_interface::runtime_metric dropped_trxs{ chain::plugin_interface::metric_type::counter, "dropped_trxs", "dropped_trxs", 0 }; - - vector metrics() final { - vector metrics { - num_peers, - num_clients, - dropped_trxs - }; - - return metrics; - } - }; - struct connection_status { string peer; bool connecting = false; @@ -51,7 +34,14 @@ namespace eosio { std::optional status( const string& endpoint )const; vector connections()const; - void register_metrics_listener(chain::plugin_interface::metrics_listener listener); + struct p2p_connections_metrics { + std::size_t num_peers; + std::size_t num_clients; + }; + + void register_update_p2p_connection_metrics(std::function&&); + void register_increment_failed_p2p_connections(std::function&&); + void register_increment_dropped_trxs(std::function&&); private: std::shared_ptr my; diff --git a/plugins/net_plugin/net_plugin.cpp b/plugins/net_plugin/net_plugin.cpp index f32e72ed8a..7fe781ad65 100644 --- a/plugins/net_plugin/net_plugin.cpp +++ b/plugins/net_plugin/net_plugin.cpp @@ -395,7 +395,6 @@ namespace eosio { boost::asio::deadline_timer accept_error_timer{thread_pool.get_executor()}; - net_plugin_metrics metrics; struct chain_info_t { uint32_t lib_num = 0; @@ -404,6 +403,11 @@ namespace eosio { block_id_type head_id; }; + + std::function update_p2p_connection_metrics; + std::function increment_failed_p2p_connections; + std::function increment_dropped_trxs; + private: alignas(hardware_destructive_interference_size) mutable std::mutex chain_info_mtx; // protects chain_info_t @@ -2470,6 +2474,9 @@ namespace eosio { } else { fc_elog( logger, "connection failed to ${a}, ${error}", ("a", c->peer_address())( "error", err.message())); c->close( false ); + if (my_impl->increment_failed_p2p_connections) { + my_impl->increment_failed_p2p_connections(); + } } } ) ); } @@ -2798,13 +2805,14 @@ namespace eosio { shared_ptr ptr = std::make_shared(); fc::raw::unpack( ds, *ptr ); if( trx_in_progress_sz > def_max_trx_in_progress_size) { - ++my_impl->metrics.dropped_trxs.value; char reason[72]; snprintf(reason, 72, "Dropping trx, too many trx in progress %lu bytes", trx_in_progress_sz); my_impl->producer_plug->log_failed_transaction(ptr->id(), ptr, reason); if (fc::time_point::now() - fc::seconds(1) >= last_dropped_trx_msg_time) { last_dropped_trx_msg_time = fc::time_point::now(); - my_impl->metrics.post_metrics(); + if (my_impl->increment_dropped_trxs) { + my_impl->increment_dropped_trxs(); + } peer_wlog(this, reason); } return true; @@ -3584,9 +3592,9 @@ namespace eosio { } g.unlock(); - metrics.num_clients.value = num_clients; - metrics.num_peers.value = num_peers; - metrics.post_metrics(); + if (update_p2p_connection_metrics) { + update_p2p_connection_metrics({.num_peers = num_peers, .num_clients = num_clients}); + } if( num_clients > 0 || num_peers > 0 ) fc_ilog( logger, "p2p client connections: ${num}/${max}, peer connections: ${pnum}/${pmax}, block producer peers: ${num_bp_peers}", @@ -4031,10 +4039,6 @@ namespace eosio { FC_CAPTURE_AND_RETHROW() } - void net_plugin::register_metrics_listener(metrics_listener listener) { - my->metrics.register_listener(std::move(listener)); - } - /** * Used to trigger a new connection from RPC API */ @@ -4111,4 +4115,16 @@ namespace eosio { return sync_master->is_in_sync(); } + void net_plugin::register_update_p2p_connection_metrics(std::function&& fun){ + my->update_p2p_connection_metrics = std::move(fun); + } + + void net_plugin::register_increment_failed_p2p_connections(std::function&& fun){ + my->increment_failed_p2p_connections = std::move(fun); + } + + void net_plugin::register_increment_dropped_trxs(std::function&& fun){ + my->increment_dropped_trxs = std::move(fun); + } + } diff --git a/plugins/producer_plugin/include/eosio/producer_plugin/producer_plugin.hpp b/plugins/producer_plugin/include/eosio/producer_plugin/producer_plugin.hpp index 82a9284581..be16e1902e 100644 --- a/plugins/producer_plugin/include/eosio/producer_plugin/producer_plugin.hpp +++ b/plugins/producer_plugin/include/eosio/producer_plugin/producer_plugin.hpp @@ -1,6 +1,5 @@ #pragma once -#include #include #include @@ -10,37 +9,6 @@ namespace eosio { using boost::signals2::signal; -using chain::plugin_interface::runtime_metric; -using chain::plugin_interface::metric_type; -using chain::plugin_interface::metrics_listener; -using chain::plugin_interface::plugin_metrics; - -struct producer_plugin_metrics : public plugin_metrics { - runtime_metric unapplied_transactions{metric_type::gauge, "unapplied_transactions", "unapplied_transactions", 0}; - runtime_metric blacklisted_transactions{metric_type::gauge, "blacklisted_transactions", "blacklisted_transactions", 0}; - runtime_metric blocks_produced{metric_type::counter, "blocks_produced", "blocks_produced", 0}; - runtime_metric trxs_produced{metric_type::counter, "trxs_produced", "trxs_produced", 0}; - runtime_metric last_irreversible{metric_type::gauge, "last_irreversible", "last_irreversible", 0}; - runtime_metric head_block_num{metric_type::gauge, "head_block_num", "head_block_num", 0}; - runtime_metric subjective_bill_account_size{metric_type::gauge, "subjective_bill_account_size", "subjective_bill_account_size", 0}; - runtime_metric scheduled_trxs{metric_type::gauge, "scheduled_trxs", "scheduled_trxs", 0}; - - vector metrics() final { - vector metrics{ - unapplied_transactions, - blacklisted_transactions, - blocks_produced, - trxs_produced, - last_irreversible, - head_block_num, - subjective_bill_account_size, - scheduled_trxs - }; - - return metrics; - } -}; - class producer_plugin : public appbase::plugin { public: APPBASE_PLUGIN_REQUIRES((chain_plugin)(signature_provider_plugin)) @@ -201,7 +169,6 @@ class producer_plugin : public appbase::plugin { void log_failed_transaction(const transaction_id_type& trx_id, const chain::packed_transaction_ptr& packed_trx_ptr, const char* reason) const; - void register_metrics_listener(metrics_listener listener); // thread-safe, called when a new block is received void received_block(uint32_t block_num); @@ -209,6 +176,33 @@ class producer_plugin : public appbase::plugin { const std::set& producer_accounts() const; static void set_test_mode(bool m) { test_mode_ = m; } + + struct produced_block_metrics { + std::size_t unapplied_transactions_total; + std::size_t blacklisted_transactions_total; + std::size_t subjective_bill_account_size_total; + std::size_t scheduled_trxs_total; + std::size_t trxs_produced_total; + uint64_t cpu_usage_us; + uint64_t net_usage_us; + + + uint32_t last_irreversible; + uint32_t head_block_num; + }; + + struct incoming_block_metrics { + std::size_t trxs_incoming_total; + uint64_t cpu_usage_us; + uint64_t net_usage_us; + + uint32_t last_irreversible; + uint32_t head_block_num; + }; + + void register_update_produced_block_metrics(std::function&&); + void register_update_incoming_block_metrics(std::function&&); + private: inline static bool test_mode_{false}; // to be moved into appbase (application_base) diff --git a/plugins/producer_plugin/producer_plugin.cpp b/plugins/producer_plugin/producer_plugin.cpp index ce3f662778..468c263bb6 100644 --- a/plugins/producer_plugin/producer_plugin.cpp +++ b/plugins/producer_plugin/producer_plugin.cpp @@ -400,8 +400,6 @@ class producer_plugin_impl : public std::enable_shared_from_this _irreversible_block_connection; std::optional _block_start_connection; - producer_plugin_metrics _metrics; - /* * HACK ALERT * Boost timers can be in a state where a handler has not yet executed but is not abortable. @@ -423,6 +421,9 @@ class producer_plugin_impl : public std::enable_shared_from_this _update_produced_block_metrics; + std::function _update_incoming_block_metrics; + // ro for read-only struct ro_trx_t { transaction_metadata_ptr trx; @@ -534,24 +535,6 @@ class producer_plugin_impl : public std::enable_shared_from_thischain(); - _metrics.last_irreversible.value = chain.last_irreversible_block_num(); - _metrics.head_block_num.value = chain.head_block_num(); - - const auto& sch_idx = chain.db().get_index(); - _metrics.scheduled_trxs.value = sch_idx.size(); - - _metrics.post_metrics(); - } - } - void abort_block() { auto& chain = chain_plug->chain(); @@ -656,8 +639,13 @@ class producer_plugin_impl : public std::enable_shared_from_thisblock->timestamp).count()/1000 ) ); } } - - update_block_metrics(); + if (_update_incoming_block_metrics) { + _update_incoming_block_metrics({.trxs_incoming_total = block->transactions.size(), + .cpu_usage_us = br.total_cpu_usage_us, + .net_usage_us = br.total_net_usage, + .last_irreversible = chain.last_irreversible_block_num(), + .head_block_num = chain.head_block_num()}); + } return true; } @@ -1807,7 +1795,6 @@ bool producer_plugin_impl::should_interrupt_start_block( const fc::time_point& d producer_plugin_impl::start_block_result producer_plugin_impl::start_block() { chain::controller& chain = chain_plug->chain(); - update_block_metrics(); if( !chain_plug->accept_transactions() ) return start_block_result::waiting_for_block; @@ -2760,8 +2747,18 @@ void producer_plugin_impl::produce_block() { br.total_time += fc::time_point::now() - start; - ++_metrics.blocks_produced.value; - _metrics.trxs_produced.value += new_bs->block->transactions.size(); + if (_update_produced_block_metrics) { + _update_produced_block_metrics( + {.unapplied_transactions_total = _unapplied_transactions.size(), + .blacklisted_transactions_total = _blacklisted_transactions.size(), + .subjective_bill_account_size_total = _subjective_billing.get_account_cache_size(), + .scheduled_trxs_total = chain.db().get_index().size(), + .trxs_produced_total = new_bs->block->transactions.size(), + .cpu_usage_us = br.total_cpu_usage_us, + .net_usage_us = br.total_net_usage, + .last_irreversible = chain.last_irreversible_block_num(), + .head_block_num = chain.head_block_num()}); + } ilog("Produced block ${id}... #${n} @ ${t} signed by ${p} " "[trxs: ${count}, lib: ${lib}, confirmed: ${confs}, net: ${net}, cpu: ${cpu}, elapsed: ${et}, time: ${tt}]", @@ -2995,7 +2992,12 @@ const std::set& producer_plugin::producer_accounts() const { return my->_producers; } -void producer_plugin::register_metrics_listener(metrics_listener listener) { - my->_metrics.register_listener(listener); +void producer_plugin::register_update_produced_block_metrics(std::function&& fun){ + my->_update_produced_block_metrics = std::move(fun); } + +void producer_plugin::register_update_incoming_block_metrics(std::function&& fun){ + my->_update_incoming_block_metrics = std::move(fun); +} + } // namespace eosio diff --git a/plugins/prometheus_plugin/metrics.hpp b/plugins/prometheus_plugin/metrics.hpp new file mode 100644 index 0000000000..7d0e6535c7 --- /dev/null +++ b/plugins/prometheus_plugin/metrics.hpp @@ -0,0 +1,171 @@ +#pragma once + +#include +#include +#include + +#include +#include +#include + +namespace eosio::metrics { + +struct catalog_type { + + using Gauge = prometheus::Gauge; + using Counter = prometheus::Counter; + + template + prometheus::Family& family(const std::string& name, const std::string& help) { + return prometheus::detail::Builder{}.Name(name).Help(help).Register(registry); + } + + template + T& build(const std::string& name, const std::string& help) { + return family(name, help).Add({}); + } + + prometheus::Registry registry; + // http plugin + prometheus::Family& http_request_counts; + + // net plugin p2p-connections + prometheus::Family& p2p_connections; + + Gauge& num_peers; + Gauge& num_clients; + + // net plugin failed p2p connection + Counter& failed_p2p_connections; + + // net plugin dropped_trxs + Counter& dropped_trxs_total; + + // producer plugin + prometheus::Family& cpu_usage_us; + prometheus::Family& net_usage_us; + + Gauge& last_irreversible; + Gauge& head_block_num; + + // produced blocks + Counter& unapplied_transactions_total; + Counter& blacklisted_transactions_total; + Counter& subjective_bill_account_size_total; + Counter& scheduled_trxs_total; + Counter& trxs_produced_total; + Counter& cpu_usage_us_produced_block; + Counter& net_usage_us_produced_block; + Counter& blocks_produced; + + // incoming blocks + Counter& trxs_incoming_total; + Counter& cpu_usage_us_incoming_block; + Counter& net_usage_us_incoming_block; + Counter& blocks_incoming; + + // prometheus exporter + Counter& bytes_transferred; + Counter& num_scrapes; + + + catalog_type() + : http_request_counts(family("http_requests_total", "number of HTTP requests")) + , p2p_connections(family("p2p_connections", "current number of connected p2p connections")) + , num_peers(p2p_connections.Add({{"direction", "out"}})) + , num_clients(p2p_connections.Add({{"direction", "in"}})) + , failed_p2p_connections( + build("failed_p2p_connections", "total number of failed out-going p2p connections")) + , dropped_trxs_total(build("dropped_trxs_total", "total number of dropped transactions by net plugin")) + , cpu_usage_us(family("cpu_usage_us_total", "total cpu usage in microseconds for blocks")) + , net_usage_us(family("net_usage_us_total", "total net usage in microseconds for blocks")) + , last_irreversible(build("last_irreversible", "last irreversible block number")) + , head_block_num(build("head_block_num", "head block number")) + , unapplied_transactions_total(build("unapplied_transactions_total", + "total number of unapplied transactions from produced blocks")) + , blacklisted_transactions_total(build("blacklisted_transactions_total", + "total number of blacklisted transactions from produced blocks")) + , subjective_bill_account_size_total(build( + "subjective_bill_account_size_total", "total number of subjective bill account size from produced blocks")) + , scheduled_trxs_total( + build("scheduled_trxs_total", "total number of scheduled transactions from produced blocks")) + , trxs_produced_total(build("trxs_produced_total", "number of transactions produced")) + , cpu_usage_us_produced_block(cpu_usage_us.Add({{"block_type", "produced"}})) + , net_usage_us_produced_block(net_usage_us.Add({{"block_type", "produced"}})) + , blocks_produced(build("blocks_produced", "number of blocks produced")) + , trxs_incoming_total(build("trxs_incoming_total", "number of incoming transactions")) + , cpu_usage_us_incoming_block(cpu_usage_us.Add({{"block_type", "incoming"}})) + , net_usage_us_incoming_block(net_usage_us.Add({{"block_type", "incoming"}})) + , blocks_incoming(build("blocks_incoming", "number of incoming blocks")) + , bytes_transferred(build("exposer_transferred_bytes_total", + "total number of bytes for responses to prometheus scape requests")) + , num_scrapes(build("exposer_scrapes_total", "total number of prometheus scape requests received")) {} + + std::string report() { + const prometheus::TextSerializer serializer; + auto result = serializer.Serialize(registry.Collect()); + bytes_transferred.Increment(result.size()); + num_scrapes.Increment(1); + return result; + } + + void update(const http_plugin::metrics& metrics) { + http_request_counts.Add({{"handler", metrics.target}}).Increment(1); + } + + void update(const net_plugin::p2p_connections_metrics& metrics) { + num_peers.Set(metrics.num_peers); + num_clients.Set(metrics.num_clients); + } + + void update(const producer_plugin::produced_block_metrics& metrics) { + unapplied_transactions_total.Increment(metrics.unapplied_transactions_total); + blacklisted_transactions_total.Increment(metrics.blacklisted_transactions_total); + subjective_bill_account_size_total.Increment(metrics.subjective_bill_account_size_total); + scheduled_trxs_total.Increment(metrics.scheduled_trxs_total); + trxs_produced_total.Increment(metrics.trxs_produced_total); + blocks_produced.Increment(1); + cpu_usage_us_produced_block.Increment(metrics.cpu_usage_us); + net_usage_us_produced_block.Increment(metrics.net_usage_us); + + last_irreversible.Set(metrics.last_irreversible); + head_block_num.Set(metrics.head_block_num); + } + + void update(const producer_plugin::incoming_block_metrics& metrics) { + trxs_incoming_total.Increment(metrics.trxs_incoming_total); + blocks_incoming.Increment(1); + cpu_usage_us_incoming_block.Increment(metrics.cpu_usage_us); + net_usage_us_incoming_block.Increment(metrics.net_usage_us); + + last_irreversible.Set(metrics.last_irreversible); + head_block_num.Set(metrics.head_block_num); + } + + void register_update_handlers(boost::asio::io_context::strand& strand) { + auto& http = app().get_plugin(); + http.register_update_metrics( + [&strand, this](http_plugin::metrics metrics) { strand.post([metrics, this]() { update(metrics); }); }); + + auto& net = app().get_plugin(); + + net.register_update_p2p_connection_metrics([&strand, this](net_plugin::p2p_connections_metrics metrics) { + strand.post([metrics, this]() { update(metrics); }); + }); + + net.register_increment_failed_p2p_connections([this]() { failed_p2p_connections.Increment(1); }); + net.register_increment_dropped_trxs([this]() { dropped_trxs_total.Increment(1); }); + + auto& producer = app().get_plugin(); + producer.register_update_produced_block_metrics( + [&strand, this](const producer_plugin::produced_block_metrics& metrics) { + strand.post([metrics, this]() { update(metrics); }); + }); + producer.register_update_incoming_block_metrics( + [&strand, this](const producer_plugin::incoming_block_metrics& metrics) { + strand.post([metrics, this]() { update(metrics); }); + }); + } +}; + +} // namespace eosio::metrics \ No newline at end of file diff --git a/plugins/prometheus_plugin/prometheus_plugin.cpp b/plugins/prometheus_plugin/prometheus_plugin.cpp index 33cc3cbd5a..c02f2cfc4b 100644 --- a/plugins/prometheus_plugin/prometheus_plugin.cpp +++ b/plugins/prometheus_plugin/prometheus_plugin.cpp @@ -10,108 +10,15 @@ #include -#include -#include -#include -#include -#include -#include +#include "metrics.hpp" + +namespace eosio { -namespace eosio { static const char* prometheus_api_name = "/v1/prometheus/metrics"; using namespace prometheus; using namespace chain::plugin_interface; static auto _prometheus_plugin = application::register_plugin(); - using metric_type=chain::plugin_interface::metric_type; - - struct prometheus_plugin_metrics : plugin_metrics { - runtime_metric bytes_transferred{metric_type::counter, "exposer_transferred_bytes_total", "exposer_transferred_bytes_total"}; - runtime_metric num_scrapes{metric_type::counter, "exposer_scrapes_total", "exposer_scrapes_total"}; - - std::vector metrics() final { - return std::vector{ - bytes_transferred, - num_scrapes - }; - } - }; - - struct metrics_model { - std::vector> _collectables; - std::shared_ptr _registry; - std::vector>> _gauges; - std::vector>> _counters; - - void add_gauge_metric(const runtime_metric& plugin_metric) { - auto& gauge_family = BuildGauge() - .Name(plugin_metric.family) - .Help("") - .Register(*_registry); - auto& gauge = gauge_family.Add({}); - gauge.Set(plugin_metric.value); - - _gauges.push_back(gauge_family); - - tlog("Added gauge metric ${f}:${l}", ("f", plugin_metric.family) ("l", plugin_metric.label)); - } - - void add_counter_metric(const runtime_metric& plugin_metric) { - auto& counter_family = BuildCounter() - .Name(plugin_metric.family) - .Help("") - .Register(*_registry); - auto& counter = counter_family.Add({}); - counter.Increment(plugin_metric.value); - _counters.push_back(counter_family); - - tlog("Added counter metric ${f}:${l}", ("f", plugin_metric.family) ("l", plugin_metric.label)); - } - - void add_runtime_metric(const runtime_metric& plugin_metric) { - switch(plugin_metric.type) { - case metric_type::gauge: - add_gauge_metric(plugin_metric); - break; - case metric_type::counter: - add_counter_metric(plugin_metric); - break; - - default: - break; - } - } - - void add_runtime_metrics(const std::vector& metrics){ - for (auto const& m : metrics) { - add_runtime_metric(m); - } - } - - metrics_model() { - _registry = std::make_shared(); - _collectables.push_back(_registry); - } - - std::vector collect_metrics() { - auto collected_metrics = std::vector{}; - - for (auto&& collectable : _collectables) { - - auto&& metrics = collectable->Collect(); - collected_metrics.insert(collected_metrics.end(), - std::make_move_iterator(metrics.begin()), - std::make_move_iterator(metrics.end())); - } - - return collected_metrics; - } - - std::string serialize() { - const prometheus::TextSerializer serializer; - return serializer.Serialize(collect_metrics()); - } - }; namespace http = boost::beast::http; struct prometheus_plugin_impl : rest::simple_server { @@ -137,78 +44,19 @@ namespace eosio { res.set(http::field::server, server_header()); res.set(http::field::content_type, "text/plain"); res.keep_alive(req.keep_alive()); - res.body() = metrics(); + res.body() = _catalog.report(); res.prepare_payload(); return res; } eosio::chain::named_thread_pool _prometheus_thread_pool; boost::asio::io_context::strand _prometheus_strand; - prometheus_plugin_metrics _metrics; + metrics::catalog_type _catalog; - map> _plugin_metrics; boost::asio::ip::tcp::endpoint _endpoint; - prometheus_plugin_impl(): _prometheus_strand(_prometheus_thread_pool.get_executor()){ } - - void update_metrics(const std::string& plugin_name, vector metrics) { - auto plugin_metrics = _plugin_metrics.find(plugin_name); - if (plugin_metrics != _plugin_metrics.end()) { - plugin_metrics->second = std::move(metrics); - } - } - - metrics_listener create_metrics_listener(std::string plugin_name) { - return [plugin_name{std::move(plugin_name)}, self=this] (vector metrics) { - self->_prometheus_strand.post( - [self, plugin_name, metrics{std::move(metrics)}]() mutable { - self->update_metrics(plugin_name, std::move(metrics)); - }); - }; - } - - - void initialize_metrics() { - net_plugin* np = app().find_plugin(); - if (nullptr != np) { - _plugin_metrics.emplace(std::pair{"net", std::vector()}); - np->register_metrics_listener(create_metrics_listener("net")); - } else { - wlog("net_plugin not found -- metrics not added"); - } - - producer_plugin* pp = app().find_plugin(); - if (nullptr != pp) { - _plugin_metrics.emplace(std::pair{"prod", std::vector()}); - pp->register_metrics_listener(create_metrics_listener("prod")); - } else { - wlog("producer_plugin not found -- metrics not added"); - } - - http_plugin* hp = app().find_plugin(); - if (nullptr != pp) { - _plugin_metrics.emplace(std::pair{"http", std::vector()}); - hp->register_metrics_listener(create_metrics_listener("http")); - } else { - wlog("http_plugin not found -- metrics not added"); - } - } - - std::string metrics() { - metrics_model mm; - vector prometheus_metrics = _metrics.metrics(); - mm.add_runtime_metrics(prometheus_metrics); - - for (auto& pm : _plugin_metrics) { - mm.add_runtime_metrics(pm.second); - } - - std::string body = mm.serialize(); - - _metrics.bytes_transferred.value += body.length(); - _metrics.num_scrapes.value++; - - return body; + prometheus_plugin_impl(): _prometheus_strand(_prometheus_thread_pool.get_executor()){ + _catalog.register_update_handlers(_prometheus_strand); } void start() { @@ -231,7 +79,6 @@ namespace eosio { } void prometheus_plugin::plugin_initialize(const variables_map& options) { - my->initialize_metrics(); string lipstr = options.at("prometheus-exporter-address").as(); EOS_ASSERT(lipstr.size() > 0, chain::plugin_config_exception, "prometheus-exporter-address must have a value"); From 663fc516d1f7f0af5f46f2851a002e583639f9e6 Mon Sep 17 00:00:00 2001 From: greg7mdp Date: Mon, 24 Apr 2023 13:50:14 -0400 Subject: [PATCH 20/58] Fix test failure on fast machine (release builds) When I build in release mode, I have the following tests failing (both gcc-12 and clang-16, no problem in debug) /home/greg/github/enf/leap/unittests/api_tests.cpp(1078): error: in "api_tests/checktime_pause_max_trx_cpu_extended_test": elapsed 26100us /home/greg/github/enf/leap/unittests/api_tests.cpp(1136): error: in "api_tests/checktime_pause_max_trx_extended_test": elapsed 26056us This is the first failure in the code: // This assumes that loading the WASM takes at least 1.5 ms // If this check fails but duration is >= 24'999 (previous check did not fail), then the check here is likely // because WASM took less than 1.5 ms to load. BOOST_CHECK_MESSAGE( dur > 26'500, "elapsed " << dur << "us" ); --- unittests/api_tests.cpp | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/unittests/api_tests.cpp b/unittests/api_tests.cpp index 624e457ead..e28d455e73 100644 --- a/unittests/api_tests.cpp +++ b/unittests/api_tests.cpp @@ -1072,10 +1072,10 @@ BOOST_AUTO_TEST_CASE(checktime_pause_max_trx_cpu_extended_test) { try { dlog("elapsed ${e}us", ("e", dur) ); BOOST_CHECK( dur >= 24'999 ); // should never fail BOOST_TEST( t.is_code_cached("pause"_n) ); - // This assumes that loading the WASM takes at least 1.5 ms + // This assumes that loading the WASM takes at least 0.750 ms // If this check fails but duration is >= 24'999 (previous check did not fail), then the check here is likely - // because WASM took less than 1.5 ms to load. - BOOST_CHECK_MESSAGE( dur > 26'500, "elapsed " << dur << "us" ); + // because WASM took less than 0.750 ms to load. + BOOST_CHECK_MESSAGE( dur > 25'750, "elapsed " << dur << "us" ); BOOST_CHECK_MESSAGE( dur < 150'000, "elapsed " << dur << "us" ); // Should not run to block_cpu_usage deadline // Test hitting max_transaction_time throws tx_cpu_usage_exceeded @@ -1130,10 +1130,10 @@ BOOST_AUTO_TEST_CASE(checktime_pause_max_trx_extended_test) { try { dlog("elapsed ${e}us", ("e", dur) ); BOOST_CHECK( dur >= 25'000 ); // should never fail BOOST_TEST( t.is_code_cached("pause"_n) ); - // This assumes that loading the WASM takes at least 1.5 ms + // This assumes that loading the WASM takes at least 0.750 ms // If this check fails but duration is >= 25'000 (previous check did not fail), then the check here is likely - // because WASM took less than 1.5 ms to load. - BOOST_CHECK_MESSAGE( dur > 26'500, "elapsed " << dur << "us" ); + // because WASM took less than 0.750 ms to load. + BOOST_CHECK_MESSAGE( dur > 25'750, "elapsed " << dur << "us" ); BOOST_CHECK_MESSAGE( dur < 250'000, "elapsed " << dur << "us" ); // Should not run to max_transaction_cpu_usage deadline BOOST_REQUIRE_EQUAL( t.validate(), true ); From f5cb026ee1e18c0d5997552cf147d53bb60196b2 Mon Sep 17 00:00:00 2001 From: greg7mdp Date: Mon, 24 Apr 2023 14:25:08 -0400 Subject: [PATCH 21/58] Lower another timeout which I hit occasionally when running tests. --- unittests/api_tests.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/unittests/api_tests.cpp b/unittests/api_tests.cpp index e28d455e73..727992cb0e 100644 --- a/unittests/api_tests.cpp +++ b/unittests/api_tests.cpp @@ -1349,7 +1349,7 @@ BOOST_FIXTURE_TEST_CASE(checktime_hashing_fail, TESTER) { try { //hit deadline exception, but cache the contract BOOST_CHECK_EXCEPTION( call_test( *this, test_api_action{}, - 5000, 10, 10 ), + 5000, 8, 8 ), deadline_exception, is_deadline_exception ); BOOST_TEST( is_code_cached("testapi"_n) ); From a7d268df10188811e152720f576dc94d8ddf8119 Mon Sep 17 00:00:00 2001 From: Kevin Heifner Date: Mon, 24 Apr 2023 13:40:37 -0500 Subject: [PATCH 22/58] GH-668 Move subjective_billing to chain library. Break circular dependency between producer_plugin and chain_plugin. --- libraries/chain/controller.cpp | 11 +++ .../chain/include/eosio/chain/controller.hpp | 3 + .../eosio/chain}/subjective_billing.hpp | 9 +-- plugins/chain_plugin/CMakeLists.txt | 4 +- plugins/chain_plugin/chain_plugin.cpp | 18 ++--- .../eosio/chain_plugin/chain_plugin.hpp | 5 -- .../eosio/producer_plugin/producer_plugin.hpp | 1 - plugins/producer_plugin/producer_plugin.cpp | 72 +++++++++---------- plugins/producer_plugin/test/CMakeLists.txt | 5 -- tests/CMakeLists.txt | 2 +- tests/chain_plugin_tests.cpp | 6 +- tests/get_producers_tests.cpp | 4 +- tests/get_table_seckey_tests.cpp | 2 +- tests/get_table_tests.cpp | 8 +-- tests/test_chain_plugin.cpp | 2 +- .../test_subjective_billing.cpp | 9 +-- 16 files changed, 72 insertions(+), 89 deletions(-) rename {plugins/producer_plugin/include/eosio/producer_plugin => libraries/chain/include/eosio/chain}/subjective_billing.hpp (98%) rename {plugins/producer_plugin/test => unittests}/test_subjective_billing.cpp (95%) diff --git a/libraries/chain/controller.cpp b/libraries/chain/controller.cpp index 9b38edc753..b4af185d91 100644 --- a/libraries/chain/controller.cpp +++ b/libraries/chain/controller.cpp @@ -21,6 +21,7 @@ #include #include #include +#include #include #include #include @@ -232,6 +233,7 @@ struct controller_impl { block_state_ptr head; fork_database fork_db; resource_limits_manager resource_limits; + subjective_billing subjective_bill; authorization_manager authorization; protocol_feature_manager protocol_features; controller::config conf; @@ -2767,6 +2769,15 @@ const protocol_feature_manager& controller::get_protocol_feature_manager()const return my->protocol_features; } +const subjective_billing& controller::get_subjective_billing()const { + return my->subjective_bill; +} + +subjective_billing& controller::get_mutable_subjective_billing() { + return my->subjective_bill; +} + + uint32_t controller::get_max_nonprivileged_inline_action_size()const { return my->conf.max_nonprivileged_inline_action_size; diff --git a/libraries/chain/include/eosio/chain/controller.hpp b/libraries/chain/include/eosio/chain/controller.hpp index b697845482..c10aff3e53 100644 --- a/libraries/chain/include/eosio/chain/controller.hpp +++ b/libraries/chain/include/eosio/chain/controller.hpp @@ -39,6 +39,7 @@ namespace eosio { namespace chain { class permission_object; class account_object; class deep_mind_handler; + class subjective_billing; using resource_limits::resource_limits_manager; using apply_handler = std::function; using forked_branch_callback = std::function; @@ -197,6 +198,8 @@ namespace eosio { namespace chain { const authorization_manager& get_authorization_manager()const; authorization_manager& get_mutable_authorization_manager(); const protocol_feature_manager& get_protocol_feature_manager()const; + const subjective_billing& get_subjective_billing()const; + subjective_billing& get_mutable_subjective_billing(); uint32_t get_max_nonprivileged_inline_action_size()const; const flat_set& get_actor_whitelist() const; diff --git a/plugins/producer_plugin/include/eosio/producer_plugin/subjective_billing.hpp b/libraries/chain/include/eosio/chain/subjective_billing.hpp similarity index 98% rename from plugins/producer_plugin/include/eosio/producer_plugin/subjective_billing.hpp rename to libraries/chain/include/eosio/chain/subjective_billing.hpp index 07b432068c..2a51b10713 100644 --- a/plugins/producer_plugin/include/eosio/producer_plugin/subjective_billing.hpp +++ b/libraries/chain/include/eosio/chain/subjective_billing.hpp @@ -13,7 +13,7 @@ #include #include -namespace eosio { +namespace eosio::chain { class subjective_billing { private: @@ -92,7 +92,7 @@ class subjective_billing { public: // public for tests static constexpr uint32_t subjective_time_interval_ms = 5'000; - size_t get_account_cache_size() {return _account_subjective_bill_cache.size();} + size_t get_account_cache_size() const {return _account_subjective_bill_cache.size();} void remove_subjective_billing( const chain::transaction_id_type& trx_id, uint32_t time_ordinal ) { auto& idx = _trx_cache_index.get(); auto itr = idx.find( trx_id ); @@ -149,9 +149,6 @@ class subjective_billing { } } - void abort_block() { - } - void on_block( fc::logger& log, const chain::block_state_ptr& bsp, const fc::time_point& now ) { if( bsp == nullptr || _disabled ) return; const auto time_ordinal = time_ordinal_for(now); @@ -200,4 +197,4 @@ class subjective_billing { } }; -} //eosio +} //eosio::chain diff --git a/plugins/chain_plugin/CMakeLists.txt b/plugins/chain_plugin/CMakeLists.txt index 4d2723bc97..0648d20fb4 100644 --- a/plugins/chain_plugin/CMakeLists.txt +++ b/plugins/chain_plugin/CMakeLists.txt @@ -11,7 +11,7 @@ if(EOSIO_ENABLE_DEVELOPER_OPTIONS) target_compile_definitions(chain_plugin PUBLIC EOSIO_DEVELOPER) endif() -target_link_libraries( chain_plugin producer_plugin eosio_chain appbase resource_monitor_plugin ) -target_include_directories( chain_plugin PUBLIC "${CMAKE_CURRENT_SOURCE_DIR}/include" "${CMAKE_CURRENT_SOURCE_DIR}/../chain_interface/include" "${CMAKE_CURRENT_SOURCE_DIR}/../../libraries/appbase/include" "${CMAKE_CURRENT_SOURCE_DIR}/../producer_plugin/include" "${CMAKE_CURRENT_SOURCE_DIR}/../resource_monitor_plugin/include") +target_link_libraries( chain_plugin eosio_chain custom_appbase appbase resource_monitor_plugin ) +target_include_directories( chain_plugin PUBLIC "${CMAKE_CURRENT_SOURCE_DIR}/include" "${CMAKE_CURRENT_SOURCE_DIR}/../chain_interface/include" "${CMAKE_CURRENT_SOURCE_DIR}/../../libraries/appbase/include" "${CMAKE_CURRENT_SOURCE_DIR}/../resource_monitor_plugin/include") add_subdirectory( test ) \ No newline at end of file diff --git a/plugins/chain_plugin/chain_plugin.cpp b/plugins/chain_plugin/chain_plugin.cpp index c71dd2e2df..e85963f821 100644 --- a/plugins/chain_plugin/chain_plugin.cpp +++ b/plugins/chain_plugin/chain_plugin.cpp @@ -1,6 +1,5 @@ #include #include -#include #include #include #include @@ -12,6 +11,7 @@ #include #include #include +#include #include #include #include @@ -184,7 +184,6 @@ class chain_plugin_impl { std::optional _account_query_db; - const producer_plugin* producer_plug; std::optional _trx_retry_db; chain_apis::trx_finality_status_processing_ptr _trx_finality_status_processing; }; @@ -197,7 +196,7 @@ chain_plugin::chain_plugin() app().register_config_type(); } -chain_plugin::~chain_plugin(){} +chain_plugin::~chain_plugin() = default; void chain_plugin::set_program_options(options_description& cli, options_description& cfg) { @@ -1092,9 +1091,6 @@ void chain_plugin::plugin_startup() EOS_ASSERT( my->chain_config->read_mode != db_read_mode::IRREVERSIBLE || !accept_transactions(), plugin_config_exception, "read-mode = irreversible. transactions should not be enabled by enable_accept_transactions" ); try { - my->producer_plug = app().find_plugin(); - EOS_ASSERT(my->producer_plug, plugin_exception, "Failed to find producer_plugin"); - auto shutdown = [](){ return app().quit(); }; auto check_shutdown = [](){ return app().is_quiting(); }; if (my->snapshot_path) { @@ -1179,7 +1175,7 @@ chain_apis::read_write chain_plugin::get_read_write_api(const fc::microseconds& } chain_apis::read_only chain_plugin::get_read_only_api(const fc::microseconds& http_max_response_time) const { - return chain_apis::read_only(chain(), my->_account_query_db, get_abi_serializer_max_time(), http_max_response_time, my->producer_plug, my->_trx_finality_status_processing.get()); + return chain_apis::read_only(chain(), my->_account_query_db, get_abi_serializer_max_time(), http_max_response_time, my->_trx_finality_status_processing.get()); } @@ -2401,11 +2397,9 @@ read_only::get_account_return_t read_only::get_account( const get_account_params } result.ram_usage = rm.get_account_ram_usage( result.account_name ); - if ( producer_plug ) { // producer_plug is null when called from chain_plugin_tests.cpp and get_table_tests.cpp - eosio::chain::resource_limits::account_resource_limit subjective_cpu_bill_limit; - subjective_cpu_bill_limit.used = producer_plug->get_subjective_bill( result.account_name, fc::time_point::now() ); - result.subjective_cpu_bill_limit = subjective_cpu_bill_limit; - } + eosio::chain::resource_limits::account_resource_limit subjective_cpu_bill_limit; + subjective_cpu_bill_limit.used = db.get_subjective_billing().get_subjective_bill( result.account_name, fc::time_point::now() ); + result.subjective_cpu_bill_limit = subjective_cpu_bill_limit; const auto linked_action_map = ([&](){ const auto& links = d.get_index(); diff --git a/plugins/chain_plugin/include/eosio/chain_plugin/chain_plugin.hpp b/plugins/chain_plugin/include/eosio/chain_plugin/chain_plugin.hpp index bf359d1578..8ab6517ae8 100644 --- a/plugins/chain_plugin/include/eosio/chain_plugin/chain_plugin.hpp +++ b/plugins/chain_plugin/include/eosio/chain_plugin/chain_plugin.hpp @@ -47,8 +47,6 @@ namespace eosio { using chain::abi_resolver; using chain::packed_transaction; -class producer_plugin; - namespace chain_apis { struct empty{}; @@ -128,7 +126,6 @@ class read_only : public api_base { const fc::microseconds abi_serializer_max_time; const fc::microseconds http_max_response_time; bool shorten_abi_errors = true; - const producer_plugin* producer_plug; const trx_finality_status_processing* trx_finality_status_proc; friend class api_base; @@ -137,13 +134,11 @@ class read_only : public api_base { read_only(const controller& db, const std::optional& aqdb, const fc::microseconds& abi_serializer_max_time, const fc::microseconds& http_max_response_time, - const producer_plugin* producer_plug, const trx_finality_status_processing* trx_finality_status_proc) : db(db) , aqdb(aqdb) , abi_serializer_max_time(abi_serializer_max_time) , http_max_response_time(http_max_response_time) - , producer_plug(producer_plug) , trx_finality_status_proc(trx_finality_status_proc) { } diff --git a/plugins/producer_plugin/include/eosio/producer_plugin/producer_plugin.hpp b/plugins/producer_plugin/include/eosio/producer_plugin/producer_plugin.hpp index 82a9284581..be326fe060 100644 --- a/plugins/producer_plugin/include/eosio/producer_plugin/producer_plugin.hpp +++ b/plugins/producer_plugin/include/eosio/producer_plugin/producer_plugin.hpp @@ -138,7 +138,6 @@ class producer_plugin : public appbase::plugin { bool is_producer_key(const chain::public_key_type& key) const; chain::signature_type sign_compact(const chain::public_key_type& key, const fc::sha256& digest) const; - int64_t get_subjective_bill( const account_name& first_auth, const fc::time_point& now ) const; virtual void plugin_initialize(const boost::program_options::variables_map& options); virtual void plugin_startup(); diff --git a/plugins/producer_plugin/producer_plugin.cpp b/plugins/producer_plugin/producer_plugin.cpp index ce3f662778..8507406d27 100644 --- a/plugins/producer_plugin/producer_plugin.cpp +++ b/plugins/producer_plugin/producer_plugin.cpp @@ -1,12 +1,12 @@ #include #include -#include #include #include #include #include #include #include +#include #include #include #include @@ -141,12 +141,7 @@ namespace { // track multiple failures on unapplied transactions class account_failures { public: - - //lifetime of sb must outlive account_failures - explicit account_failures( const eosio::subjective_billing& sb ) - : subjective_billing(sb) - { - } + account_failures() = default; void set_max_failures_per_account( uint32_t max_failures, uint32_t size ) { max_failures_per_account = max_failures; @@ -162,17 +157,16 @@ class account_failures { // return true if exceeds max_failures_per_account and should be dropped bool failure_limit( const account_name& n ) { auto fitr = failed_accounts.find( n ); - bool is_whitelisted = subjective_billing.is_account_disabled( n ); - if( !is_whitelisted && fitr != failed_accounts.end() && fitr->second.num_failures >= max_failures_per_account ) { + if( fitr != failed_accounts.end() && fitr->second.num_failures >= max_failures_per_account ) { ++fitr->second.num_failures; return true; } return false; } - void report_and_clear(uint32_t block_num) { + void report_and_clear(uint32_t block_num, const chain::subjective_billing& sub_bill) { if (last_reset_block_num != block_num && (block_num % reset_window_size_in_num_blocks == 0) ) { - report(block_num); + report(block_num, sub_bill); failed_accounts.clear(); last_reset_block_num = block_num; } @@ -184,7 +178,7 @@ class account_failures { } private: - void report(uint32_t block_num) const { + void report(uint32_t block_num, const chain::subjective_billing& sub_bill) const { if( _log.is_enabled(fc::log_level::debug)) { auto now = fc::time_point::now(); for ( const auto& e : failed_accounts ) { @@ -203,7 +197,7 @@ class account_failures { reason += "other"; } fc_dlog( _log, "Failed ${n} trxs, account: ${a}, sub bill: ${b}us, reason: ${r}", - ("n", e.second.num_failures)("b", subjective_billing.get_subjective_bill(e.first, now)) + ("n", e.second.num_failures)("b", sub_bill.get_subjective_bill(e.first, now)) ("a", e.first)("r", reason) ); } } @@ -245,7 +239,6 @@ class account_failures { uint32_t max_failures_per_account = 3; uint32_t last_reset_block_num = 0; uint32_t reset_window_size_in_num_blocks = 1; - const eosio::subjective_billing& subjective_billing; }; struct block_time_tracker { @@ -339,7 +332,7 @@ class producer_plugin_impl : public std::enable_shared_from_this& next, const fc::time_point& start, - const chain::controller& chain, + chain::controller& chain, const transaction_trace_ptr& trace, bool return_failure_trace, bool disable_subjective_enforcement, @@ -391,8 +384,7 @@ class producer_plugin_impl : public std::enable_shared_from_this _accepted_block_connection; @@ -500,9 +492,10 @@ class producer_plugin_impl : public std::enable_shared_from_thischain(); auto before = _unapplied_transactions.size(); _unapplied_transactions.clear_applied( bsp ); - _subjective_billing.on_block( _log, bsp, fc::time_point::now() ); + chain.get_mutable_subjective_billing().on_block( _log, bsp, fc::time_point::now() ); if (before > 0) { fc_dlog( _log, "Removed applied transactions before: ${before}, after: ${after}", ("before", before)("after", _unapplied_transactions.size()) ); @@ -536,12 +529,12 @@ class producer_plugin_impl : public std::enable_shared_from_thischain(); _metrics.unapplied_transactions.value = _unapplied_transactions.size(); - _metrics.subjective_bill_account_size.value = _subjective_billing.get_account_cache_size(); + _metrics.subjective_bill_account_size.value = chain.get_subjective_billing().get_account_cache_size(); _metrics.blacklisted_transactions.value = _blacklisted_transactions.size(); _metrics.unapplied_transactions.value = _unapplied_transactions.size(); - auto &chain = chain_plug->chain(); _metrics.last_irreversible.value = chain.last_irreversible_block_num(); _metrics.head_block_num.value = chain.head_block_num(); @@ -559,7 +552,6 @@ class producer_plugin_impl : public std::enable_shared_from_this_subjective_billing.get_subjective_bill( first_auth, now ); -} - chain::signature_type producer_plugin::sign_compact(const chain::public_key_type& key, const fc::sha256& digest) const { if(key != chain::public_key_type()) { @@ -1026,7 +1013,7 @@ void producer_plugin::plugin_initialize(const boost::program_options::variables_ fc::microseconds subjective_account_decay_time = fc::minutes(options.at( "subjective-account-decay-time-minutes" ).as()); EOS_ASSERT( subjective_account_decay_time.count() > 0, plugin_config_exception, "subjective-account-decay-time-minutes ${dt} must be greater than 0", ("dt", subjective_account_decay_time.to_seconds() / 60)); - my->_subjective_billing.set_expired_accumulator_average_window( subjective_account_decay_time ); + chain.get_mutable_subjective_billing().set_expired_accumulator_average_window( subjective_account_decay_time ); my->_max_transaction_time_ms = options.at("max-transaction-time").as(); @@ -1052,7 +1039,7 @@ void producer_plugin::plugin_initialize(const boost::program_options::variables_ disable_subjective_billing = false; } if( disable_subjective_billing ) { - my->_subjective_billing.disable(); + chain.get_mutable_subjective_billing().disable(); ilog( "Subjective CPU billing disabled" ); } else if( !my->_disable_subjective_p2p_billing && !my->_disable_subjective_api_billing ) { ilog( "Subjective CPU billing enabled" ); @@ -1179,7 +1166,7 @@ void producer_plugin::plugin_initialize(const boost::program_options::variables_ if( options.count("disable-subjective-account-billing") ) { std::vector accounts = options["disable-subjective-account-billing"].as>(); for( const auto& a : accounts ) { - my->_subjective_billing.disable_account( account_name(a) ); + chain.get_mutable_subjective_billing().disable_account( account_name(a) ); } } @@ -1979,15 +1966,16 @@ producer_plugin_impl::start_block_result producer_plugin_impl::start_block() { } try { - _account_fails.report_and_clear(hbs->block_num); + chain::subjective_billing& subjective_bill = chain.get_mutable_subjective_billing(); + _account_fails.report_and_clear(hbs->block_num, subjective_bill); _time_tracker.clear(); if( !remove_expired_trxs( preprocess_deadline ) ) return start_block_result::exhausted; if( !remove_expired_blacklisted_trxs( preprocess_deadline ) ) return start_block_result::exhausted; - if( !_subjective_billing.remove_expired( _log, chain.pending_block_time(), fc::time_point::now(), - [&](){ return should_interrupt_start_block( preprocess_deadline, pending_block_num ); } ) ) { + if( !subjective_bill.remove_expired( _log, chain.pending_block_time(), fc::time_point::now(), + [&](){ return should_interrupt_start_block( preprocess_deadline, pending_block_num ); } ) ) { return start_block_result::exhausted; } @@ -2220,12 +2208,16 @@ producer_plugin_impl::push_transaction( const fc::time_point& block_deadline, auto start = fc::time_point::now(); EOS_ASSERT(!trx->is_read_only(), producer_exception, "Unexpected read-only trx"); + chain::controller& chain = chain_plug->chain(); + chain::subjective_billing& subjective_bill = chain.get_mutable_subjective_billing(); + + auto first_auth = trx->packed_trx()->get_transaction().first_authorizer(); + bool disable_subjective_enforcement = (api_trx && _disable_subjective_api_billing) || (!api_trx && _disable_subjective_p2p_billing) + || subjective_bill.is_account_disabled( first_auth ) || trx->is_transient(); - chain::controller& chain = chain_plug->chain(); - auto first_auth = trx->packed_trx()->get_transaction().first_authorizer(); if( !disable_subjective_enforcement && _account_fails.failure_limit( first_auth ) ) { if( next ) { auto except_ptr = std::static_pointer_cast( std::make_shared( @@ -2244,12 +2236,12 @@ producer_plugin_impl::push_transaction( const fc::time_point& block_deadline, int64_t sub_bill = 0; if( !disable_subjective_enforcement ) - sub_bill = _subjective_billing.get_subjective_bill( first_auth, fc::time_point::now() ); + sub_bill = subjective_bill.get_subjective_bill( first_auth, fc::time_point::now() ); auto prev_billed_cpu_time_us = trx->billed_cpu_time_us; if( _pending_block_mode == pending_block_mode::producing && prev_billed_cpu_time_us > 0 ) { const auto& rl = chain.get_resource_limits_manager(); - if ( !_subjective_billing.is_account_disabled( first_auth ) && !rl.is_unlimited_cpu( first_auth ) ) { + if ( !subjective_bill.is_account_disabled( first_auth ) && !rl.is_unlimited_cpu( first_auth ) ) { int64_t prev_billed_plus100_us = prev_billed_cpu_time_us + EOS_PERCENT( prev_billed_cpu_time_us, 100 * config::percent_1 ); if( prev_billed_plus100_us < max_trx_time.count() ) max_trx_time = fc::microseconds( prev_billed_plus100_us ); } @@ -2264,7 +2256,7 @@ producer_plugin_impl::push_result producer_plugin_impl::handle_push_result( const transaction_metadata_ptr& trx, const next_function& next, const fc::time_point& start, - const chain::controller& chain, + chain::controller& chain, const transaction_trace_ptr& trace, bool return_failure_trace, bool disable_subjective_enforcement, @@ -2272,6 +2264,8 @@ producer_plugin_impl::handle_push_result( const transaction_metadata_ptr& trx, int64_t sub_bill, uint32_t prev_billed_cpu_time_us) { auto end = fc::time_point::now(); + chain::subjective_billing& subjective_bill = chain.get_mutable_subjective_billing(); + push_result pr; if( trace->except ) { // Transient trxs are dry-run or read-only. @@ -2299,7 +2293,7 @@ producer_plugin_impl::handle_push_result( const transaction_metadata_ptr& trx, fc_tlog( _log, "Subjective bill for failed ${a}: ${b} elapsed ${t}us, time ${r}us", ("a",first_auth)("b",sub_bill)("t",trace->elapsed)("r", end - start)); if (!disable_subjective_enforcement) // subjectively bill failure when producing since not in objective cpu account billing - _subjective_billing.subjective_bill_failure( first_auth, trace->elapsed, fc::time_point::now() ); + subjective_bill.subjective_bill_failure( first_auth, trace->elapsed, fc::time_point::now() ); log_trx_results( trx, trace, start ); // this failed our configured maximum transaction time, we don't want to replay it @@ -2332,7 +2326,7 @@ producer_plugin_impl::handle_push_result( const transaction_metadata_ptr& trx, log_trx_results( trx, trace, start ); // if producing then trx is in objective cpu account billing if (!disable_subjective_enforcement && _pending_block_mode != pending_block_mode::producing) { - _subjective_billing.subjective_bill( trx->id(), trx->packed_trx()->expiration(), first_auth, trace->elapsed ); + subjective_bill.subjective_bill( trx->id(), trx->packed_trx()->expiration(), first_auth, trace->elapsed ); } if( next ) next( trace ); } diff --git a/plugins/producer_plugin/test/CMakeLists.txt b/plugins/producer_plugin/test/CMakeLists.txt index 19dc9d567a..c364b9627e 100644 --- a/plugins/producer_plugin/test/CMakeLists.txt +++ b/plugins/producer_plugin/test/CMakeLists.txt @@ -1,8 +1,3 @@ -add_executable( test_subjective_billing test_subjective_billing.cpp ) -target_link_libraries( test_subjective_billing producer_plugin eosio_testing ) - -add_test(NAME test_subjective_billing COMMAND plugins/producer_plugin/test/test_subjective_billing WORKING_DIRECTORY ${CMAKE_BINARY_DIR}) - add_executable( test_trx_full test_trx_full.cpp ) target_link_libraries( test_trx_full producer_plugin eosio_testing ) diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 810d0520df..0f868a8f6b 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -5,7 +5,7 @@ list(REMOVE_ITEM UNIT_TESTS ship_client.cpp) list(REMOVE_ITEM UNIT_TESTS ship_streamer.cpp) add_executable( plugin_test ${UNIT_TESTS} ) -target_link_libraries( plugin_test eosio_testing eosio_chain chainbase chain_plugin wallet_plugin fc state_history ${PLATFORM_SPECIFIC_LIBS} ) +target_link_libraries( plugin_test eosio_testing eosio_chain chainbase chain_plugin producer_plugin wallet_plugin fc state_history ${PLATFORM_SPECIFIC_LIBS} ) target_include_directories( plugin_test PUBLIC ${CMAKE_SOURCE_DIR}/plugins/net_plugin/include diff --git a/tests/chain_plugin_tests.cpp b/tests/chain_plugin_tests.cpp index 9a29e803e2..a2f23fb2c1 100644 --- a/tests/chain_plugin_tests.cpp +++ b/tests/chain_plugin_tests.cpp @@ -95,7 +95,7 @@ BOOST_FIXTURE_TEST_CASE( get_block_with_invalid_abi, TESTER ) try { char headnumstr[20]; sprintf(headnumstr, "%d", headnum); chain_apis::read_only::get_raw_block_params param{headnumstr}; - chain_apis::read_only plugin(*(this->control), {}, fc::microseconds::maximum(), fc::microseconds::maximum(), {}, {}); + chain_apis::read_only plugin(*(this->control), {}, fc::microseconds::maximum(), fc::microseconds::maximum(), {}); // block should be decoded successfully auto block = plugin.get_raw_block(param, fc::time_point::maximum()); @@ -139,7 +139,7 @@ BOOST_FIXTURE_TEST_CASE( get_block_with_invalid_abi, TESTER ) try { BOOST_FIXTURE_TEST_CASE( get_consensus_parameters, TESTER ) try { produce_blocks(1); - chain_apis::read_only plugin(*(this->control), {}, fc::microseconds::maximum(), fc::microseconds::maximum(), nullptr, nullptr); + chain_apis::read_only plugin(*(this->control), {}, fc::microseconds::maximum(), fc::microseconds::maximum(), nullptr); auto parms = plugin.get_consensus_parameters({}, fc::time_point::maximum()); @@ -186,7 +186,7 @@ BOOST_FIXTURE_TEST_CASE( get_account, TESTER ) try { produce_block(); - chain_apis::read_only plugin(*(this->control), {}, fc::microseconds::maximum(), fc::microseconds::maximum(), nullptr, nullptr); + chain_apis::read_only plugin(*(this->control), {}, fc::microseconds::maximum(), fc::microseconds::maximum(), nullptr); chain_apis::read_only::get_account_params p{"alice"_n}; diff --git a/tests/get_producers_tests.cpp b/tests/get_producers_tests.cpp index b3c24c720f..97646f8b8b 100644 --- a/tests/get_producers_tests.cpp +++ b/tests/get_producers_tests.cpp @@ -16,7 +16,7 @@ using namespace eosio::testing; BOOST_AUTO_TEST_CASE( get_producers) { try { tester chain; - eosio::chain_apis::read_only plugin(*(chain.control), {}, fc::microseconds::maximum(), fc::microseconds::maximum(), {}, {}); + eosio::chain_apis::read_only plugin(*(chain.control), {}, fc::microseconds::maximum(), fc::microseconds::maximum(), {}); eosio::chain_apis::read_only::get_producers_params params = { .json = true, .lower_bound = "", .limit = 21 }; auto results = plugin.get_producers(params, fc::time_point::maximum()); @@ -52,7 +52,7 @@ BOOST_AUTO_TEST_CASE( get_producers_from_table) { try { // ensure that enough voting is occurring so that producer1111 is elected as the producer chain.cross_15_percent_threshold(); - eosio::chain_apis::read_only plugin(*(chain.control), {}, fc::microseconds::maximum(), fc::microseconds::maximum(), {}, {}); + eosio::chain_apis::read_only plugin(*(chain.control), {}, fc::microseconds::maximum(), fc::microseconds::maximum(), {}); eosio::chain_apis::read_only::get_producers_params params = { .json = true, .lower_bound = "", .limit = 21 }; auto results = plugin.get_producers(params, fc::time_point::maximum()); diff --git a/tests/get_table_seckey_tests.cpp b/tests/get_table_seckey_tests.cpp index 9ba62fc9a3..748b1b8bb7 100644 --- a/tests/get_table_seckey_tests.cpp +++ b/tests/get_table_seckey_tests.cpp @@ -49,7 +49,7 @@ BOOST_FIXTURE_TEST_CASE( get_table_next_key_test, TESTER ) try { set_abi( "test"_n, test_contracts::get_table_seckey_test_abi().data() ); produce_block(); - chain_apis::read_only plugin(*(this->control), {}, fc::microseconds::maximum(), fc::microseconds::maximum(), {}, {}); + chain_apis::read_only plugin(*(this->control), {}, fc::microseconds::maximum(), fc::microseconds::maximum(), {}); chain_apis::read_only::get_table_rows_params params = []{ chain_apis::read_only::get_table_rows_params params{}; params.json=true; diff --git a/tests/get_table_tests.cpp b/tests/get_table_tests.cpp index a7f711be8e..cb236551e7 100644 --- a/tests/get_table_tests.cpp +++ b/tests/get_table_tests.cpp @@ -96,7 +96,7 @@ BOOST_FIXTURE_TEST_CASE( get_scope_test, TESTER ) try { produce_blocks(1); // iterate over scope - eosio::chain_apis::read_only plugin(*(this->control), {}, fc::microseconds::maximum(), fc::microseconds::maximum(), {}, {}); + eosio::chain_apis::read_only plugin(*(this->control), {}, fc::microseconds::maximum(), fc::microseconds::maximum(), {}); eosio::chain_apis::read_only::get_table_by_scope_params param{"eosio.token"_n, "accounts"_n, "inita", "", 10}; eosio::chain_apis::read_only::get_table_by_scope_result result = plugin.read_only::get_table_by_scope(param, fc::time_point::maximum()); @@ -201,7 +201,7 @@ BOOST_FIXTURE_TEST_CASE( get_table_test, TESTER ) try { produce_blocks(1); // get table: normal case - eosio::chain_apis::read_only plugin(*(this->control), {}, fc::microseconds::maximum(), fc::microseconds::maximum(), {}, {}); + eosio::chain_apis::read_only plugin(*(this->control), {}, fc::microseconds::maximum(), fc::microseconds::maximum(), {}); eosio::chain_apis::read_only::get_table_rows_params p; p.code = "eosio.token"_n; p.scope = "inita"; @@ -371,7 +371,7 @@ BOOST_FIXTURE_TEST_CASE( get_table_by_seckey_test, TESTER ) try { produce_blocks(1); // get table: normal case - eosio::chain_apis::read_only plugin(*(this->control), {}, fc::microseconds::maximum(), fc::microseconds::maximum(), {}, {}); + eosio::chain_apis::read_only plugin(*(this->control), {}, fc::microseconds::maximum(), fc::microseconds::maximum(), {}); eosio::chain_apis::read_only::get_table_rows_params p; p.code = "eosio"_n; p.scope = "eosio"; @@ -523,7 +523,7 @@ BOOST_FIXTURE_TEST_CASE( get_table_next_key_test, TESTER ) try { // } - chain_apis::read_only plugin(*(this->control), {}, fc::microseconds::maximum(), fc::microseconds::maximum(), {}, {}); + chain_apis::read_only plugin(*(this->control), {}, fc::microseconds::maximum(), fc::microseconds::maximum(), {}); chain_apis::read_only::get_table_rows_params params = []{ chain_apis::read_only::get_table_rows_params params{}; params.json=true; diff --git a/tests/test_chain_plugin.cpp b/tests/test_chain_plugin.cpp index a93e756c33..1822206f09 100644 --- a/tests/test_chain_plugin.cpp +++ b/tests/test_chain_plugin.cpp @@ -232,7 +232,7 @@ class chain_plugin_tester : public TESTER { read_only::get_account_results get_account_info(const account_name acct){ auto account_object = control->get_account(acct); read_only::get_account_params params = { account_object.name }; - chain_apis::read_only plugin(*(control.get()), {}, fc::microseconds::maximum(), fc::microseconds::maximum(), {}, {}); + chain_apis::read_only plugin(*(control.get()), {}, fc::microseconds::maximum(), fc::microseconds::maximum(), {}); auto res = plugin.get_account(params, fc::time_point::maximum())(); BOOST_REQUIRE(!std::holds_alternative(res)); return std::get(std::move(res)); diff --git a/plugins/producer_plugin/test/test_subjective_billing.cpp b/unittests/test_subjective_billing.cpp similarity index 95% rename from plugins/producer_plugin/test/test_subjective_billing.cpp rename to unittests/test_subjective_billing.cpp index 9149041eec..3f39284913 100644 --- a/plugins/producer_plugin/test/test_subjective_billing.cpp +++ b/unittests/test_subjective_billing.cpp @@ -1,8 +1,6 @@ -#define BOOST_TEST_MODULE subjective_billing -#include - -#include +#include +#include "eosio/chain/subjective_billing.hpp" #include namespace { @@ -41,7 +39,6 @@ BOOST_AUTO_TEST_CASE( subjective_bill_test ) { BOOST_CHECK_EQUAL( 9, sub_bill.get_subjective_bill(b, now) ); sub_bill.on_block(log, {}, now); - sub_bill.abort_block(); // they all failed so nothing in aborted block BOOST_CHECK_EQUAL( 13+11, sub_bill.get_subjective_bill(a, now) ); BOOST_CHECK_EQUAL( 9, sub_bill.get_subjective_bill(b, now) ); @@ -69,14 +66,12 @@ BOOST_AUTO_TEST_CASE( subjective_bill_test ) { BOOST_CHECK_EQUAL( 7, sub_bill.get_subjective_bill(b, now) ); sub_bill.on_block(log, {}, now); // have not seen any of the transactions come back yet - sub_bill.abort_block(); BOOST_CHECK_EQUAL( 23+19, sub_bill.get_subjective_bill(a, now) ); BOOST_CHECK_EQUAL( 7, sub_bill.get_subjective_bill(b, now) ); sub_bill.on_block(log, {}, now); sub_bill.remove_subjective_billing( id1, 0 ); // simulate seeing id1 come back in block (this is what on_block would do) - sub_bill.abort_block(); BOOST_CHECK_EQUAL( 19, sub_bill.get_subjective_bill(a, now) ); BOOST_CHECK_EQUAL( 7, sub_bill.get_subjective_bill(b, now) ); From af8383c34ced8344ebd5fd519fc84db526963b52 Mon Sep 17 00:00:00 2001 From: Lin Huang Date: Mon, 24 Apr 2023 15:13:50 -0400 Subject: [PATCH 23/58] [4.0] Bump Leap version to 4.0.0 --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 0b07f43caa..c821ef5092 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -16,7 +16,7 @@ set( CXX_STANDARD_REQUIRED ON) set(VERSION_MAJOR 4) set(VERSION_MINOR 0) set(VERSION_PATCH 0) -set(VERSION_SUFFIX rc3) +#set(VERSION_SUFFIX rc3) if(VERSION_SUFFIX) set(VERSION_FULL "${VERSION_MAJOR}.${VERSION_MINOR}.${VERSION_PATCH}-${VERSION_SUFFIX}") From c5774f69113bdf0c3de586637970a1f9fb810e85 Mon Sep 17 00:00:00 2001 From: Kevin Heifner Date: Mon, 24 Apr 2023 14:55:03 -0500 Subject: [PATCH 24/58] GH-668 Remove space which confuses CMakeLists.txt processing --- ...test_subjective_billing.cpp => subjective_billing_tests.cpp} | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) rename unittests/{test_subjective_billing.cpp => subjective_billing_tests.cpp} (99%) diff --git a/unittests/test_subjective_billing.cpp b/unittests/subjective_billing_tests.cpp similarity index 99% rename from unittests/test_subjective_billing.cpp rename to unittests/subjective_billing_tests.cpp index 3f39284913..3e417e36a8 100644 --- a/unittests/test_subjective_billing.cpp +++ b/unittests/subjective_billing_tests.cpp @@ -8,7 +8,7 @@ namespace { using namespace eosio; using namespace eosio::chain; -BOOST_AUTO_TEST_SUITE( subjective_billing_test ) +BOOST_AUTO_TEST_SUITE(subjective_billing_test) BOOST_AUTO_TEST_CASE( subjective_bill_test ) { From 16685ab12df4d5483c892fb3a4bdae114f04df21 Mon Sep 17 00:00:00 2001 From: Matt Witherspoon <32485495+spoonincode@users.noreply.github.com> Date: Mon, 24 Apr 2023 17:31:27 -0400 Subject: [PATCH 25/58] fix ambiguous to_variant()s --- libraries/libfc/test/io/test_json.cpp | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/libraries/libfc/test/io/test_json.cpp b/libraries/libfc/test/io/test_json.cpp index f5e08c03f6..4794ec7ed0 100644 --- a/libraries/libfc/test/io/test_json.cpp +++ b/libraries/libfc/test/io/test_json.cpp @@ -72,23 +72,23 @@ BOOST_AUTO_TEST_CASE(to_string_test) BOOST_CHECK_EQUAL(length_exception_in_mid_str, "\"" + json_test_util::repeat_chars + "\""); } { - variant v(4294967296); // 0xFFFFFFFF + 1 + variant v(4294967296LL); // 0xFFFFFFFF + 1 std::string large_int = json::to_string( v, fc::time_point::maximum(), json::output_formatting::stringify_large_ints_and_doubles, json::max_length_limit); BOOST_CHECK_EQUAL(large_int, "\"4294967296\""); - variant v1(4294967295); // 0xFFFFFFFF + variant v1(4294967295LL); // 0xFFFFFFFF std::string normal_int = json::to_string( v1, fc::time_point::maximum(), json::output_formatting::stringify_large_ints_and_doubles, json::max_length_limit); BOOST_CHECK_EQUAL(normal_int, "4294967295"); - variant v2(-4294967296); + variant v2(-4294967296LL); std::string large_int_neg = json::to_string( v2, fc::time_point::maximum(), json::output_formatting::stringify_large_ints_and_doubles, json::max_length_limit); BOOST_CHECK_EQUAL(large_int_neg, "\"-4294967296\""); - variant v3(-4294967295); + variant v3(-4294967295LL); std::string normal_int_neg = json::to_string( v3, fc::time_point::maximum(), json::output_formatting::stringify_large_ints_and_doubles, json::max_length_limit); BOOST_CHECK_EQUAL(normal_int_neg, "-4294967295"); - variant v4(-90909090909090909); + variant v4(-90909090909090909LL); std::string super_neg = json::to_string( v4, fc::time_point::maximum(), json::output_formatting::stringify_large_ints_and_doubles, json::max_length_limit); BOOST_CHECK_EQUAL(super_neg, "\"-90909090909090909\""); } From a6b95519d032d0d9b0f41496be9dd54cde877075 Mon Sep 17 00:00:00 2001 From: Pinelliaw Date: Tue, 25 Apr 2023 21:26:28 +0800 Subject: [PATCH 26/58] fix: not fail on signature checks for read-only txns --- libraries/chain/authorization_manager.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libraries/chain/authorization_manager.cpp b/libraries/chain/authorization_manager.cpp index 6f3e598d19..69312b4c10 100644 --- a/libraries/chain/authorization_manager.cpp +++ b/libraries/chain/authorization_manager.cpp @@ -559,7 +559,7 @@ namespace eosio { namespace chain { } - if( !allow_unused_keys || check_but_dont_fail) { + if( !allow_unused_keys && !check_but_dont_fail) { EOS_ASSERT( checker.all_keys_used(), tx_irrelevant_sig, "transaction bears irrelevant signatures from these keys: ${keys}", ("keys", checker.unused_keys()) ); From 02590a54d79b652d0262bd0571ef98bc42ebec4a Mon Sep 17 00:00:00 2001 From: 766C6164 Date: Tue, 25 Apr 2023 12:31:48 -0400 Subject: [PATCH 27/58] Addressng feedback --- .../include/eosio/chain/pending_snapshot.hpp | 22 ++++---- .../include/eosio/chain/snapshot_db_json.hpp | 12 ++--- .../eosio/chain/snapshot_scheduler.hpp | 38 +++++++------- libraries/chain/snapshot_scheduler.cpp | 52 +++++++++---------- .../test/test_snapshot_scheduler.cpp | 17 +++--- 5 files changed, 67 insertions(+), 74 deletions(-) diff --git a/libraries/chain/include/eosio/chain/pending_snapshot.hpp b/libraries/chain/include/eosio/chain/pending_snapshot.hpp index 667e1cb802..d31c118888 100644 --- a/libraries/chain/include/eosio/chain/pending_snapshot.hpp +++ b/libraries/chain/include/eosio/chain/pending_snapshot.hpp @@ -5,32 +5,31 @@ #include -namespace eosio { -namespace chain { +namespace eosio::chain { -namespace bfs = std::filesystem; +namespace fs = std::filesystem; template class pending_snapshot { public: using next_t = eosio::chain::next_function; - pending_snapshot(const chain::block_id_type& block_id, next_t& next, std::string pending_path, std::string final_path) - : block_id(block_id), next(next), pending_path(pending_path), final_path(final_path) {} + pending_snapshot(const chain::block_id_type& block_id, const next_t& next, std::string pending_path, std::string final_path) + : block_id(block_id), next(next), pending_path(std::move(pending_path)), final_path(std::move(final_path)) {} uint32_t get_height() const { return chain::block_header::num_from_id(block_id); } - static bfs::path get_final_path(const chain::block_id_type& block_id, const bfs::path& snapshots_dir) { + static fs::path get_final_path(const chain::block_id_type& block_id, const fs::path& snapshots_dir) { return snapshots_dir / fc::format_string("snapshot-${id}.bin", fc::mutable_variant_object()("id", block_id)); } - static bfs::path get_pending_path(const chain::block_id_type& block_id, const bfs::path& snapshots_dir) { + static fs::path get_pending_path(const chain::block_id_type& block_id, const fs::path& snapshots_dir) { return snapshots_dir / fc::format_string(".pending-snapshot-${id}.bin", fc::mutable_variant_object()("id", block_id)); } - static bfs::path get_temp_path(const chain::block_id_type& block_id, const bfs::path& snapshots_dir) { + static fs::path get_temp_path(const chain::block_id_type& block_id, const fs::path& snapshots_dir) { return snapshots_dir / fc::format_string(".incomplete-snapshot-${id}.bin", fc::mutable_variant_object()("id", block_id)); } @@ -40,13 +39,13 @@ class pending_snapshot { std::error_code ec; if(!in_chain) { - bfs::remove(bfs::path(pending_path), ec); + fs::remove(fs::path(pending_path), ec); EOS_THROW(chain::snapshot_finalization_exception, "Snapshotted block was forked out of the chain. ID: ${block_id}", ("block_id", block_id)); } - bfs::rename(bfs::path(pending_path), bfs::path(final_path), ec); + fs::rename(fs::path(pending_path), fs::path(final_path), ec); EOS_ASSERT(!ec, chain::snapshot_finalization_exception, "Unable to finalize valid snapshot of block number ${bn}: [code: ${ec}] ${message}", ("bn", get_height())("ec", ec.value())("message", ec.message())); @@ -59,5 +58,4 @@ class pending_snapshot { std::string pending_path; std::string final_path; }; -}// namespace chain -}// namespace eosio +}// namespace eosio::chain diff --git a/libraries/chain/include/eosio/chain/snapshot_db_json.hpp b/libraries/chain/include/eosio/chain/snapshot_db_json.hpp index 42a4858908..e34ba1e8b2 100644 --- a/libraries/chain/include/eosio/chain/snapshot_db_json.hpp +++ b/libraries/chain/include/eosio/chain/snapshot_db_json.hpp @@ -7,10 +7,9 @@ #include #include -namespace eosio { -namespace chain { +namespace eosio::chain { -namespace bfs = std::filesystem; +namespace fs = std::filesystem; /// this class designed to serialize/deserialize snapshot schedule to a filesystem so it can be restored after restart class snapshot_db_json { @@ -48,7 +47,6 @@ class snapshot_db_json { } catch(std::ifstream::failure& e) { elog("unable to restore snapshots schedule from filesystem ${jsonpath}, details: ${details}", ("jsonpath", get_json_path().string())("details", e.what())); - // appbase::app().quit(); } return *this; @@ -78,15 +76,13 @@ class snapshot_db_json { } catch(std::ofstream::failure& e) { elog("unable to store snapshots schedule to filesystem to ${jsonpath}, details: ${details}", ("jsonpath", get_json_path().string())("details", e.what())); - // appbase::app().quit(); } return *this; } private: - std::filesystem::path db_path; + fs::path db_path; }; -}// namespace chain -}// namespace eosio +}// namespace eosio::chain diff --git a/libraries/chain/include/eosio/chain/snapshot_scheduler.hpp b/libraries/chain/include/eosio/chain/snapshot_scheduler.hpp index c391f885a6..d5ce481d6d 100644 --- a/libraries/chain/include/eosio/chain/snapshot_scheduler.hpp +++ b/libraries/chain/include/eosio/chain/snapshot_scheduler.hpp @@ -18,11 +18,10 @@ #include -namespace eosio { -namespace chain { +namespace eosio::chain { namespace bmi = boost::multi_index; -namespace bfs = std::filesystem; +namespace fs = std::filesystem; class snapshot_scheduler { public: @@ -74,15 +73,15 @@ class snapshot_scheduler { struct as_vector; using snapshot_requests = bmi::multi_index_container< - snapshot_scheduler::snapshot_schedule_information, + snapshot_schedule_information, indexed_by< - bmi::hashed_unique, BOOST_MULTI_INDEX_MEMBER(snapshot_scheduler::snapshot_request_id_information, uint32_t, snapshot_request_id)>, + bmi::hashed_unique, BOOST_MULTI_INDEX_MEMBER(snapshot_request_id_information, uint32_t, snapshot_request_id)>, bmi::random_access>, bmi::ordered_unique, - composite_key>>>; + composite_key>>>; snapshot_requests _snapshot_requests; snapshot_db_json _snapshot_db; pending_snapshot_index _pending_snapshot_index; @@ -91,11 +90,11 @@ class snapshot_scheduler { uint32_t _inflight_sid = 0; // path to write the snapshots to - bfs::path _snapshots_dir; + fs::path _snapshots_dir; void x_serialize() { auto& vec = _snapshot_requests.get(); - std::vector sr(vec.begin(), vec.end()); + std::vector sr(vec.begin(), vec.end()); _snapshot_db << sr; }; @@ -109,27 +108,26 @@ class snapshot_scheduler { void on_irreversible_block(const signed_block_ptr& lib, const chain::controller& chain); // snapshot scheduler handlers - snapshot_scheduler::snapshot_schedule_result schedule_snapshot(const snapshot_scheduler::snapshot_request_information& sri); - snapshot_scheduler::snapshot_schedule_result unschedule_snapshot(uint32_t sri); - snapshot_scheduler::get_snapshot_requests_result get_snapshot_requests(); + snapshot_schedule_result schedule_snapshot(const snapshot_request_information& sri); + snapshot_schedule_result unschedule_snapshot(uint32_t sri); + get_snapshot_requests_result get_snapshot_requests(); // initialize with storage - void set_db_path(bfs::path db_path); + void set_db_path(fs::path db_path); // set snapshot path - void set_snapshots_path(bfs::path sn_path); + void set_snapshots_path(fs::path sn_path); // add pending snapshot info to inflight snapshot request - void add_pending_snapshot_info(const snapshot_scheduler::snapshot_information& si); + void add_pending_snapshot_info(const snapshot_information& si); // execute snapshot void execute_snapshot(uint32_t srid, chain::controller& chain); // former producer_plugin snapshot fn - void create_snapshot(snapshot_scheduler::next_function next, chain::controller& chain, std::function predicate); + void create_snapshot(next_function next, chain::controller& chain, std::function predicate); }; -}// namespace chain -}// namespace eosio +}// namespace eosio::chain FC_REFLECT(eosio::chain::snapshot_scheduler::snapshot_information, (head_block_id)(head_block_num)(head_block_time)(version)(snapshot_name)) FC_REFLECT(eosio::chain::snapshot_scheduler::snapshot_request_information, (block_spacing)(start_block_num)(end_block_num)(snapshot_description)) diff --git a/libraries/chain/snapshot_scheduler.cpp b/libraries/chain/snapshot_scheduler.cpp index 05f906a900..ee7a356fe0 100644 --- a/libraries/chain/snapshot_scheduler.cpp +++ b/libraries/chain/snapshot_scheduler.cpp @@ -4,8 +4,7 @@ #include #include -namespace eosio { -namespace chain { +namespace eosio::chain { // snapshot_scheduler_listener void snapshot_scheduler::on_start_block(uint32_t height, chain::controller& chain) { @@ -77,7 +76,7 @@ void snapshot_scheduler::on_irreversible_block(const signed_block_ptr& lib, cons } } -snapshot_scheduler::snapshot_schedule_result snapshot_scheduler::schedule_snapshot(const snapshot_scheduler::snapshot_request_information& sri) { +snapshot_scheduler::snapshot_schedule_result snapshot_scheduler::schedule_snapshot(const snapshot_request_information& sri) { auto& snapshot_by_value = _snapshot_requests.get(); auto existing = snapshot_by_value.find(std::make_tuple(sri.block_spacing, sri.start_block_num, sri.end_block_num)); EOS_ASSERT(existing == snapshot_by_value.end(), chain::duplicate_snapshot_request, "Duplicate snapshot request"); @@ -91,11 +90,11 @@ snapshot_scheduler::snapshot_schedule_result snapshot_scheduler::schedule_snapsh } } - _snapshot_requests.emplace(snapshot_scheduler::snapshot_schedule_information{{_snapshot_id++}, {sri.block_spacing, sri.start_block_num, sri.end_block_num, sri.snapshot_description}, {}}); + _snapshot_requests.emplace(snapshot_schedule_information{{_snapshot_id++}, {sri.block_spacing, sri.start_block_num, sri.end_block_num, sri.snapshot_description}, {}}); x_serialize(); // returning snapshot_schedule_result - return snapshot_scheduler::snapshot_schedule_result{{_snapshot_id - 1}, {sri.block_spacing, sri.start_block_num, sri.end_block_num, sri.snapshot_description}}; + return snapshot_schedule_result{{_snapshot_id - 1}, {sri.block_spacing, sri.start_block_num, sri.end_block_num, sri.snapshot_description}}; } snapshot_scheduler::snapshot_schedule_result snapshot_scheduler::unschedule_snapshot(uint32_t sri) { @@ -103,7 +102,7 @@ snapshot_scheduler::snapshot_schedule_result snapshot_scheduler::unschedule_snap auto existing = snapshot_by_id.find(sri); EOS_ASSERT(existing != snapshot_by_id.end(), chain::snapshot_request_not_found, "Snapshot request not found"); - snapshot_scheduler::snapshot_schedule_result result{{existing->snapshot_request_id}, {existing->block_spacing, existing->start_block_num, existing->end_block_num, existing->snapshot_description}}; + snapshot_schedule_result result{{existing->snapshot_request_id}, {existing->block_spacing, existing->start_block_num, existing->end_block_num, existing->snapshot_description}}; _snapshot_requests.erase(existing); x_serialize(); @@ -112,18 +111,18 @@ snapshot_scheduler::snapshot_schedule_result snapshot_scheduler::unschedule_snap } snapshot_scheduler::get_snapshot_requests_result snapshot_scheduler::get_snapshot_requests() { - snapshot_scheduler::get_snapshot_requests_result result; + get_snapshot_requests_result result; auto& asvector = _snapshot_requests.get(); result.snapshot_requests.reserve(asvector.size()); result.snapshot_requests.insert(result.snapshot_requests.begin(), asvector.begin(), asvector.end()); return result; } -void snapshot_scheduler::set_db_path(bfs::path db_path) { +void snapshot_scheduler::set_db_path(fs::path db_path) { _snapshot_db.set_path(std::move(db_path)); // init from db if(std::filesystem::exists(_snapshot_db.get_json_path())) { - std::vector sr; + std::vector sr; _snapshot_db >> sr; // if db read succeeded, clear/load _snapshot_requests.get().clear(); @@ -131,11 +130,11 @@ void snapshot_scheduler::set_db_path(bfs::path db_path) { } } -void snapshot_scheduler::set_snapshots_path(bfs::path sn_path) { +void snapshot_scheduler::set_snapshots_path(fs::path sn_path) { _snapshots_dir = std::move(sn_path); } -void snapshot_scheduler::add_pending_snapshot_info(const snapshot_scheduler::snapshot_information& si) { +void snapshot_scheduler::add_pending_snapshot_info(const snapshot_information& si) { auto& snapshot_by_id = _snapshot_requests.get(); auto snapshot_req = snapshot_by_id.find(_inflight_sid); if(snapshot_req != snapshot_by_id.end()) { @@ -147,7 +146,7 @@ void snapshot_scheduler::add_pending_snapshot_info(const snapshot_scheduler::sna void snapshot_scheduler::execute_snapshot(uint32_t srid, chain::controller& chain) { _inflight_sid = srid; - auto next = [srid, this](const chain::next_function_variant& result) { + auto next = [srid, this](const chain::next_function_variant& result) { if(std::holds_alternative(result)) { try { std::get(result)->dynamic_rethrow_exception(); @@ -162,14 +161,14 @@ void snapshot_scheduler::execute_snapshot(uint32_t srid, chain::controller& chai } } else { // success, snapshot finalized - auto snapshot_info = std::get(result); + auto snapshot_info = std::get(result); auto& snapshot_by_id = _snapshot_requests.get(); auto snapshot_req = snapshot_by_id.find(srid); if(snapshot_req != snapshot_by_id.end()) { _snapshot_requests.modify(snapshot_req, [&](auto& p) { auto& pending = p.pending_snapshots; - auto it = std::remove_if(pending.begin(), pending.end(), [&snapshot_info](const snapshot_scheduler::snapshot_information& s) { return s.head_block_num <= snapshot_info.head_block_num; }); + auto it = std::remove_if(pending.begin(), pending.end(), [&snapshot_info](const snapshot_information& s) { return s.head_block_num <= snapshot_info.head_block_num; }); pending.erase(it, pending.end()); }); } @@ -178,23 +177,23 @@ void snapshot_scheduler::execute_snapshot(uint32_t srid, chain::controller& chai create_snapshot(next, chain, {}); } -void snapshot_scheduler::create_snapshot(snapshot_scheduler::next_function next, chain::controller& chain, std::function predicate) { +void snapshot_scheduler::create_snapshot(next_function next, chain::controller& chain, std::function predicate) { auto head_id = chain.head_block_id(); const auto head_block_num = chain.head_block_num(); const auto head_block_time = chain.head_block_time(); - const auto& snapshot_path = pending_snapshot::get_final_path(head_id, _snapshots_dir); - const auto& temp_path = pending_snapshot::get_temp_path(head_id, _snapshots_dir); + const auto& snapshot_path = pending_snapshot::get_final_path(head_id, _snapshots_dir); + const auto& temp_path = pending_snapshot::get_temp_path(head_id, _snapshots_dir); // maintain legacy exception if the snapshot exists - if(bfs::is_regular_file(snapshot_path)) { + if(fs::is_regular_file(snapshot_path)) { auto ex = snapshot_exists_exception(FC_LOG_MESSAGE(error, "snapshot named ${name} already exists", ("name", _snapshots_dir))); next(ex.dynamic_copy_exception()); return; } - auto write_snapshot = [&](const bfs::path& p) -> void { + auto write_snapshot = [&](const fs::path& p) -> void { if(predicate) predicate(); - bfs::create_directory(p.parent_path()); + fs::create_directory(p.parent_path()); auto snap_out = std::ofstream(p.generic_string(), (std::ios::out | std::ios::binary)); auto writer = std::make_shared(snap_out); chain.write_snapshot(writer); @@ -208,7 +207,7 @@ void snapshot_scheduler::create_snapshot(snapshot_scheduler::next_function& res) { + entry.next = [prev = entry.next, next](const next_function_variant& res) { prev(res); next(res); }; }); } else { - const auto& pending_path = pending_snapshot::get_pending_path(head_id, _snapshots_dir); + const auto& pending_path = pending_snapshot::get_pending_path(head_id, _snapshots_dir); try { write_snapshot(temp_path);// create a new pending snapshot std::error_code ec; - bfs::rename(temp_path, pending_path, ec); + fs::rename(temp_path, pending_path, ec); EOS_ASSERT(!ec, snapshot_finalization_exception, "Unable to promote temp snapshot to pending for block number ${bn}: [code: ${ec}] ${message}", ("bn", head_block_num)("ec", ec.value())("message", ec.message())); _pending_snapshot_index.emplace(head_id, next, pending_path.generic_string(), snapshot_path.generic_string()); - add_pending_snapshot_info(snapshot_scheduler::snapshot_information{head_id, head_block_num, head_block_time, chain_snapshot_header::current_version, pending_path.generic_string()}); + add_pending_snapshot_info(snapshot_information{head_id, head_block_num, head_block_time, chain_snapshot_header::current_version, pending_path.generic_string()}); } CATCH_AND_CALL(next); } } -}// namespace chain -}// namespace eosio +}// namespace eosio::chain diff --git a/plugins/producer_plugin/test/test_snapshot_scheduler.cpp b/plugins/producer_plugin/test/test_snapshot_scheduler.cpp index 3c59c0e9ca..aefbae3835 100644 --- a/plugins/producer_plugin/test/test_snapshot_scheduler.cpp +++ b/plugins/producer_plugin/test/test_snapshot_scheduler.cpp @@ -10,6 +10,9 @@ namespace { using namespace eosio; using namespace eosio::chain; +using snapshot_request_information = snapshot_scheduler::snapshot_request_information; +using snapshot_request_id_information = snapshot_scheduler::snapshot_request_id_information; + BOOST_AUTO_TEST_SUITE(snapshot_scheduler_test) BOOST_AUTO_TEST_CASE(snapshot_scheduler_test) { @@ -19,8 +22,8 @@ BOOST_AUTO_TEST_CASE(snapshot_scheduler_test) { { // add/remove test - snapshot_scheduler::snapshot_request_information sri1 = {.block_spacing = 100, .start_block_num = 5000, .end_block_num = 10000, .snapshot_description = "Example of recurring snapshot"}; - snapshot_scheduler::snapshot_request_information sri2 = {.block_spacing = 0, .start_block_num = 5200, .end_block_num = 5200, .snapshot_description = "Example of one-time snapshot"}; + snapshot_request_information sri1 = {.block_spacing = 100, .start_block_num = 5000, .end_block_num = 10000, .snapshot_description = "Example of recurring snapshot"}; + snapshot_request_information sri2 = {.block_spacing = 0, .start_block_num = 5200, .end_block_num = 5200, .snapshot_description = "Example of one-time snapshot"}; scheduler.schedule_snapshot(sri1); scheduler.schedule_snapshot(sri2); @@ -31,27 +34,27 @@ BOOST_AUTO_TEST_CASE(snapshot_scheduler_test) { return e.to_detail_string().find("Duplicate snapshot request") != std::string::npos; }); - snapshot_scheduler::snapshot_request_id_information sri_delete_1 = {.snapshot_request_id = 0}; + snapshot_request_id_information sri_delete_1 = {.snapshot_request_id = 0}; scheduler.unschedule_snapshot(sri_delete_1); BOOST_CHECK_EQUAL(1, scheduler.get_snapshot_requests().snapshot_requests.size()); - snapshot_scheduler::snapshot_request_id_information sri_delete_none = {.snapshot_request_id = 2}; + snapshot_request_id_information sri_delete_none = {.snapshot_request_id = 2}; BOOST_CHECK_EXCEPTION(scheduler.unschedule_snapshot(sri_delete_none), snapshot_request_not_found, [](const fc::assert_exception& e) { return e.to_detail_string().find("Snapshot request not found") != std::string::npos; }); - snapshot_scheduler::snapshot_request_id_information sri_delete_2 = {.snapshot_request_id = 1}; + snapshot_request_id_information sri_delete_2 = {.snapshot_request_id = 1}; scheduler.unschedule_snapshot(sri_delete_2); BOOST_CHECK_EQUAL(0, scheduler.get_snapshot_requests().snapshot_requests.size()); - snapshot_scheduler::snapshot_request_information sri_large_spacing = {.block_spacing = 1000, .start_block_num = 5000, .end_block_num = 5010}; + snapshot_request_information sri_large_spacing = {.block_spacing = 1000, .start_block_num = 5000, .end_block_num = 5010}; BOOST_CHECK_EXCEPTION(scheduler.schedule_snapshot(sri_large_spacing), invalid_snapshot_request, [](const fc::assert_exception& e) { return e.to_detail_string().find("Block spacing exceeds defined by start and end range") != std::string::npos; }); - snapshot_scheduler::snapshot_request_information sri_start_end = {.block_spacing = 1000, .start_block_num = 50000, .end_block_num = 5000}; + snapshot_request_information sri_start_end = {.block_spacing = 1000, .start_block_num = 50000, .end_block_num = 5000}; BOOST_CHECK_EXCEPTION(scheduler.schedule_snapshot(sri_start_end), invalid_snapshot_request, [](const fc::assert_exception& e) { return e.to_detail_string().find("End block number should be greater or equal to start block number") != std::string::npos; }); From 79a2f281c5ede18c31c5c4dd7540fb92eb37e549 Mon Sep 17 00:00:00 2001 From: 766C6164 Date: Tue, 25 Apr 2023 12:50:39 -0400 Subject: [PATCH 28/58] Removed a separate header related to json db --- .../include/eosio/chain/snapshot_db_json.hpp | 88 ------------------- .../eosio/chain/snapshot_scheduler.hpp | 80 ++++++++++++++++- .../test/test_snapshot_scheduler.cpp | 4 +- 3 files changed, 79 insertions(+), 93 deletions(-) delete mode 100644 libraries/chain/include/eosio/chain/snapshot_db_json.hpp diff --git a/libraries/chain/include/eosio/chain/snapshot_db_json.hpp b/libraries/chain/include/eosio/chain/snapshot_db_json.hpp deleted file mode 100644 index e34ba1e8b2..0000000000 --- a/libraries/chain/include/eosio/chain/snapshot_db_json.hpp +++ /dev/null @@ -1,88 +0,0 @@ -#pragma once - -#include -#include - -#include -#include -#include - -namespace eosio::chain { - -namespace fs = std::filesystem; - -/// this class designed to serialize/deserialize snapshot schedule to a filesystem so it can be restored after restart -class snapshot_db_json { -public: - snapshot_db_json() = default; - ~snapshot_db_json() = default; - - void set_path(std::filesystem::path path) { - db_path = std::move(path); - } - - std::filesystem::path get_json_path() const { - return db_path / "snapshot-schedule.json"; - } - - template - const snapshot_db_json& operator>>(std::vector& sr) { - boost::property_tree::ptree root; - - try { - std::ifstream file(get_json_path().string()); - file.exceptions(std::istream::failbit | std::istream::eofbit); - boost::property_tree::read_json(file, root); - - // parse ptree - for(boost::property_tree::ptree::value_type& req: root.get_child("snapshot_requests")) { - F ssi; - ssi.snapshot_request_id = req.second.get("snapshot_request_id"); - ssi.snapshot_description = req.second.get("snapshot_description"); - ssi.block_spacing = req.second.get("block_spacing"); - ssi.start_block_num = req.second.get("start_block_num"); - ssi.end_block_num = req.second.get("end_block_num"); - sr.push_back(ssi); - } - } catch(std::ifstream::failure& e) { - elog("unable to restore snapshots schedule from filesystem ${jsonpath}, details: ${details}", - ("jsonpath", get_json_path().string())("details", e.what())); - } - - return *this; - } - - template - const snapshot_db_json& operator<<(std::vector& sr) const { - boost::property_tree::ptree root; - boost::property_tree::ptree node_srs; - - for(const auto& key: sr) { - boost::property_tree::ptree node; - node.put("snapshot_request_id", key.snapshot_request_id); - node.put("snapshot_description", key.snapshot_description); - node.put("block_spacing", key.block_spacing); - node.put("start_block_num", key.start_block_num); - node.put("end_block_num", key.end_block_num); - node_srs.push_back(std::make_pair("", node)); - } - - root.push_back(std::make_pair("snapshot_requests", node_srs)); - - try { - std::ofstream file(get_json_path().string()); - file.exceptions(std::istream::failbit | std::istream::eofbit); - boost::property_tree::write_json(file, root); - } catch(std::ofstream::failure& e) { - elog("unable to store snapshots schedule to filesystem to ${jsonpath}, details: ${details}", - ("jsonpath", get_json_path().string())("details", e.what())); - } - - return *this; - } - -private: - fs::path db_path; -}; - -}// namespace eosio::chain diff --git a/libraries/chain/include/eosio/chain/snapshot_scheduler.hpp b/libraries/chain/include/eosio/chain/snapshot_scheduler.hpp index d5ce481d6d..2aebf62c9b 100644 --- a/libraries/chain/include/eosio/chain/snapshot_scheduler.hpp +++ b/libraries/chain/include/eosio/chain/snapshot_scheduler.hpp @@ -1,7 +1,6 @@ #pragma once #include -#include #include #include @@ -17,6 +16,9 @@ #include #include +#include +#include +#include namespace eosio::chain { @@ -66,6 +68,76 @@ class snapshot_scheduler { bmi::hashed_unique, BOOST_MULTI_INDEX_MEMBER(pending_snapshot, block_id_type, block_id)>, bmi::ordered_non_unique, BOOST_MULTI_INDEX_CONST_MEM_FUN(pending_snapshot, uint32_t, get_height)>>>; + class snapshot_db_json { + public: + snapshot_db_json() = default; + ~snapshot_db_json() = default; + + void set_path(std::filesystem::path path) { + db_path = std::move(path); + } + + std::filesystem::path get_json_path() const { + return db_path / "snapshot-schedule.json"; + } + + const snapshot_db_json& operator>>(std::vector& sr) { + boost::property_tree::ptree root; + + try { + std::ifstream file(get_json_path().string()); + file.exceptions(std::istream::failbit | std::istream::eofbit); + boost::property_tree::read_json(file, root); + + // parse ptree + for(boost::property_tree::ptree::value_type& req: root.get_child("snapshot_requests")) { + snapshot_schedule_information ssi; + ssi.snapshot_request_id = req.second.get("snapshot_request_id"); + ssi.snapshot_description = req.second.get("snapshot_description"); + ssi.block_spacing = req.second.get("block_spacing"); + ssi.start_block_num = req.second.get("start_block_num"); + ssi.end_block_num = req.second.get("end_block_num"); + sr.push_back(ssi); + } + } catch(std::ifstream::failure& e) { + elog("unable to restore snapshots schedule from filesystem ${jsonpath}, details: ${details}", + ("jsonpath", get_json_path().string())("details", e.what())); + } + + return *this; + } + + const snapshot_db_json& operator<<(const std::vector& sr) const { + boost::property_tree::ptree root; + boost::property_tree::ptree node_srs; + + for(const auto& key: sr) { + boost::property_tree::ptree node; + node.put("snapshot_request_id", key.snapshot_request_id); + node.put("snapshot_description", key.snapshot_description); + node.put("block_spacing", key.block_spacing); + node.put("start_block_num", key.start_block_num); + node.put("end_block_num", key.end_block_num); + node_srs.push_back(std::make_pair("", node)); + } + + root.push_back(std::make_pair("snapshot_requests", node_srs)); + + try { + std::ofstream file(get_json_path().string()); + file.exceptions(std::istream::failbit | std::istream::eofbit); + boost::property_tree::write_json(file, root); + } catch(std::ofstream::failure& e) { + elog("unable to store snapshots schedule to filesystem to ${jsonpath}, details: ${details}", + ("jsonpath", get_json_path().string())("details", e.what())); + } + + return *this; + } + + private: + fs::path db_path; + }; private: struct by_snapshot_id; @@ -127,10 +199,12 @@ class snapshot_scheduler { // former producer_plugin snapshot fn void create_snapshot(next_function next, chain::controller& chain, std::function predicate); }; + + }// namespace eosio::chain -FC_REFLECT(eosio::chain::snapshot_scheduler::snapshot_information, (head_block_id)(head_block_num)(head_block_time)(version)(snapshot_name)) -FC_REFLECT(eosio::chain::snapshot_scheduler::snapshot_request_information, (block_spacing)(start_block_num)(end_block_num)(snapshot_description)) +FC_REFLECT(eosio::chain::snapshot_scheduler::snapshot_information, (head_block_id) (head_block_num) (head_block_time) (version) (snapshot_name)) +FC_REFLECT(eosio::chain::snapshot_scheduler::snapshot_request_information, (block_spacing) (start_block_num) (end_block_num) (snapshot_description)) FC_REFLECT(eosio::chain::snapshot_scheduler::snapshot_request_id_information, (snapshot_request_id)) FC_REFLECT(eosio::chain::snapshot_scheduler::get_snapshot_requests_result, (snapshot_requests)) FC_REFLECT_DERIVED(eosio::chain::snapshot_scheduler::snapshot_schedule_information, (eosio::chain::snapshot_scheduler::snapshot_request_id_information)(eosio::chain::snapshot_scheduler::snapshot_request_information), (pending_snapshots)) diff --git a/plugins/producer_plugin/test/test_snapshot_scheduler.cpp b/plugins/producer_plugin/test/test_snapshot_scheduler.cpp index aefbae3835..ad6eb78a8b 100644 --- a/plugins/producer_plugin/test/test_snapshot_scheduler.cpp +++ b/plugins/producer_plugin/test/test_snapshot_scheduler.cpp @@ -3,8 +3,8 @@ #include #include -#include #include + namespace { using namespace eosio; @@ -137,7 +137,7 @@ BOOST_AUTO_TEST_CASE(snapshot_scheduler_test) { app_thread.join(); // lets check whether schedule can be read back after restart - snapshot_db_json db; + snapshot_scheduler::snapshot_db_json db; std::vector ssi; db.set_path(temp / "snapshots"); db >> ssi; From a3589123ec4405cbeb853e3aa0d97df2d6c46c2c Mon Sep 17 00:00:00 2001 From: Pinelliaw Date: Wed, 26 Apr 2023 01:04:02 +0800 Subject: [PATCH 29/58] refactor: include the time it takes for checker.all_keys_used() --- libraries/chain/authorization_manager.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/libraries/chain/authorization_manager.cpp b/libraries/chain/authorization_manager.cpp index 69312b4c10..9cb4580a0c 100644 --- a/libraries/chain/authorization_manager.cpp +++ b/libraries/chain/authorization_manager.cpp @@ -559,8 +559,8 @@ namespace eosio { namespace chain { } - if( !allow_unused_keys && !check_but_dont_fail) { - EOS_ASSERT( checker.all_keys_used(), tx_irrelevant_sig, + if( !allow_unused_keys ) { + EOS_ASSERT( checker.all_keys_used() || check_but_dont_fail, tx_irrelevant_sig, "transaction bears irrelevant signatures from these keys: ${keys}", ("keys", checker.unused_keys()) ); } From 6bf4c8881f1ed197f6cd53af09e611066e3b5e49 Mon Sep 17 00:00:00 2001 From: greg7mdp Date: Tue, 25 Apr 2023 13:06:43 -0400 Subject: [PATCH 30/58] remove no longer used `noncopyable` struct --- libraries/chain/include/eosio/chain/types.hpp | 8 -------- 1 file changed, 8 deletions(-) diff --git a/libraries/chain/include/eosio/chain/types.hpp b/libraries/chain/include/eosio/chain/types.hpp index 0a71ea9fdf..b0089f3a15 100644 --- a/libraries/chain/include/eosio/chain/types.hpp +++ b/libraries/chain/include/eosio/chain/types.hpp @@ -87,14 +87,6 @@ namespace eosio::chain { #endif struct void_t{}; - struct noncopyable { - noncopyable() = default; - noncopyable(noncopyable&&) = default; - noncopyable& operator=(noncopyable&&) = default; - noncopyable(const noncopyable&) = delete; - noncopyable& operator=(const noncopyable&) = delete; - }; - using chainbase::allocator; using shared_string = chainbase::shared_string; template From ef0950216286eb12a1e3307c2e3ce2b2e81e737b Mon Sep 17 00:00:00 2001 From: 766C6164 Date: Tue, 25 Apr 2023 15:01:11 -0400 Subject: [PATCH 31/58] Missed one more spot --- plugins/producer_plugin/test/test_snapshot_scheduler.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/plugins/producer_plugin/test/test_snapshot_scheduler.cpp b/plugins/producer_plugin/test/test_snapshot_scheduler.cpp index ad6eb78a8b..60b35bf20a 100644 --- a/plugins/producer_plugin/test/test_snapshot_scheduler.cpp +++ b/plugins/producer_plugin/test/test_snapshot_scheduler.cpp @@ -108,9 +108,9 @@ BOOST_AUTO_TEST_CASE(snapshot_scheduler_test) { } }); - snapshot_scheduler::snapshot_request_information sri1 = {.block_spacing = 8, .start_block_num = 1, .end_block_num = 300000, .snapshot_description = "Example of recurring snapshot 1"}; - snapshot_scheduler::snapshot_request_information sri2 = {.block_spacing = 5000, .start_block_num = 100000, .end_block_num = 300000, .snapshot_description = "Example of recurring snapshot 2 that will never happen"}; - snapshot_scheduler::snapshot_request_information sri3 = {.block_spacing = 2, .start_block_num = 0, .end_block_num = 3, .snapshot_description = "Example of recurring snapshot 3 that will expire"}; + snapshot_request_information sri1 = {.block_spacing = 8, .start_block_num = 1, .end_block_num = 300000, .snapshot_description = "Example of recurring snapshot 1"}; + snapshot_request_information sri2 = {.block_spacing = 5000, .start_block_num = 100000, .end_block_num = 300000, .snapshot_description = "Example of recurring snapshot 2 that will never happen"}; + snapshot_request_information sri3 = {.block_spacing = 2, .start_block_num = 0, .end_block_num = 3, .snapshot_description = "Example of recurring snapshot 3 that will expire"}; pp->schedule_snapshot(sri1); pp->schedule_snapshot(sri2); From 5f01a2adac975884052e6119c7b3c29d4e279598 Mon Sep 17 00:00:00 2001 From: greg7mdp Date: Tue, 25 Apr 2023 16:59:26 -0400 Subject: [PATCH 32/58] Update submodules to the version building with `-std=c++20` --- libraries/cli11/cli11 | 2 +- libraries/eos-vm | 2 +- libraries/libfc/libraries/bn256 | 2 +- tests/abieos | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/libraries/cli11/cli11 b/libraries/cli11/cli11 index 90b0d6720d..f325cc66b4 160000 --- a/libraries/cli11/cli11 +++ b/libraries/cli11/cli11 @@ -1 +1 @@ -Subproject commit 90b0d6720d199be4503c7f8fdbbc31ffb7d58c1a +Subproject commit f325cc66b40cf05e35b457561e0fa1adfedfc973 diff --git a/libraries/eos-vm b/libraries/eos-vm index 1592261e96..329db27d88 160000 --- a/libraries/eos-vm +++ b/libraries/eos-vm @@ -1 +1 @@ -Subproject commit 1592261e96a5ebb4a5f261d7167c0723ca941b9b +Subproject commit 329db27d888dce32c96b4f209cdea45f1d07e5e7 diff --git a/libraries/libfc/libraries/bn256 b/libraries/libfc/libraries/bn256 index 4fe51c043d..63c6c9919c 160000 --- a/libraries/libfc/libraries/bn256 +++ b/libraries/libfc/libraries/bn256 @@ -1 +1 @@ -Subproject commit 4fe51c043d86798fa9bab7015e4d48d1722a9fa5 +Subproject commit 63c6c9919c98a76c23209a321a7d006c4f44ce53 diff --git a/tests/abieos b/tests/abieos index 06fac05851..08145090b6 160000 --- a/tests/abieos +++ b/tests/abieos @@ -1 +1 @@ -Subproject commit 06fac058514378aa491056f8613d4c739ff89fdb +Subproject commit 08145090b6407b91fbab5721b624d2b3001ef84f From 925ad2069019e0c107fc00479a8a7783e978140b Mon Sep 17 00:00:00 2001 From: 766C6164 Date: Tue, 25 Apr 2023 17:39:07 -0400 Subject: [PATCH 33/58] Addressed merge conflict --- plugins/producer_plugin/producer_plugin.cpp | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/plugins/producer_plugin/producer_plugin.cpp b/plugins/producer_plugin/producer_plugin.cpp index 72f086a1af..fb0901a1bc 100644 --- a/plugins/producer_plugin/producer_plugin.cpp +++ b/plugins/producer_plugin/producer_plugin.cpp @@ -1,19 +1,15 @@ #include -#include -#include #include - #include #include #include #include +#include +#include #include #include #include #include -#include -#include - #include #include @@ -376,7 +372,6 @@ class producer_plugin_impl : public std::enable_shared_from_this Date: Tue, 25 Apr 2023 17:41:38 -0400 Subject: [PATCH 34/58] Removed not needed anymore header --- plugins/producer_plugin/producer_plugin.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/plugins/producer_plugin/producer_plugin.cpp b/plugins/producer_plugin/producer_plugin.cpp index fb0901a1bc..5ea0f0e55f 100644 --- a/plugins/producer_plugin/producer_plugin.cpp +++ b/plugins/producer_plugin/producer_plugin.cpp @@ -4,7 +4,6 @@ #include #include #include -#include #include #include #include From 4f431014f11c6d2cd0c224c24a8dd45ab06f4c75 Mon Sep 17 00:00:00 2001 From: Peter Oschwald Date: Tue, 25 Apr 2023 17:08:58 -0500 Subject: [PATCH 35/58] Refactor how cluster is configured to give higher level understanding of the node types being configured. Instead of simply relying on cluster's pnodes and totalnodes specification, add higher level definition of producerNodeCount and validationNodeCount (can be added to as additional node types are supported) that in sum make totalNodes. Also pre-populate the nodeIds that cluster will configure for each node type. Remove support for -s (topo) as currently only supporting default of mesh anyway. --- tests/performance_tests/CMakeLists.txt | 10 ++--- .../performance_test_basic.py | 41 ++++++++++--------- 2 files changed, 27 insertions(+), 24 deletions(-) diff --git a/tests/performance_tests/CMakeLists.txt b/tests/performance_tests/CMakeLists.txt index 1a176019e2..7f6b653512 100644 --- a/tests/performance_tests/CMakeLists.txt +++ b/tests/performance_tests/CMakeLists.txt @@ -15,11 +15,11 @@ endif() add_test(NAME performance_test COMMAND tests/performance_tests/performance_test.py testBpOpMode --max-tps-to-test 50 --test-iteration-min-step 10 --test-iteration-duration-sec 10 --final-iterations-duration-sec 10 --calc-chain-threads lmax overrideBasicTestConfig -v --clean-run --tps-limit-per-generator 25 --chain-state-db-size-mb 200 ${UNSHARE} WORKING_DIRECTORY ${CMAKE_BINARY_DIR}) add_test(NAME performance_test_ex_cpu_trx_spec COMMAND tests/performance_tests/performance_test.py testBpOpMode --max-tps-to-test 50 --test-iteration-min-step 10 --test-iteration-duration-sec 10 --final-iterations-duration-sec 10 overrideBasicTestConfig -v --clean-run --tps-limit-per-generator 25 --chain-state-db-size-mb 200 --account-name "c" --abi-file eosmechanics.abi --wasm-file eosmechanics.wasm --contract-dir unittests/contracts/eosio.mechanics --user-trx-data-file tests/performance_tests/cpuTrxData.json ${UNSHARE} WORKING_DIRECTORY ${CMAKE_BINARY_DIR}) -add_test(NAME performance_test_basic COMMAND tests/performance_tests/performance_test_basic.py -v -p 1 -n 1 --target-tps 20 --tps-limit-per-generator 10 --test-duration-sec 5 --clean-run --chain-state-db-size-mb 200 ${UNSHARE} WORKING_DIRECTORY ${CMAKE_BINARY_DIR}) -add_test(NAME performance_test_basic_ex_transfer_trx_spec COMMAND tests/performance_tests/performance_test_basic.py -v -p 1 -n 1 --target-tps 20 --tps-limit-per-generator 10 --test-duration-sec 5 --clean-run --chain-state-db-size-mb 200 --user-trx-data-file tests/performance_tests/userTrxDataTransfer.json ${UNSHARE} WORKING_DIRECTORY ${CMAKE_BINARY_DIR}) -add_test(NAME performance_test_basic_ex_new_acct_trx_spec COMMAND tests/performance_tests/performance_test_basic.py -v -p 1 -n 1 --target-tps 20 --tps-limit-per-generator 10 --test-duration-sec 5 --clean-run --chain-state-db-size-mb 200 --user-trx-data-file tests/performance_tests/userTrxDataNewAccount.json ${UNSHARE} WORKING_DIRECTORY ${CMAKE_BINARY_DIR}) -add_test(NAME performance_test_basic_ex_cpu_trx_spec COMMAND tests/performance_tests/performance_test_basic.py -v -p 1 -n 1 --target-tps 20 --tps-limit-per-generator 10 --test-duration-sec 5 --clean-run --chain-state-db-size-mb 200 --account-name "c" --abi-file eosmechanics.abi --wasm-file eosmechanics.wasm --contract-dir unittests/contracts/eosio.mechanics --user-trx-data-file tests/performance_tests/cpuTrxData.json ${UNSHARE} WORKING_DIRECTORY ${CMAKE_BINARY_DIR}) -add_test(NAME performance_test_basic_ex_ram_trx_spec COMMAND tests/performance_tests/performance_test_basic.py -v -p 1 -n 1 --target-tps 20 --tps-limit-per-generator 10 --test-duration-sec 5 --clean-run --chain-state-db-size-mb 200 --account-name "r" --abi-file eosmechanics.abi --wasm-file eosmechanics.wasm --contract-dir unittests/contracts/eosio.mechanics --user-trx-data-file tests/performance_tests/ramTrxData.json ${UNSHARE} WORKING_DIRECTORY ${CMAKE_BINARY_DIR}) +add_test(NAME performance_test_basic COMMAND tests/performance_tests/performance_test_basic.py -v --producer-nodes 1 --validation-nodes 1 --target-tps 20 --tps-limit-per-generator 10 --test-duration-sec 5 --clean-run --chain-state-db-size-mb 200 ${UNSHARE} WORKING_DIRECTORY ${CMAKE_BINARY_DIR}) +add_test(NAME performance_test_basic_ex_transfer_trx_spec COMMAND tests/performance_tests/performance_test_basic.py -v --producer-nodes 1 --validation-nodes 1 --target-tps 20 --tps-limit-per-generator 10 --test-duration-sec 5 --clean-run --chain-state-db-size-mb 200 --user-trx-data-file tests/performance_tests/userTrxDataTransfer.json ${UNSHARE} WORKING_DIRECTORY ${CMAKE_BINARY_DIR}) +add_test(NAME performance_test_basic_ex_new_acct_trx_spec COMMAND tests/performance_tests/performance_test_basic.py -v --producer-nodes 1 --validation-nodes 1 --target-tps 20 --tps-limit-per-generator 10 --test-duration-sec 5 --clean-run --chain-state-db-size-mb 200 --user-trx-data-file tests/performance_tests/userTrxDataNewAccount.json ${UNSHARE} WORKING_DIRECTORY ${CMAKE_BINARY_DIR}) +add_test(NAME performance_test_basic_ex_cpu_trx_spec COMMAND tests/performance_tests/performance_test_basic.py -v --producer-nodes 1 --validation-nodes 1 --target-tps 20 --tps-limit-per-generator 10 --test-duration-sec 5 --clean-run --chain-state-db-size-mb 200 --account-name "c" --abi-file eosmechanics.abi --wasm-file eosmechanics.wasm --contract-dir unittests/contracts/eosio.mechanics --user-trx-data-file tests/performance_tests/cpuTrxData.json ${UNSHARE} WORKING_DIRECTORY ${CMAKE_BINARY_DIR}) +add_test(NAME performance_test_basic_ex_ram_trx_spec COMMAND tests/performance_tests/performance_test_basic.py -v --producer-nodes 1 --validation-nodes 1 --target-tps 20 --tps-limit-per-generator 10 --test-duration-sec 5 --clean-run --chain-state-db-size-mb 200 --account-name "r" --abi-file eosmechanics.abi --wasm-file eosmechanics.wasm --contract-dir unittests/contracts/eosio.mechanics --user-trx-data-file tests/performance_tests/ramTrxData.json ${UNSHARE} WORKING_DIRECTORY ${CMAKE_BINARY_DIR}) set_property(TEST performance_test PROPERTY LABELS long_running_tests) set_property(TEST performance_test_ex_cpu_trx_spec PROPERTY LABELS long_running_tests) set_property(TEST performance_test_basic PROPERTY LABELS nonparallelizable_tests) diff --git a/tests/performance_tests/performance_test_basic.py b/tests/performance_tests/performance_test_basic.py index ac1f4822d7..5b5f973345 100755 --- a/tests/performance_tests/performance_test_basic.py +++ b/tests/performance_tests/performance_test_basic.py @@ -76,9 +76,8 @@ class SpecifiedContract: abiFile: str = "eosio.system.abi" account: Account = Account("eosio") - pnodes: int = 1 - totalNodes: int = 2 - topo: str = "mesh" + producerNodeCount: int = 1 + validationNodeCount: int = 1 extraNodeosArgs: ExtraNodeosArgs = field(default_factory=ExtraNodeosArgs) specifiedContract: SpecifiedContract = field(default_factory=SpecifiedContract) genesisPath: Path = Path("tests")/"performance_tests"/"genesis.json" @@ -90,20 +89,27 @@ class SpecifiedContract: nodeosVers: str = "" specificExtraNodeosArgs: dict = field(default_factory=dict) _totalNodes: int = 2 + _pNodes: int = 1 + _producerNodeIds: list = field(default_factory=list) + _validationNodeIds: list = field(default_factory=list) nonProdsEosVmOcEnable: bool = False def __post_init__(self): - self._totalNodes = self.pnodes + 1 if self.totalNodes <= self.pnodes else self.totalNodes + self._totalNodes = self.producerNodeCount + self.validationNodeCount + # Setup Expectations for Producer and Validation Node IDs + # Producer Nodes are index [0, producerNodeCount) and validation nodes (validationNodeCount)/non-producer nodes follow the producer nodes [producerNodeCount, _totalNodes) + self._producerNodeIds = list(range(0, self.producerNodeCount)) + self._validationNodeIds = list(range(self.producerNodeCount, self.producerNodeCount + self.validationNodeCount)) nonProdsSpecificNodeosStr = "" if not self.prodsEnableTraceApi: nonProdsSpecificNodeosStr += "--plugin eosio::trace_api_plugin " if self.nonProdsEosVmOcEnable: nonProdsSpecificNodeosStr += "--eos-vm-oc-enable " - self.specificExtraNodeosArgs.update({f"{node}" : nonProdsSpecificNodeosStr for node in range(self.pnodes, self._totalNodes)}) + self.specificExtraNodeosArgs.update({f"{nodeId}" : nonProdsSpecificNodeosStr for nodeId in self._validationNodeIds}) assert self.nodeosVers != "v1" and self.nodeosVers != "v0", f"nodeos version {Utils.getNodeosVersion().split('.')[0]} is unsupported by performance test" if self.nodeosVers == "v2": self.writeTrx = lambda trxDataFile, blockNum, trx: [trxDataFile.write(f"{trx['trx']['id']},{blockNum},{trx['cpu_usage_us']},{trx['net_usage_words']}\n")] - self.specificExtraNodeosArgs.update({f"{node}" : '--plugin eosio::history_api_plugin --filter-on "*"' for node in range(self.pnodes, self._totalNodes)}) + self.specificExtraNodeosArgs.update({f"{nodeId}" : '--plugin eosio::history_api_plugin --filter-on "*"' for nodeId in self._validationNodeIds}) self.createBlockData = lambda block, blockTransactionTotal, blockNetTotal, blockCpuTotal: log_reader.blockData(blockId=block["payload"]["id"], blockNum=block['payload']['block_num'], transactions=blockTransactionTotal, net=blockNetTotal, cpu=blockCpuTotal, producer=block["payload"]["producer"], status=block["payload"]["confirmed"], _timestamp=block["payload"]["timestamp"]) self.updateTrxDict = lambda blockNum, transaction, trxDict: trxDict.update(dict([(transaction['trx']['id'], log_reader.trxData(blockNum, transaction['cpu_usage_us'],transaction['net_usage_words']))])) else: @@ -164,11 +170,9 @@ def __init__(self, testHelperConfig: TestHelperConfig=TestHelperConfig(), cluste self.blockTrxDataPath = self.blockDataLogDirPath/Path("blockTrxData.txt") self.reportPath = self.loggingConfig.logDirPath/Path("data.json") - # Setup Expectations for Producer and Validation Node IDs - # Producer Nodes are index [0, pnodes) and validation nodes/non-producer nodes [pnodes, _totalNodes) # Use first producer node and first non-producer node - self.producerNodeId = 0 - self.validationNodeId = self.clusterConfig.pnodes + self.producerNodeId = self.clusterConfig._producerNodeIds[0] + self.validationNodeId = self.clusterConfig._validationNodeIds[0] pid = os.getpid() self.nodeosLogDir = Path(self.loggingConfig.logDirPath)/"var"/f"{self.testNamePath}{pid}" self.nodeosLogPath = self.nodeosLogDir/f"node_{str(self.validationNodeId).zfill(2)}"/"stderr.txt" @@ -276,9 +280,8 @@ def waitForEmptyBlocks(self, node, numEmptyToWaitOn): def launchCluster(self): return self.cluster.launch( - pnodes=self.clusterConfig.pnodes, + pnodes=self.clusterConfig._pNodes, totalNodes=self.clusterConfig._totalNodes, - topo=self.clusterConfig.topo, genesisPath=self.clusterConfig.genesisPath, maximumP2pPerHost=self.clusterConfig.maximumP2pPerHost, maximumClients=self.clusterConfig.maximumClients, @@ -343,8 +346,8 @@ def runTpsTest(self) -> PtbTpsTestResult: completedRun = False self.producerNode = self.cluster.getNode(self.producerNodeId) self.connectionPairList = [] - for producer in range(0, self.clusterConfig.pnodes): - self.connectionPairList.append(f"{self.cluster.getNode(producer).host}:{self.cluster.getNodeP2pPort(producer)}") + for producerId in self.clusterConfig._producerNodeIds: + self.connectionPairList.append(f"{self.cluster.getNode(producerId).host}:{self.cluster.getNodeP2pPort(producerId)}") self.validationNode = self.cluster.getNode(self.validationNodeId) self.wallet = self.walletMgr.create('default') self.setupContract() @@ -534,8 +537,7 @@ def runTest(self) -> bool: def setupTestHelperConfig(args) -> TestHelperConfig: return PerformanceTestBasic.TestHelperConfig(killAll=args.clean_run, dontKill=args.leave_running, keepLogs=not args.del_perf_logs, - dumpErrorDetails=args.dump_error_details, delay=args.d, nodesFile=args.nodes_file, - verbose=args.v) + dumpErrorDetails=args.dump_error_details, delay=args.d, verbose=args.v) def setupClusterConfig(args) -> ClusterConfig: @@ -559,7 +561,7 @@ def setupClusterConfig(args) -> ClusterConfig: resourceMonitorPluginArgs=resourceMonitorPluginArgs) SC = PerformanceTestBasic.ClusterConfig.SpecifiedContract specifiedContract=SC(contractDir=args.contract_dir, wasmFile=args.wasm_file, abiFile=args.abi_file, account=Account(args.account_name)) - return PerformanceTestBasic.ClusterConfig(pnodes=args.p, totalNodes=args.n, topo=args.s, genesisPath=args.genesis, + return PerformanceTestBasic.ClusterConfig(producerNodeCount=args.producer_nodes, validationNodeCount=args.validation_nodes, genesisPath=args.genesis, prodsEnableTraceApi=args.prods_enable_trace_api, extraNodeosArgs=extraNodeosArgs, specifiedContract=specifiedContract, loggingLevel=args.cluster_log_lvl, nodeosVers=nodeosVers, nonProdsEosVmOcEnable=args.non_prods_eos_vm_oc_enable) @@ -567,8 +569,7 @@ def setupClusterConfig(args) -> ClusterConfig: class PtbArgumentsHandler(object): @staticmethod def createBaseArgumentParser(suppressHelp: bool=False): - testHelperArgParser=TestHelper.createArgumentParser(includeArgs={"-p","-n","-d","-s","--nodes-file" - ,"--dump-error-details","-v","--leave-running" + testHelperArgParser=TestHelper.createArgumentParser(includeArgs={"-d","--dump-error-details","-v","--leave-running" ,"--clean-run","--unshared"}, suppressHelp=suppressHelp) ptbBaseParser = argparse.ArgumentParser(parents=[testHelperArgParser], add_help=False, formatter_class=argparse.ArgumentDefaultsHelpFormatter) @@ -576,6 +577,8 @@ def createBaseArgumentParser(suppressHelp: bool=False): ptbBaseGrpDescription="Performance Test Basic base configuration items." ptbBaseParserGroup = ptbBaseParser.add_argument_group(title=None if suppressHelp else ptbBaseGrpTitle, description=None if suppressHelp else ptbBaseGrpDescription) + ptbBaseParserGroup.add_argument("--producer-nodes", type=int, help=argparse.SUPPRESS if suppressHelp else "Producing nodes count", default=1) + ptbBaseParserGroup.add_argument("--validation-nodes", type=int, help=argparse.SUPPRESS if suppressHelp else "Validation nodes count", default=1) ptbBaseParserGroup.add_argument("--tps-limit-per-generator", type=int, help=argparse.SUPPRESS if suppressHelp else "Maximum amount of transactions per second a single generator can have.", default=4000) ptbBaseParserGroup.add_argument("--genesis", type=str, help=argparse.SUPPRESS if suppressHelp else "Path to genesis.json", default="tests/performance_tests/genesis.json") ptbBaseParserGroup.add_argument("--num-blocks-to-prune", type=int, help=argparse.SUPPRESS if suppressHelp else ("The number of potentially non-empty blocks, in addition to leading and trailing size 0 blocks, " From e07fd0a0c841d3a0b0c9323a807a047aa6ce4d3d Mon Sep 17 00:00:00 2001 From: Pinelliaw Date: Wed, 26 Apr 2023 01:04:02 +0800 Subject: [PATCH 36/58] Fix not fail on signature checks for dry-run, include the time it takes for checker.all_keys_used() --- libraries/chain/authorization_manager.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/libraries/chain/authorization_manager.cpp b/libraries/chain/authorization_manager.cpp index 7f2b6375dd..4a51e5dc73 100644 --- a/libraries/chain/authorization_manager.cpp +++ b/libraries/chain/authorization_manager.cpp @@ -557,8 +557,8 @@ namespace eosio { namespace chain { } - if( !allow_unused_keys || check_but_dont_fail) { - EOS_ASSERT( checker.all_keys_used(), tx_irrelevant_sig, + if( !allow_unused_keys ) { + EOS_ASSERT( checker.all_keys_used() || check_but_dont_fail, tx_irrelevant_sig, "transaction bears irrelevant signatures from these keys: ${keys}", ("keys", checker.unused_keys()) ); } From 847fec0f13f829087ec9a8ce9537c5145b45fe15 Mon Sep 17 00:00:00 2001 From: greg7mdp Date: Wed, 26 Apr 2023 11:38:25 -0400 Subject: [PATCH 37/58] Update chainbase to tip (-Wfree-nonheap-object suppressed got gcc-12) --- libraries/chainbase | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libraries/chainbase b/libraries/chainbase index 2b5b800a31..c1d30da95c 160000 --- a/libraries/chainbase +++ b/libraries/chainbase @@ -1 +1 @@ -Subproject commit 2b5b800a317a4e3edab14115788cff0907bb5f3d +Subproject commit c1d30da95c9f5e2e80d32732d3063671ff23b123 From ee930e01a7b7b57151a343bbb30ef469fb223d06 Mon Sep 17 00:00:00 2001 From: Peter Oschwald Date: Wed, 26 Apr 2023 09:09:01 -0500 Subject: [PATCH 38/58] Encapsulate validation node configuration. --- .../performance_test_basic.py | 23 +++++++++++++------ 1 file changed, 16 insertions(+), 7 deletions(-) diff --git a/tests/performance_tests/performance_test_basic.py b/tests/performance_tests/performance_test_basic.py index 5b5f973345..ea4182b4c2 100755 --- a/tests/performance_tests/performance_test_basic.py +++ b/tests/performance_tests/performance_test_basic.py @@ -100,16 +100,25 @@ def __post_init__(self): # Producer Nodes are index [0, producerNodeCount) and validation nodes (validationNodeCount)/non-producer nodes follow the producer nodes [producerNodeCount, _totalNodes) self._producerNodeIds = list(range(0, self.producerNodeCount)) self._validationNodeIds = list(range(self.producerNodeCount, self.producerNodeCount + self.validationNodeCount)) - nonProdsSpecificNodeosStr = "" - if not self.prodsEnableTraceApi: - nonProdsSpecificNodeosStr += "--plugin eosio::trace_api_plugin " - if self.nonProdsEosVmOcEnable: - nonProdsSpecificNodeosStr += "--eos-vm-oc-enable " - self.specificExtraNodeosArgs.update({f"{nodeId}" : nonProdsSpecificNodeosStr for nodeId in self._validationNodeIds}) + + def configureValidationNodes(): + validationNodeSpecificNodeosStr = "" + if self.nodeosVers == "v2": + validationNodeSpecificNodeosStr += '--plugin eosio::history_api_plugin --filter-on "*" ' + else: + #If prodsEnableTraceApi, then Cluster configures all nodes with trace_api_plugin so no need to duplicate here + if not self.prodsEnableTraceApi: + validationNodeSpecificNodeosStr += "--plugin eosio::trace_api_plugin " + if self.nonProdsEosVmOcEnable: + validationNodeSpecificNodeosStr += "--eos-vm-oc-enable " + if validationNodeSpecificNodeosStr: + self.specificExtraNodeosArgs.update({f"{nodeId}" : validationNodeSpecificNodeosStr for nodeId in self._validationNodeIds}) + + configureValidationNodes() + assert self.nodeosVers != "v1" and self.nodeosVers != "v0", f"nodeos version {Utils.getNodeosVersion().split('.')[0]} is unsupported by performance test" if self.nodeosVers == "v2": self.writeTrx = lambda trxDataFile, blockNum, trx: [trxDataFile.write(f"{trx['trx']['id']},{blockNum},{trx['cpu_usage_us']},{trx['net_usage_words']}\n")] - self.specificExtraNodeosArgs.update({f"{nodeId}" : '--plugin eosio::history_api_plugin --filter-on "*"' for nodeId in self._validationNodeIds}) self.createBlockData = lambda block, blockTransactionTotal, blockNetTotal, blockCpuTotal: log_reader.blockData(blockId=block["payload"]["id"], blockNum=block['payload']['block_num'], transactions=blockTransactionTotal, net=blockNetTotal, cpu=blockCpuTotal, producer=block["payload"]["producer"], status=block["payload"]["confirmed"], _timestamp=block["payload"]["timestamp"]) self.updateTrxDict = lambda blockNum, transaction, trxDict: trxDict.update(dict([(transaction['trx']['id'], log_reader.trxData(blockNum, transaction['cpu_usage_us'],transaction['net_usage_words']))])) else: From 22a5aa1f084a464416db4118540afa7e59e017bf Mon Sep 17 00:00:00 2001 From: Peter Oschwald Date: Wed, 26 Apr 2023 11:52:56 -0500 Subject: [PATCH 39/58] Add initial support for configuring and running an API node. --- .../performance_test_basic.py | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/tests/performance_tests/performance_test_basic.py b/tests/performance_tests/performance_test_basic.py index ea4182b4c2..f586a91911 100755 --- a/tests/performance_tests/performance_test_basic.py +++ b/tests/performance_tests/performance_test_basic.py @@ -78,6 +78,7 @@ class SpecifiedContract: producerNodeCount: int = 1 validationNodeCount: int = 1 + apiNodeCount: int = 0 extraNodeosArgs: ExtraNodeosArgs = field(default_factory=ExtraNodeosArgs) specifiedContract: SpecifiedContract = field(default_factory=SpecifiedContract) genesisPath: Path = Path("tests")/"performance_tests"/"genesis.json" @@ -92,14 +93,16 @@ class SpecifiedContract: _pNodes: int = 1 _producerNodeIds: list = field(default_factory=list) _validationNodeIds: list = field(default_factory=list) + _apiNodeIds: list = field(default_factory=list) nonProdsEosVmOcEnable: bool = False def __post_init__(self): - self._totalNodes = self.producerNodeCount + self.validationNodeCount + self._totalNodes = self.producerNodeCount + self.validationNodeCount + self.apiNodeCount # Setup Expectations for Producer and Validation Node IDs - # Producer Nodes are index [0, producerNodeCount) and validation nodes (validationNodeCount)/non-producer nodes follow the producer nodes [producerNodeCount, _totalNodes) + # Producer Nodes are index [0, producerNodeCount) and non-producer nodes (validationNodeCount, apiNodeCount) nodes follow the producer nodes [producerNodeCount, _totalNodes) self._producerNodeIds = list(range(0, self.producerNodeCount)) self._validationNodeIds = list(range(self.producerNodeCount, self.producerNodeCount + self.validationNodeCount)) + self._apiNodeIds = list(range(self.producerNodeCount + self.validationNodeCount, self.producerNodeCount + self.validationNodeCount + self.validationNodeCount)) def configureValidationNodes(): validationNodeSpecificNodeosStr = "" @@ -114,7 +117,14 @@ def configureValidationNodes(): if validationNodeSpecificNodeosStr: self.specificExtraNodeosArgs.update({f"{nodeId}" : validationNodeSpecificNodeosStr for nodeId in self._validationNodeIds}) + def configureApiNodes(): + apiNodeSpecificNodeosStr = "" + apiNodeSpecificNodeosStr += "--plugin eosio::chain_api_plugin " + if apiNodeSpecificNodeosStr: + self.specificExtraNodeosArgs.update({f"{nodeId}" : apiNodeSpecificNodeosStr for nodeId in self._apiNodeIds}) + configureValidationNodes() + configureApiNodes() assert self.nodeosVers != "v1" and self.nodeosVers != "v0", f"nodeos version {Utils.getNodeosVersion().split('.')[0]} is unsupported by performance test" if self.nodeosVers == "v2": @@ -570,8 +580,8 @@ def setupClusterConfig(args) -> ClusterConfig: resourceMonitorPluginArgs=resourceMonitorPluginArgs) SC = PerformanceTestBasic.ClusterConfig.SpecifiedContract specifiedContract=SC(contractDir=args.contract_dir, wasmFile=args.wasm_file, abiFile=args.abi_file, account=Account(args.account_name)) - return PerformanceTestBasic.ClusterConfig(producerNodeCount=args.producer_nodes, validationNodeCount=args.validation_nodes, genesisPath=args.genesis, - prodsEnableTraceApi=args.prods_enable_trace_api, extraNodeosArgs=extraNodeosArgs, + return PerformanceTestBasic.ClusterConfig(producerNodeCount=args.producer_nodes, validationNodeCount=args.validation_nodes, apiNodeCount=args.api_nodes, + genesisPath=args.genesis, prodsEnableTraceApi=args.prods_enable_trace_api, extraNodeosArgs=extraNodeosArgs, specifiedContract=specifiedContract, loggingLevel=args.cluster_log_lvl, nodeosVers=nodeosVers, nonProdsEosVmOcEnable=args.non_prods_eos_vm_oc_enable) @@ -588,6 +598,7 @@ def createBaseArgumentParser(suppressHelp: bool=False): ptbBaseParserGroup.add_argument("--producer-nodes", type=int, help=argparse.SUPPRESS if suppressHelp else "Producing nodes count", default=1) ptbBaseParserGroup.add_argument("--validation-nodes", type=int, help=argparse.SUPPRESS if suppressHelp else "Validation nodes count", default=1) + ptbBaseParserGroup.add_argument("--api-nodes", type=int, help=argparse.SUPPRESS if suppressHelp else "API nodes count", default=0) ptbBaseParserGroup.add_argument("--tps-limit-per-generator", type=int, help=argparse.SUPPRESS if suppressHelp else "Maximum amount of transactions per second a single generator can have.", default=4000) ptbBaseParserGroup.add_argument("--genesis", type=str, help=argparse.SUPPRESS if suppressHelp else "Path to genesis.json", default="tests/performance_tests/genesis.json") ptbBaseParserGroup.add_argument("--num-blocks-to-prune", type=int, help=argparse.SUPPRESS if suppressHelp else ("The number of potentially non-empty blocks, in addition to leading and trailing size 0 blocks, " From 18fd82fd7e5dee8075ec932f30143a635de77902 Mon Sep 17 00:00:00 2001 From: Peter Oschwald Date: Wed, 26 Apr 2023 12:13:43 -0500 Subject: [PATCH 40/58] Update README documentation. --- tests/performance_tests/README.md | 40 +++++++++++++++++++------------ 1 file changed, 25 insertions(+), 15 deletions(-) diff --git a/tests/performance_tests/README.md b/tests/performance_tests/README.md index bffcaf2a0e..4d20121e0d 100644 --- a/tests/performance_tests/README.md +++ b/tests/performance_tests/README.md @@ -440,9 +440,9 @@ Advanced Configuration Options: ``` usage: performance_test.py testBpOpMode overrideBasicTestConfig - [-h] [-p P] [-n N] [-d D] [--nodes-file NODES_FILE] [-s {mesh}] - [--dump-error-details] [-v] [--leave-running] [--clean-run] - [--unshared] [--tps-limit-per-generator TPS_LIMIT_PER_GENERATOR] + [-h] [-d D] [--dump-error-details] [-v] [--leave-running] [--clean-run] [--unshared] + [--producer-nodes PRODUCER_NODES] [--validation-nodes VALIDATION_NODES] [--api-nodes API_NODES] + [--tps-limit-per-generator TPS_LIMIT_PER_GENERATOR] [--genesis GENESIS] [--num-blocks-to-prune NUM_BLOCKS_TO_PRUNE] [--signature-cpu-billable-pct SIGNATURE_CPU_BILLABLE_PCT] [--chain-threads CHAIN_THREADS] @@ -452,6 +452,7 @@ usage: performance_test.py testBpOpMode overrideBasicTestConfig [--disable-subjective-billing DISABLE_SUBJECTIVE_BILLING] [--cpu-effort-percent CPU_EFFORT_PERCENT] [--producer-threads PRODUCER_THREADS] + [--http-max-in-flight-requests HTTP_MAX_IN_FLIGHT_REQUESTS] [--http-max-response-time-ms HTTP_MAX_RESPONSE_TIME_MS] [--http-max-bytes-in-flight-mb HTTP_MAX_BYTES_IN_FLIGHT_MB] [--del-perf-logs] [--del-report] [--quiet] [--prods-enable-trace-api] @@ -479,12 +480,7 @@ optional arguments: Test Helper Arguments: Test Helper configuration items used to configure and spin up the regression test framework and blockchain environment. - -p P producing nodes count - -n N total nodes -d D delay between nodes startup - --nodes-file NODES_FILE - File containing nodes info in JSON format. - -s {mesh} topology --dump-error-details Upon error print etc/eosio/node_*/config.ini and /node_*/stderr.log to stdout -v verbose logging --leave-running Leave cluster running after test finishes @@ -494,6 +490,12 @@ Test Helper Arguments: Performance Test Basic Base: Performance Test Basic base configuration items. + --producer-nodes PRODUCER_NODES + Producing nodes count + --validation-nodes VALIDATION_NODES + Validation nodes count + --api-nodes API_NODES + API nodes count --tps-limit-per-generator TPS_LIMIT_PER_GENERATOR Maximum amount of transactions per second a single generator can have. --genesis GENESIS Path to genesis.json @@ -522,6 +524,8 @@ Performance Test Basic Base: Percentage of cpu block production time used to produce block. Whole number percentages, e.g. 80 for 80% --producer-threads PRODUCER_THREADS Number of worker threads in producer thread pool + --http-max-in-flight-requests HTTP_MAX_IN_FLIGHT_REQUESTS + Maximum number of requests http_plugin should use for processing http requests. 429 error response when exceeded. -1 for unlimited --http-max-response-time-ms HTTP_MAX_RESPONSE_TIME_MS Maximum time for processing a request, -1 for unlimited --http-max-bytes-in-flight-mb HTTP_MAX_BYTES_IN_FLIGHT_MB @@ -576,10 +580,12 @@ The following scripts are typically used by the Performance Harness main script Usage ``` - usage: performance_test_basic.py [-h] [-p P] [-n N] [-d D] - [--nodes-file NODES_FILE] [-s {mesh}] + usage: performance_test_basic.py [-h] [-d D] [--dump-error-details] [-v] [--leave-running] [--clean-run] [--unshared] + [--producer-nodes PRODUCER_NODES] + [--validation-nodes VALIDATION_NODES] + [--api-nodes API_NODES] [--tps-limit-per-generator TPS_LIMIT_PER_GENERATOR] [--genesis GENESIS] [--num-blocks-to-prune NUM_BLOCKS_TO_PRUNE] @@ -591,6 +597,7 @@ The following scripts are typically used by the Performance Harness main script [--disable-subjective-billing DISABLE_SUBJECTIVE_BILLING] [--cpu-effort-percent CPU_EFFORT_PERCENT] [--producer-threads PRODUCER_THREADS] + [--http-max-in-flight-requests HTTP_MAX_IN_FLIGHT_REQUESTS] [--http-max-response-time-ms HTTP_MAX_RESPONSE_TIME_MS] [--http-max-bytes-in-flight-mb HTTP_MAX_BYTES_IN_FLIGHT_MB] [--del-perf-logs] [--del-report] [--quiet] @@ -624,12 +631,7 @@ optional arguments: Test Helper Arguments: Test Helper configuration items used to configure and spin up the regression test framework and blockchain environment. - -p P producing nodes count (default: 1) - -n N total nodes (default: 0) -d D delay between nodes startup (default: 1) - --nodes-file NODES_FILE - File containing nodes info in JSON format. (default: None) - -s {mesh} topology (default: mesh) --dump-error-details Upon error print etc/eosio/node_*/config.ini and /node_*/stderr.log to stdout (default: False) -v verbose logging (default: False) --leave-running Leave cluster running after test finishes (default: False) @@ -639,6 +641,12 @@ Test Helper Arguments: Performance Test Basic Base: Performance Test Basic base configuration items. + --producer-nodes PRODUCER_NODES + Producing nodes count (default: 1) + --validation-nodes VALIDATION_NODES + Validation nodes count (default: 1) + --api-nodes API_NODES + API nodes count (default: 0) --tps-limit-per-generator TPS_LIMIT_PER_GENERATOR Maximum amount of transactions per second a single generator can have. (default: 4000) --genesis GENESIS Path to genesis.json (default: tests/performance_tests/genesis.json) @@ -666,6 +674,8 @@ Performance Test Basic Base: Percentage of cpu block production time used to produce block. Whole number percentages, e.g. 80 for 80% (default: 100) --producer-threads PRODUCER_THREADS Number of worker threads in producer thread pool (default: 2) + --http-max-in-flight-requests HTTP_MAX_IN_FLIGHT_REQUESTS + Maximum number of requests http_plugin should use for processing http requests. 429 error response when exceeded. -1 for unlimited (default: -1) --http-max-response-time-ms HTTP_MAX_RESPONSE_TIME_MS Maximum time for processing a request, -1 for unlimited (default: -1) --http-max-bytes-in-flight-mb HTTP_MAX_BYTES_IN_FLIGHT_MB From c410c7c943665e322419c54268e9ef6acf5faf2b Mon Sep 17 00:00:00 2001 From: Kevin Heifner Date: Thu, 27 Apr 2023 09:07:31 -0500 Subject: [PATCH 41/58] GH-641 Refactor libfc tests into one executable --- libraries/libfc/test/CMakeLists.txt | 23 ++++++++++--------- libraries/libfc/test/crypto/test_blake2.cpp | 3 +-- .../libfc/test/crypto/test_cypher_suites.cpp | 3 +-- .../libfc/test/crypto/test_hash_functions.cpp | 3 +-- .../libfc/test/crypto/test_k1_recover.cpp | 3 +-- .../test/crypto/test_modular_arithmetic.cpp | 3 +-- libraries/libfc/test/crypto/test_utils.hpp | 4 ++-- libraries/libfc/test/crypto/test_webauthn.cpp | 3 +-- libraries/libfc/test/io/test_cfile.cpp | 3 +-- libraries/libfc/test/io/test_json.cpp | 3 +-- .../libfc/test/io/test_tracked_storage.cpp | 3 +-- .../test/network/test_message_buffer.cpp | 3 +-- .../test/scoped_exit/test_scoped_exit.cpp | 3 +-- .../static_variant/test_static_variant.cpp | 3 +-- libraries/libfc/test/test_base64.cpp | 2 +- libraries/libfc/test/variant/test_variant.cpp | 3 +-- .../test_variant_estimated_size.cpp | 3 +-- 17 files changed, 29 insertions(+), 42 deletions(-) diff --git a/libraries/libfc/test/CMakeLists.txt b/libraries/libfc/test/CMakeLists.txt index b71efc19df..4dc210cde4 100644 --- a/libraries/libfc/test/CMakeLists.txt +++ b/libraries/libfc/test/CMakeLists.txt @@ -1,12 +1,13 @@ -add_subdirectory( crypto ) -add_subdirectory( io ) -add_subdirectory( network ) -add_subdirectory( scoped_exit ) -add_subdirectory( static_variant ) -add_subdirectory( variant ) -add_subdirectory( variant_estimated_size ) +file(GLOB UNIT_TESTS + "*.cpp" + "crypto/*.cpp" + "io/*.cpp" + "network/*.cpp" + "scoped_exit/*.cpp" + "static_variant/*.cpp" + "variant/*.cpp" + "variant_estimated_size/*.cpp") +add_executable( test_fc ${UNIT_TESTS} ) +target_link_libraries( test_fc fc ) -add_executable( test_base64 test_base64.cpp ) -target_link_libraries( test_base64 fc ) - -add_test(NAME test_base64 COMMAND test_base64 WORKING_DIRECTORY ${CMAKE_BINARY_DIR}) +add_test(NAME test_fc COMMAND test_fc WORKING_DIRECTORY ${CMAKE_BINARY_DIR}) diff --git a/libraries/libfc/test/crypto/test_blake2.cpp b/libraries/libfc/test/crypto/test_blake2.cpp index bd0284f64e..ce55b95cc3 100644 --- a/libraries/libfc/test/crypto/test_blake2.cpp +++ b/libraries/libfc/test/crypto/test_blake2.cpp @@ -1,5 +1,4 @@ -#define BOOST_TEST_MODULE blake2 -#include +#include #include #include diff --git a/libraries/libfc/test/crypto/test_cypher_suites.cpp b/libraries/libfc/test/crypto/test_cypher_suites.cpp index 2a421a822b..9189cd6385 100644 --- a/libraries/libfc/test/crypto/test_cypher_suites.cpp +++ b/libraries/libfc/test/crypto/test_cypher_suites.cpp @@ -1,5 +1,4 @@ -#define BOOST_TEST_MODULE cypher_suites -#include +#include #include #include diff --git a/libraries/libfc/test/crypto/test_hash_functions.cpp b/libraries/libfc/test/crypto/test_hash_functions.cpp index 7612e497d0..a77b927256 100644 --- a/libraries/libfc/test/crypto/test_hash_functions.cpp +++ b/libraries/libfc/test/crypto/test_hash_functions.cpp @@ -1,5 +1,4 @@ -#define BOOST_TEST_MODULE hash_functions -#include +#include #include #include diff --git a/libraries/libfc/test/crypto/test_k1_recover.cpp b/libraries/libfc/test/crypto/test_k1_recover.cpp index a93311ef49..cc16451cb4 100644 --- a/libraries/libfc/test/crypto/test_k1_recover.cpp +++ b/libraries/libfc/test/crypto/test_k1_recover.cpp @@ -1,5 +1,4 @@ -#define BOOST_TEST_MODULE blake2 -#include +#include #include #include diff --git a/libraries/libfc/test/crypto/test_modular_arithmetic.cpp b/libraries/libfc/test/crypto/test_modular_arithmetic.cpp index 0e029dd1e7..f0bf5bd4f9 100644 --- a/libraries/libfc/test/crypto/test_modular_arithmetic.cpp +++ b/libraries/libfc/test/crypto/test_modular_arithmetic.cpp @@ -1,5 +1,4 @@ -#define BOOST_TEST_MODULE modular_arithmetic -#include +#include #include #include diff --git a/libraries/libfc/test/crypto/test_utils.hpp b/libraries/libfc/test/crypto/test_utils.hpp index f70dbc84c7..25cdafbf21 100644 --- a/libraries/libfc/test/crypto/test_utils.hpp +++ b/libraries/libfc/test/crypto/test_utils.hpp @@ -1,9 +1,9 @@ -uint32_t to_uint32(const std::string& s) { +inline uint32_t to_uint32(const std::string& s) { size_t l = s.size(); return (uint32_t)std::stoul(s.c_str(), &l, 16); } -bytes to_bytes(const std::string& source) { +inline bytes to_bytes(const std::string& source) { BOOST_REQUIRE(!(source.length() % 2)); bytes output(source.length()/2); fc::from_hex(source, output.data(), output.size()); diff --git a/libraries/libfc/test/crypto/test_webauthn.cpp b/libraries/libfc/test/crypto/test_webauthn.cpp index ba5a5bd0ff..600d543f5c 100644 --- a/libraries/libfc/test/crypto/test_webauthn.cpp +++ b/libraries/libfc/test/crypto/test_webauthn.cpp @@ -1,5 +1,4 @@ -#define BOOST_TEST_MODULE webauthn_test_mod -#include +#include #include #include diff --git a/libraries/libfc/test/io/test_cfile.cpp b/libraries/libfc/test/io/test_cfile.cpp index 4dc329ae20..58546bff46 100644 --- a/libraries/libfc/test/io/test_cfile.cpp +++ b/libraries/libfc/test/io/test_cfile.cpp @@ -1,5 +1,4 @@ -#define BOOST_TEST_MODULE io -#include +#include #include diff --git a/libraries/libfc/test/io/test_json.cpp b/libraries/libfc/test/io/test_json.cpp index 4794ec7ed0..372e909980 100644 --- a/libraries/libfc/test/io/test_json.cpp +++ b/libraries/libfc/test/io/test_json.cpp @@ -1,5 +1,4 @@ -#define BOOST_TEST_MODULE io_json -#include +#include #include #include diff --git a/libraries/libfc/test/io/test_tracked_storage.cpp b/libraries/libfc/test/io/test_tracked_storage.cpp index 2c478d687f..e9d4fafea0 100644 --- a/libraries/libfc/test/io/test_tracked_storage.cpp +++ b/libraries/libfc/test/io/test_tracked_storage.cpp @@ -1,5 +1,4 @@ -#define BOOST_TEST_MODULE tracked_storage -#include +#include #include #include #include diff --git a/libraries/libfc/test/network/test_message_buffer.cpp b/libraries/libfc/test/network/test_message_buffer.cpp index de872f43cb..c1578d9f65 100644 --- a/libraries/libfc/test/network/test_message_buffer.cpp +++ b/libraries/libfc/test/network/test_message_buffer.cpp @@ -2,8 +2,7 @@ #include -#define BOOST_TEST_MODULE message_buffer -#include +#include namespace { size_t mb_size(boost::asio::mutable_buffer& mb) { diff --git a/libraries/libfc/test/scoped_exit/test_scoped_exit.cpp b/libraries/libfc/test/scoped_exit/test_scoped_exit.cpp index d143cae4cf..32356c00f3 100644 --- a/libraries/libfc/test/scoped_exit/test_scoped_exit.cpp +++ b/libraries/libfc/test/scoped_exit/test_scoped_exit.cpp @@ -1,7 +1,6 @@ #include -#define BOOST_TEST_MODULE scoped_exit -#include +#include using namespace fc; diff --git a/libraries/libfc/test/static_variant/test_static_variant.cpp b/libraries/libfc/test/static_variant/test_static_variant.cpp index 77cbaa7dd9..9ebc9da07c 100644 --- a/libraries/libfc/test/static_variant/test_static_variant.cpp +++ b/libraries/libfc/test/static_variant/test_static_variant.cpp @@ -1,5 +1,4 @@ -#define BOOST_TEST_MODULE static_variant -#include +#include #include #include #include diff --git a/libraries/libfc/test/test_base64.cpp b/libraries/libfc/test/test_base64.cpp index f1a7cc2803..64e977dc81 100644 --- a/libraries/libfc/test/test_base64.cpp +++ b/libraries/libfc/test/test_base64.cpp @@ -1,4 +1,4 @@ -#define BOOST_TEST_MODULE base64 +#define BOOST_TEST_MODULE libfc #include #include diff --git a/libraries/libfc/test/variant/test_variant.cpp b/libraries/libfc/test/variant/test_variant.cpp index 252a49f482..9d3df2df07 100644 --- a/libraries/libfc/test/variant/test_variant.cpp +++ b/libraries/libfc/test/variant/test_variant.cpp @@ -1,5 +1,4 @@ -#define BOOST_TEST_MODULE variant -#include +#include #include #include diff --git a/libraries/libfc/test/variant_estimated_size/test_variant_estimated_size.cpp b/libraries/libfc/test/variant_estimated_size/test_variant_estimated_size.cpp index 9027f73dd5..9e7e7b26a0 100644 --- a/libraries/libfc/test/variant_estimated_size/test_variant_estimated_size.cpp +++ b/libraries/libfc/test/variant_estimated_size/test_variant_estimated_size.cpp @@ -1,5 +1,4 @@ -#define BOOST_TEST_MODULE variant -#include +#include #include #include From 72d47e0a217b86b92d1f9d9a71e6de24bfac8657 Mon Sep 17 00:00:00 2001 From: Kevin Heifner Date: Thu, 27 Apr 2023 09:13:39 -0500 Subject: [PATCH 42/58] GH-641 Remove unneeded CMakeLists.txt files --- libraries/libfc/test/crypto/CMakeLists.txt | 24 ------------------- libraries/libfc/test/io/CMakeLists.txt | 14 ----------- libraries/libfc/test/network/CMakeLists.txt | 4 ---- .../libfc/test/scoped_exit/CMakeLists.txt | 4 ---- .../libfc/test/static_variant/CMakeLists.txt | 4 ---- libraries/libfc/test/variant/CMakeLists.txt | 4 ---- .../variant_estimated_size/CMakeLists.txt | 4 ---- 7 files changed, 58 deletions(-) delete mode 100644 libraries/libfc/test/crypto/CMakeLists.txt delete mode 100644 libraries/libfc/test/io/CMakeLists.txt delete mode 100644 libraries/libfc/test/network/CMakeLists.txt delete mode 100644 libraries/libfc/test/scoped_exit/CMakeLists.txt delete mode 100644 libraries/libfc/test/static_variant/CMakeLists.txt delete mode 100644 libraries/libfc/test/variant/CMakeLists.txt delete mode 100644 libraries/libfc/test/variant_estimated_size/CMakeLists.txt diff --git a/libraries/libfc/test/crypto/CMakeLists.txt b/libraries/libfc/test/crypto/CMakeLists.txt deleted file mode 100644 index 14fb28bdfd..0000000000 --- a/libraries/libfc/test/crypto/CMakeLists.txt +++ /dev/null @@ -1,24 +0,0 @@ -add_executable( test_cypher_suites test_cypher_suites.cpp ) -target_link_libraries( test_cypher_suites fc ) - -add_executable( test_webauthn test_webauthn.cpp ) -target_link_libraries( test_webauthn fc ) - -add_executable( test_hash_functions test_hash_functions.cpp ) -target_link_libraries( test_hash_functions fc ) - -add_executable( test_blake2 test_blake2.cpp ) -target_link_libraries( test_blake2 fc ) - -add_executable( test_modular_arithmetic test_modular_arithmetic.cpp ) -target_link_libraries( test_modular_arithmetic fc ) - -add_executable( test_k1_recover test_k1_recover.cpp ) -target_link_libraries( test_k1_recover fc ) - -add_test(NAME test_cypher_suites COMMAND test_cypher_suites WORKING_DIRECTORY ${CMAKE_BINARY_DIR}) -add_test(NAME test_webauthn COMMAND test_webauthn WORKING_DIRECTORY ${CMAKE_BINARY_DIR}) -add_test(NAME test_hash_functions COMMAND test_hash_functions WORKING_DIRECTORY ${CMAKE_BINARY_DIR}) -add_test(NAME test_blake2 COMMAND test_blake2 WORKING_DIRECTORY ${CMAKE_BINARY_DIR}) -add_test(NAME test_modular_arithmetic COMMAND test_modular_arithmetic WORKING_DIRECTORY ${CMAKE_BINARY_DIR}) -add_test(NAME test_k1_recover COMMAND test_k1_recover WORKING_DIRECTORY ${CMAKE_BINARY_DIR}) \ No newline at end of file diff --git a/libraries/libfc/test/io/CMakeLists.txt b/libraries/libfc/test/io/CMakeLists.txt deleted file mode 100644 index 6041a6686c..0000000000 --- a/libraries/libfc/test/io/CMakeLists.txt +++ /dev/null @@ -1,14 +0,0 @@ -add_executable( test_cfile test_cfile.cpp ) -target_link_libraries( test_cfile fc ) - -add_executable( test_json test_json.cpp ) -target_link_libraries( test_json fc ) - -add_executable( test_tracked_storage test_tracked_storage.cpp ) -target_link_libraries( test_tracked_storage fc ) - -add_test(NAME test_cfile COMMAND test_cfile WORKING_DIRECTORY ${CMAKE_BINARY_DIR}) -add_test(NAME test_json COMMAND test_json WORKING_DIRECTORY ${CMAKE_BINARY_DIR}) -add_test(NAME test_tracked_storage COMMAND test_tracked_storage WORKING_DIRECTORY ${CMAKE_BINARY_DIR}) - - diff --git a/libraries/libfc/test/network/CMakeLists.txt b/libraries/libfc/test/network/CMakeLists.txt deleted file mode 100644 index 31c2d229db..0000000000 --- a/libraries/libfc/test/network/CMakeLists.txt +++ /dev/null @@ -1,4 +0,0 @@ -add_executable( test_message_buffer test_message_buffer.cpp) -target_link_libraries( test_message_buffer fc ) - -add_test(NAME test_message_buffer COMMAND test_message_buffer WORKING_DIRECTORY ${CMAKE_BINARY_DIR}) diff --git a/libraries/libfc/test/scoped_exit/CMakeLists.txt b/libraries/libfc/test/scoped_exit/CMakeLists.txt deleted file mode 100644 index bd627de3fa..0000000000 --- a/libraries/libfc/test/scoped_exit/CMakeLists.txt +++ /dev/null @@ -1,4 +0,0 @@ -add_executable( test_scoped_exit test_scoped_exit.cpp ) -target_link_libraries( test_scoped_exit fc ) - -add_test(NAME test_scope_exit COMMAND test_scoped_exit WORKING_DIRECTORY ${CMAKE_BINARY_DIR}) diff --git a/libraries/libfc/test/static_variant/CMakeLists.txt b/libraries/libfc/test/static_variant/CMakeLists.txt deleted file mode 100644 index f07097c6c9..0000000000 --- a/libraries/libfc/test/static_variant/CMakeLists.txt +++ /dev/null @@ -1,4 +0,0 @@ -add_executable( test_static_variant test_static_variant.cpp) -target_link_libraries( test_static_variant fc ) - -add_test(NAME test_static_variant COMMAND test_static_variant WORKING_DIRECTORY ${CMAKE_BINARY_DIR}) diff --git a/libraries/libfc/test/variant/CMakeLists.txt b/libraries/libfc/test/variant/CMakeLists.txt deleted file mode 100644 index 4114d1a4c1..0000000000 --- a/libraries/libfc/test/variant/CMakeLists.txt +++ /dev/null @@ -1,4 +0,0 @@ -add_executable( test_variant test_variant.cpp ) -target_link_libraries( test_variant fc ) - -add_test(NAME test_variant COMMAND test_variant WORKING_DIRECTORY ${CMAKE_BINARY_DIR}) diff --git a/libraries/libfc/test/variant_estimated_size/CMakeLists.txt b/libraries/libfc/test/variant_estimated_size/CMakeLists.txt deleted file mode 100644 index f7c46f4b1e..0000000000 --- a/libraries/libfc/test/variant_estimated_size/CMakeLists.txt +++ /dev/null @@ -1,4 +0,0 @@ -add_executable( variant_estimated_size test_variant_estimated_size.cpp ) -target_link_libraries( variant_estimated_size fc ) - -add_test(NAME test_variant_estimated_size COMMAND variant_estimated_size) From 8db6fd8654a159ab6ba0f12647472362aafa1223 Mon Sep 17 00:00:00 2001 From: Kevin Heifner Date: Thu, 27 Apr 2023 11:32:09 -0500 Subject: [PATCH 43/58] GH-641 Explicitly list source files to avoid cmake issues with GLOB --- libraries/libfc/test/CMakeLists.txt | 27 +++++++++++++++++---------- 1 file changed, 17 insertions(+), 10 deletions(-) diff --git a/libraries/libfc/test/CMakeLists.txt b/libraries/libfc/test/CMakeLists.txt index 4dc210cde4..bacd693aef 100644 --- a/libraries/libfc/test/CMakeLists.txt +++ b/libraries/libfc/test/CMakeLists.txt @@ -1,13 +1,20 @@ -file(GLOB UNIT_TESTS - "*.cpp" - "crypto/*.cpp" - "io/*.cpp" - "network/*.cpp" - "scoped_exit/*.cpp" - "static_variant/*.cpp" - "variant/*.cpp" - "variant_estimated_size/*.cpp") -add_executable( test_fc ${UNIT_TESTS} ) +add_executable( test_fc + crypto/test_blake2.cpp + crypto/test_cypher_suites.cpp + crypto/test_hash_functions.cpp + crypto/test_k1_recover.cpp + crypto/test_modular_arithmetic.cpp + crypto/test_webauthn.cpp + io/test_cfile.cpp + io/test_json.cpp + io/test_tracked_storage.cpp + network/test_message_buffer.cpp + scoped_exit/test_scoped_exit.cpp + static_variant/test_static_variant.cpp + variant/test_variant.cpp + variant_estimated_size/test_variant_estimated_size.cpp + test_base64.cpp + ) target_link_libraries( test_fc fc ) add_test(NAME test_fc COMMAND test_fc WORKING_DIRECTORY ${CMAKE_BINARY_DIR}) From 22dd72753ecb530266c059759b08c35c26862e72 Mon Sep 17 00:00:00 2001 From: Huang-Ming Huang Date: Thu, 27 Apr 2023 11:33:50 -0500 Subject: [PATCH 44/58] address PR comments --- .../include/eosio/net_plugin/net_plugin.hpp | 4 +-- .../eosio/producer_plugin/producer_plugin.hpp | 31 +++++++++---------- plugins/prometheus_plugin/metrics.hpp | 12 +++++-- 3 files changed, 26 insertions(+), 21 deletions(-) diff --git a/plugins/net_plugin/include/eosio/net_plugin/net_plugin.hpp b/plugins/net_plugin/include/eosio/net_plugin/net_plugin.hpp index 488b9bcf6a..8eafaba2e5 100644 --- a/plugins/net_plugin/include/eosio/net_plugin/net_plugin.hpp +++ b/plugins/net_plugin/include/eosio/net_plugin/net_plugin.hpp @@ -35,8 +35,8 @@ namespace eosio { vector connections()const; struct p2p_connections_metrics { - std::size_t num_peers; - std::size_t num_clients; + std::size_t num_peers = 0; + std::size_t num_clients = 0; }; void register_update_p2p_connection_metrics(std::function&&); diff --git a/plugins/producer_plugin/include/eosio/producer_plugin/producer_plugin.hpp b/plugins/producer_plugin/include/eosio/producer_plugin/producer_plugin.hpp index be16e1902e..edfafed8a8 100644 --- a/plugins/producer_plugin/include/eosio/producer_plugin/producer_plugin.hpp +++ b/plugins/producer_plugin/include/eosio/producer_plugin/producer_plugin.hpp @@ -178,26 +178,25 @@ class producer_plugin : public appbase::plugin { static void set_test_mode(bool m) { test_mode_ = m; } struct produced_block_metrics { - std::size_t unapplied_transactions_total; - std::size_t blacklisted_transactions_total; - std::size_t subjective_bill_account_size_total; - std::size_t scheduled_trxs_total; - std::size_t trxs_produced_total; - uint64_t cpu_usage_us; - uint64_t net_usage_us; + std::size_t unapplied_transactions_total = 0; + std::size_t blacklisted_transactions_total = 0; + std::size_t subjective_bill_account_size_total = 0; + std::size_t scheduled_trxs_total = 0; + std::size_t trxs_produced_total = 0; + uint64_t cpu_usage_us = 0; + uint64_t net_usage_us = 0; - - uint32_t last_irreversible; - uint32_t head_block_num; + uint32_t last_irreversible = 0; + uint32_t head_block_num = 0; }; struct incoming_block_metrics { - std::size_t trxs_incoming_total; - uint64_t cpu_usage_us; - uint64_t net_usage_us; - - uint32_t last_irreversible; - uint32_t head_block_num; + std::size_t trxs_incoming_total = 0; + uint64_t cpu_usage_us = 0; + uint64_t net_usage_us = 0; + + uint32_t last_irreversible = 0; + uint32_t head_block_num = 0; }; void register_update_produced_block_metrics(std::function&&); diff --git a/plugins/prometheus_plugin/metrics.hpp b/plugins/prometheus_plugin/metrics.hpp index 7d0e6535c7..de697ec7c3 100644 --- a/plugins/prometheus_plugin/metrics.hpp +++ b/plugins/prometheus_plugin/metrics.hpp @@ -145,7 +145,7 @@ struct catalog_type { void register_update_handlers(boost::asio::io_context::strand& strand) { auto& http = app().get_plugin(); http.register_update_metrics( - [&strand, this](http_plugin::metrics metrics) { strand.post([metrics, this]() { update(metrics); }); }); + [&strand, this](http_plugin::metrics metrics) { strand.post([metrics = std::move(metrics), this]() { update(metrics); }); }); auto& net = app().get_plugin(); @@ -153,8 +153,14 @@ struct catalog_type { strand.post([metrics, this]() { update(metrics); }); }); - net.register_increment_failed_p2p_connections([this]() { failed_p2p_connections.Increment(1); }); - net.register_increment_dropped_trxs([this]() { dropped_trxs_total.Increment(1); }); + net.register_increment_failed_p2p_connections([this]() { + // Increment is thread safe + failed_p2p_connections.Increment(1); + }); + net.register_increment_dropped_trxs([this]() { + // Increment is thread safe + dropped_trxs_total.Increment(1); + }); auto& producer = app().get_plugin(); producer.register_update_produced_block_metrics( From 90235832311dd6cbd28be4e863b7fc08f36c969e Mon Sep 17 00:00:00 2001 From: Matt Witherspoon <32485495+spoonincode@users.noreply.github.com> Date: Thu, 27 Apr 2023 13:07:39 -0400 Subject: [PATCH 45/58] fix parallel-ctest-containers creation of log tarballs on failure --- .github/actions/parallel-ctest-containers/dist/index.mjs | 2 +- .github/actions/parallel-ctest-containers/main.mjs | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/actions/parallel-ctest-containers/dist/index.mjs b/.github/actions/parallel-ctest-containers/dist/index.mjs index 460fad6f5f..7ebdf382a7 100644 --- a/.github/actions/parallel-ctest-containers/dist/index.mjs +++ b/.github/actions/parallel-ctest-containers/dist/index.mjs @@ -1,3 +1,3 @@ import{createRequire as e}from"module";var t={7351:function(e,t,r){var n=this&&this.__createBinding||(Object.create?function(e,t,r,n){if(n===undefined)n=r;Object.defineProperty(e,n,{enumerable:true,get:function(){return t[r]}})}:function(e,t,r,n){if(n===undefined)n=r;e[n]=t[r]});var i=this&&this.__setModuleDefault||(Object.create?function(e,t){Object.defineProperty(e,"default",{enumerable:true,value:t})}:function(e,t){e["default"]=t});var a=this&&this.__importStar||function(e){if(e&&e.__esModule)return e;var t={};if(e!=null)for(var r in e)if(r!=="default"&&Object.hasOwnProperty.call(e,r))n(t,e,r);i(t,e);return t};Object.defineProperty(t,"__esModule",{value:true});t.issue=t.issueCommand=void 0;const o=a(r(2037));const s=r(6321);function issueCommand(e,t,r){const n=new Command(e,t,r);process.stdout.write(n.toString()+o.EOL)}t.issueCommand=issueCommand;function issue(e,t=""){issueCommand(e,{},t)}t.issue=issue;const u="::";class Command{constructor(e,t,r){if(!e){e="missing.command"}this.command=e;this.properties=t;this.message=r}toString(){let e=u+this.command;if(this.properties&&Object.keys(this.properties).length>0){e+=" ";let t=true;for(const r in this.properties){if(this.properties.hasOwnProperty(r)){const n=this.properties[r];if(n){if(t){t=false}else{e+=","}e+=`${r}=${escapeProperty(n)}`}}}}e+=`${u}${escapeData(this.message)}`;return e}}function escapeData(e){return s.toCommandValue(e).replace(/%/g,"%25").replace(/\r/g,"%0D").replace(/\n/g,"%0A")}function escapeProperty(e){return s.toCommandValue(e).replace(/%/g,"%25").replace(/\r/g,"%0D").replace(/\n/g,"%0A").replace(/:/g,"%3A").replace(/,/g,"%2C")}},2186:function(e,t,r){var n=this&&this.__createBinding||(Object.create?function(e,t,r,n){if(n===undefined)n=r;Object.defineProperty(e,n,{enumerable:true,get:function(){return t[r]}})}:function(e,t,r,n){if(n===undefined)n=r;e[n]=t[r]});var i=this&&this.__setModuleDefault||(Object.create?function(e,t){Object.defineProperty(e,"default",{enumerable:true,value:t})}:function(e,t){e["default"]=t});var a=this&&this.__importStar||function(e){if(e&&e.__esModule)return e;var t={};if(e!=null)for(var r in e)if(r!=="default"&&Object.hasOwnProperty.call(e,r))n(t,e,r);i(t,e);return t};var o=this&&this.__awaiter||function(e,t,r,n){function adopt(e){return e instanceof r?e:new r((function(t){t(e)}))}return new(r||(r=Promise))((function(r,i){function fulfilled(e){try{step(n.next(e))}catch(e){i(e)}}function rejected(e){try{step(n["throw"](e))}catch(e){i(e)}}function step(e){e.done?r(e.value):adopt(e.value).then(fulfilled,rejected)}step((n=n.apply(e,t||[])).next())}))};Object.defineProperty(t,"__esModule",{value:true});t.getIDToken=t.getState=t.saveState=t.group=t.endGroup=t.startGroup=t.info=t.notice=t.warning=t.error=t.debug=t.isDebug=t.setFailed=t.setCommandEcho=t.setOutput=t.getBooleanInput=t.getMultilineInput=t.getInput=t.addPath=t.setSecret=t.exportVariable=t.ExitCode=void 0;const s=r(7351);const u=r(717);const l=r(6321);const c=a(r(2037));const d=a(r(1017));const h=r(5840);const p=r(8041);var g;(function(e){e[e["Success"]=0]="Success";e[e["Failure"]=1]="Failure"})(g=t.ExitCode||(t.ExitCode={}));function exportVariable(e,t){const r=l.toCommandValue(t);process.env[e]=r;const n=process.env["GITHUB_ENV"]||"";if(n){const t=`ghadelimiter_${h.v4()}`;if(e.includes(t)){throw new Error(`Unexpected input: name should not contain the delimiter "${t}"`)}if(r.includes(t)){throw new Error(`Unexpected input: value should not contain the delimiter "${t}"`)}const n=`${e}<<${t}${c.EOL}${r}${c.EOL}${t}`;u.issueCommand("ENV",n)}else{s.issueCommand("set-env",{name:e},r)}}t.exportVariable=exportVariable;function setSecret(e){s.issueCommand("add-mask",{},e)}t.setSecret=setSecret;function addPath(e){const t=process.env["GITHUB_PATH"]||"";if(t){u.issueCommand("PATH",e)}else{s.issueCommand("add-path",{},e)}process.env["PATH"]=`${e}${d.delimiter}${process.env["PATH"]}`}t.addPath=addPath;function getInput(e,t){const r=process.env[`INPUT_${e.replace(/ /g,"_").toUpperCase()}`]||"";if(t&&t.required&&!r){throw new Error(`Input required and not supplied: ${e}`)}if(t&&t.trimWhitespace===false){return r}return r.trim()}t.getInput=getInput;function getMultilineInput(e,t){const r=getInput(e,t).split("\n").filter((e=>e!==""));return r}t.getMultilineInput=getMultilineInput;function getBooleanInput(e,t){const r=["true","True","TRUE"];const n=["false","False","FALSE"];const i=getInput(e,t);if(r.includes(i))return true;if(n.includes(i))return false;throw new TypeError(`Input does not meet YAML 1.2 "Core Schema" specification: ${e}\n`+`Support boolean input list: \`true | True | TRUE | false | False | FALSE\``)}t.getBooleanInput=getBooleanInput;function setOutput(e,t){process.stdout.write(c.EOL);s.issueCommand("set-output",{name:e},t)}t.setOutput=setOutput;function setCommandEcho(e){s.issue("echo",e?"on":"off")}t.setCommandEcho=setCommandEcho;function setFailed(e){process.exitCode=g.Failure;error(e)}t.setFailed=setFailed;function isDebug(){return process.env["RUNNER_DEBUG"]==="1"}t.isDebug=isDebug;function debug(e){s.issueCommand("debug",{},e)}t.debug=debug;function error(e,t={}){s.issueCommand("error",l.toCommandProperties(t),e instanceof Error?e.toString():e)}t.error=error;function warning(e,t={}){s.issueCommand("warning",l.toCommandProperties(t),e instanceof Error?e.toString():e)}t.warning=warning;function notice(e,t={}){s.issueCommand("notice",l.toCommandProperties(t),e instanceof Error?e.toString():e)}t.notice=notice;function info(e){process.stdout.write(e+c.EOL)}t.info=info;function startGroup(e){s.issue("group",e)}t.startGroup=startGroup;function endGroup(){s.issue("endgroup")}t.endGroup=endGroup;function group(e,t){return o(this,void 0,void 0,(function*(){startGroup(e);let r;try{r=yield t()}finally{endGroup()}return r}))}t.group=group;function saveState(e,t){s.issueCommand("save-state",{name:e},t)}t.saveState=saveState;function getState(e){return process.env[`STATE_${e}`]||""}t.getState=getState;function getIDToken(e){return o(this,void 0,void 0,(function*(){return yield p.OidcClient.getIDToken(e)}))}t.getIDToken=getIDToken;var m=r(1327);Object.defineProperty(t,"summary",{enumerable:true,get:function(){return m.summary}});var b=r(1327);Object.defineProperty(t,"markdownSummary",{enumerable:true,get:function(){return b.markdownSummary}});var v=r(2981);Object.defineProperty(t,"toPosixPath",{enumerable:true,get:function(){return v.toPosixPath}});Object.defineProperty(t,"toWin32Path",{enumerable:true,get:function(){return v.toWin32Path}});Object.defineProperty(t,"toPlatformPath",{enumerable:true,get:function(){return v.toPlatformPath}})},717:function(e,t,r){var n=this&&this.__createBinding||(Object.create?function(e,t,r,n){if(n===undefined)n=r;Object.defineProperty(e,n,{enumerable:true,get:function(){return t[r]}})}:function(e,t,r,n){if(n===undefined)n=r;e[n]=t[r]});var i=this&&this.__setModuleDefault||(Object.create?function(e,t){Object.defineProperty(e,"default",{enumerable:true,value:t})}:function(e,t){e["default"]=t});var a=this&&this.__importStar||function(e){if(e&&e.__esModule)return e;var t={};if(e!=null)for(var r in e)if(r!=="default"&&Object.hasOwnProperty.call(e,r))n(t,e,r);i(t,e);return t};Object.defineProperty(t,"__esModule",{value:true});t.issueCommand=void 0;const o=a(r(7147));const s=a(r(2037));const u=r(6321);function issueCommand(e,t){const r=process.env[`GITHUB_${e}`];if(!r){throw new Error(`Unable to find environment variable for file command ${e}`)}if(!o.existsSync(r)){throw new Error(`Missing file at path: ${r}`)}o.appendFileSync(r,`${u.toCommandValue(t)}${s.EOL}`,{encoding:"utf8"})}t.issueCommand=issueCommand},8041:function(e,t,r){var n=this&&this.__awaiter||function(e,t,r,n){function adopt(e){return e instanceof r?e:new r((function(t){t(e)}))}return new(r||(r=Promise))((function(r,i){function fulfilled(e){try{step(n.next(e))}catch(e){i(e)}}function rejected(e){try{step(n["throw"](e))}catch(e){i(e)}}function step(e){e.done?r(e.value):adopt(e.value).then(fulfilled,rejected)}step((n=n.apply(e,t||[])).next())}))};Object.defineProperty(t,"__esModule",{value:true});t.OidcClient=void 0;const i=r(6255);const a=r(5526);const o=r(2186);class OidcClient{static createHttpClient(e=true,t=10){const r={allowRetries:e,maxRetries:t};return new i.HttpClient("actions/oidc-client",[new a.BearerCredentialHandler(OidcClient.getRequestToken())],r)}static getRequestToken(){const e=process.env["ACTIONS_ID_TOKEN_REQUEST_TOKEN"];if(!e){throw new Error("Unable to get ACTIONS_ID_TOKEN_REQUEST_TOKEN env variable")}return e}static getIDTokenUrl(){const e=process.env["ACTIONS_ID_TOKEN_REQUEST_URL"];if(!e){throw new Error("Unable to get ACTIONS_ID_TOKEN_REQUEST_URL env variable")}return e}static getCall(e){var t;return n(this,void 0,void 0,(function*(){const r=OidcClient.createHttpClient();const n=yield r.getJson(e).catch((e=>{throw new Error(`Failed to get ID Token. \n \n Error Code : ${e.statusCode}\n \n Error Message: ${e.result.message}`)}));const i=(t=n.result)===null||t===void 0?void 0:t.value;if(!i){throw new Error("Response json body do not have ID Token field")}return i}))}static getIDToken(e){return n(this,void 0,void 0,(function*(){try{let t=OidcClient.getIDTokenUrl();if(e){const r=encodeURIComponent(e);t=`${t}&audience=${r}`}o.debug(`ID token url is ${t}`);const r=yield OidcClient.getCall(t);o.setSecret(r);return r}catch(e){throw new Error(`Error message: ${e.message}`)}}))}}t.OidcClient=OidcClient},2981:function(e,t,r){var n=this&&this.__createBinding||(Object.create?function(e,t,r,n){if(n===undefined)n=r;Object.defineProperty(e,n,{enumerable:true,get:function(){return t[r]}})}:function(e,t,r,n){if(n===undefined)n=r;e[n]=t[r]});var i=this&&this.__setModuleDefault||(Object.create?function(e,t){Object.defineProperty(e,"default",{enumerable:true,value:t})}:function(e,t){e["default"]=t});var a=this&&this.__importStar||function(e){if(e&&e.__esModule)return e;var t={};if(e!=null)for(var r in e)if(r!=="default"&&Object.hasOwnProperty.call(e,r))n(t,e,r);i(t,e);return t};Object.defineProperty(t,"__esModule",{value:true});t.toPlatformPath=t.toWin32Path=t.toPosixPath=void 0;const o=a(r(1017));function toPosixPath(e){return e.replace(/[\\]/g,"/")}t.toPosixPath=toPosixPath;function toWin32Path(e){return e.replace(/[/]/g,"\\")}t.toWin32Path=toWin32Path;function toPlatformPath(e){return e.replace(/[/\\]/g,o.sep)}t.toPlatformPath=toPlatformPath},1327:function(e,t,r){var n=this&&this.__awaiter||function(e,t,r,n){function adopt(e){return e instanceof r?e:new r((function(t){t(e)}))}return new(r||(r=Promise))((function(r,i){function fulfilled(e){try{step(n.next(e))}catch(e){i(e)}}function rejected(e){try{step(n["throw"](e))}catch(e){i(e)}}function step(e){e.done?r(e.value):adopt(e.value).then(fulfilled,rejected)}step((n=n.apply(e,t||[])).next())}))};Object.defineProperty(t,"__esModule",{value:true});t.summary=t.markdownSummary=t.SUMMARY_DOCS_URL=t.SUMMARY_ENV_VAR=void 0;const i=r(2037);const a=r(7147);const{access:o,appendFile:s,writeFile:u}=a.promises;t.SUMMARY_ENV_VAR="GITHUB_STEP_SUMMARY";t.SUMMARY_DOCS_URL="https://docs.github.com/actions/using-workflows/workflow-commands-for-github-actions#adding-a-job-summary";class Summary{constructor(){this._buffer=""}filePath(){return n(this,void 0,void 0,(function*(){if(this._filePath){return this._filePath}const e=process.env[t.SUMMARY_ENV_VAR];if(!e){throw new Error(`Unable to find environment variable for $${t.SUMMARY_ENV_VAR}. Check if your runtime environment supports job summaries.`)}try{yield o(e,a.constants.R_OK|a.constants.W_OK)}catch(t){throw new Error(`Unable to access summary file: '${e}'. Check if the file has correct read/write permissions.`)}this._filePath=e;return this._filePath}))}wrap(e,t,r={}){const n=Object.entries(r).map((([e,t])=>` ${e}="${t}"`)).join("");if(!t){return`<${e}${n}>`}return`<${e}${n}>${t}`}write(e){return n(this,void 0,void 0,(function*(){const t=!!(e===null||e===void 0?void 0:e.overwrite);const r=yield this.filePath();const n=t?u:s;yield n(r,this._buffer,{encoding:"utf8"});return this.emptyBuffer()}))}clear(){return n(this,void 0,void 0,(function*(){return this.emptyBuffer().write({overwrite:true})}))}stringify(){return this._buffer}isEmptyBuffer(){return this._buffer.length===0}emptyBuffer(){this._buffer="";return this}addRaw(e,t=false){this._buffer+=e;return t?this.addEOL():this}addEOL(){return this.addRaw(i.EOL)}addCodeBlock(e,t){const r=Object.assign({},t&&{lang:t});const n=this.wrap("pre",this.wrap("code",e),r);return this.addRaw(n).addEOL()}addList(e,t=false){const r=t?"ol":"ul";const n=e.map((e=>this.wrap("li",e))).join("");const i=this.wrap(r,n);return this.addRaw(i).addEOL()}addTable(e){const t=e.map((e=>{const t=e.map((e=>{if(typeof e==="string"){return this.wrap("td",e)}const{header:t,data:r,colspan:n,rowspan:i}=e;const a=t?"th":"td";const o=Object.assign(Object.assign({},n&&{colspan:n}),i&&{rowspan:i});return this.wrap(a,r,o)})).join("");return this.wrap("tr",t)})).join("");const r=this.wrap("table",t);return this.addRaw(r).addEOL()}addDetails(e,t){const r=this.wrap("details",this.wrap("summary",e)+t);return this.addRaw(r).addEOL()}addImage(e,t,r){const{width:n,height:i}=r||{};const a=Object.assign(Object.assign({},n&&{width:n}),i&&{height:i});const o=this.wrap("img",null,Object.assign({src:e,alt:t},a));return this.addRaw(o).addEOL()}addHeading(e,t){const r=`h${t}`;const n=["h1","h2","h3","h4","h5","h6"].includes(r)?r:"h1";const i=this.wrap(n,e);return this.addRaw(i).addEOL()}addSeparator(){const e=this.wrap("hr",null);return this.addRaw(e).addEOL()}addBreak(){const e=this.wrap("br",null);return this.addRaw(e).addEOL()}addQuote(e,t){const r=Object.assign({},t&&{cite:t});const n=this.wrap("blockquote",e,r);return this.addRaw(n).addEOL()}addLink(e,t){const r=this.wrap("a",e,{href:t});return this.addRaw(r).addEOL()}}const l=new Summary;t.markdownSummary=l;t.summary=l},6321:(e,t)=>{Object.defineProperty(t,"__esModule",{value:true});t.toCommandProperties=t.toCommandValue=void 0;function toCommandValue(e){if(e===null||e===undefined){return""}else if(typeof e==="string"||e instanceof String){return e}return JSON.stringify(e)}t.toCommandValue=toCommandValue;function toCommandProperties(e){if(!Object.keys(e).length){return{}}return{title:e.title,file:e.file,line:e.startLine,endLine:e.endLine,col:e.startColumn,endColumn:e.endColumn}}t.toCommandProperties=toCommandProperties},5526:function(e,t){var r=this&&this.__awaiter||function(e,t,r,n){function adopt(e){return e instanceof r?e:new r((function(t){t(e)}))}return new(r||(r=Promise))((function(r,i){function fulfilled(e){try{step(n.next(e))}catch(e){i(e)}}function rejected(e){try{step(n["throw"](e))}catch(e){i(e)}}function step(e){e.done?r(e.value):adopt(e.value).then(fulfilled,rejected)}step((n=n.apply(e,t||[])).next())}))};Object.defineProperty(t,"__esModule",{value:true});t.PersonalAccessTokenCredentialHandler=t.BearerCredentialHandler=t.BasicCredentialHandler=void 0;class BasicCredentialHandler{constructor(e,t){this.username=e;this.password=t}prepareRequest(e){if(!e.headers){throw Error("The request has no headers")}e.headers["Authorization"]=`Basic ${Buffer.from(`${this.username}:${this.password}`).toString("base64")}`}canHandleAuthentication(){return false}handleAuthentication(){return r(this,void 0,void 0,(function*(){throw new Error("not implemented")}))}}t.BasicCredentialHandler=BasicCredentialHandler;class BearerCredentialHandler{constructor(e){this.token=e}prepareRequest(e){if(!e.headers){throw Error("The request has no headers")}e.headers["Authorization"]=`Bearer ${this.token}`}canHandleAuthentication(){return false}handleAuthentication(){return r(this,void 0,void 0,(function*(){throw new Error("not implemented")}))}}t.BearerCredentialHandler=BearerCredentialHandler;class PersonalAccessTokenCredentialHandler{constructor(e){this.token=e}prepareRequest(e){if(!e.headers){throw Error("The request has no headers")}e.headers["Authorization"]=`Basic ${Buffer.from(`PAT:${this.token}`).toString("base64")}`}canHandleAuthentication(){return false}handleAuthentication(){return r(this,void 0,void 0,(function*(){throw new Error("not implemented")}))}}t.PersonalAccessTokenCredentialHandler=PersonalAccessTokenCredentialHandler},6255:function(e,t,r){var n=this&&this.__createBinding||(Object.create?function(e,t,r,n){if(n===undefined)n=r;Object.defineProperty(e,n,{enumerable:true,get:function(){return t[r]}})}:function(e,t,r,n){if(n===undefined)n=r;e[n]=t[r]});var i=this&&this.__setModuleDefault||(Object.create?function(e,t){Object.defineProperty(e,"default",{enumerable:true,value:t})}:function(e,t){e["default"]=t});var a=this&&this.__importStar||function(e){if(e&&e.__esModule)return e;var t={};if(e!=null)for(var r in e)if(r!=="default"&&Object.hasOwnProperty.call(e,r))n(t,e,r);i(t,e);return t};var o=this&&this.__awaiter||function(e,t,r,n){function adopt(e){return e instanceof r?e:new r((function(t){t(e)}))}return new(r||(r=Promise))((function(r,i){function fulfilled(e){try{step(n.next(e))}catch(e){i(e)}}function rejected(e){try{step(n["throw"](e))}catch(e){i(e)}}function step(e){e.done?r(e.value):adopt(e.value).then(fulfilled,rejected)}step((n=n.apply(e,t||[])).next())}))};Object.defineProperty(t,"__esModule",{value:true});t.HttpClient=t.isHttps=t.HttpClientResponse=t.HttpClientError=t.getProxyUrl=t.MediaTypes=t.Headers=t.HttpCodes=void 0;const s=a(r(3685));const u=a(r(5687));const l=a(r(9835));const c=a(r(4294));var d;(function(e){e[e["OK"]=200]="OK";e[e["MultipleChoices"]=300]="MultipleChoices";e[e["MovedPermanently"]=301]="MovedPermanently";e[e["ResourceMoved"]=302]="ResourceMoved";e[e["SeeOther"]=303]="SeeOther";e[e["NotModified"]=304]="NotModified";e[e["UseProxy"]=305]="UseProxy";e[e["SwitchProxy"]=306]="SwitchProxy";e[e["TemporaryRedirect"]=307]="TemporaryRedirect";e[e["PermanentRedirect"]=308]="PermanentRedirect";e[e["BadRequest"]=400]="BadRequest";e[e["Unauthorized"]=401]="Unauthorized";e[e["PaymentRequired"]=402]="PaymentRequired";e[e["Forbidden"]=403]="Forbidden";e[e["NotFound"]=404]="NotFound";e[e["MethodNotAllowed"]=405]="MethodNotAllowed";e[e["NotAcceptable"]=406]="NotAcceptable";e[e["ProxyAuthenticationRequired"]=407]="ProxyAuthenticationRequired";e[e["RequestTimeout"]=408]="RequestTimeout";e[e["Conflict"]=409]="Conflict";e[e["Gone"]=410]="Gone";e[e["TooManyRequests"]=429]="TooManyRequests";e[e["InternalServerError"]=500]="InternalServerError";e[e["NotImplemented"]=501]="NotImplemented";e[e["BadGateway"]=502]="BadGateway";e[e["ServiceUnavailable"]=503]="ServiceUnavailable";e[e["GatewayTimeout"]=504]="GatewayTimeout"})(d=t.HttpCodes||(t.HttpCodes={}));var h;(function(e){e["Accept"]="accept";e["ContentType"]="content-type"})(h=t.Headers||(t.Headers={}));var p;(function(e){e["ApplicationJson"]="application/json"})(p=t.MediaTypes||(t.MediaTypes={}));function getProxyUrl(e){const t=l.getProxyUrl(new URL(e));return t?t.href:""}t.getProxyUrl=getProxyUrl;const g=[d.MovedPermanently,d.ResourceMoved,d.SeeOther,d.TemporaryRedirect,d.PermanentRedirect];const m=[d.BadGateway,d.ServiceUnavailable,d.GatewayTimeout];const b=["OPTIONS","GET","DELETE","HEAD"];const v=10;const _=5;class HttpClientError extends Error{constructor(e,t){super(e);this.name="HttpClientError";this.statusCode=t;Object.setPrototypeOf(this,HttpClientError.prototype)}}t.HttpClientError=HttpClientError;class HttpClientResponse{constructor(e){this.message=e}readBody(){return o(this,void 0,void 0,(function*(){return new Promise((e=>o(this,void 0,void 0,(function*(){let t=Buffer.alloc(0);this.message.on("data",(e=>{t=Buffer.concat([t,e])}));this.message.on("end",(()=>{e(t.toString())}))}))))}))}}t.HttpClientResponse=HttpClientResponse;function isHttps(e){const t=new URL(e);return t.protocol==="https:"}t.isHttps=isHttps;class HttpClient{constructor(e,t,r){this._ignoreSslError=false;this._allowRedirects=true;this._allowRedirectDowngrade=false;this._maxRedirects=50;this._allowRetries=false;this._maxRetries=1;this._keepAlive=false;this._disposed=false;this.userAgent=e;this.handlers=t||[];this.requestOptions=r;if(r){if(r.ignoreSslError!=null){this._ignoreSslError=r.ignoreSslError}this._socketTimeout=r.socketTimeout;if(r.allowRedirects!=null){this._allowRedirects=r.allowRedirects}if(r.allowRedirectDowngrade!=null){this._allowRedirectDowngrade=r.allowRedirectDowngrade}if(r.maxRedirects!=null){this._maxRedirects=Math.max(r.maxRedirects,0)}if(r.keepAlive!=null){this._keepAlive=r.keepAlive}if(r.allowRetries!=null){this._allowRetries=r.allowRetries}if(r.maxRetries!=null){this._maxRetries=r.maxRetries}}}options(e,t){return o(this,void 0,void 0,(function*(){return this.request("OPTIONS",e,null,t||{})}))}get(e,t){return o(this,void 0,void 0,(function*(){return this.request("GET",e,null,t||{})}))}del(e,t){return o(this,void 0,void 0,(function*(){return this.request("DELETE",e,null,t||{})}))}post(e,t,r){return o(this,void 0,void 0,(function*(){return this.request("POST",e,t,r||{})}))}patch(e,t,r){return o(this,void 0,void 0,(function*(){return this.request("PATCH",e,t,r||{})}))}put(e,t,r){return o(this,void 0,void 0,(function*(){return this.request("PUT",e,t,r||{})}))}head(e,t){return o(this,void 0,void 0,(function*(){return this.request("HEAD",e,null,t||{})}))}sendStream(e,t,r,n){return o(this,void 0,void 0,(function*(){return this.request(e,t,r,n)}))}getJson(e,t={}){return o(this,void 0,void 0,(function*(){t[h.Accept]=this._getExistingOrDefaultHeader(t,h.Accept,p.ApplicationJson);const r=yield this.get(e,t);return this._processResponse(r,this.requestOptions)}))}postJson(e,t,r={}){return o(this,void 0,void 0,(function*(){const n=JSON.stringify(t,null,2);r[h.Accept]=this._getExistingOrDefaultHeader(r,h.Accept,p.ApplicationJson);r[h.ContentType]=this._getExistingOrDefaultHeader(r,h.ContentType,p.ApplicationJson);const i=yield this.post(e,n,r);return this._processResponse(i,this.requestOptions)}))}putJson(e,t,r={}){return o(this,void 0,void 0,(function*(){const n=JSON.stringify(t,null,2);r[h.Accept]=this._getExistingOrDefaultHeader(r,h.Accept,p.ApplicationJson);r[h.ContentType]=this._getExistingOrDefaultHeader(r,h.ContentType,p.ApplicationJson);const i=yield this.put(e,n,r);return this._processResponse(i,this.requestOptions)}))}patchJson(e,t,r={}){return o(this,void 0,void 0,(function*(){const n=JSON.stringify(t,null,2);r[h.Accept]=this._getExistingOrDefaultHeader(r,h.Accept,p.ApplicationJson);r[h.ContentType]=this._getExistingOrDefaultHeader(r,h.ContentType,p.ApplicationJson);const i=yield this.patch(e,n,r);return this._processResponse(i,this.requestOptions)}))}request(e,t,r,n){return o(this,void 0,void 0,(function*(){if(this._disposed){throw new Error("Client has already been disposed.")}const i=new URL(t);let a=this._prepareRequest(e,i,n);const o=this._allowRetries&&b.includes(e)?this._maxRetries+1:1;let s=0;let u;do{u=yield this.requestRaw(a,r);if(u&&u.message&&u.message.statusCode===d.Unauthorized){let e;for(const t of this.handlers){if(t.canHandleAuthentication(u)){e=t;break}}if(e){return e.handleAuthentication(this,a,r)}else{return u}}let t=this._maxRedirects;while(u.message.statusCode&&g.includes(u.message.statusCode)&&this._allowRedirects&&t>0){const o=u.message.headers["location"];if(!o){break}const s=new URL(o);if(i.protocol==="https:"&&i.protocol!==s.protocol&&!this._allowRedirectDowngrade){throw new Error("Redirect from HTTPS to HTTP protocol. This downgrade is not allowed for security reasons. If you want to allow this behavior, set the allowRedirectDowngrade option to true.")}yield u.readBody();if(s.hostname!==i.hostname){for(const e in n){if(e.toLowerCase()==="authorization"){delete n[e]}}}a=this._prepareRequest(e,s,n);u=yield this.requestRaw(a,r);t--}if(!u.message.statusCode||!m.includes(u.message.statusCode)){return u}s+=1;if(s{function callbackForResult(e,t){if(e){n(e)}else if(!t){n(new Error("Unknown error"))}else{r(t)}}this.requestRawWithCallback(e,t,callbackForResult)}))}))}requestRawWithCallback(e,t,r){if(typeof t==="string"){if(!e.options.headers){e.options.headers={}}e.options.headers["Content-Length"]=Buffer.byteLength(t,"utf8")}let n=false;function handleResult(e,t){if(!n){n=true;r(e,t)}}const i=e.httpModule.request(e.options,(e=>{const t=new HttpClientResponse(e);handleResult(undefined,t)}));let a;i.on("socket",(e=>{a=e}));i.setTimeout(this._socketTimeout||3*6e4,(()=>{if(a){a.end()}handleResult(new Error(`Request timeout: ${e.options.path}`))}));i.on("error",(function(e){handleResult(e)}));if(t&&typeof t==="string"){i.write(t,"utf8")}if(t&&typeof t!=="string"){t.on("close",(function(){i.end()}));t.pipe(i)}else{i.end()}}getAgent(e){const t=new URL(e);return this._getAgent(t)}_prepareRequest(e,t,r){const n={};n.parsedUrl=t;const i=n.parsedUrl.protocol==="https:";n.httpModule=i?u:s;const a=i?443:80;n.options={};n.options.host=n.parsedUrl.hostname;n.options.port=n.parsedUrl.port?parseInt(n.parsedUrl.port):a;n.options.path=(n.parsedUrl.pathname||"")+(n.parsedUrl.search||"");n.options.method=e;n.options.headers=this._mergeHeaders(r);if(this.userAgent!=null){n.options.headers["user-agent"]=this.userAgent}n.options.agent=this._getAgent(n.parsedUrl);if(this.handlers){for(const e of this.handlers){e.prepareRequest(n.options)}}return n}_mergeHeaders(e){if(this.requestOptions&&this.requestOptions.headers){return Object.assign({},lowercaseKeys(this.requestOptions.headers),lowercaseKeys(e||{}))}return lowercaseKeys(e||{})}_getExistingOrDefaultHeader(e,t,r){let n;if(this.requestOptions&&this.requestOptions.headers){n=lowercaseKeys(this.requestOptions.headers)[t]}return e[t]||n||r}_getAgent(e){let t;const r=l.getProxyUrl(e);const n=r&&r.hostname;if(this._keepAlive&&n){t=this._proxyAgent}if(this._keepAlive&&!n){t=this._agent}if(t){return t}const i=e.protocol==="https:";let a=100;if(this.requestOptions){a=this.requestOptions.maxSockets||s.globalAgent.maxSockets}if(r&&r.hostname){const e={maxSockets:a,keepAlive:this._keepAlive,proxy:Object.assign(Object.assign({},(r.username||r.password)&&{proxyAuth:`${r.username}:${r.password}`}),{host:r.hostname,port:r.port})};let n;const o=r.protocol==="https:";if(i){n=o?c.httpsOverHttps:c.httpsOverHttp}else{n=o?c.httpOverHttps:c.httpOverHttp}t=n(e);this._proxyAgent=t}if(this._keepAlive&&!t){const e={keepAlive:this._keepAlive,maxSockets:a};t=i?new u.Agent(e):new s.Agent(e);this._agent=t}if(!t){t=i?u.globalAgent:s.globalAgent}if(i&&this._ignoreSslError){t.options=Object.assign(t.options||{},{rejectUnauthorized:false})}return t}_performExponentialBackoff(e){return o(this,void 0,void 0,(function*(){e=Math.min(v,e);const t=_*Math.pow(2,e);return new Promise((e=>setTimeout((()=>e()),t)))}))}_processResponse(e,t){return o(this,void 0,void 0,(function*(){return new Promise(((r,n)=>o(this,void 0,void 0,(function*(){const i=e.message.statusCode||0;const a={statusCode:i,result:null,headers:{}};if(i===d.NotFound){r(a)}function dateTimeDeserializer(e,t){if(typeof t==="string"){const e=new Date(t);if(!isNaN(e.valueOf())){return e}}return t}let o;let s;try{s=yield e.readBody();if(s&&s.length>0){if(t&&t.deserializeDates){o=JSON.parse(s,dateTimeDeserializer)}else{o=JSON.parse(s)}a.result=o}a.headers=e.message.headers}catch(e){}if(i>299){let e;if(o&&o.message){e=o.message}else if(s&&s.length>0){e=s}else{e=`Failed request: (${i})`}const t=new HttpClientError(e,i);t.result=a.result;n(t)}else{r(a)}}))))}))}}t.HttpClient=HttpClient;const lowercaseKeys=e=>Object.keys(e).reduce(((t,r)=>(t[r.toLowerCase()]=e[r],t)),{})},9835:(e,t)=>{Object.defineProperty(t,"__esModule",{value:true});t.checkBypass=t.getProxyUrl=void 0;function getProxyUrl(e){const t=e.protocol==="https:";if(checkBypass(e)){return undefined}const r=(()=>{if(t){return process.env["https_proxy"]||process.env["HTTPS_PROXY"]}else{return process.env["http_proxy"]||process.env["HTTP_PROXY"]}})();if(r){return new URL(r)}else{return undefined}}t.getProxyUrl=getProxyUrl;function checkBypass(e){if(!e.hostname){return false}const t=process.env["no_proxy"]||process.env["NO_PROXY"]||"";if(!t){return false}let r;if(e.port){r=Number(e.port)}else if(e.protocol==="http:"){r=80}else if(e.protocol==="https:"){r=443}const n=[e.hostname.toUpperCase()];if(typeof r==="number"){n.push(`${n[0]}:${r}`)}for(const e of t.split(",").map((e=>e.trim().toUpperCase())).filter((e=>e))){if(n.some((t=>t===e))){return true}}return false}t.checkBypass=checkBypass},3664:(e,t,r)=>{const{Buffer:n}=r(4300);const i=Symbol.for("BufferList");function BufferList(e){if(!(this instanceof BufferList)){return new BufferList(e)}BufferList._init.call(this,e)}BufferList._init=function _init(e){Object.defineProperty(this,i,{value:true});this._bufs=[];this.length=0;if(e){this.append(e)}};BufferList.prototype._new=function _new(e){return new BufferList(e)};BufferList.prototype._offset=function _offset(e){if(e===0){return[0,0]}let t=0;for(let r=0;rthis.length||e<0){return undefined}const t=this._offset(e);return this._bufs[t[0]][t[1]]};BufferList.prototype.slice=function slice(e,t){if(typeof e==="number"&&e<0){e+=this.length}if(typeof t==="number"&&t<0){t+=this.length}return this.copy(null,0,e,t)};BufferList.prototype.copy=function copy(e,t,r,i){if(typeof r!=="number"||r<0){r=0}if(typeof i!=="number"||i>this.length){i=this.length}if(r>=this.length){return e||n.alloc(0)}if(i<=0){return e||n.alloc(0)}const copy=!!e;const a=this._offset(r);const o=i-r;let s=o;let u=copy&&t||0;let l=a[1];if(r===0&&i===this.length){if(!copy){return this._bufs.length===1?this._bufs[0]:n.concat(this._bufs,this.length)}for(let t=0;tr){this._bufs[t].copy(e,u,l);u+=r}else{this._bufs[t].copy(e,u,l,l+s);u+=r;break}s-=r;if(l){l=0}}if(e.length>u)return e.slice(0,u);return e};BufferList.prototype.shallowSlice=function shallowSlice(e,t){e=e||0;t=typeof t!=="number"?this.length:t;if(e<0){e+=this.length}if(t<0){t+=this.length}if(e===t){return this._new()}const r=this._offset(e);const n=this._offset(t);const i=this._bufs.slice(r[0],n[0]+1);if(n[1]===0){i.pop()}else{i[i.length-1]=i[i.length-1].slice(0,n[1])}if(r[1]!==0){i[0]=i[0].slice(r[1])}return this._new(i)};BufferList.prototype.toString=function toString(e,t,r){return this.slice(t,r).toString(e)};BufferList.prototype.consume=function consume(e){e=Math.trunc(e);if(Number.isNaN(e)||e<=0)return this;while(this._bufs.length){if(e>=this._bufs[0].length){e-=this._bufs[0].length;this.length-=this._bufs[0].length;this._bufs.shift()}else{this._bufs[0]=this._bufs[0].slice(e);this.length-=e;break}}return this};BufferList.prototype.duplicate=function duplicate(){const e=this._new();for(let t=0;tthis.length?this.length:t}const i=this._offset(t);let a=i[0];let o=i[1];for(;a=e.length){const r=t.indexOf(e,o);if(r!==-1){return this._reverseOffset([a,r])}o=t.length-e.length+1}else{const t=this._reverseOffset([a,o]);if(this._match(t,e)){return t}o++}}o=0}return-1};BufferList.prototype._match=function(e,t){if(this.length-e{const n=r(1642).Duplex;const i=r(4124);const a=r(3664);function BufferListStream(e){if(!(this instanceof BufferListStream)){return new BufferListStream(e)}if(typeof e==="function"){this._callback=e;const t=function piper(e){if(this._callback){this._callback(e);this._callback=null}}.bind(this);this.on("pipe",(function onPipe(e){e.on("error",t)}));this.on("unpipe",(function onUnpipe(e){e.removeListener("error",t)}));e=null}a._init.call(this,e);n.call(this)}i(BufferListStream,n);Object.assign(BufferListStream.prototype,a.prototype);BufferListStream.prototype._new=function _new(e){return new BufferListStream(e)};BufferListStream.prototype._write=function _write(e,t,r){this._appendBuffer(e);if(typeof r==="function"){r()}};BufferListStream.prototype._read=function _read(e){if(!this.length){return this.push(null)}e=Math.min(e,this.length);this.push(this.slice(0,e));this.consume(e)};BufferListStream.prototype.end=function end(e){n.prototype.end.call(this,e);if(this._callback){this._callback(null,this.slice());this._callback=null}};BufferListStream.prototype._destroy=function _destroy(e,t){this._bufs.length=0;this.length=0;t(e)};BufferListStream.prototype._isBufferList=function _isBufferList(e){return e instanceof BufferListStream||e instanceof a||BufferListStream.isBufferList(e)};BufferListStream.isBufferList=a.isBufferList;e.exports=BufferListStream;e.exports.BufferListStream=BufferListStream;e.exports.BufferList=a},1205:(e,t,r)=>{var n=r(1223);var noop=function(){};var isRequest=function(e){return e.setHeader&&typeof e.abort==="function"};var isChildProcess=function(e){return e.stdio&&Array.isArray(e.stdio)&&e.stdio.length===3};var eos=function(e,t,r){if(typeof t==="function")return eos(e,null,t);if(!t)t={};r=n(r||noop);var i=e._writableState;var a=e._readableState;var o=t.readable||t.readable!==false&&e.readable;var s=t.writable||t.writable!==false&&e.writable;var u=false;var onlegacyfinish=function(){if(!e.writable)onfinish()};var onfinish=function(){s=false;if(!o)r.call(e)};var onend=function(){o=false;if(!s)r.call(e)};var onexit=function(t){r.call(e,t?new Error("exited with error code: "+t):null)};var onerror=function(t){r.call(e,t)};var onclose=function(){process.nextTick(onclosenexttick)};var onclosenexttick=function(){if(u)return;if(o&&!(a&&(a.ended&&!a.destroyed)))return r.call(e,new Error("premature close"));if(s&&!(i&&(i.ended&&!i.destroyed)))return r.call(e,new Error("premature close"))};var onrequest=function(){e.req.on("finish",onfinish)};if(isRequest(e)){e.on("complete",onfinish);e.on("abort",onclose);if(e.req)onrequest();else e.on("request",onrequest)}else if(s&&!i){e.on("end",onlegacyfinish);e.on("close",onlegacyfinish)}if(isChildProcess(e))e.on("exit",onexit);e.on("end",onend);e.on("finish",onfinish);if(t.error!==false)e.on("error",onerror);e.on("close",onclose);return function(){u=true;e.removeListener("complete",onfinish);e.removeListener("abort",onclose);e.removeListener("request",onrequest);if(e.req)e.req.removeListener("finish",onfinish);e.removeListener("end",onlegacyfinish);e.removeListener("close",onlegacyfinish);e.removeListener("finish",onfinish);e.removeListener("exit",onexit);e.removeListener("end",onend);e.removeListener("error",onerror);e.removeListener("close",onclose)}};e.exports=eos},3186:(e,t,r)=>{e.exports=r(7147).constants||r(2057)},4124:(e,t,r)=>{try{var n=r(3837);if(typeof n.inherits!=="function")throw"";e.exports=n.inherits}catch(t){e.exports=r(8544)}},8544:e=>{if(typeof Object.create==="function"){e.exports=function inherits(e,t){if(t){e.super_=t;e.prototype=Object.create(t.prototype,{constructor:{value:e,enumerable:false,writable:true,configurable:true}})}}}else{e.exports=function inherits(e,t){if(t){e.super_=t;var TempCtor=function(){};TempCtor.prototype=t.prototype;e.prototype=new TempCtor;e.prototype.constructor=e}}}},1223:(e,t,r)=>{var n=r(2940);e.exports=n(once);e.exports.strict=n(onceStrict);once.proto=once((function(){Object.defineProperty(Function.prototype,"once",{value:function(){return once(this)},configurable:true});Object.defineProperty(Function.prototype,"onceStrict",{value:function(){return onceStrict(this)},configurable:true})}));function once(e){var f=function(){if(f.called)return f.value;f.called=true;return f.value=e.apply(this,arguments)};f.called=false;return f}function onceStrict(e){var f=function(){if(f.called)throw new Error(f.onceError);f.called=true;return f.value=e.apply(this,arguments)};var t=e.name||"Function wrapped with `once`";f.onceError=t+" shouldn't be called more than once";f.called=false;return f}},7214:e=>{const t={};function createErrorType(e,r,n){if(!n){n=Error}function getMessage(e,t,n){if(typeof r==="string"){return r}else{return r(e,t,n)}}class NodeError extends n{constructor(e,t,r){super(getMessage(e,t,r))}}NodeError.prototype.name=n.name;NodeError.prototype.code=e;t[e]=NodeError}function oneOf(e,t){if(Array.isArray(e)){const r=e.length;e=e.map((e=>String(e)));if(r>2){return`one of ${t} ${e.slice(0,r-1).join(", ")}, or `+e[r-1]}else if(r===2){return`one of ${t} ${e[0]} or ${e[1]}`}else{return`of ${t} ${e[0]}`}}else{return`of ${t} ${String(e)}`}}function startsWith(e,t,r){return e.substr(!r||r<0?0:+r,t.length)===t}function endsWith(e,t,r){if(r===undefined||r>e.length){r=e.length}return e.substring(r-t.length,r)===t}function includes(e,t,r){if(typeof r!=="number"){r=0}if(r+t.length>e.length){return false}else{return e.indexOf(t,r)!==-1}}createErrorType("ERR_INVALID_OPT_VALUE",(function(e,t){return'The value "'+t+'" is invalid for option "'+e+'"'}),TypeError);createErrorType("ERR_INVALID_ARG_TYPE",(function(e,t,r){let n;if(typeof t==="string"&&startsWith(t,"not ")){n="must not be";t=t.replace(/^not /,"")}else{n="must be"}let i;if(endsWith(e," argument")){i=`The ${e} ${n} ${oneOf(t,"type")}`}else{const r=includes(e,".")?"property":"argument";i=`The "${e}" ${r} ${n} ${oneOf(t,"type")}`}i+=`. Received type ${typeof r}`;return i}),TypeError);createErrorType("ERR_STREAM_PUSH_AFTER_EOF","stream.push() after EOF");createErrorType("ERR_METHOD_NOT_IMPLEMENTED",(function(e){return"The "+e+" method is not implemented"}));createErrorType("ERR_STREAM_PREMATURE_CLOSE","Premature close");createErrorType("ERR_STREAM_DESTROYED",(function(e){return"Cannot call "+e+" after a stream was destroyed"}));createErrorType("ERR_MULTIPLE_CALLBACK","Callback called multiple times");createErrorType("ERR_STREAM_CANNOT_PIPE","Cannot pipe, not readable");createErrorType("ERR_STREAM_WRITE_AFTER_END","write after end");createErrorType("ERR_STREAM_NULL_VALUES","May not write null values to stream",TypeError);createErrorType("ERR_UNKNOWN_ENCODING",(function(e){return"Unknown encoding: "+e}),TypeError);createErrorType("ERR_STREAM_UNSHIFT_AFTER_END_EVENT","stream.unshift() after end event");e.exports.q=t},1359:(e,t,r)=>{var n=Object.keys||function(e){var t=[];for(var r in e){t.push(r)}return t};e.exports=Duplex;var i=r(1433);var a=r(6993);r(4124)(Duplex,i);{var o=n(a.prototype);for(var s=0;s{e.exports=PassThrough;var n=r(4415);r(4124)(PassThrough,n);function PassThrough(e){if(!(this instanceof PassThrough))return new PassThrough(e);n.call(this,e)}PassThrough.prototype._transform=function(e,t,r){r(null,e)}},1433:(e,t,r)=>{e.exports=Readable;var n;Readable.ReadableState=ReadableState;var i=r(2361).EventEmitter;var a=function EElistenerCount(e,t){return e.listeners(t).length};var o=r(2387);var s=r(4300).Buffer;var u=global.Uint8Array||function(){};function _uint8ArrayToBuffer(e){return s.from(e)}function _isUint8Array(e){return s.isBuffer(e)||e instanceof u}var l=r(3837);var c;if(l&&l.debuglog){c=l.debuglog("stream")}else{c=function debug(){}}var d=r(2746);var h=r(7049);var p=r(9948),g=p.getHighWaterMark;var m=r(7214).q,b=m.ERR_INVALID_ARG_TYPE,v=m.ERR_STREAM_PUSH_AFTER_EOF,_=m.ERR_METHOD_NOT_IMPLEMENTED,y=m.ERR_STREAM_UNSHIFT_AFTER_END_EVENT;var w;var R;var S;r(4124)(Readable,o);var E=h.errorOrDestroy;var k=["error","close","destroy","pause","resume"];function prependListener(e,t,r){if(typeof e.prependListener==="function")return e.prependListener(t,r);if(!e._events||!e._events[t])e.on(t,r);else if(Array.isArray(e._events[t]))e._events[t].unshift(r);else e._events[t]=[r,e._events[t]]}function ReadableState(e,t,i){n=n||r(1359);e=e||{};if(typeof i!=="boolean")i=t instanceof n;this.objectMode=!!e.objectMode;if(i)this.objectMode=this.objectMode||!!e.readableObjectMode;this.highWaterMark=g(this,e,"readableHighWaterMark",i);this.buffer=new d;this.length=0;this.pipes=null;this.pipesCount=0;this.flowing=null;this.ended=false;this.endEmitted=false;this.reading=false;this.sync=true;this.needReadable=false;this.emittedReadable=false;this.readableListening=false;this.resumeScheduled=false;this.paused=true;this.emitClose=e.emitClose!==false;this.autoDestroy=!!e.autoDestroy;this.destroyed=false;this.defaultEncoding=e.defaultEncoding||"utf8";this.awaitDrain=0;this.readingMore=false;this.decoder=null;this.encoding=null;if(e.encoding){if(!w)w=r(4841).s;this.decoder=new w(e.encoding);this.encoding=e.encoding}}function Readable(e){n=n||r(1359);if(!(this instanceof Readable))return new Readable(e);var t=this instanceof n;this._readableState=new ReadableState(e,this,t);this.readable=true;if(e){if(typeof e.read==="function")this._read=e.read;if(typeof e.destroy==="function")this._destroy=e.destroy}o.call(this)}Object.defineProperty(Readable.prototype,"destroyed",{enumerable:false,get:function get(){if(this._readableState===undefined){return false}return this._readableState.destroyed},set:function set(e){if(!this._readableState){return}this._readableState.destroyed=e}});Readable.prototype.destroy=h.destroy;Readable.prototype._undestroy=h.undestroy;Readable.prototype._destroy=function(e,t){t(e)};Readable.prototype.push=function(e,t){var r=this._readableState;var n;if(!r.objectMode){if(typeof e==="string"){t=t||r.defaultEncoding;if(t!==r.encoding){e=s.from(e,t);t=""}n=true}}else{n=true}return readableAddChunk(this,e,t,false,n)};Readable.prototype.unshift=function(e){return readableAddChunk(this,e,null,true,false)};function readableAddChunk(e,t,r,n,i){c("readableAddChunk",t);var a=e._readableState;if(t===null){a.reading=false;onEofChunk(e,a)}else{var o;if(!i)o=chunkInvalid(a,t);if(o){E(e,o)}else if(a.objectMode||t&&t.length>0){if(typeof t!=="string"&&!a.objectMode&&Object.getPrototypeOf(t)!==s.prototype){t=_uint8ArrayToBuffer(t)}if(n){if(a.endEmitted)E(e,new y);else addChunk(e,a,t,true)}else if(a.ended){E(e,new v)}else if(a.destroyed){return false}else{a.reading=false;if(a.decoder&&!r){t=a.decoder.write(t);if(a.objectMode||t.length!==0)addChunk(e,a,t,false);else maybeReadMore(e,a)}else{addChunk(e,a,t,false)}}}else if(!n){a.reading=false;maybeReadMore(e,a)}}return!a.ended&&(a.length=O){e=O}else{e--;e|=e>>>1;e|=e>>>2;e|=e>>>4;e|=e>>>8;e|=e>>>16;e++}return e}function howMuchToRead(e,t){if(e<=0||t.length===0&&t.ended)return 0;if(t.objectMode)return 1;if(e!==e){if(t.flowing&&t.length)return t.buffer.head.data.length;else return t.length}if(e>t.highWaterMark)t.highWaterMark=computeNewHighWaterMark(e);if(e<=t.length)return e;if(!t.ended){t.needReadable=true;return 0}return t.length}Readable.prototype.read=function(e){c("read",e);e=parseInt(e,10);var t=this._readableState;var r=e;if(e!==0)t.emittedReadable=false;if(e===0&&t.needReadable&&((t.highWaterMark!==0?t.length>=t.highWaterMark:t.length>0)||t.ended)){c("read: emitReadable",t.length,t.ended);if(t.length===0&&t.ended)endReadable(this);else emitReadable(this);return null}e=howMuchToRead(e,t);if(e===0&&t.ended){if(t.length===0)endReadable(this);return null}var n=t.needReadable;c("need readable",n);if(t.length===0||t.length-e0)i=fromList(e,t);else i=null;if(i===null){t.needReadable=t.length<=t.highWaterMark;e=0}else{t.length-=e;t.awaitDrain=0}if(t.length===0){if(!t.ended)t.needReadable=true;if(r!==e&&t.ended)endReadable(this)}if(i!==null)this.emit("data",i);return i};function onEofChunk(e,t){c("onEofChunk");if(t.ended)return;if(t.decoder){var r=t.decoder.end();if(r&&r.length){t.buffer.push(r);t.length+=t.objectMode?1:r.length}}t.ended=true;if(t.sync){emitReadable(e)}else{t.needReadable=false;if(!t.emittedReadable){t.emittedReadable=true;emitReadable_(e)}}}function emitReadable(e){var t=e._readableState;c("emitReadable",t.needReadable,t.emittedReadable);t.needReadable=false;if(!t.emittedReadable){c("emitReadable",t.flowing);t.emittedReadable=true;process.nextTick(emitReadable_,e)}}function emitReadable_(e){var t=e._readableState;c("emitReadable_",t.destroyed,t.length,t.ended);if(!t.destroyed&&(t.length||t.ended)){e.emit("readable");t.emittedReadable=false}t.needReadable=!t.flowing&&!t.ended&&t.length<=t.highWaterMark;flow(e)}function maybeReadMore(e,t){if(!t.readingMore){t.readingMore=true;process.nextTick(maybeReadMore_,e,t)}}function maybeReadMore_(e,t){while(!t.reading&&!t.ended&&(t.length1&&indexOf(n.pipes,e)!==-1)&&!u){c("false write response, pause",n.awaitDrain);n.awaitDrain++}r.pause()}}function onerror(t){c("onerror",t);unpipe();e.removeListener("error",onerror);if(a(e,"error")===0)E(e,t)}prependListener(e,"error",onerror);function onclose(){e.removeListener("finish",onfinish);unpipe()}e.once("close",onclose);function onfinish(){c("onfinish");e.removeListener("close",onclose);unpipe()}e.once("finish",onfinish);function unpipe(){c("unpipe");r.unpipe(e)}e.emit("pipe",r);if(!n.flowing){c("pipe resume");r.resume()}return e};function pipeOnDrain(e){return function pipeOnDrainFunctionResult(){var t=e._readableState;c("pipeOnDrain",t.awaitDrain);if(t.awaitDrain)t.awaitDrain--;if(t.awaitDrain===0&&a(e,"data")){t.flowing=true;flow(e)}}}Readable.prototype.unpipe=function(e){var t=this._readableState;var r={hasUnpiped:false};if(t.pipesCount===0)return this;if(t.pipesCount===1){if(e&&e!==t.pipes)return this;if(!e)e=t.pipes;t.pipes=null;t.pipesCount=0;t.flowing=false;if(e)e.emit("unpipe",this,r);return this}if(!e){var n=t.pipes;var i=t.pipesCount;t.pipes=null;t.pipesCount=0;t.flowing=false;for(var a=0;a0;if(n.flowing!==false)this.resume()}else if(e==="readable"){if(!n.endEmitted&&!n.readableListening){n.readableListening=n.needReadable=true;n.flowing=false;n.emittedReadable=false;c("on readable",n.length,n.reading);if(n.length){emitReadable(this)}else if(!n.reading){process.nextTick(nReadingNextTick,this)}}}return r};Readable.prototype.addListener=Readable.prototype.on;Readable.prototype.removeListener=function(e,t){var r=o.prototype.removeListener.call(this,e,t);if(e==="readable"){process.nextTick(updateReadableListening,this)}return r};Readable.prototype.removeAllListeners=function(e){var t=o.prototype.removeAllListeners.apply(this,arguments);if(e==="readable"||e===undefined){process.nextTick(updateReadableListening,this)}return t};function updateReadableListening(e){var t=e._readableState;t.readableListening=e.listenerCount("readable")>0;if(t.resumeScheduled&&!t.paused){t.flowing=true}else if(e.listenerCount("data")>0){e.resume()}}function nReadingNextTick(e){c("readable nexttick read 0");e.read(0)}Readable.prototype.resume=function(){var e=this._readableState;if(!e.flowing){c("resume");e.flowing=!e.readableListening;resume(this,e)}e.paused=false;return this};function resume(e,t){if(!t.resumeScheduled){t.resumeScheduled=true;process.nextTick(resume_,e,t)}}function resume_(e,t){c("resume",t.reading);if(!t.reading){e.read(0)}t.resumeScheduled=false;e.emit("resume");flow(e);if(t.flowing&&!t.reading)e.read(0)}Readable.prototype.pause=function(){c("call pause flowing=%j",this._readableState.flowing);if(this._readableState.flowing!==false){c("pause");this._readableState.flowing=false;this.emit("pause")}this._readableState.paused=true;return this};function flow(e){var t=e._readableState;c("flow",t.flowing);while(t.flowing&&e.read()!==null){}}Readable.prototype.wrap=function(e){var t=this;var r=this._readableState;var n=false;e.on("end",(function(){c("wrapped end");if(r.decoder&&!r.ended){var e=r.decoder.end();if(e&&e.length)t.push(e)}t.push(null)}));e.on("data",(function(i){c("wrapped data");if(r.decoder)i=r.decoder.write(i);if(r.objectMode&&(i===null||i===undefined))return;else if(!r.objectMode&&(!i||!i.length))return;var a=t.push(i);if(!a){n=true;e.pause()}}));for(var i in e){if(this[i]===undefined&&typeof e[i]==="function"){this[i]=function methodWrap(t){return function methodWrapReturnFunction(){return e[t].apply(e,arguments)}}(i)}}for(var a=0;a=t.length){if(t.decoder)r=t.buffer.join("");else if(t.buffer.length===1)r=t.buffer.first();else r=t.buffer.concat(t.length);t.buffer.clear()}else{r=t.buffer.consume(e,t.decoder)}return r}function endReadable(e){var t=e._readableState;c("endReadable",t.endEmitted);if(!t.endEmitted){t.ended=true;process.nextTick(endReadableNT,t,e)}}function endReadableNT(e,t){c("endReadableNT",e.endEmitted,e.length);if(!e.endEmitted&&e.length===0){e.endEmitted=true;t.readable=false;t.emit("end");if(e.autoDestroy){var r=t._writableState;if(!r||r.autoDestroy&&r.finished){t.destroy()}}}}if(typeof Symbol==="function"){Readable.from=function(e,t){if(S===undefined){S=r(9082)}return S(Readable,e,t)}}function indexOf(e,t){for(var r=0,n=e.length;r{e.exports=Transform;var n=r(7214).q,i=n.ERR_METHOD_NOT_IMPLEMENTED,a=n.ERR_MULTIPLE_CALLBACK,o=n.ERR_TRANSFORM_ALREADY_TRANSFORMING,s=n.ERR_TRANSFORM_WITH_LENGTH_0;var u=r(1359);r(4124)(Transform,u);function afterTransform(e,t){var r=this._transformState;r.transforming=false;var n=r.writecb;if(n===null){return this.emit("error",new a)}r.writechunk=null;r.writecb=null;if(t!=null)this.push(t);n(e);var i=this._readableState;i.reading=false;if(i.needReadable||i.length{e.exports=Writable;function WriteReq(e,t,r){this.chunk=e;this.encoding=t;this.callback=r;this.next=null}function CorkedRequest(e){var t=this;this.next=null;this.entry=null;this.finish=function(){onCorkedFinish(t,e)}}var n;Writable.WritableState=WritableState;var i={deprecate:r(5278)};var a=r(2387);var o=r(4300).Buffer;var s=global.Uint8Array||function(){};function _uint8ArrayToBuffer(e){return o.from(e)}function _isUint8Array(e){return o.isBuffer(e)||e instanceof s}var u=r(7049);var l=r(9948),c=l.getHighWaterMark;var d=r(7214).q,h=d.ERR_INVALID_ARG_TYPE,p=d.ERR_METHOD_NOT_IMPLEMENTED,g=d.ERR_MULTIPLE_CALLBACK,m=d.ERR_STREAM_CANNOT_PIPE,b=d.ERR_STREAM_DESTROYED,v=d.ERR_STREAM_NULL_VALUES,_=d.ERR_STREAM_WRITE_AFTER_END,y=d.ERR_UNKNOWN_ENCODING;var w=u.errorOrDestroy;r(4124)(Writable,a);function nop(){}function WritableState(e,t,i){n=n||r(1359);e=e||{};if(typeof i!=="boolean")i=t instanceof n;this.objectMode=!!e.objectMode;if(i)this.objectMode=this.objectMode||!!e.writableObjectMode;this.highWaterMark=c(this,e,"writableHighWaterMark",i);this.finalCalled=false;this.needDrain=false;this.ending=false;this.ended=false;this.finished=false;this.destroyed=false;var a=e.decodeStrings===false;this.decodeStrings=!a;this.defaultEncoding=e.defaultEncoding||"utf8";this.length=0;this.writing=false;this.corked=0;this.sync=true;this.bufferProcessing=false;this.onwrite=function(e){onwrite(t,e)};this.writecb=null;this.writelen=0;this.bufferedRequest=null;this.lastBufferedRequest=null;this.pendingcb=0;this.prefinished=false;this.errorEmitted=false;this.emitClose=e.emitClose!==false;this.autoDestroy=!!e.autoDestroy;this.bufferedRequestCount=0;this.corkedRequestsFree=new CorkedRequest(this)}WritableState.prototype.getBuffer=function getBuffer(){var e=this.bufferedRequest;var t=[];while(e){t.push(e);e=e.next}return t};(function(){try{Object.defineProperty(WritableState.prototype,"buffer",{get:i.deprecate((function writableStateBufferGetter(){return this.getBuffer()}),"_writableState.buffer is deprecated. Use _writableState.getBuffer "+"instead.","DEP0003")})}catch(e){}})();var R;if(typeof Symbol==="function"&&Symbol.hasInstance&&typeof Function.prototype[Symbol.hasInstance]==="function"){R=Function.prototype[Symbol.hasInstance];Object.defineProperty(Writable,Symbol.hasInstance,{value:function value(e){if(R.call(this,e))return true;if(this!==Writable)return false;return e&&e._writableState instanceof WritableState}})}else{R=function realHasInstance(e){return e instanceof this}}function Writable(e){n=n||r(1359);var t=this instanceof n;if(!t&&!R.call(Writable,this))return new Writable(e);this._writableState=new WritableState(e,this,t);this.writable=true;if(e){if(typeof e.write==="function")this._write=e.write;if(typeof e.writev==="function")this._writev=e.writev;if(typeof e.destroy==="function")this._destroy=e.destroy;if(typeof e.final==="function")this._final=e.final}a.call(this)}Writable.prototype.pipe=function(){w(this,new m)};function writeAfterEnd(e,t){var r=new _;w(e,r);process.nextTick(t,r)}function validChunk(e,t,r,n){var i;if(r===null){i=new v}else if(typeof r!=="string"&&!t.objectMode){i=new h("chunk",["string","Buffer"],r)}if(i){w(e,i);process.nextTick(n,i);return false}return true}Writable.prototype.write=function(e,t,r){var n=this._writableState;var i=false;var a=!n.objectMode&&_isUint8Array(e);if(a&&!o.isBuffer(e)){e=_uint8ArrayToBuffer(e)}if(typeof t==="function"){r=t;t=null}if(a)t="buffer";else if(!t)t=n.defaultEncoding;if(typeof r!=="function")r=nop;if(n.ending)writeAfterEnd(this,r);else if(a||validChunk(this,n,e,r)){n.pendingcb++;i=writeOrBuffer(this,n,a,e,t,r)}return i};Writable.prototype.cork=function(){this._writableState.corked++};Writable.prototype.uncork=function(){var e=this._writableState;if(e.corked){e.corked--;if(!e.writing&&!e.corked&&!e.bufferProcessing&&e.bufferedRequest)clearBuffer(this,e)}};Writable.prototype.setDefaultEncoding=function setDefaultEncoding(e){if(typeof e==="string")e=e.toLowerCase();if(!(["hex","utf8","utf-8","ascii","binary","base64","ucs2","ucs-2","utf16le","utf-16le","raw"].indexOf((e+"").toLowerCase())>-1))throw new y(e);this._writableState.defaultEncoding=e;return this};Object.defineProperty(Writable.prototype,"writableBuffer",{enumerable:false,get:function get(){return this._writableState&&this._writableState.getBuffer()}});function decodeChunk(e,t,r){if(!e.objectMode&&e.decodeStrings!==false&&typeof t==="string"){t=o.from(t,r)}return t}Object.defineProperty(Writable.prototype,"writableHighWaterMark",{enumerable:false,get:function get(){return this._writableState.highWaterMark}});function writeOrBuffer(e,t,r,n,i,a){if(!r){var o=decodeChunk(t,n,i);if(n!==o){r=true;i="buffer";n=o}}var s=t.objectMode?1:n.length;t.length+=s;var u=t.length{var n;function _defineProperty(e,t,r){if(t in e){Object.defineProperty(e,t,{value:r,enumerable:true,configurable:true,writable:true})}else{e[t]=r}return e}var i=r(6080);var a=Symbol("lastResolve");var o=Symbol("lastReject");var s=Symbol("error");var u=Symbol("ended");var l=Symbol("lastPromise");var c=Symbol("handlePromise");var d=Symbol("stream");function createIterResult(e,t){return{value:e,done:t}}function readAndResolve(e){var t=e[a];if(t!==null){var r=e[d].read();if(r!==null){e[l]=null;e[a]=null;e[o]=null;t(createIterResult(r,false))}}}function onReadable(e){process.nextTick(readAndResolve,e)}function wrapForNext(e,t){return function(r,n){e.then((function(){if(t[u]){r(createIterResult(undefined,true));return}t[c](r,n)}),n)}}var h=Object.getPrototypeOf((function(){}));var p=Object.setPrototypeOf((n={get stream(){return this[d]},next:function next(){var e=this;var t=this[s];if(t!==null){return Promise.reject(t)}if(this[u]){return Promise.resolve(createIterResult(undefined,true))}if(this[d].destroyed){return new Promise((function(t,r){process.nextTick((function(){if(e[s]){r(e[s])}else{t(createIterResult(undefined,true))}}))}))}var r=this[l];var n;if(r){n=new Promise(wrapForNext(r,this))}else{var i=this[d].read();if(i!==null){return Promise.resolve(createIterResult(i,false))}n=new Promise(this[c])}this[l]=n;return n}},_defineProperty(n,Symbol.asyncIterator,(function(){return this})),_defineProperty(n,"return",(function _return(){var e=this;return new Promise((function(t,r){e[d].destroy(null,(function(e){if(e){r(e);return}t(createIterResult(undefined,true))}))}))})),n),h);var g=function createReadableStreamAsyncIterator(e){var t;var r=Object.create(p,(t={},_defineProperty(t,d,{value:e,writable:true}),_defineProperty(t,a,{value:null,writable:true}),_defineProperty(t,o,{value:null,writable:true}),_defineProperty(t,s,{value:null,writable:true}),_defineProperty(t,u,{value:e._readableState.endEmitted,writable:true}),_defineProperty(t,c,{value:function value(e,t){var n=r[d].read();if(n){r[l]=null;r[a]=null;r[o]=null;e(createIterResult(n,false))}else{r[a]=e;r[o]=t}},writable:true}),t));r[l]=null;i(e,(function(e){if(e&&e.code!=="ERR_STREAM_PREMATURE_CLOSE"){var t=r[o];if(t!==null){r[l]=null;r[a]=null;r[o]=null;t(e)}r[s]=e;return}var n=r[a];if(n!==null){r[l]=null;r[a]=null;r[o]=null;n(createIterResult(undefined,true))}r[u]=true}));e.on("readable",onReadable.bind(null,r));return r};e.exports=g},2746:(e,t,r)=>{function ownKeys(e,t){var r=Object.keys(e);if(Object.getOwnPropertySymbols){var n=Object.getOwnPropertySymbols(e);if(t)n=n.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}));r.push.apply(r,n)}return r}function _objectSpread(e){for(var t=1;t0)this.tail.next=t;else this.head=t;this.tail=t;++this.length}},{key:"unshift",value:function unshift(e){var t={data:e,next:this.head};if(this.length===0)this.tail=t;this.head=t;++this.length}},{key:"shift",value:function shift(){if(this.length===0)return;var e=this.head.data;if(this.length===1)this.head=this.tail=null;else this.head=this.head.next;--this.length;return e}},{key:"clear",value:function clear(){this.head=this.tail=null;this.length=0}},{key:"join",value:function join(e){if(this.length===0)return"";var t=this.head;var r=""+t.data;while(t=t.next){r+=e+t.data}return r}},{key:"concat",value:function concat(e){if(this.length===0)return i.alloc(0);var t=i.allocUnsafe(e>>>0);var r=this.head;var n=0;while(r){copyBuffer(r.data,t,n);n+=r.data.length;r=r.next}return t}},{key:"consume",value:function consume(e,t){var r;if(ei.length?i.length:e;if(a===i.length)n+=i;else n+=i.slice(0,e);e-=a;if(e===0){if(a===i.length){++r;if(t.next)this.head=t.next;else this.head=this.tail=null}else{this.head=t;t.data=i.slice(a)}break}++r}this.length-=r;return n}},{key:"_getBuffer",value:function _getBuffer(e){var t=i.allocUnsafe(e);var r=this.head;var n=1;r.data.copy(t);e-=r.data.length;while(r=r.next){var a=r.data;var o=e>a.length?a.length:e;a.copy(t,t.length-e,0,o);e-=o;if(e===0){if(o===a.length){++n;if(r.next)this.head=r.next;else this.head=this.tail=null}else{this.head=r;r.data=a.slice(o)}break}++n}this.length-=n;return t}},{key:s,value:function value(e,t){return o(this,_objectSpread({},t,{depth:0,customInspect:false}))}}]);return BufferList}()},7049:e=>{function destroy(e,t){var r=this;var n=this._readableState&&this._readableState.destroyed;var i=this._writableState&&this._writableState.destroyed;if(n||i){if(t){t(e)}else if(e){if(!this._writableState){process.nextTick(emitErrorNT,this,e)}else if(!this._writableState.errorEmitted){this._writableState.errorEmitted=true;process.nextTick(emitErrorNT,this,e)}}return this}if(this._readableState){this._readableState.destroyed=true}if(this._writableState){this._writableState.destroyed=true}this._destroy(e||null,(function(e){if(!t&&e){if(!r._writableState){process.nextTick(emitErrorAndCloseNT,r,e)}else if(!r._writableState.errorEmitted){r._writableState.errorEmitted=true;process.nextTick(emitErrorAndCloseNT,r,e)}else{process.nextTick(emitCloseNT,r)}}else if(t){process.nextTick(emitCloseNT,r);t(e)}else{process.nextTick(emitCloseNT,r)}}));return this}function emitErrorAndCloseNT(e,t){emitErrorNT(e,t);emitCloseNT(e)}function emitCloseNT(e){if(e._writableState&&!e._writableState.emitClose)return;if(e._readableState&&!e._readableState.emitClose)return;e.emit("close")}function undestroy(){if(this._readableState){this._readableState.destroyed=false;this._readableState.reading=false;this._readableState.ended=false;this._readableState.endEmitted=false}if(this._writableState){this._writableState.destroyed=false;this._writableState.ended=false;this._writableState.ending=false;this._writableState.finalCalled=false;this._writableState.prefinished=false;this._writableState.finished=false;this._writableState.errorEmitted=false}}function emitErrorNT(e,t){e.emit("error",t)}function errorOrDestroy(e,t){var r=e._readableState;var n=e._writableState;if(r&&r.autoDestroy||n&&n.autoDestroy)e.destroy(t);else e.emit("error",t)}e.exports={destroy:destroy,undestroy:undestroy,errorOrDestroy:errorOrDestroy}},6080:(e,t,r)=>{var n=r(7214).q.ERR_STREAM_PREMATURE_CLOSE;function once(e){var t=false;return function(){if(t)return;t=true;for(var r=arguments.length,n=new Array(r),i=0;i{function asyncGeneratorStep(e,t,r,n,i,a,o){try{var s=e[a](o);var u=s.value}catch(e){r(e);return}if(s.done){t(u)}else{Promise.resolve(u).then(n,i)}}function _asyncToGenerator(e){return function(){var t=this,r=arguments;return new Promise((function(n,i){var a=e.apply(t,r);function _next(e){asyncGeneratorStep(a,n,i,_next,_throw,"next",e)}function _throw(e){asyncGeneratorStep(a,n,i,_next,_throw,"throw",e)}_next(undefined)}))}}function ownKeys(e,t){var r=Object.keys(e);if(Object.getOwnPropertySymbols){var n=Object.getOwnPropertySymbols(e);if(t)n=n.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}));r.push.apply(r,n)}return r}function _objectSpread(e){for(var t=1;t{var n;function once(e){var t=false;return function(){if(t)return;t=true;e.apply(void 0,arguments)}}var i=r(7214).q,a=i.ERR_MISSING_ARGS,o=i.ERR_STREAM_DESTROYED;function noop(e){if(e)throw e}function isRequest(e){return e.setHeader&&typeof e.abort==="function"}function destroyer(e,t,i,a){a=once(a);var s=false;e.on("close",(function(){s=true}));if(n===undefined)n=r(6080);n(e,{readable:t,writable:i},(function(e){if(e)return a(e);s=true;a()}));var u=false;return function(t){if(s)return;if(u)return;u=true;if(isRequest(e))return e.abort();if(typeof e.destroy==="function")return e.destroy();a(t||new o("pipe"))}}function call(e){e()}function pipe(e,t){return e.pipe(t)}function popCallback(e){if(!e.length)return noop;if(typeof e[e.length-1]!=="function")return noop;return e.pop()}function pipeline(){for(var e=arguments.length,t=new Array(e),r=0;r0;return destroyer(e,a,s,(function(e){if(!i)i=e;if(e)o.forEach(call);if(a)return;o.forEach(call);n(i)}))}));return t.reduce(pipe)}e.exports=pipeline},9948:(e,t,r)=>{var n=r(7214).q.ERR_INVALID_OPT_VALUE;function highWaterMarkFrom(e,t,r){return e.highWaterMark!=null?e.highWaterMark:t?e[r]:null}function getHighWaterMark(e,t,r,i){var a=highWaterMarkFrom(t,i,r);if(a!=null){if(!(isFinite(a)&&Math.floor(a)===a)||a<0){var o=i?r:"highWaterMark";throw new n(o,a)}return Math.floor(a)}return e.objectMode?16:16*1024}e.exports={getHighWaterMark:getHighWaterMark}},2387:(e,t,r)=>{e.exports=r(2781)},1642:(e,t,r)=>{var n=r(2781);if(process.env.READABLE_STREAM==="disable"&&n){e.exports=n.Readable;Object.assign(e.exports,n);e.exports.Stream=n}else{t=e.exports=r(1433);t.Stream=n||t;t.Readable=t;t.Writable=r(6993);t.Duplex=r(1359);t.Transform=r(4415);t.PassThrough=r(1542);t.finished=r(6080);t.pipeline=r(6989)}},1867:(e,t,r)=>{ /*! safe-buffer. MIT License. Feross Aboukhadijeh */ -var n=r(4300);var i=n.Buffer;function copyProps(e,t){for(var r in e){t[r]=e[r]}}if(i.from&&i.alloc&&i.allocUnsafe&&i.allocUnsafeSlow){e.exports=n}else{copyProps(n,t);t.Buffer=SafeBuffer}function SafeBuffer(e,t,r){return i(e,t,r)}SafeBuffer.prototype=Object.create(i.prototype);copyProps(i,SafeBuffer);SafeBuffer.from=function(e,t,r){if(typeof e==="number"){throw new TypeError("Argument must not be a number")}return i(e,t,r)};SafeBuffer.alloc=function(e,t,r){if(typeof e!=="number"){throw new TypeError("Argument must be a number")}var n=i(e);if(t!==undefined){if(typeof r==="string"){n.fill(t,r)}else{n.fill(t)}}else{n.fill(0)}return n};SafeBuffer.allocUnsafe=function(e){if(typeof e!=="number"){throw new TypeError("Argument must be a number")}return i(e)};SafeBuffer.allocUnsafeSlow=function(e){if(typeof e!=="number"){throw new TypeError("Argument must be a number")}return n.SlowBuffer(e)}},4841:(e,t,r)=>{var n=r(1867).Buffer;var i=n.isEncoding||function(e){e=""+e;switch(e&&e.toLowerCase()){case"hex":case"utf8":case"utf-8":case"ascii":case"binary":case"base64":case"ucs2":case"ucs-2":case"utf16le":case"utf-16le":case"raw":return true;default:return false}};function _normalizeEncoding(e){if(!e)return"utf8";var t;while(true){switch(e){case"utf8":case"utf-8":return"utf8";case"ucs2":case"ucs-2":case"utf16le":case"utf-16le":return"utf16le";case"latin1":case"binary":return"latin1";case"base64":case"ascii":case"hex":return e;default:if(t)return;e=(""+e).toLowerCase();t=true}}}function normalizeEncoding(e){var t=_normalizeEncoding(e);if(typeof t!=="string"&&(n.isEncoding===i||!i(e)))throw new Error("Unknown encoding: "+e);return t||e}t.s=StringDecoder;function StringDecoder(e){this.encoding=normalizeEncoding(e);var t;switch(this.encoding){case"utf16le":this.text=utf16Text;this.end=utf16End;t=4;break;case"utf8":this.fillLast=utf8FillLast;t=4;break;case"base64":this.text=base64Text;this.end=base64End;t=3;break;default:this.write=simpleWrite;this.end=simpleEnd;return}this.lastNeed=0;this.lastTotal=0;this.lastChar=n.allocUnsafe(t)}StringDecoder.prototype.write=function(e){if(e.length===0)return"";var t;var r;if(this.lastNeed){t=this.fillLast(e);if(t===undefined)return"";r=this.lastNeed;this.lastNeed=0}else{r=0}if(r>5===6)return 2;else if(e>>4===14)return 3;else if(e>>3===30)return 4;return e>>6===2?-1:-2}function utf8CheckIncomplete(e,t,r){var n=t.length-1;if(n=0){if(i>0)e.lastNeed=i-1;return i}if(--n=0){if(i>0)e.lastNeed=i-2;return i}if(--n=0){if(i>0){if(i===2)i=0;else e.lastNeed=i-3}return i}return 0}function utf8CheckExtraBytes(e,t,r){if((t[0]&192)!==128){e.lastNeed=0;return"�"}if(e.lastNeed>1&&t.length>1){if((t[1]&192)!==128){e.lastNeed=1;return"�"}if(e.lastNeed>2&&t.length>2){if((t[2]&192)!==128){e.lastNeed=2;return"�"}}}}function utf8FillLast(e){var t=this.lastTotal-this.lastNeed;var r=utf8CheckExtraBytes(this,e,t);if(r!==undefined)return r;if(this.lastNeed<=e.length){e.copy(this.lastChar,t,0,this.lastNeed);return this.lastChar.toString(this.encoding,0,this.lastTotal)}e.copy(this.lastChar,t,0,e.length);this.lastNeed-=e.length}function utf8Text(e,t){var r=utf8CheckIncomplete(this,e,t);if(!this.lastNeed)return e.toString("utf8",t);this.lastTotal=r;var n=e.length-(r-this.lastNeed);e.copy(this.lastChar,0,n);return e.toString("utf8",t,n)}function utf8End(e){var t=e&&e.length?this.write(e):"";if(this.lastNeed)return t+"�";return t}function utf16Text(e,t){if((e.length-t)%2===0){var r=e.toString("utf16le",t);if(r){var n=r.charCodeAt(r.length-1);if(n>=55296&&n<=56319){this.lastNeed=2;this.lastTotal=4;this.lastChar[0]=e[e.length-2];this.lastChar[1]=e[e.length-1];return r.slice(0,-1)}}return r}this.lastNeed=1;this.lastTotal=2;this.lastChar[0]=e[e.length-1];return e.toString("utf16le",t,e.length-1)}function utf16End(e){var t=e&&e.length?this.write(e):"";if(this.lastNeed){var r=this.lastTotal-this.lastNeed;return t+this.lastChar.toString("utf16le",0,r)}return t}function base64Text(e,t){var r=(e.length-t)%3;if(r===0)return e.toString("base64",t);this.lastNeed=3-r;this.lastTotal=3;if(r===1){this.lastChar[0]=e[e.length-1]}else{this.lastChar[0]=e[e.length-2];this.lastChar[1]=e[e.length-1]}return e.toString("base64",t,e.length-r)}function base64End(e){var t=e&&e.length?this.write(e):"";if(this.lastNeed)return t+this.lastChar.toString("base64",0,3-this.lastNeed);return t}function simpleWrite(e){return e.toString(this.encoding)}function simpleEnd(e){return e&&e.length?this.write(e):""}},7882:(e,t,r)=>{var n=r(3837);var i=r(336);var a=r(8860);var o=r(1642).Writable;var s=r(1642).PassThrough;var noop=function(){};var overflow=function(e){e&=511;return e&&512-e};var emptyStream=function(e,t){var r=new Source(e,t);r.end();return r};var mixinPax=function(e,t){if(t.path)e.name=t.path;if(t.linkpath)e.linkname=t.linkpath;if(t.size)e.size=parseInt(t.size,10);e.pax=t;return e};var Source=function(e,t){this._parent=e;this.offset=t;s.call(this,{autoDestroy:false})};n.inherits(Source,s);Source.prototype.destroy=function(e){this._parent.destroy(e)};var Extract=function(e){if(!(this instanceof Extract))return new Extract(e);o.call(this,e);e=e||{};this._offset=0;this._buffer=i();this._missing=0;this._partial=false;this._onparse=noop;this._header=null;this._stream=null;this._overflow=null;this._cb=null;this._locked=false;this._destroyed=false;this._pax=null;this._paxGlobal=null;this._gnuLongPath=null;this._gnuLongLinkPath=null;var t=this;var r=t._buffer;var oncontinue=function(){t._continue()};var onunlock=function(e){t._locked=false;if(e)return t.destroy(e);if(!t._stream)oncontinue()};var onstreamend=function(){t._stream=null;var e=overflow(t._header.size);if(e)t._parse(e,ondrain);else t._parse(512,onheader);if(!t._locked)oncontinue()};var ondrain=function(){t._buffer.consume(overflow(t._header.size));t._parse(512,onheader);oncontinue()};var onpaxglobalheader=function(){var e=t._header.size;t._paxGlobal=a.decodePax(r.slice(0,e));r.consume(e);onstreamend()};var onpaxheader=function(){var e=t._header.size;t._pax=a.decodePax(r.slice(0,e));if(t._paxGlobal)t._pax=Object.assign({},t._paxGlobal,t._pax);r.consume(e);onstreamend()};var ongnulongpath=function(){var n=t._header.size;this._gnuLongPath=a.decodeLongPath(r.slice(0,n),e.filenameEncoding);r.consume(n);onstreamend()};var ongnulonglinkpath=function(){var n=t._header.size;this._gnuLongLinkPath=a.decodeLongPath(r.slice(0,n),e.filenameEncoding);r.consume(n);onstreamend()};var onheader=function(){var n=t._offset;var i;try{i=t._header=a.decode(r.slice(0,512),e.filenameEncoding,e.allowUnknownFormat)}catch(e){t.emit("error",e)}r.consume(512);if(!i){t._parse(512,onheader);oncontinue();return}if(i.type==="gnu-long-path"){t._parse(i.size,ongnulongpath);oncontinue();return}if(i.type==="gnu-long-link-path"){t._parse(i.size,ongnulonglinkpath);oncontinue();return}if(i.type==="pax-global-header"){t._parse(i.size,onpaxglobalheader);oncontinue();return}if(i.type==="pax-header"){t._parse(i.size,onpaxheader);oncontinue();return}if(t._gnuLongPath){i.name=t._gnuLongPath;t._gnuLongPath=null}if(t._gnuLongLinkPath){i.linkname=t._gnuLongLinkPath;t._gnuLongLinkPath=null}if(t._pax){t._header=i=mixinPax(i,t._pax);t._pax=null}t._locked=true;if(!i.size||i.type==="directory"){t._parse(512,onheader);t.emit("entry",i,emptyStream(t,n),onunlock);return}t._stream=new Source(t,n);t.emit("entry",i,t._stream,onunlock);t._parse(i.size,onstreamend);oncontinue()};this._onheader=onheader;this._parse(512,onheader)};n.inherits(Extract,o);Extract.prototype.destroy=function(e){if(this._destroyed)return;this._destroyed=true;if(e)this.emit("error",e);this.emit("close");if(this._stream)this._stream.emit("close")};Extract.prototype._parse=function(e,t){if(this._destroyed)return;this._offset+=e;this._missing=e;if(t===this._onheader)this._partial=false;this._onparse=t};Extract.prototype._continue=function(){if(this._destroyed)return;var e=this._cb;this._cb=noop;if(this._overflow)this._write(this._overflow,undefined,e);else e()};Extract.prototype._write=function(e,t,r){if(this._destroyed)return;var n=this._stream;var i=this._buffer;var a=this._missing;if(e.length)this._partial=true;if(e.lengtha){o=e.slice(a);e=e.slice(0,a)}if(n)n.end(e);else i.append(e);this._overflow=o;this._onparse()};Extract.prototype._final=function(e){if(this._partial)return this.destroy(new Error("Unexpected end of data"));e()};e.exports=Extract},8860:(e,t)=>{var r=Buffer.alloc;var n="0000000000000000000";var i="7777777777777777777";var a="0".charCodeAt(0);var o=Buffer.from("ustar\0","binary");var s=Buffer.from("00","binary");var u=Buffer.from("ustar ","binary");var l=Buffer.from(" \0","binary");var c=parseInt("7777",8);var d=257;var h=263;var clamp=function(e,t,r){if(typeof e!=="number")return r;e=~~e;if(e>=t)return t;if(e>=0)return e;e+=t;if(e>=0)return e;return 0};var toType=function(e){switch(e){case 0:return"file";case 1:return"link";case 2:return"symlink";case 3:return"character-device";case 4:return"block-device";case 5:return"directory";case 6:return"fifo";case 7:return"contiguous-file";case 72:return"pax-header";case 55:return"pax-global-header";case 27:return"gnu-long-link-path";case 28:case 30:return"gnu-long-path"}return null};var toTypeflag=function(e){switch(e){case"file":return 0;case"link":return 1;case"symlink":return 2;case"character-device":return 3;case"block-device":return 4;case"directory":return 5;case"fifo":return 6;case"contiguous-file":return 7;case"pax-header":return 72}return 0};var indexOf=function(e,t,r,n){for(;rt)return i.slice(0,t)+" ";else return n.slice(0,t-e.length)+e+" "};function parse256(e){var t;if(e[0]===128)t=true;else if(e[0]===255)t=false;else return null;var r=[];for(var n=e.length-1;n>0;n--){var i=e[n];if(t)r.push(i);else r.push(255-i)}var a=0;var o=r.length;for(n=0;n=Math.pow(10,r))r++;return t+r+e};t.decodeLongPath=function(e,t){return decodeStr(e,0,e.length,t)};t.encodePax=function(e){var t="";if(e.name)t+=addLength(" path="+e.name+"\n");if(e.linkname)t+=addLength(" linkpath="+e.linkname+"\n");var r=e.pax;if(r){for(var n in r){t+=addLength(" "+n+"="+r[n]+"\n")}}return Buffer.from(t)};t.decodePax=function(e){var t={};while(e.length){var r=0;while(r100){var u=n.indexOf("/");if(u===-1)return null;i+=i?"/"+n.slice(0,u):n.slice(0,u);n=n.slice(u+1)}if(Buffer.byteLength(n)>100||Buffer.byteLength(i)>155)return null;if(e.linkname&&Buffer.byteLength(e.linkname)>100)return null;t.write(n);t.write(encodeOct(e.mode&c,6),100);t.write(encodeOct(e.uid,6),108);t.write(encodeOct(e.gid,6),116);t.write(encodeOct(e.size,11),124);t.write(encodeOct(e.mtime.getTime()/1e3|0,11),136);t[156]=a+toTypeflag(e.type);if(e.linkname)t.write(e.linkname,157);o.copy(t,d);s.copy(t,h);if(e.uname)t.write(e.uname,265);if(e.gname)t.write(e.gname,297);t.write(encodeOct(e.devmajor||0,6),329);t.write(encodeOct(e.devminor||0,6),337);if(i)t.write(i,345);t.write(encodeOct(cksum(t),6),148);return t};t.decode=function(e,t,r){var n=e[156]===0?0:e[156]-a;var i=decodeStr(e,0,100,t);var s=decodeOct(e,100,8);var c=decodeOct(e,108,8);var p=decodeOct(e,116,8);var g=decodeOct(e,124,12);var m=decodeOct(e,136,12);var b=toType(n);var v=e[157]===0?null:decodeStr(e,157,100,t);var _=decodeStr(e,265,32);var y=decodeStr(e,297,32);var w=decodeOct(e,329,8);var R=decodeOct(e,337,8);var S=cksum(e);if(S===8*32)return null;if(S!==decodeOct(e,148,8))throw new Error("Invalid tar header. Maybe the tar is corrupted or it needs to be gunzipped?");if(o.compare(e,d,d+6)===0){if(e[345])i=decodeStr(e,345,155,t)+"/"+i}else if(u.compare(e,d,d+6)===0&&l.compare(e,h,h+2)===0){}else{if(!r){throw new Error("Invalid tar header: unknown format.")}}if(n===0&&i&&i[i.length-1]==="/")n=5;return{name:i,mode:s,uid:c,gid:p,size:g,mtime:new Date(1e3*m),type:b,linkname:v,uname:_,gname:y,devmajor:w,devminor:R}}},2283:(e,t,r)=>{t.extract=r(7882);t.pack=r(4930)},4930:(e,t,r)=>{var n=r(3186);var i=r(1205);var a=r(4124);var o=Buffer.alloc;var s=r(1642).Readable;var u=r(1642).Writable;var l=r(1576).StringDecoder;var c=r(8860);var d=parseInt("755",8);var h=parseInt("644",8);var p=o(1024);var noop=function(){};var overflow=function(e,t){t&=511;if(t)e.push(p.slice(0,512-t))};function modeToType(e){switch(e&n.S_IFMT){case n.S_IFBLK:return"block-device";case n.S_IFCHR:return"character-device";case n.S_IFDIR:return"directory";case n.S_IFIFO:return"fifo";case n.S_IFLNK:return"symlink"}return"file"}var Sink=function(e){u.call(this);this.written=0;this._to=e;this._destroyed=false};a(Sink,u);Sink.prototype._write=function(e,t,r){this.written+=e.length;if(this._to.push(e))return r();this._to._drain=r};Sink.prototype.destroy=function(){if(this._destroyed)return;this._destroyed=true;this.emit("close")};var LinkSink=function(){u.call(this);this.linkname="";this._decoder=new l("utf-8");this._destroyed=false};a(LinkSink,u);LinkSink.prototype._write=function(e,t,r){this.linkname+=this._decoder.write(e);r()};LinkSink.prototype.destroy=function(){if(this._destroyed)return;this._destroyed=true;this.emit("close")};var Void=function(){u.call(this);this._destroyed=false};a(Void,u);Void.prototype._write=function(e,t,r){r(new Error("No body allowed for this entry"))};Void.prototype.destroy=function(){if(this._destroyed)return;this._destroyed=true;this.emit("close")};var Pack=function(e){if(!(this instanceof Pack))return new Pack(e);s.call(this,e);this._drain=noop;this._finalized=false;this._finalizing=false;this._destroyed=false;this._stream=null};a(Pack,s);Pack.prototype.entry=function(e,t,r){if(this._stream)throw new Error("already piping an entry");if(this._finalized||this._destroyed)return;if(typeof t==="function"){r=t;t=null}if(!r)r=noop;var n=this;if(!e.size||e.type==="symlink")e.size=0;if(!e.type)e.type=modeToType(e.mode);if(!e.mode)e.mode=e.type==="directory"?d:h;if(!e.uid)e.uid=0;if(!e.gid)e.gid=0;if(!e.mtime)e.mtime=new Date;if(typeof t==="string")t=Buffer.from(t);if(Buffer.isBuffer(t)){e.size=t.length;this._encode(e);var a=this.push(t);overflow(n,e.size);if(a)process.nextTick(r);else this._drain=r;return new Void}if(e.type==="symlink"&&!e.linkname){var o=new LinkSink;i(o,(function(t){if(t){n.destroy();return r(t)}e.linkname=o.linkname;n._encode(e);r()}));return o}this._encode(e);if(e.type!=="file"&&e.type!=="contiguous-file"){process.nextTick(r);return new Void}var s=new Sink(this);this._stream=s;i(s,(function(t){n._stream=null;if(t){n.destroy();return r(t)}if(s.written!==e.size){n.destroy();return r(new Error("size mismatch"))}overflow(n,e.size);if(n._finalizing)n.finalize();r()}));return s};Pack.prototype.finalize=function(){if(this._stream){this._finalizing=true;return}if(this._finalized)return;this._finalized=true;this.push(p);this.push(null)};Pack.prototype.destroy=function(e){if(this._destroyed)return;this._destroyed=true;if(e)this.emit("error",e);this.emit("close");if(this._stream&&this._stream.destroy)this._stream.destroy()};Pack.prototype._encode=function(e){if(!e.pax){var t=c.encode(e);if(t){this.push(t);return}}this._encodePax(e)};Pack.prototype._encodePax=function(e){var t=c.encodePax({name:e.name,linkname:e.linkname,pax:e.pax});var r={name:"PaxHeader",mode:e.mode,uid:e.uid,gid:e.gid,size:t.length,mtime:e.mtime,type:"pax-header",linkname:e.linkname&&"PaxHeader",uname:e.uname,gname:e.gname,devmajor:e.devmajor,devminor:e.devminor};this.push(c.encode(r));this.push(t);overflow(this,t.length);r.size=e.size;r.type=e.type;this.push(c.encode(r))};Pack.prototype._read=function(e){var t=this._drain;this._drain=noop;t()};e.exports=Pack},4294:(e,t,r)=>{e.exports=r(4219)},4219:(e,t,r)=>{var n=r(1808);var i=r(4404);var a=r(3685);var o=r(5687);var s=r(2361);var u=r(9491);var l=r(3837);t.httpOverHttp=httpOverHttp;t.httpsOverHttp=httpsOverHttp;t.httpOverHttps=httpOverHttps;t.httpsOverHttps=httpsOverHttps;function httpOverHttp(e){var t=new TunnelingAgent(e);t.request=a.request;return t}function httpsOverHttp(e){var t=new TunnelingAgent(e);t.request=a.request;t.createSocket=createSecureSocket;t.defaultPort=443;return t}function httpOverHttps(e){var t=new TunnelingAgent(e);t.request=o.request;return t}function httpsOverHttps(e){var t=new TunnelingAgent(e);t.request=o.request;t.createSocket=createSecureSocket;t.defaultPort=443;return t}function TunnelingAgent(e){var t=this;t.options=e||{};t.proxyOptions=t.options.proxy||{};t.maxSockets=t.options.maxSockets||a.Agent.defaultMaxSockets;t.requests=[];t.sockets=[];t.on("free",(function onFree(e,r,n,i){var a=toOptions(r,n,i);for(var o=0,s=t.requests.length;o=this.maxSockets){i.requests.push(a);return}i.createSocket(a,(function(t){t.on("free",onFree);t.on("close",onCloseOrRemove);t.on("agentRemove",onCloseOrRemove);e.onSocket(t);function onFree(){i.emit("free",t,a)}function onCloseOrRemove(e){i.removeSocket(t);t.removeListener("free",onFree);t.removeListener("close",onCloseOrRemove);t.removeListener("agentRemove",onCloseOrRemove)}}))};TunnelingAgent.prototype.createSocket=function createSocket(e,t){var r=this;var n={};r.sockets.push(n);var i=mergeOptions({},r.proxyOptions,{method:"CONNECT",path:e.host+":"+e.port,agent:false,headers:{host:e.host+":"+e.port}});if(e.localAddress){i.localAddress=e.localAddress}if(i.proxyAuth){i.headers=i.headers||{};i.headers["Proxy-Authorization"]="Basic "+new Buffer(i.proxyAuth).toString("base64")}c("making CONNECT request");var a=r.request(i);a.useChunkedEncodingByDefault=false;a.once("response",onResponse);a.once("upgrade",onUpgrade);a.once("connect",onConnect);a.once("error",onError);a.end();function onResponse(e){e.upgrade=true}function onUpgrade(e,t,r){process.nextTick((function(){onConnect(e,t,r)}))}function onConnect(i,o,s){a.removeAllListeners();o.removeAllListeners();if(i.statusCode!==200){c("tunneling socket could not be established, statusCode=%d",i.statusCode);o.destroy();var u=new Error("tunneling socket could not be established, "+"statusCode="+i.statusCode);u.code="ECONNRESET";e.request.emit("error",u);r.removeSocket(n);return}if(s.length>0){c("got illegal response body from proxy");o.destroy();var u=new Error("got illegal response body from proxy");u.code="ECONNRESET";e.request.emit("error",u);r.removeSocket(n);return}c("tunneling connection has established");r.sockets[r.sockets.indexOf(n)]=o;return t(o)}function onError(t){a.removeAllListeners();c("tunneling socket could not be established, cause=%s\n",t.message,t.stack);var i=new Error("tunneling socket could not be established, "+"cause="+t.message);i.code="ECONNRESET";e.request.emit("error",i);r.removeSocket(n)}};TunnelingAgent.prototype.removeSocket=function removeSocket(e){var t=this.sockets.indexOf(e);if(t===-1){return}this.sockets.splice(t,1);var r=this.requests.shift();if(r){this.createSocket(r,(function(e){r.request.onSocket(e)}))}};function createSecureSocket(e,t){var r=this;TunnelingAgent.prototype.createSocket.call(r,e,(function(n){var a=e.request.getHeader("host");var o=mergeOptions({},r.options,{socket:n,servername:a?a.replace(/:.*$/,""):e.host});var s=i.connect(0,o);r.sockets[r.sockets.indexOf(n)]=s;t(s)}))}function toOptions(e,t,r){if(typeof e==="string"){return{host:e,port:t,localAddress:r}}return e}function mergeOptions(e){for(var t=1,r=arguments.length;t{e.exports=r(3837).deprecate},5840:(e,t,r)=>{Object.defineProperty(t,"__esModule",{value:true});Object.defineProperty(t,"v1",{enumerable:true,get:function(){return n.default}});Object.defineProperty(t,"v3",{enumerable:true,get:function(){return i.default}});Object.defineProperty(t,"v4",{enumerable:true,get:function(){return a.default}});Object.defineProperty(t,"v5",{enumerable:true,get:function(){return o.default}});Object.defineProperty(t,"NIL",{enumerable:true,get:function(){return s.default}});Object.defineProperty(t,"version",{enumerable:true,get:function(){return u.default}});Object.defineProperty(t,"validate",{enumerable:true,get:function(){return l.default}});Object.defineProperty(t,"stringify",{enumerable:true,get:function(){return c.default}});Object.defineProperty(t,"parse",{enumerable:true,get:function(){return d.default}});var n=_interopRequireDefault(r(8628));var i=_interopRequireDefault(r(6409));var a=_interopRequireDefault(r(5122));var o=_interopRequireDefault(r(9120));var s=_interopRequireDefault(r(5332));var u=_interopRequireDefault(r(1595));var l=_interopRequireDefault(r(6900));var c=_interopRequireDefault(r(8950));var d=_interopRequireDefault(r(4848));function _interopRequireDefault(e){return e&&e.__esModule?e:{default:e}}},4569:(e,t,r)=>{Object.defineProperty(t,"__esModule",{value:true});t["default"]=void 0;var n=_interopRequireDefault(r(6113));function _interopRequireDefault(e){return e&&e.__esModule?e:{default:e}}function md5(e){if(Array.isArray(e)){e=Buffer.from(e)}else if(typeof e==="string"){e=Buffer.from(e,"utf8")}return n.default.createHash("md5").update(e).digest()}var i=md5;t["default"]=i},5332:(e,t)=>{Object.defineProperty(t,"__esModule",{value:true});t["default"]=void 0;var r="00000000-0000-0000-0000-000000000000";t["default"]=r},4848:(e,t,r)=>{Object.defineProperty(t,"__esModule",{value:true});t["default"]=void 0;var n=_interopRequireDefault(r(6900));function _interopRequireDefault(e){return e&&e.__esModule?e:{default:e}}function parse(e){if(!(0,n.default)(e)){throw TypeError("Invalid UUID")}let t;const r=new Uint8Array(16);r[0]=(t=parseInt(e.slice(0,8),16))>>>24;r[1]=t>>>16&255;r[2]=t>>>8&255;r[3]=t&255;r[4]=(t=parseInt(e.slice(9,13),16))>>>8;r[5]=t&255;r[6]=(t=parseInt(e.slice(14,18),16))>>>8;r[7]=t&255;r[8]=(t=parseInt(e.slice(19,23),16))>>>8;r[9]=t&255;r[10]=(t=parseInt(e.slice(24,36),16))/1099511627776&255;r[11]=t/4294967296&255;r[12]=t>>>24&255;r[13]=t>>>16&255;r[14]=t>>>8&255;r[15]=t&255;return r}var i=parse;t["default"]=i},814:(e,t)=>{Object.defineProperty(t,"__esModule",{value:true});t["default"]=void 0;var r=/^(?:[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}|00000000-0000-0000-0000-000000000000)$/i;t["default"]=r},807:(e,t,r)=>{Object.defineProperty(t,"__esModule",{value:true});t["default"]=rng;var n=_interopRequireDefault(r(6113));function _interopRequireDefault(e){return e&&e.__esModule?e:{default:e}}const i=new Uint8Array(256);let a=i.length;function rng(){if(a>i.length-16){n.default.randomFillSync(i);a=0}return i.slice(a,a+=16)}},5274:(e,t,r)=>{Object.defineProperty(t,"__esModule",{value:true});t["default"]=void 0;var n=_interopRequireDefault(r(6113));function _interopRequireDefault(e){return e&&e.__esModule?e:{default:e}}function sha1(e){if(Array.isArray(e)){e=Buffer.from(e)}else if(typeof e==="string"){e=Buffer.from(e,"utf8")}return n.default.createHash("sha1").update(e).digest()}var i=sha1;t["default"]=i},8950:(e,t,r)=>{Object.defineProperty(t,"__esModule",{value:true});t["default"]=void 0;var n=_interopRequireDefault(r(6900));function _interopRequireDefault(e){return e&&e.__esModule?e:{default:e}}const i=[];for(let e=0;e<256;++e){i.push((e+256).toString(16).substr(1))}function stringify(e,t=0){const r=(i[e[t+0]]+i[e[t+1]]+i[e[t+2]]+i[e[t+3]]+"-"+i[e[t+4]]+i[e[t+5]]+"-"+i[e[t+6]]+i[e[t+7]]+"-"+i[e[t+8]]+i[e[t+9]]+"-"+i[e[t+10]]+i[e[t+11]]+i[e[t+12]]+i[e[t+13]]+i[e[t+14]]+i[e[t+15]]).toLowerCase();if(!(0,n.default)(r)){throw TypeError("Stringified UUID is invalid")}return r}var a=stringify;t["default"]=a},8628:(e,t,r)=>{Object.defineProperty(t,"__esModule",{value:true});t["default"]=void 0;var n=_interopRequireDefault(r(807));var i=_interopRequireDefault(r(8950));function _interopRequireDefault(e){return e&&e.__esModule?e:{default:e}}let a;let o;let s=0;let u=0;function v1(e,t,r){let l=t&&r||0;const c=t||new Array(16);e=e||{};let d=e.node||a;let h=e.clockseq!==undefined?e.clockseq:o;if(d==null||h==null){const t=e.random||(e.rng||n.default)();if(d==null){d=a=[t[0]|1,t[1],t[2],t[3],t[4],t[5]]}if(h==null){h=o=(t[6]<<8|t[7])&16383}}let p=e.msecs!==undefined?e.msecs:Date.now();let g=e.nsecs!==undefined?e.nsecs:u+1;const m=p-s+(g-u)/1e4;if(m<0&&e.clockseq===undefined){h=h+1&16383}if((m<0||p>s)&&e.nsecs===undefined){g=0}if(g>=1e4){throw new Error("uuid.v1(): Can't create more than 10M uuids/sec")}s=p;u=g;o=h;p+=122192928e5;const b=((p&268435455)*1e4+g)%4294967296;c[l++]=b>>>24&255;c[l++]=b>>>16&255;c[l++]=b>>>8&255;c[l++]=b&255;const v=p/4294967296*1e4&268435455;c[l++]=v>>>8&255;c[l++]=v&255;c[l++]=v>>>24&15|16;c[l++]=v>>>16&255;c[l++]=h>>>8|128;c[l++]=h&255;for(let e=0;e<6;++e){c[l+e]=d[e]}return t||(0,i.default)(c)}var l=v1;t["default"]=l},6409:(e,t,r)=>{Object.defineProperty(t,"__esModule",{value:true});t["default"]=void 0;var n=_interopRequireDefault(r(5998));var i=_interopRequireDefault(r(4569));function _interopRequireDefault(e){return e&&e.__esModule?e:{default:e}}const a=(0,n.default)("v3",48,i.default);var o=a;t["default"]=o},5998:(e,t,r)=>{Object.defineProperty(t,"__esModule",{value:true});t["default"]=_default;t.URL=t.DNS=void 0;var n=_interopRequireDefault(r(8950));var i=_interopRequireDefault(r(4848));function _interopRequireDefault(e){return e&&e.__esModule?e:{default:e}}function stringToBytes(e){e=unescape(encodeURIComponent(e));const t=[];for(let r=0;r{Object.defineProperty(t,"__esModule",{value:true});t["default"]=void 0;var n=_interopRequireDefault(r(807));var i=_interopRequireDefault(r(8950));function _interopRequireDefault(e){return e&&e.__esModule?e:{default:e}}function v4(e,t,r){e=e||{};const a=e.random||(e.rng||n.default)();a[6]=a[6]&15|64;a[8]=a[8]&63|128;if(t){r=r||0;for(let e=0;e<16;++e){t[r+e]=a[e]}return t}return(0,i.default)(a)}var a=v4;t["default"]=a},9120:(e,t,r)=>{Object.defineProperty(t,"__esModule",{value:true});t["default"]=void 0;var n=_interopRequireDefault(r(5998));var i=_interopRequireDefault(r(5274));function _interopRequireDefault(e){return e&&e.__esModule?e:{default:e}}const a=(0,n.default)("v5",80,i.default);var o=a;t["default"]=o},6900:(e,t,r)=>{Object.defineProperty(t,"__esModule",{value:true});t["default"]=void 0;var n=_interopRequireDefault(r(814));function _interopRequireDefault(e){return e&&e.__esModule?e:{default:e}}function validate(e){return typeof e==="string"&&n.default.test(e)}var i=validate;t["default"]=i},1595:(e,t,r)=>{Object.defineProperty(t,"__esModule",{value:true});t["default"]=void 0;var n=_interopRequireDefault(r(6900));function _interopRequireDefault(e){return e&&e.__esModule?e:{default:e}}function version(e){if(!(0,n.default)(e)){throw TypeError("Invalid UUID")}return parseInt(e.substr(14,1),16)}var i=version;t["default"]=i},2940:e=>{e.exports=wrappy;function wrappy(e,t){if(e&&t)return wrappy(e)(t);if(typeof e!=="function")throw new TypeError("need wrapper function");Object.keys(e).forEach((function(t){wrapper[t]=e[t]}));return wrapper;function wrapper(){var t=new Array(arguments.length);for(var r=0;r{t.exports=e(import.meta.url)("assert")},4300:t=>{t.exports=e(import.meta.url)("buffer")},2057:t=>{t.exports=e(import.meta.url)("constants")},6113:t=>{t.exports=e(import.meta.url)("crypto")},2361:t=>{t.exports=e(import.meta.url)("events")},7147:t=>{t.exports=e(import.meta.url)("fs")},3685:t=>{t.exports=e(import.meta.url)("http")},5687:t=>{t.exports=e(import.meta.url)("https")},1808:t=>{t.exports=e(import.meta.url)("net")},7718:t=>{t.exports=e(import.meta.url)("node:child_process")},7561:t=>{t.exports=e(import.meta.url)("node:fs")},7742:t=>{t.exports=e(import.meta.url)("node:process")},4492:t=>{t.exports=e(import.meta.url)("node:stream")},5628:t=>{t.exports=e(import.meta.url)("node:zlib")},2037:t=>{t.exports=e(import.meta.url)("os")},1017:t=>{t.exports=e(import.meta.url)("path")},2781:t=>{t.exports=e(import.meta.url)("stream")},1576:t=>{t.exports=e(import.meta.url)("string_decoder")},4404:t=>{t.exports=e(import.meta.url)("tls")},3837:t=>{t.exports=e(import.meta.url)("util")},6955:(e,t,r)=>{r.a(e,(async e=>{var t=r(7718);var n=r(7742);var i=r(4492);var a=r(7561);var o=r(5628);var s=r(2283);var u=r(2186);const l=u.getInput("container",{required:true});const c=JSON.parse(u.getInput("error-log-paths",{required:true}));const d=u.getInput("log-tarball-prefix",{required:true});const h=u.getInput("tests-label",{required:true});const p=u.getInput("test-timeout",{required:true});try{if(t.spawnSync("docker",["run","--name","base","-v",`${n.cwd()}/build.tar.zst:/build.tar.zst`,"--workdir","/__w/leap/leap",l,"sh","-c","zstdcat /build.tar.zst | tar x"],{stdio:"inherit"}).status)throw new Error("Failed to create base container");if(t.spawnSync("docker",["commit","base","baseimage"],{stdio:"inherit"}).status)throw new Error("Failed to create base image");if(t.spawnSync("docker",["rm","base"],{stdio:"inherit"}).status)throw new Error("Failed to remove base container");const e=t.spawnSync("docker",["run","--rm","baseimage","bash","-e","-o","pipefail","-c",`cd build; ctest -L '${h}' --show-only=json-v1`]);if(e.status)throw new Error("Failed to discover tests with label");const r=JSON.parse(e.stdout).tests;let g=[];r.forEach((e=>{g.push(new Promise((r=>{t.spawn("docker",["run","--security-opt","seccomp=unconfined","-e","GITHUB_ACTIONS=True","--name",e.name,"--init","baseimage","bash","-c",`cd build; ctest --output-on-failure -R '^${e.name}$' --timeout ${p}`],{stdio:"inherit"}).on("close",(e=>r(e)))})))}));const m=await Promise.all(g);for(let e=0;e{if(!e.name.startsWith(`__w/leap/leap/build`)){t.on("end",(()=>r()));t.resume();return}e.name=e.name.substring(`__w/leap/leap/`.length);if(e.name!=="build/"&&c.filter((t=>e.name.startsWith(t))).length===0){t.on("end",(()=>r()));t.resume();return}t.pipe(l.entry(e,r))})).on("finish",(()=>{l.finalize()}));t.spawn("docker",["export",r[e]]).stdout.pipe(n);i.promises.pipeline(l,o.createGzip(),a.createWriteStream(`${d}-${r[e]}-logs.tar.gz`))}}catch(e){u.setFailed(`Uncaught exception ${e.message}`)}e()}),1)}};var r={};function __nccwpck_require__(e){var n=r[e];if(n!==undefined){return n.exports}var i=r[e]={exports:{}};var a=true;try{t[e].call(i.exports,i,i.exports,__nccwpck_require__);a=false}finally{if(a)delete r[e]}return i.exports}(()=>{var e=typeof Symbol==="function"?Symbol("webpack then"):"__webpack_then__";var t=typeof Symbol==="function"?Symbol("webpack exports"):"__webpack_exports__";var completeQueue=e=>{if(e){e.forEach((e=>e.r--));e.forEach((e=>e.r--?e.r++:e()))}};var completeFunction=e=>!--e.r&&e();var queueFunction=(e,t)=>e?e.push(t):completeFunction(t);var wrapDeps=r=>r.map((r=>{if(r!==null&&typeof r==="object"){if(r[e])return r;if(r.then){var n=[];r.then((e=>{i[t]=e;completeQueue(n);n=0}));var i={};i[e]=(e,t)=>(queueFunction(n,e),r["catch"](t));return i}}var a={};a[e]=e=>completeFunction(e);a[t]=r;return a}));__nccwpck_require__.a=(r,n,i)=>{var a=i&&[];var o=r.exports;var s;var u;var l;var c=true;var d=false;var whenAll=(t,r,n)=>{if(d)return;d=true;r.r+=t.length;t.map(((t,i)=>t[e](r,n)));d=false};var h=new Promise(((e,t)=>{l=t;u=()=>(e(o),completeQueue(a),a=0)}));h[t]=o;h[e]=(e,t)=>{if(c){return completeFunction(e)}if(s)whenAll(s,e,t);queueFunction(a,e);h["catch"](t)};r.exports=h;n((e=>{if(!e)return u();s=wrapDeps(e);var r,n;var i=new Promise(((e,i)=>{r=()=>e(n=s.map((e=>e[t])));r.r=0;whenAll(s,r,i)}));return r.r?i:n})).then(u,l);c=false}})();if(typeof __nccwpck_require__!=="undefined")__nccwpck_require__.ab=new URL(".",import.meta.url).pathname.slice(import.meta.url.match(/^file:\/\/\/\w:/)?1:0,-1)+"/";var n=__nccwpck_require__(6955);n=await n; \ No newline at end of file +var n=r(4300);var i=n.Buffer;function copyProps(e,t){for(var r in e){t[r]=e[r]}}if(i.from&&i.alloc&&i.allocUnsafe&&i.allocUnsafeSlow){e.exports=n}else{copyProps(n,t);t.Buffer=SafeBuffer}function SafeBuffer(e,t,r){return i(e,t,r)}SafeBuffer.prototype=Object.create(i.prototype);copyProps(i,SafeBuffer);SafeBuffer.from=function(e,t,r){if(typeof e==="number"){throw new TypeError("Argument must not be a number")}return i(e,t,r)};SafeBuffer.alloc=function(e,t,r){if(typeof e!=="number"){throw new TypeError("Argument must be a number")}var n=i(e);if(t!==undefined){if(typeof r==="string"){n.fill(t,r)}else{n.fill(t)}}else{n.fill(0)}return n};SafeBuffer.allocUnsafe=function(e){if(typeof e!=="number"){throw new TypeError("Argument must be a number")}return i(e)};SafeBuffer.allocUnsafeSlow=function(e){if(typeof e!=="number"){throw new TypeError("Argument must be a number")}return n.SlowBuffer(e)}},4841:(e,t,r)=>{var n=r(1867).Buffer;var i=n.isEncoding||function(e){e=""+e;switch(e&&e.toLowerCase()){case"hex":case"utf8":case"utf-8":case"ascii":case"binary":case"base64":case"ucs2":case"ucs-2":case"utf16le":case"utf-16le":case"raw":return true;default:return false}};function _normalizeEncoding(e){if(!e)return"utf8";var t;while(true){switch(e){case"utf8":case"utf-8":return"utf8";case"ucs2":case"ucs-2":case"utf16le":case"utf-16le":return"utf16le";case"latin1":case"binary":return"latin1";case"base64":case"ascii":case"hex":return e;default:if(t)return;e=(""+e).toLowerCase();t=true}}}function normalizeEncoding(e){var t=_normalizeEncoding(e);if(typeof t!=="string"&&(n.isEncoding===i||!i(e)))throw new Error("Unknown encoding: "+e);return t||e}t.s=StringDecoder;function StringDecoder(e){this.encoding=normalizeEncoding(e);var t;switch(this.encoding){case"utf16le":this.text=utf16Text;this.end=utf16End;t=4;break;case"utf8":this.fillLast=utf8FillLast;t=4;break;case"base64":this.text=base64Text;this.end=base64End;t=3;break;default:this.write=simpleWrite;this.end=simpleEnd;return}this.lastNeed=0;this.lastTotal=0;this.lastChar=n.allocUnsafe(t)}StringDecoder.prototype.write=function(e){if(e.length===0)return"";var t;var r;if(this.lastNeed){t=this.fillLast(e);if(t===undefined)return"";r=this.lastNeed;this.lastNeed=0}else{r=0}if(r>5===6)return 2;else if(e>>4===14)return 3;else if(e>>3===30)return 4;return e>>6===2?-1:-2}function utf8CheckIncomplete(e,t,r){var n=t.length-1;if(n=0){if(i>0)e.lastNeed=i-1;return i}if(--n=0){if(i>0)e.lastNeed=i-2;return i}if(--n=0){if(i>0){if(i===2)i=0;else e.lastNeed=i-3}return i}return 0}function utf8CheckExtraBytes(e,t,r){if((t[0]&192)!==128){e.lastNeed=0;return"�"}if(e.lastNeed>1&&t.length>1){if((t[1]&192)!==128){e.lastNeed=1;return"�"}if(e.lastNeed>2&&t.length>2){if((t[2]&192)!==128){e.lastNeed=2;return"�"}}}}function utf8FillLast(e){var t=this.lastTotal-this.lastNeed;var r=utf8CheckExtraBytes(this,e,t);if(r!==undefined)return r;if(this.lastNeed<=e.length){e.copy(this.lastChar,t,0,this.lastNeed);return this.lastChar.toString(this.encoding,0,this.lastTotal)}e.copy(this.lastChar,t,0,e.length);this.lastNeed-=e.length}function utf8Text(e,t){var r=utf8CheckIncomplete(this,e,t);if(!this.lastNeed)return e.toString("utf8",t);this.lastTotal=r;var n=e.length-(r-this.lastNeed);e.copy(this.lastChar,0,n);return e.toString("utf8",t,n)}function utf8End(e){var t=e&&e.length?this.write(e):"";if(this.lastNeed)return t+"�";return t}function utf16Text(e,t){if((e.length-t)%2===0){var r=e.toString("utf16le",t);if(r){var n=r.charCodeAt(r.length-1);if(n>=55296&&n<=56319){this.lastNeed=2;this.lastTotal=4;this.lastChar[0]=e[e.length-2];this.lastChar[1]=e[e.length-1];return r.slice(0,-1)}}return r}this.lastNeed=1;this.lastTotal=2;this.lastChar[0]=e[e.length-1];return e.toString("utf16le",t,e.length-1)}function utf16End(e){var t=e&&e.length?this.write(e):"";if(this.lastNeed){var r=this.lastTotal-this.lastNeed;return t+this.lastChar.toString("utf16le",0,r)}return t}function base64Text(e,t){var r=(e.length-t)%3;if(r===0)return e.toString("base64",t);this.lastNeed=3-r;this.lastTotal=3;if(r===1){this.lastChar[0]=e[e.length-1]}else{this.lastChar[0]=e[e.length-2];this.lastChar[1]=e[e.length-1]}return e.toString("base64",t,e.length-r)}function base64End(e){var t=e&&e.length?this.write(e):"";if(this.lastNeed)return t+this.lastChar.toString("base64",0,3-this.lastNeed);return t}function simpleWrite(e){return e.toString(this.encoding)}function simpleEnd(e){return e&&e.length?this.write(e):""}},7882:(e,t,r)=>{var n=r(3837);var i=r(336);var a=r(8860);var o=r(1642).Writable;var s=r(1642).PassThrough;var noop=function(){};var overflow=function(e){e&=511;return e&&512-e};var emptyStream=function(e,t){var r=new Source(e,t);r.end();return r};var mixinPax=function(e,t){if(t.path)e.name=t.path;if(t.linkpath)e.linkname=t.linkpath;if(t.size)e.size=parseInt(t.size,10);e.pax=t;return e};var Source=function(e,t){this._parent=e;this.offset=t;s.call(this,{autoDestroy:false})};n.inherits(Source,s);Source.prototype.destroy=function(e){this._parent.destroy(e)};var Extract=function(e){if(!(this instanceof Extract))return new Extract(e);o.call(this,e);e=e||{};this._offset=0;this._buffer=i();this._missing=0;this._partial=false;this._onparse=noop;this._header=null;this._stream=null;this._overflow=null;this._cb=null;this._locked=false;this._destroyed=false;this._pax=null;this._paxGlobal=null;this._gnuLongPath=null;this._gnuLongLinkPath=null;var t=this;var r=t._buffer;var oncontinue=function(){t._continue()};var onunlock=function(e){t._locked=false;if(e)return t.destroy(e);if(!t._stream)oncontinue()};var onstreamend=function(){t._stream=null;var e=overflow(t._header.size);if(e)t._parse(e,ondrain);else t._parse(512,onheader);if(!t._locked)oncontinue()};var ondrain=function(){t._buffer.consume(overflow(t._header.size));t._parse(512,onheader);oncontinue()};var onpaxglobalheader=function(){var e=t._header.size;t._paxGlobal=a.decodePax(r.slice(0,e));r.consume(e);onstreamend()};var onpaxheader=function(){var e=t._header.size;t._pax=a.decodePax(r.slice(0,e));if(t._paxGlobal)t._pax=Object.assign({},t._paxGlobal,t._pax);r.consume(e);onstreamend()};var ongnulongpath=function(){var n=t._header.size;this._gnuLongPath=a.decodeLongPath(r.slice(0,n),e.filenameEncoding);r.consume(n);onstreamend()};var ongnulonglinkpath=function(){var n=t._header.size;this._gnuLongLinkPath=a.decodeLongPath(r.slice(0,n),e.filenameEncoding);r.consume(n);onstreamend()};var onheader=function(){var n=t._offset;var i;try{i=t._header=a.decode(r.slice(0,512),e.filenameEncoding,e.allowUnknownFormat)}catch(e){t.emit("error",e)}r.consume(512);if(!i){t._parse(512,onheader);oncontinue();return}if(i.type==="gnu-long-path"){t._parse(i.size,ongnulongpath);oncontinue();return}if(i.type==="gnu-long-link-path"){t._parse(i.size,ongnulonglinkpath);oncontinue();return}if(i.type==="pax-global-header"){t._parse(i.size,onpaxglobalheader);oncontinue();return}if(i.type==="pax-header"){t._parse(i.size,onpaxheader);oncontinue();return}if(t._gnuLongPath){i.name=t._gnuLongPath;t._gnuLongPath=null}if(t._gnuLongLinkPath){i.linkname=t._gnuLongLinkPath;t._gnuLongLinkPath=null}if(t._pax){t._header=i=mixinPax(i,t._pax);t._pax=null}t._locked=true;if(!i.size||i.type==="directory"){t._parse(512,onheader);t.emit("entry",i,emptyStream(t,n),onunlock);return}t._stream=new Source(t,n);t.emit("entry",i,t._stream,onunlock);t._parse(i.size,onstreamend);oncontinue()};this._onheader=onheader;this._parse(512,onheader)};n.inherits(Extract,o);Extract.prototype.destroy=function(e){if(this._destroyed)return;this._destroyed=true;if(e)this.emit("error",e);this.emit("close");if(this._stream)this._stream.emit("close")};Extract.prototype._parse=function(e,t){if(this._destroyed)return;this._offset+=e;this._missing=e;if(t===this._onheader)this._partial=false;this._onparse=t};Extract.prototype._continue=function(){if(this._destroyed)return;var e=this._cb;this._cb=noop;if(this._overflow)this._write(this._overflow,undefined,e);else e()};Extract.prototype._write=function(e,t,r){if(this._destroyed)return;var n=this._stream;var i=this._buffer;var a=this._missing;if(e.length)this._partial=true;if(e.lengtha){o=e.slice(a);e=e.slice(0,a)}if(n)n.end(e);else i.append(e);this._overflow=o;this._onparse()};Extract.prototype._final=function(e){if(this._partial)return this.destroy(new Error("Unexpected end of data"));e()};e.exports=Extract},8860:(e,t)=>{var r=Buffer.alloc;var n="0000000000000000000";var i="7777777777777777777";var a="0".charCodeAt(0);var o=Buffer.from("ustar\0","binary");var s=Buffer.from("00","binary");var u=Buffer.from("ustar ","binary");var l=Buffer.from(" \0","binary");var c=parseInt("7777",8);var d=257;var h=263;var clamp=function(e,t,r){if(typeof e!=="number")return r;e=~~e;if(e>=t)return t;if(e>=0)return e;e+=t;if(e>=0)return e;return 0};var toType=function(e){switch(e){case 0:return"file";case 1:return"link";case 2:return"symlink";case 3:return"character-device";case 4:return"block-device";case 5:return"directory";case 6:return"fifo";case 7:return"contiguous-file";case 72:return"pax-header";case 55:return"pax-global-header";case 27:return"gnu-long-link-path";case 28:case 30:return"gnu-long-path"}return null};var toTypeflag=function(e){switch(e){case"file":return 0;case"link":return 1;case"symlink":return 2;case"character-device":return 3;case"block-device":return 4;case"directory":return 5;case"fifo":return 6;case"contiguous-file":return 7;case"pax-header":return 72}return 0};var indexOf=function(e,t,r,n){for(;rt)return i.slice(0,t)+" ";else return n.slice(0,t-e.length)+e+" "};function parse256(e){var t;if(e[0]===128)t=true;else if(e[0]===255)t=false;else return null;var r=[];for(var n=e.length-1;n>0;n--){var i=e[n];if(t)r.push(i);else r.push(255-i)}var a=0;var o=r.length;for(n=0;n=Math.pow(10,r))r++;return t+r+e};t.decodeLongPath=function(e,t){return decodeStr(e,0,e.length,t)};t.encodePax=function(e){var t="";if(e.name)t+=addLength(" path="+e.name+"\n");if(e.linkname)t+=addLength(" linkpath="+e.linkname+"\n");var r=e.pax;if(r){for(var n in r){t+=addLength(" "+n+"="+r[n]+"\n")}}return Buffer.from(t)};t.decodePax=function(e){var t={};while(e.length){var r=0;while(r100){var u=n.indexOf("/");if(u===-1)return null;i+=i?"/"+n.slice(0,u):n.slice(0,u);n=n.slice(u+1)}if(Buffer.byteLength(n)>100||Buffer.byteLength(i)>155)return null;if(e.linkname&&Buffer.byteLength(e.linkname)>100)return null;t.write(n);t.write(encodeOct(e.mode&c,6),100);t.write(encodeOct(e.uid,6),108);t.write(encodeOct(e.gid,6),116);t.write(encodeOct(e.size,11),124);t.write(encodeOct(e.mtime.getTime()/1e3|0,11),136);t[156]=a+toTypeflag(e.type);if(e.linkname)t.write(e.linkname,157);o.copy(t,d);s.copy(t,h);if(e.uname)t.write(e.uname,265);if(e.gname)t.write(e.gname,297);t.write(encodeOct(e.devmajor||0,6),329);t.write(encodeOct(e.devminor||0,6),337);if(i)t.write(i,345);t.write(encodeOct(cksum(t),6),148);return t};t.decode=function(e,t,r){var n=e[156]===0?0:e[156]-a;var i=decodeStr(e,0,100,t);var s=decodeOct(e,100,8);var c=decodeOct(e,108,8);var p=decodeOct(e,116,8);var g=decodeOct(e,124,12);var m=decodeOct(e,136,12);var b=toType(n);var v=e[157]===0?null:decodeStr(e,157,100,t);var _=decodeStr(e,265,32);var y=decodeStr(e,297,32);var w=decodeOct(e,329,8);var R=decodeOct(e,337,8);var S=cksum(e);if(S===8*32)return null;if(S!==decodeOct(e,148,8))throw new Error("Invalid tar header. Maybe the tar is corrupted or it needs to be gunzipped?");if(o.compare(e,d,d+6)===0){if(e[345])i=decodeStr(e,345,155,t)+"/"+i}else if(u.compare(e,d,d+6)===0&&l.compare(e,h,h+2)===0){}else{if(!r){throw new Error("Invalid tar header: unknown format.")}}if(n===0&&i&&i[i.length-1]==="/")n=5;return{name:i,mode:s,uid:c,gid:p,size:g,mtime:new Date(1e3*m),type:b,linkname:v,uname:_,gname:y,devmajor:w,devminor:R}}},2283:(e,t,r)=>{t.extract=r(7882);t.pack=r(4930)},4930:(e,t,r)=>{var n=r(3186);var i=r(1205);var a=r(4124);var o=Buffer.alloc;var s=r(1642).Readable;var u=r(1642).Writable;var l=r(1576).StringDecoder;var c=r(8860);var d=parseInt("755",8);var h=parseInt("644",8);var p=o(1024);var noop=function(){};var overflow=function(e,t){t&=511;if(t)e.push(p.slice(0,512-t))};function modeToType(e){switch(e&n.S_IFMT){case n.S_IFBLK:return"block-device";case n.S_IFCHR:return"character-device";case n.S_IFDIR:return"directory";case n.S_IFIFO:return"fifo";case n.S_IFLNK:return"symlink"}return"file"}var Sink=function(e){u.call(this);this.written=0;this._to=e;this._destroyed=false};a(Sink,u);Sink.prototype._write=function(e,t,r){this.written+=e.length;if(this._to.push(e))return r();this._to._drain=r};Sink.prototype.destroy=function(){if(this._destroyed)return;this._destroyed=true;this.emit("close")};var LinkSink=function(){u.call(this);this.linkname="";this._decoder=new l("utf-8");this._destroyed=false};a(LinkSink,u);LinkSink.prototype._write=function(e,t,r){this.linkname+=this._decoder.write(e);r()};LinkSink.prototype.destroy=function(){if(this._destroyed)return;this._destroyed=true;this.emit("close")};var Void=function(){u.call(this);this._destroyed=false};a(Void,u);Void.prototype._write=function(e,t,r){r(new Error("No body allowed for this entry"))};Void.prototype.destroy=function(){if(this._destroyed)return;this._destroyed=true;this.emit("close")};var Pack=function(e){if(!(this instanceof Pack))return new Pack(e);s.call(this,e);this._drain=noop;this._finalized=false;this._finalizing=false;this._destroyed=false;this._stream=null};a(Pack,s);Pack.prototype.entry=function(e,t,r){if(this._stream)throw new Error("already piping an entry");if(this._finalized||this._destroyed)return;if(typeof t==="function"){r=t;t=null}if(!r)r=noop;var n=this;if(!e.size||e.type==="symlink")e.size=0;if(!e.type)e.type=modeToType(e.mode);if(!e.mode)e.mode=e.type==="directory"?d:h;if(!e.uid)e.uid=0;if(!e.gid)e.gid=0;if(!e.mtime)e.mtime=new Date;if(typeof t==="string")t=Buffer.from(t);if(Buffer.isBuffer(t)){e.size=t.length;this._encode(e);var a=this.push(t);overflow(n,e.size);if(a)process.nextTick(r);else this._drain=r;return new Void}if(e.type==="symlink"&&!e.linkname){var o=new LinkSink;i(o,(function(t){if(t){n.destroy();return r(t)}e.linkname=o.linkname;n._encode(e);r()}));return o}this._encode(e);if(e.type!=="file"&&e.type!=="contiguous-file"){process.nextTick(r);return new Void}var s=new Sink(this);this._stream=s;i(s,(function(t){n._stream=null;if(t){n.destroy();return r(t)}if(s.written!==e.size){n.destroy();return r(new Error("size mismatch"))}overflow(n,e.size);if(n._finalizing)n.finalize();r()}));return s};Pack.prototype.finalize=function(){if(this._stream){this._finalizing=true;return}if(this._finalized)return;this._finalized=true;this.push(p);this.push(null)};Pack.prototype.destroy=function(e){if(this._destroyed)return;this._destroyed=true;if(e)this.emit("error",e);this.emit("close");if(this._stream&&this._stream.destroy)this._stream.destroy()};Pack.prototype._encode=function(e){if(!e.pax){var t=c.encode(e);if(t){this.push(t);return}}this._encodePax(e)};Pack.prototype._encodePax=function(e){var t=c.encodePax({name:e.name,linkname:e.linkname,pax:e.pax});var r={name:"PaxHeader",mode:e.mode,uid:e.uid,gid:e.gid,size:t.length,mtime:e.mtime,type:"pax-header",linkname:e.linkname&&"PaxHeader",uname:e.uname,gname:e.gname,devmajor:e.devmajor,devminor:e.devminor};this.push(c.encode(r));this.push(t);overflow(this,t.length);r.size=e.size;r.type=e.type;this.push(c.encode(r))};Pack.prototype._read=function(e){var t=this._drain;this._drain=noop;t()};e.exports=Pack},4294:(e,t,r)=>{e.exports=r(4219)},4219:(e,t,r)=>{var n=r(1808);var i=r(4404);var a=r(3685);var o=r(5687);var s=r(2361);var u=r(9491);var l=r(3837);t.httpOverHttp=httpOverHttp;t.httpsOverHttp=httpsOverHttp;t.httpOverHttps=httpOverHttps;t.httpsOverHttps=httpsOverHttps;function httpOverHttp(e){var t=new TunnelingAgent(e);t.request=a.request;return t}function httpsOverHttp(e){var t=new TunnelingAgent(e);t.request=a.request;t.createSocket=createSecureSocket;t.defaultPort=443;return t}function httpOverHttps(e){var t=new TunnelingAgent(e);t.request=o.request;return t}function httpsOverHttps(e){var t=new TunnelingAgent(e);t.request=o.request;t.createSocket=createSecureSocket;t.defaultPort=443;return t}function TunnelingAgent(e){var t=this;t.options=e||{};t.proxyOptions=t.options.proxy||{};t.maxSockets=t.options.maxSockets||a.Agent.defaultMaxSockets;t.requests=[];t.sockets=[];t.on("free",(function onFree(e,r,n,i){var a=toOptions(r,n,i);for(var o=0,s=t.requests.length;o=this.maxSockets){i.requests.push(a);return}i.createSocket(a,(function(t){t.on("free",onFree);t.on("close",onCloseOrRemove);t.on("agentRemove",onCloseOrRemove);e.onSocket(t);function onFree(){i.emit("free",t,a)}function onCloseOrRemove(e){i.removeSocket(t);t.removeListener("free",onFree);t.removeListener("close",onCloseOrRemove);t.removeListener("agentRemove",onCloseOrRemove)}}))};TunnelingAgent.prototype.createSocket=function createSocket(e,t){var r=this;var n={};r.sockets.push(n);var i=mergeOptions({},r.proxyOptions,{method:"CONNECT",path:e.host+":"+e.port,agent:false,headers:{host:e.host+":"+e.port}});if(e.localAddress){i.localAddress=e.localAddress}if(i.proxyAuth){i.headers=i.headers||{};i.headers["Proxy-Authorization"]="Basic "+new Buffer(i.proxyAuth).toString("base64")}c("making CONNECT request");var a=r.request(i);a.useChunkedEncodingByDefault=false;a.once("response",onResponse);a.once("upgrade",onUpgrade);a.once("connect",onConnect);a.once("error",onError);a.end();function onResponse(e){e.upgrade=true}function onUpgrade(e,t,r){process.nextTick((function(){onConnect(e,t,r)}))}function onConnect(i,o,s){a.removeAllListeners();o.removeAllListeners();if(i.statusCode!==200){c("tunneling socket could not be established, statusCode=%d",i.statusCode);o.destroy();var u=new Error("tunneling socket could not be established, "+"statusCode="+i.statusCode);u.code="ECONNRESET";e.request.emit("error",u);r.removeSocket(n);return}if(s.length>0){c("got illegal response body from proxy");o.destroy();var u=new Error("got illegal response body from proxy");u.code="ECONNRESET";e.request.emit("error",u);r.removeSocket(n);return}c("tunneling connection has established");r.sockets[r.sockets.indexOf(n)]=o;return t(o)}function onError(t){a.removeAllListeners();c("tunneling socket could not be established, cause=%s\n",t.message,t.stack);var i=new Error("tunneling socket could not be established, "+"cause="+t.message);i.code="ECONNRESET";e.request.emit("error",i);r.removeSocket(n)}};TunnelingAgent.prototype.removeSocket=function removeSocket(e){var t=this.sockets.indexOf(e);if(t===-1){return}this.sockets.splice(t,1);var r=this.requests.shift();if(r){this.createSocket(r,(function(e){r.request.onSocket(e)}))}};function createSecureSocket(e,t){var r=this;TunnelingAgent.prototype.createSocket.call(r,e,(function(n){var a=e.request.getHeader("host");var o=mergeOptions({},r.options,{socket:n,servername:a?a.replace(/:.*$/,""):e.host});var s=i.connect(0,o);r.sockets[r.sockets.indexOf(n)]=s;t(s)}))}function toOptions(e,t,r){if(typeof e==="string"){return{host:e,port:t,localAddress:r}}return e}function mergeOptions(e){for(var t=1,r=arguments.length;t{e.exports=r(3837).deprecate},5840:(e,t,r)=>{Object.defineProperty(t,"__esModule",{value:true});Object.defineProperty(t,"v1",{enumerable:true,get:function(){return n.default}});Object.defineProperty(t,"v3",{enumerable:true,get:function(){return i.default}});Object.defineProperty(t,"v4",{enumerable:true,get:function(){return a.default}});Object.defineProperty(t,"v5",{enumerable:true,get:function(){return o.default}});Object.defineProperty(t,"NIL",{enumerable:true,get:function(){return s.default}});Object.defineProperty(t,"version",{enumerable:true,get:function(){return u.default}});Object.defineProperty(t,"validate",{enumerable:true,get:function(){return l.default}});Object.defineProperty(t,"stringify",{enumerable:true,get:function(){return c.default}});Object.defineProperty(t,"parse",{enumerable:true,get:function(){return d.default}});var n=_interopRequireDefault(r(8628));var i=_interopRequireDefault(r(6409));var a=_interopRequireDefault(r(5122));var o=_interopRequireDefault(r(9120));var s=_interopRequireDefault(r(5332));var u=_interopRequireDefault(r(1595));var l=_interopRequireDefault(r(6900));var c=_interopRequireDefault(r(8950));var d=_interopRequireDefault(r(4848));function _interopRequireDefault(e){return e&&e.__esModule?e:{default:e}}},4569:(e,t,r)=>{Object.defineProperty(t,"__esModule",{value:true});t["default"]=void 0;var n=_interopRequireDefault(r(6113));function _interopRequireDefault(e){return e&&e.__esModule?e:{default:e}}function md5(e){if(Array.isArray(e)){e=Buffer.from(e)}else if(typeof e==="string"){e=Buffer.from(e,"utf8")}return n.default.createHash("md5").update(e).digest()}var i=md5;t["default"]=i},5332:(e,t)=>{Object.defineProperty(t,"__esModule",{value:true});t["default"]=void 0;var r="00000000-0000-0000-0000-000000000000";t["default"]=r},4848:(e,t,r)=>{Object.defineProperty(t,"__esModule",{value:true});t["default"]=void 0;var n=_interopRequireDefault(r(6900));function _interopRequireDefault(e){return e&&e.__esModule?e:{default:e}}function parse(e){if(!(0,n.default)(e)){throw TypeError("Invalid UUID")}let t;const r=new Uint8Array(16);r[0]=(t=parseInt(e.slice(0,8),16))>>>24;r[1]=t>>>16&255;r[2]=t>>>8&255;r[3]=t&255;r[4]=(t=parseInt(e.slice(9,13),16))>>>8;r[5]=t&255;r[6]=(t=parseInt(e.slice(14,18),16))>>>8;r[7]=t&255;r[8]=(t=parseInt(e.slice(19,23),16))>>>8;r[9]=t&255;r[10]=(t=parseInt(e.slice(24,36),16))/1099511627776&255;r[11]=t/4294967296&255;r[12]=t>>>24&255;r[13]=t>>>16&255;r[14]=t>>>8&255;r[15]=t&255;return r}var i=parse;t["default"]=i},814:(e,t)=>{Object.defineProperty(t,"__esModule",{value:true});t["default"]=void 0;var r=/^(?:[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}|00000000-0000-0000-0000-000000000000)$/i;t["default"]=r},807:(e,t,r)=>{Object.defineProperty(t,"__esModule",{value:true});t["default"]=rng;var n=_interopRequireDefault(r(6113));function _interopRequireDefault(e){return e&&e.__esModule?e:{default:e}}const i=new Uint8Array(256);let a=i.length;function rng(){if(a>i.length-16){n.default.randomFillSync(i);a=0}return i.slice(a,a+=16)}},5274:(e,t,r)=>{Object.defineProperty(t,"__esModule",{value:true});t["default"]=void 0;var n=_interopRequireDefault(r(6113));function _interopRequireDefault(e){return e&&e.__esModule?e:{default:e}}function sha1(e){if(Array.isArray(e)){e=Buffer.from(e)}else if(typeof e==="string"){e=Buffer.from(e,"utf8")}return n.default.createHash("sha1").update(e).digest()}var i=sha1;t["default"]=i},8950:(e,t,r)=>{Object.defineProperty(t,"__esModule",{value:true});t["default"]=void 0;var n=_interopRequireDefault(r(6900));function _interopRequireDefault(e){return e&&e.__esModule?e:{default:e}}const i=[];for(let e=0;e<256;++e){i.push((e+256).toString(16).substr(1))}function stringify(e,t=0){const r=(i[e[t+0]]+i[e[t+1]]+i[e[t+2]]+i[e[t+3]]+"-"+i[e[t+4]]+i[e[t+5]]+"-"+i[e[t+6]]+i[e[t+7]]+"-"+i[e[t+8]]+i[e[t+9]]+"-"+i[e[t+10]]+i[e[t+11]]+i[e[t+12]]+i[e[t+13]]+i[e[t+14]]+i[e[t+15]]).toLowerCase();if(!(0,n.default)(r)){throw TypeError("Stringified UUID is invalid")}return r}var a=stringify;t["default"]=a},8628:(e,t,r)=>{Object.defineProperty(t,"__esModule",{value:true});t["default"]=void 0;var n=_interopRequireDefault(r(807));var i=_interopRequireDefault(r(8950));function _interopRequireDefault(e){return e&&e.__esModule?e:{default:e}}let a;let o;let s=0;let u=0;function v1(e,t,r){let l=t&&r||0;const c=t||new Array(16);e=e||{};let d=e.node||a;let h=e.clockseq!==undefined?e.clockseq:o;if(d==null||h==null){const t=e.random||(e.rng||n.default)();if(d==null){d=a=[t[0]|1,t[1],t[2],t[3],t[4],t[5]]}if(h==null){h=o=(t[6]<<8|t[7])&16383}}let p=e.msecs!==undefined?e.msecs:Date.now();let g=e.nsecs!==undefined?e.nsecs:u+1;const m=p-s+(g-u)/1e4;if(m<0&&e.clockseq===undefined){h=h+1&16383}if((m<0||p>s)&&e.nsecs===undefined){g=0}if(g>=1e4){throw new Error("uuid.v1(): Can't create more than 10M uuids/sec")}s=p;u=g;o=h;p+=122192928e5;const b=((p&268435455)*1e4+g)%4294967296;c[l++]=b>>>24&255;c[l++]=b>>>16&255;c[l++]=b>>>8&255;c[l++]=b&255;const v=p/4294967296*1e4&268435455;c[l++]=v>>>8&255;c[l++]=v&255;c[l++]=v>>>24&15|16;c[l++]=v>>>16&255;c[l++]=h>>>8|128;c[l++]=h&255;for(let e=0;e<6;++e){c[l+e]=d[e]}return t||(0,i.default)(c)}var l=v1;t["default"]=l},6409:(e,t,r)=>{Object.defineProperty(t,"__esModule",{value:true});t["default"]=void 0;var n=_interopRequireDefault(r(5998));var i=_interopRequireDefault(r(4569));function _interopRequireDefault(e){return e&&e.__esModule?e:{default:e}}const a=(0,n.default)("v3",48,i.default);var o=a;t["default"]=o},5998:(e,t,r)=>{Object.defineProperty(t,"__esModule",{value:true});t["default"]=_default;t.URL=t.DNS=void 0;var n=_interopRequireDefault(r(8950));var i=_interopRequireDefault(r(4848));function _interopRequireDefault(e){return e&&e.__esModule?e:{default:e}}function stringToBytes(e){e=unescape(encodeURIComponent(e));const t=[];for(let r=0;r{Object.defineProperty(t,"__esModule",{value:true});t["default"]=void 0;var n=_interopRequireDefault(r(807));var i=_interopRequireDefault(r(8950));function _interopRequireDefault(e){return e&&e.__esModule?e:{default:e}}function v4(e,t,r){e=e||{};const a=e.random||(e.rng||n.default)();a[6]=a[6]&15|64;a[8]=a[8]&63|128;if(t){r=r||0;for(let e=0;e<16;++e){t[r+e]=a[e]}return t}return(0,i.default)(a)}var a=v4;t["default"]=a},9120:(e,t,r)=>{Object.defineProperty(t,"__esModule",{value:true});t["default"]=void 0;var n=_interopRequireDefault(r(5998));var i=_interopRequireDefault(r(5274));function _interopRequireDefault(e){return e&&e.__esModule?e:{default:e}}const a=(0,n.default)("v5",80,i.default);var o=a;t["default"]=o},6900:(e,t,r)=>{Object.defineProperty(t,"__esModule",{value:true});t["default"]=void 0;var n=_interopRequireDefault(r(814));function _interopRequireDefault(e){return e&&e.__esModule?e:{default:e}}function validate(e){return typeof e==="string"&&n.default.test(e)}var i=validate;t["default"]=i},1595:(e,t,r)=>{Object.defineProperty(t,"__esModule",{value:true});t["default"]=void 0;var n=_interopRequireDefault(r(6900));function _interopRequireDefault(e){return e&&e.__esModule?e:{default:e}}function version(e){if(!(0,n.default)(e)){throw TypeError("Invalid UUID")}return parseInt(e.substr(14,1),16)}var i=version;t["default"]=i},2940:e=>{e.exports=wrappy;function wrappy(e,t){if(e&&t)return wrappy(e)(t);if(typeof e!=="function")throw new TypeError("need wrapper function");Object.keys(e).forEach((function(t){wrapper[t]=e[t]}));return wrapper;function wrapper(){var t=new Array(arguments.length);for(var r=0;r{t.exports=e(import.meta.url)("assert")},4300:t=>{t.exports=e(import.meta.url)("buffer")},2057:t=>{t.exports=e(import.meta.url)("constants")},6113:t=>{t.exports=e(import.meta.url)("crypto")},2361:t=>{t.exports=e(import.meta.url)("events")},7147:t=>{t.exports=e(import.meta.url)("fs")},3685:t=>{t.exports=e(import.meta.url)("http")},5687:t=>{t.exports=e(import.meta.url)("https")},1808:t=>{t.exports=e(import.meta.url)("net")},7718:t=>{t.exports=e(import.meta.url)("node:child_process")},7561:t=>{t.exports=e(import.meta.url)("node:fs")},7742:t=>{t.exports=e(import.meta.url)("node:process")},4492:t=>{t.exports=e(import.meta.url)("node:stream")},5628:t=>{t.exports=e(import.meta.url)("node:zlib")},2037:t=>{t.exports=e(import.meta.url)("os")},1017:t=>{t.exports=e(import.meta.url)("path")},2781:t=>{t.exports=e(import.meta.url)("stream")},1576:t=>{t.exports=e(import.meta.url)("string_decoder")},4404:t=>{t.exports=e(import.meta.url)("tls")},3837:t=>{t.exports=e(import.meta.url)("util")},6955:(e,t,r)=>{r.a(e,(async e=>{var t=r(7718);var n=r(7742);var i=r(4492);var a=r(7561);var o=r(5628);var s=r(2283);var u=r(2186);const l=u.getInput("container",{required:true});const c=JSON.parse(u.getInput("error-log-paths",{required:true}));const d=u.getInput("log-tarball-prefix",{required:true});const h=u.getInput("tests-label",{required:true});const p=u.getInput("test-timeout",{required:true});try{if(t.spawnSync("docker",["run","--name","base","-v",`${n.cwd()}/build.tar.zst:/build.tar.zst`,"--workdir","/__w/leap/leap",l,"sh","-c","zstdcat /build.tar.zst | tar x"],{stdio:"inherit"}).status)throw new Error("Failed to create base container");if(t.spawnSync("docker",["commit","base","baseimage"],{stdio:"inherit"}).status)throw new Error("Failed to create base image");if(t.spawnSync("docker",["rm","base"],{stdio:"inherit"}).status)throw new Error("Failed to remove base container");const e=t.spawnSync("docker",["run","--rm","baseimage","bash","-e","-o","pipefail","-c",`cd build; ctest -L '${h}' --show-only=json-v1`]);if(e.status)throw new Error("Failed to discover tests with label");const r=JSON.parse(e.stdout).tests;let g=[];r.forEach((e=>{g.push(new Promise((r=>{t.spawn("docker",["run","--security-opt","seccomp=unconfined","-e","GITHUB_ACTIONS=True","--name",e.name,"--init","baseimage","bash","-c",`cd build; ctest --output-on-failure -R '^${e.name}$' --timeout ${p}`],{stdio:"inherit"}).on("close",(e=>r(e)))})))}));const m=await Promise.all(g);for(let e=0;e{if(!e.name.startsWith(`__w/leap/leap/build`)){t.on("end",(()=>r()));t.resume();return}e.name=e.name.substring(`__w/leap/leap/`.length);if(e.name!=="build/"&&c.filter((t=>e.name.startsWith(t))).length===0){t.on("end",(()=>r()));t.resume();return}t.pipe(l.entry(e,r))})).on("finish",(()=>{l.finalize()}));t.spawn("docker",["export",r[e].name]).stdout.pipe(n);i.promises.pipeline(l,o.createGzip(),a.createWriteStream(`${d}-${r[e].name}-logs.tar.gz`))}}catch(e){u.setFailed(`Uncaught exception ${e.message}`)}e()}),1)}};var r={};function __nccwpck_require__(e){var n=r[e];if(n!==undefined){return n.exports}var i=r[e]={exports:{}};var a=true;try{t[e].call(i.exports,i,i.exports,__nccwpck_require__);a=false}finally{if(a)delete r[e]}return i.exports}(()=>{var e=typeof Symbol==="function"?Symbol("webpack then"):"__webpack_then__";var t=typeof Symbol==="function"?Symbol("webpack exports"):"__webpack_exports__";var completeQueue=e=>{if(e){e.forEach((e=>e.r--));e.forEach((e=>e.r--?e.r++:e()))}};var completeFunction=e=>!--e.r&&e();var queueFunction=(e,t)=>e?e.push(t):completeFunction(t);var wrapDeps=r=>r.map((r=>{if(r!==null&&typeof r==="object"){if(r[e])return r;if(r.then){var n=[];r.then((e=>{i[t]=e;completeQueue(n);n=0}));var i={};i[e]=(e,t)=>(queueFunction(n,e),r["catch"](t));return i}}var a={};a[e]=e=>completeFunction(e);a[t]=r;return a}));__nccwpck_require__.a=(r,n,i)=>{var a=i&&[];var o=r.exports;var s;var u;var l;var c=true;var d=false;var whenAll=(t,r,n)=>{if(d)return;d=true;r.r+=t.length;t.map(((t,i)=>t[e](r,n)));d=false};var h=new Promise(((e,t)=>{l=t;u=()=>(e(o),completeQueue(a),a=0)}));h[t]=o;h[e]=(e,t)=>{if(c){return completeFunction(e)}if(s)whenAll(s,e,t);queueFunction(a,e);h["catch"](t)};r.exports=h;n((e=>{if(!e)return u();s=wrapDeps(e);var r,n;var i=new Promise(((e,i)=>{r=()=>e(n=s.map((e=>e[t])));r.r=0;whenAll(s,r,i)}));return r.r?i:n})).then(u,l);c=false}})();if(typeof __nccwpck_require__!=="undefined")__nccwpck_require__.ab=new URL(".",import.meta.url).pathname.slice(import.meta.url.match(/^file:\/\/\/\w:/)?1:0,-1)+"/";var n=__nccwpck_require__(6955);n=await n; \ No newline at end of file diff --git a/.github/actions/parallel-ctest-containers/main.mjs b/.github/actions/parallel-ctest-containers/main.mjs index ff4e8330da..e88014bb51 100644 --- a/.github/actions/parallel-ctest-containers/main.mjs +++ b/.github/actions/parallel-ctest-containers/main.mjs @@ -61,8 +61,8 @@ try { stream.pipe(packer.entry(header, next)); }).on('finish', () => {packer.finalize()}); - child_process.spawn("docker", ["export", tests[i]]).stdout.pipe(extractor); - stream.promises.pipeline(packer, zlib.createGzip(), fs.createWriteStream(`${log_tarball_prefix}-${tests[i]}-logs.tar.gz`)); + child_process.spawn("docker", ["export", tests[i].name]).stdout.pipe(extractor); + stream.promises.pipeline(packer, zlib.createGzip(), fs.createWriteStream(`${log_tarball_prefix}-${tests[i].name}-logs.tar.gz`)); } } catch(e) { core.setFailed(`Uncaught exception ${e.message}`); From a4681a3c04872a5a5b04b14026dad17b5efdfd3c Mon Sep 17 00:00:00 2001 From: greg7mdp Date: Thu, 27 Apr 2023 15:36:51 -0400 Subject: [PATCH 46/58] Declare `WASM::check_limits` variable is Logging library to fix g++ issue Within `libraries/wasm-jit/Source`, `WASM` depends on `IR` and `Logging`, and `IR` depends on `Logging`. With the current declaration of `WASM::check_limits` in `WASM`, we have the following g++-10 link error when building with `-std=c++20`: ``` _64-linux-gnu/libz.so -ldl libraries/libfc/secp256k1/libsecp256k1.a libraries/chainbase/libchainbase.a -lpthread libraries/softfloat/libsoftfloat.a && : /usr/bin/ld: libraries/wasm-jit/Source/IR/libIR.a(DisassemblyNames.cpp.o): warning: relocation against `_ZN4WASM12check_limitsE' in read-only section `.text' /usr/bin/ld: libraries/wasm-jit/Source/IR/libIR.a(DisassemblyNames.cpp.o): in function `void Serialization::serialize(Serialization::MemoryInputStream&, std::__cxx11::basic_string, std::allocator >&)': /home/greg/github/enf/leap/libraries/wasm-jit/Include/Inline/Serialization.h:274: undefined reference to `WASM::check_limits' /usr/bin/ld: warning: creating DT_TEXTREL in a PIE collect2: error: ld returned 1 exit status ``` Moving the declaration of `WASM::check_limits` in `Logging` (end leaf of dependency graph) fixes the issue. --- libraries/wasm-jit/Source/Logging/Logging.cpp | 4 ++++ libraries/wasm-jit/Source/WASM/WASMSerialization.cpp | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/libraries/wasm-jit/Source/Logging/Logging.cpp b/libraries/wasm-jit/Source/Logging/Logging.cpp index 80d368ec15..6ca27c2850 100644 --- a/libraries/wasm-jit/Source/Logging/Logging.cpp +++ b/libraries/wasm-jit/Source/Logging/Logging.cpp @@ -5,6 +5,10 @@ #include #include +namespace WASM +{ + bool check_limits = true; +} namespace Log { static bool categoryEnabled[(Uptr)Category::num] = diff --git a/libraries/wasm-jit/Source/WASM/WASMSerialization.cpp b/libraries/wasm-jit/Source/WASM/WASMSerialization.cpp index 0ad7d4963d..218350444c 100644 --- a/libraries/wasm-jit/Source/WASM/WASMSerialization.cpp +++ b/libraries/wasm-jit/Source/WASM/WASMSerialization.cpp @@ -168,7 +168,7 @@ namespace WASM using namespace IR; using namespace Serialization; - bool check_limits = true; + extern bool check_limits; enum { From 67edf65502c61dc19d24df1cf04b07809f37a3be Mon Sep 17 00:00:00 2001 From: Huang-Ming Huang Date: Thu, 27 Apr 2023 16:24:14 -0500 Subject: [PATCH 47/58] remove unused parameter --- plugins/http_plugin/http_plugin.cpp | 4 ++-- plugins/http_plugin/include/eosio/http_plugin/common.hpp | 1 - 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/plugins/http_plugin/http_plugin.cpp b/plugins/http_plugin/http_plugin.cpp index 40fa7ab9f3..96a43c4a1f 100644 --- a/plugins/http_plugin/http_plugin.cpp +++ b/plugins/http_plugin/http_plugin.cpp @@ -74,7 +74,7 @@ namespace eosio { * @return the constructed internal_url_handler */ static detail::internal_url_handler make_app_thread_url_handler(const string& url, appbase::exec_queue to_queue, int priority, url_handler next, http_plugin_impl_ptr my, http_content_type content_type ) { - detail::internal_url_handler handler{url}; + detail::internal_url_handler handler; handler.content_type = content_type; auto next_ptr = std::make_shared(std::move(next)); handler.fn = [my=std::move(my), priority, to_queue, next_ptr=std::move(next_ptr)] @@ -112,7 +112,7 @@ namespace eosio { * @return the constructed internal_url_handler */ static detail::internal_url_handler make_http_thread_url_handler(const string& url, url_handler next, http_content_type content_type) { - detail::internal_url_handler handler{url}; + detail::internal_url_handler handler; handler.content_type = content_type; handler.fn = [next=std::move(next)]( const detail::abstract_conn_ptr& conn, string&& r, string&& b, url_response_callback&& then ) mutable { try { diff --git a/plugins/http_plugin/include/eosio/http_plugin/common.hpp b/plugins/http_plugin/include/eosio/http_plugin/common.hpp index 1974733a98..d39e257adc 100644 --- a/plugins/http_plugin/include/eosio/http_plugin/common.hpp +++ b/plugins/http_plugin/include/eosio/http_plugin/common.hpp @@ -71,7 +71,6 @@ using internal_url_handler_fn = std::function Date: Thu, 27 Apr 2023 18:32:43 -0400 Subject: [PATCH 48/58] Move variable declaration from `Logging` to `IR` as per PR comment. --- libraries/wasm-jit/Source/IR/DisassemblyNames.cpp | 4 ++++ libraries/wasm-jit/Source/Logging/Logging.cpp | 4 ---- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/libraries/wasm-jit/Source/IR/DisassemblyNames.cpp b/libraries/wasm-jit/Source/IR/DisassemblyNames.cpp index d3c755e80f..62ad4ba92a 100644 --- a/libraries/wasm-jit/Source/IR/DisassemblyNames.cpp +++ b/libraries/wasm-jit/Source/IR/DisassemblyNames.cpp @@ -6,6 +6,10 @@ using namespace Serialization; +namespace WASM +{ + bool check_limits = true; +} namespace IR { void getDisassemblyNames(const Module& module,DisassemblyNames& outNames) diff --git a/libraries/wasm-jit/Source/Logging/Logging.cpp b/libraries/wasm-jit/Source/Logging/Logging.cpp index 6ca27c2850..80d368ec15 100644 --- a/libraries/wasm-jit/Source/Logging/Logging.cpp +++ b/libraries/wasm-jit/Source/Logging/Logging.cpp @@ -5,10 +5,6 @@ #include #include -namespace WASM -{ - bool check_limits = true; -} namespace Log { static bool categoryEnabled[(Uptr)Category::num] = From ca7e9b1ae4b788aea17cf2b4e44eb4ca9b2924dc Mon Sep 17 00:00:00 2001 From: Kevin Heifner Date: Fri, 28 Apr 2023 07:12:00 -0500 Subject: [PATCH 49/58] GH-641 Add an explicit main --- libraries/libfc/test/CMakeLists.txt | 1 + libraries/libfc/test/main.cpp | 2 ++ libraries/libfc/test/test_base64.cpp | 3 +-- 3 files changed, 4 insertions(+), 2 deletions(-) create mode 100644 libraries/libfc/test/main.cpp diff --git a/libraries/libfc/test/CMakeLists.txt b/libraries/libfc/test/CMakeLists.txt index bacd693aef..5f9d7c48b0 100644 --- a/libraries/libfc/test/CMakeLists.txt +++ b/libraries/libfc/test/CMakeLists.txt @@ -14,6 +14,7 @@ add_executable( test_fc variant/test_variant.cpp variant_estimated_size/test_variant_estimated_size.cpp test_base64.cpp + main.cpp ) target_link_libraries( test_fc fc ) diff --git a/libraries/libfc/test/main.cpp b/libraries/libfc/test/main.cpp new file mode 100644 index 0000000000..8d4461639a --- /dev/null +++ b/libraries/libfc/test/main.cpp @@ -0,0 +1,2 @@ +#define BOOST_TEST_MODULE libfc +#include diff --git a/libraries/libfc/test/test_base64.cpp b/libraries/libfc/test/test_base64.cpp index 64e977dc81..ff6ac6a0ec 100644 --- a/libraries/libfc/test/test_base64.cpp +++ b/libraries/libfc/test/test_base64.cpp @@ -1,5 +1,4 @@ -#define BOOST_TEST_MODULE libfc -#include +#include #include #include From ac2c37c3499817d8a838feb9fb2e1ad0e81813b5 Mon Sep 17 00:00:00 2001 From: greg7mdp Date: Fri, 28 Apr 2023 10:17:42 -0400 Subject: [PATCH 50/58] `std::result_of` was deprecated in C++17, removed in C++20. --- .../chain_plugin/include/eosio/chain_plugin/chain_plugin.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/chain_plugin/include/eosio/chain_plugin/chain_plugin.hpp b/plugins/chain_plugin/include/eosio/chain_plugin/chain_plugin.hpp index 8ab6517ae8..82fd082e48 100644 --- a/plugins/chain_plugin/include/eosio/chain_plugin/chain_plugin.hpp +++ b/plugins/chain_plugin/include/eosio/chain_plugin/chain_plugin.hpp @@ -587,7 +587,7 @@ class read_only : public api_base { const auto* t_id = d.find(boost::make_tuple(p.code, scope, p.table)); const auto* index_t_id = d.find(boost::make_tuple(p.code, scope, name(table_with_index))); if( t_id != nullptr && index_t_id != nullptr ) { - using secondary_key_type = std::result_of_t; + using secondary_key_type = std::invoke_result_t; static_assert( std::is_same::value, "Return type of conv does not match type of secondary key for IndexType" ); const auto& secidx = d.get_index(); From 8375d5e63435b0831ff7646670c6e4a05f91223f Mon Sep 17 00:00:00 2001 From: Kevin Heifner Date: Fri, 28 Apr 2023 09:35:35 -0500 Subject: [PATCH 51/58] GH-641 Add escape_str function --- libraries/libfc/include/fc/string.hpp | 19 +++++ libraries/libfc/src/string.cpp | 41 ++++++++-- libraries/libfc/test/test_escape_str.cpp | 98 ++++++++++++++++++++++++ 3 files changed, 153 insertions(+), 5 deletions(-) create mode 100644 libraries/libfc/test/test_escape_str.cpp diff --git a/libraries/libfc/include/fc/string.hpp b/libraries/libfc/include/fc/string.hpp index 43b8484d73..cf405ad1b6 100644 --- a/libraries/libfc/include/fc/string.hpp +++ b/libraries/libfc/include/fc/string.hpp @@ -1,5 +1,6 @@ #pragma once #include +#include #include namespace fc @@ -20,4 +21,22 @@ namespace fc class variant_object; std::string format_string( const std::string&, const variant_object&, bool minimize = false ); std::string trim( const std::string& ); + + /** + * Convert '\t', '\r', '\n', '\\' and '"' to "\t\r\n\\\"" if escape_control_chars == true + * Convert all other < 32 & 127 ascii to escaped unicode "\u00xx" + * Removes invalid utf8 characters + * Escapes Control sequence Introducer 0x9b to \u009b + * All other characters unmolested. + * + * @param str input/output string to escape/truncate + * @param escape_control_chars if true escapes control chars in str + * @param max_len truncate string to max_len + * @param add_truncate_str if truncated by max_len, add add_truncate_str to end of any truncated string, + * new length with be max_len + strlen(add_truncate_str), nullptr allowed if no append wanted + * @return pair + */ + std::pair escape_str( std::string& str, bool escape_control_chars = true, + std::size_t max_len = std::numeric_limits::max(), + const char* add_truncate_str = "..." ); } diff --git a/libraries/libfc/src/string.cpp b/libraries/libfc/src/string.cpp index cc536ee666..c2dff7cadf 100644 --- a/libraries/libfc/src/string.cpp +++ b/libraries/libfc/src/string.cpp @@ -1,4 +1,6 @@ #include +#include +#include #include #include #include @@ -83,13 +85,42 @@ namespace fc { std::string trim( const std::string& s ) { return boost::algorithm::trim_copy(s); - /* - std::string cpy(s); - boost::algorithm::trim(cpy); - return cpy; - */ } + std::pair escape_str( std::string& str, bool escape_control_chars, + std::size_t max_len, const char* add_truncate_str ) + { + bool modified = false, truncated = false; + // truncate early to speed up escape + if (str.size() > max_len) { + str.resize(max_len); + modified = truncated = true; + } + auto itr = escape_control_chars + ? std::find_if(str.begin(), str.end(), + [](const auto& c) { + return c == '\x7f' || c == '\\' || c == '\"' || (c >= '\x00' && c <= '\x1f'); } ) + : std::find_if(str.begin(), str.end(), + [](const auto& c) { // x09 = \t, x0a = \n, x0d = \r + return c == '\x7f' || (c >= '\x00' && c <= '\x08') || c == '\x0b' || c == '\x0c' || (c >= '\x0e' && c <= '\x1f'); } ); + + if (itr != str.end() || !fc::is_valid_utf8( str )) { + str = escape_string(str, nullptr, escape_control_chars); + modified = true; + if (str.size() > max_len) { + str.resize(max_len); + truncated = true; + } + } + + if (truncated && add_truncate_str != nullptr ) { + str += add_truncate_str; + } + + return std::make_pair(std::ref(str), modified); + } + + } // namespace fc diff --git a/libraries/libfc/test/test_escape_str.cpp b/libraries/libfc/test/test_escape_str.cpp new file mode 100644 index 0000000000..b2077aef05 --- /dev/null +++ b/libraries/libfc/test/test_escape_str.cpp @@ -0,0 +1,98 @@ +#include + +#include +#include + +using namespace fc; +using namespace std::literals; + +BOOST_AUTO_TEST_SUITE(escape_str_test) + +BOOST_AUTO_TEST_CASE(escape_control_chars) try { + const std::string escape_input_str = "\b\f\n\r\t\\\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f" + "\x10\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f"; + std::string escaped_str = "\\u0008\\u000c\\n\\r\\t\\\\\\u0001\\u0002\\u0003\\u0004\\u0005\\u0006\\u0007\\u0008\\t\\n\\u000b\\u000c\\r\\u000e\\u000f" + "\\u0010\\u0011\\u0012\\u0013\\u0014\\u0015\\u0016\\u0017\\u0018\\u0019\\u001a\\u001b\\u001c\\u001d\\u001e\\u001f"; + + std::string input = escape_input_str; + BOOST_CHECK_EQUAL(escape_str(input).first, escaped_str); + + input = escape_input_str; + escaped_str = "\\u0008\\u000c\n" + "\r\t\\\\u0001\\u0002\\u0003\\u0004\\u0005\\u0006\\u0007\\u0008\t\n" + "\\u000b\\u000c\r\\u000e\\u000f\\u0010\\u0011\\u0012\\u0013\\u0014\\u0015\\u0016\\u0017\\u0018\\u0019\\u001a\\u001b\\u001c\\u001d\\u001e\\u001f"; + BOOST_CHECK_EQUAL(escape_str(input, false).first, escaped_str); + +} FC_LOG_AND_RETHROW(); + +BOOST_AUTO_TEST_CASE(empty) try { + std::string input; + BOOST_CHECK_EQUAL(escape_str(input, true, 256, "").first, ""); + + input = ""; + BOOST_CHECK_EQUAL(escape_str(input, false, 512, nullptr).first, ""); +} FC_LOG_AND_RETHROW(); + +BOOST_AUTO_TEST_CASE(truncate) try { + const std::string repeat_512_chars(512, 'a'); + const std::string repeat_256_chars(256, 'a'); + + std::string input = repeat_512_chars; + BOOST_CHECK_EQUAL(escape_str(input, true, 256, "").first, repeat_256_chars); + + input = repeat_512_chars; + BOOST_CHECK_EQUAL(escape_str(input, true, 256, nullptr).first, repeat_256_chars); + + input = repeat_512_chars; + BOOST_CHECK_EQUAL(escape_str(input, true, 256).first, repeat_256_chars + "..."); + + input = repeat_512_chars; + BOOST_CHECK_EQUAL(escape_str(input, true, 256, "<-the end->").first, repeat_256_chars + "<-the end->"); +} FC_LOG_AND_RETHROW(); + +BOOST_AUTO_TEST_CASE(modify) try { + const std::string repeat_512_chars(512, 'a'); + const std::string repeat_256_chars(256, 'a'); + + std::string input = repeat_512_chars; + BOOST_CHECK(escape_str(input, true, 256, "").second); + + input = repeat_512_chars; + BOOST_CHECK(escape_str(input, true, 256, nullptr).second); + + input = repeat_512_chars; + BOOST_CHECK(escape_str(input, true, 256).second); + + input = repeat_512_chars; + BOOST_CHECK(!escape_str(input, true, 512).second); + + input = repeat_512_chars; + BOOST_CHECK(!escape_str(input, true).second); + + input = repeat_512_chars; + BOOST_CHECK(!escape_str(input, true, 1024).second); + + input = ""; + BOOST_CHECK(!escape_str(input, true, 1024).second); + + input = "hello"; + BOOST_CHECK(!escape_str(input, true, 1024).second); + + input = "\n"; + BOOST_CHECK(escape_str(input, true, 1024).second); + + input ="\xb4"; + BOOST_CHECK(escape_str(input, true, 1024).second); + BOOST_CHECK_EQUAL(input, ""); + +} FC_LOG_AND_RETHROW(); + +BOOST_AUTO_TEST_CASE(remove_invalid_utf8) try { + auto input = "abc123$&()'?\xb4\xf5\x01\xfa~a"s; // remove invalid utf8 values, \x01 => \u0001 + auto expected_output = "abc123$&()'?\\u0001~a"s; + + BOOST_CHECK_EQUAL(escape_str(input).first, expected_output); +} FC_LOG_AND_RETHROW(); + + +BOOST_AUTO_TEST_SUITE_END() \ No newline at end of file From 94e7bd0696a890c92c4dfe00c25b987450aa75ee Mon Sep 17 00:00:00 2001 From: Kevin Heifner Date: Fri, 28 Apr 2023 09:36:15 -0500 Subject: [PATCH 52/58] GH-641 Use more efficient escape_str instead of format_string --- plugins/producer_plugin/producer_plugin.cpp | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/plugins/producer_plugin/producer_plugin.cpp b/plugins/producer_plugin/producer_plugin.cpp index 8507406d27..18cfc96e0b 100644 --- a/plugins/producer_plugin/producer_plugin.cpp +++ b/plugins/producer_plugin/producer_plugin.cpp @@ -2085,23 +2085,22 @@ inline std::string get_detailed_contract_except_info(const packed_transaction_pt { std::string contract_name; std::string act_name; - std::string details; - if( trace && !trace->action_traces.empty() ) { auto last_action_ordinal = trace->action_traces.size() - 1; contract_name = trace->action_traces[last_action_ordinal].receiver.to_string(); act_name = trace->action_traces[last_action_ordinal].act.name.to_string(); } else if ( trx ) { const auto& actions = trx->get_transaction().actions; - if( actions.empty() ) return details; // should not be possible + if( actions.empty() ) return {}; // should not be possible contract_name = actions[0].account.to_string(); act_name = actions[0].name.to_string(); } - details = except_ptr ? except_ptr->top_message() : (trace && trace->except) ? trace->except->top_message() : std::string(); - if (!details.empty()) { - details = fc::format_string("${d}", fc::mutable_variant_object() ("d", details), true); // true for limiting the formatted string size - } + std::string details = except_ptr ? except_ptr->top_message() + : (trace && trace->except) ? trace->except->top_message() + : std::string(); + const bool escape_control_chars = true; + fc::escape_str(details, escape_control_chars, 1024); // this format is parsed by external tools return "action: " + contract_name + ":" + act_name + ", " + details; From 9cbb09d0f53b5f4b736f7285d0ad44f511bf6c3a Mon Sep 17 00:00:00 2001 From: Kevin Heifner Date: Fri, 28 Apr 2023 09:36:50 -0500 Subject: [PATCH 53/58] GH-641 Optimize print_debug --- libraries/chain/apply_context.cpp | 29 ++++++++++++++++++++--------- 1 file changed, 20 insertions(+), 9 deletions(-) diff --git a/libraries/chain/apply_context.cpp b/libraries/chain/apply_context.cpp index de52aa0d99..10fa41c42d 100644 --- a/libraries/chain/apply_context.cpp +++ b/libraries/chain/apply_context.cpp @@ -19,15 +19,26 @@ namespace eosio { namespace chain { static inline void print_debug(account_name receiver, const action_trace& ar) { if (!ar.console.empty()) { - auto prefix = fc::format_string( - "\n[(${a},${n})->${r}]", - fc::mutable_variant_object() - ("a", ar.act.account) - ("n", ar.act.name) - ("r", receiver)); - dlog(prefix + ": CONSOLE OUTPUT BEGIN =====================\n" - + ar.console - + prefix + ": CONSOLE OUTPUT END =====================" ); + if (fc::logger::get(DEFAULT_LOGGER).is_enabled( fc::log_level::debug )) { + std::string prefix; + prefix.reserve(3 + 13 + 1 + 13 + 3 + 13 + 1); + prefix += "\n[("; + prefix += ar.act.account.to_string(); + prefix += ","; + prefix += ar.act.name.to_string(); + prefix += ")->"; + prefix += receiver.to_string(); + prefix += "]"; + + std::string output; + output.reserve(512); + output += prefix; + output += ": CONSOLE OUTPUT BEGIN =====================\n"; + output += ar.console; + output += prefix; + output += ": CONSOLE OUTPUT END ====================="; + dlog( std::move(output) ); + } } } From 122ad36b7dda2cee1bb3262c4aa88be4c581aea0 Mon Sep 17 00:00:00 2001 From: Kevin Heifner Date: Fri, 28 Apr 2023 10:28:19 -0500 Subject: [PATCH 54/58] GH-641 Add parens so precedence is clearer --- plugins/producer_plugin/producer_plugin.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/plugins/producer_plugin/producer_plugin.cpp b/plugins/producer_plugin/producer_plugin.cpp index 18cfc96e0b..0a3c51e45b 100644 --- a/plugins/producer_plugin/producer_plugin.cpp +++ b/plugins/producer_plugin/producer_plugin.cpp @@ -2097,8 +2097,8 @@ inline std::string get_detailed_contract_except_info(const packed_transaction_pt } std::string details = except_ptr ? except_ptr->top_message() - : (trace && trace->except) ? trace->except->top_message() - : std::string(); + : ((trace && trace->except) ? trace->except->top_message() + : std::string()); const bool escape_control_chars = true; fc::escape_str(details, escape_control_chars, 1024); From 4bbb4910fc522f812edf1f0d93f2a8ff5104ebba Mon Sep 17 00:00:00 2001 From: Kevin Heifner Date: Fri, 28 Apr 2023 11:47:39 -0500 Subject: [PATCH 55/58] GH-641 Add new test file --- libraries/libfc/test/CMakeLists.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/libraries/libfc/test/CMakeLists.txt b/libraries/libfc/test/CMakeLists.txt index 5f9d7c48b0..1e8a332473 100644 --- a/libraries/libfc/test/CMakeLists.txt +++ b/libraries/libfc/test/CMakeLists.txt @@ -14,6 +14,7 @@ add_executable( test_fc variant/test_variant.cpp variant_estimated_size/test_variant_estimated_size.cpp test_base64.cpp + test_escape_str.cpp main.cpp ) target_link_libraries( test_fc fc ) From 332e22e931b7c9d40629815b24d115fb4e6c22e2 Mon Sep 17 00:00:00 2001 From: Kevin Heifner Date: Fri, 28 Apr 2023 11:48:01 -0500 Subject: [PATCH 56/58] GH-641 Use enum instead of bool --- libraries/libfc/include/fc/string.hpp | 5 +-- libraries/libfc/src/string.cpp | 6 ++-- libraries/libfc/test/test_escape_str.cpp | 34 ++++++++++----------- plugins/producer_plugin/producer_plugin.cpp | 3 +- 4 files changed, 24 insertions(+), 24 deletions(-) diff --git a/libraries/libfc/include/fc/string.hpp b/libraries/libfc/include/fc/string.hpp index cf405ad1b6..9deaa430ad 100644 --- a/libraries/libfc/include/fc/string.hpp +++ b/libraries/libfc/include/fc/string.hpp @@ -30,13 +30,14 @@ namespace fc * All other characters unmolested. * * @param str input/output string to escape/truncate - * @param escape_control_chars if true escapes control chars in str + * @param escape_ctrl if on escapes control chars in str * @param max_len truncate string to max_len * @param add_truncate_str if truncated by max_len, add add_truncate_str to end of any truncated string, * new length with be max_len + strlen(add_truncate_str), nullptr allowed if no append wanted * @return pair */ - std::pair escape_str( std::string& str, bool escape_control_chars = true, + enum class escape_control_chars { off, on }; + std::pair escape_str( std::string& str, escape_control_chars escape_ctrl = escape_control_chars::on, std::size_t max_len = std::numeric_limits::max(), const char* add_truncate_str = "..." ); } diff --git a/libraries/libfc/src/string.cpp b/libraries/libfc/src/string.cpp index c2dff7cadf..452040e4f1 100644 --- a/libraries/libfc/src/string.cpp +++ b/libraries/libfc/src/string.cpp @@ -87,7 +87,7 @@ namespace fc { return boost::algorithm::trim_copy(s); } - std::pair escape_str( std::string& str, bool escape_control_chars, + std::pair escape_str( std::string& str, escape_control_chars escape_ctrl, std::size_t max_len, const char* add_truncate_str ) { bool modified = false, truncated = false; @@ -96,7 +96,7 @@ namespace fc { str.resize(max_len); modified = truncated = true; } - auto itr = escape_control_chars + auto itr = escape_ctrl == escape_control_chars::on ? std::find_if(str.begin(), str.end(), [](const auto& c) { return c == '\x7f' || c == '\\' || c == '\"' || (c >= '\x00' && c <= '\x1f'); } ) @@ -105,7 +105,7 @@ namespace fc { return c == '\x7f' || (c >= '\x00' && c <= '\x08') || c == '\x0b' || c == '\x0c' || (c >= '\x0e' && c <= '\x1f'); } ); if (itr != str.end() || !fc::is_valid_utf8( str )) { - str = escape_string(str, nullptr, escape_control_chars); + str = escape_string(str, nullptr, escape_ctrl == escape_control_chars::on); modified = true; if (str.size() > max_len) { str.resize(max_len); diff --git a/libraries/libfc/test/test_escape_str.cpp b/libraries/libfc/test/test_escape_str.cpp index b2077aef05..d4595c43b8 100644 --- a/libraries/libfc/test/test_escape_str.cpp +++ b/libraries/libfc/test/test_escape_str.cpp @@ -21,16 +21,16 @@ BOOST_AUTO_TEST_CASE(escape_control_chars) try { escaped_str = "\\u0008\\u000c\n" "\r\t\\\\u0001\\u0002\\u0003\\u0004\\u0005\\u0006\\u0007\\u0008\t\n" "\\u000b\\u000c\r\\u000e\\u000f\\u0010\\u0011\\u0012\\u0013\\u0014\\u0015\\u0016\\u0017\\u0018\\u0019\\u001a\\u001b\\u001c\\u001d\\u001e\\u001f"; - BOOST_CHECK_EQUAL(escape_str(input, false).first, escaped_str); + BOOST_CHECK_EQUAL(escape_str(input, fc::escape_control_chars::off).first, escaped_str); } FC_LOG_AND_RETHROW(); BOOST_AUTO_TEST_CASE(empty) try { std::string input; - BOOST_CHECK_EQUAL(escape_str(input, true, 256, "").first, ""); + BOOST_CHECK_EQUAL(escape_str(input, fc::escape_control_chars::on, 256, "").first, ""); input = ""; - BOOST_CHECK_EQUAL(escape_str(input, false, 512, nullptr).first, ""); + BOOST_CHECK_EQUAL(escape_str(input, fc::escape_control_chars::off, 512, nullptr).first, ""); } FC_LOG_AND_RETHROW(); BOOST_AUTO_TEST_CASE(truncate) try { @@ -38,16 +38,16 @@ BOOST_AUTO_TEST_CASE(truncate) try { const std::string repeat_256_chars(256, 'a'); std::string input = repeat_512_chars; - BOOST_CHECK_EQUAL(escape_str(input, true, 256, "").first, repeat_256_chars); + BOOST_CHECK_EQUAL(escape_str(input, fc::escape_control_chars::on, 256, "").first, repeat_256_chars); input = repeat_512_chars; - BOOST_CHECK_EQUAL(escape_str(input, true, 256, nullptr).first, repeat_256_chars); + BOOST_CHECK_EQUAL(escape_str(input, fc::escape_control_chars::on, 256, nullptr).first, repeat_256_chars); input = repeat_512_chars; - BOOST_CHECK_EQUAL(escape_str(input, true, 256).first, repeat_256_chars + "..."); + BOOST_CHECK_EQUAL(escape_str(input, fc::escape_control_chars::on, 256).first, repeat_256_chars + "..."); input = repeat_512_chars; - BOOST_CHECK_EQUAL(escape_str(input, true, 256, "<-the end->").first, repeat_256_chars + "<-the end->"); + BOOST_CHECK_EQUAL(escape_str(input, fc::escape_control_chars::on, 256, "<-the end->").first, repeat_256_chars + "<-the end->"); } FC_LOG_AND_RETHROW(); BOOST_AUTO_TEST_CASE(modify) try { @@ -55,34 +55,34 @@ BOOST_AUTO_TEST_CASE(modify) try { const std::string repeat_256_chars(256, 'a'); std::string input = repeat_512_chars; - BOOST_CHECK(escape_str(input, true, 256, "").second); + BOOST_CHECK(escape_str(input, fc::escape_control_chars::on, 256, "").second); input = repeat_512_chars; - BOOST_CHECK(escape_str(input, true, 256, nullptr).second); + BOOST_CHECK(escape_str(input, fc::escape_control_chars::on, 256, nullptr).second); input = repeat_512_chars; - BOOST_CHECK(escape_str(input, true, 256).second); + BOOST_CHECK(escape_str(input, fc::escape_control_chars::on, 256).second); input = repeat_512_chars; - BOOST_CHECK(!escape_str(input, true, 512).second); + BOOST_CHECK(!escape_str(input, fc::escape_control_chars::on, 512).second); input = repeat_512_chars; - BOOST_CHECK(!escape_str(input, true).second); + BOOST_CHECK(!escape_str(input, fc::escape_control_chars::on).second); input = repeat_512_chars; - BOOST_CHECK(!escape_str(input, true, 1024).second); + BOOST_CHECK(!escape_str(input, fc::escape_control_chars::on, 1024).second); input = ""; - BOOST_CHECK(!escape_str(input, true, 1024).second); + BOOST_CHECK(!escape_str(input, fc::escape_control_chars::on, 1024).second); input = "hello"; - BOOST_CHECK(!escape_str(input, true, 1024).second); + BOOST_CHECK(!escape_str(input, fc::escape_control_chars::on, 1024).second); input = "\n"; - BOOST_CHECK(escape_str(input, true, 1024).second); + BOOST_CHECK(escape_str(input, fc::escape_control_chars::on, 1024).second); input ="\xb4"; - BOOST_CHECK(escape_str(input, true, 1024).second); + BOOST_CHECK(escape_str(input, fc::escape_control_chars::on, 1024).second); BOOST_CHECK_EQUAL(input, ""); } FC_LOG_AND_RETHROW(); diff --git a/plugins/producer_plugin/producer_plugin.cpp b/plugins/producer_plugin/producer_plugin.cpp index 0a3c51e45b..c37fac9d11 100644 --- a/plugins/producer_plugin/producer_plugin.cpp +++ b/plugins/producer_plugin/producer_plugin.cpp @@ -2099,8 +2099,7 @@ inline std::string get_detailed_contract_except_info(const packed_transaction_pt std::string details = except_ptr ? except_ptr->top_message() : ((trace && trace->except) ? trace->except->top_message() : std::string()); - const bool escape_control_chars = true; - fc::escape_str(details, escape_control_chars, 1024); + fc::escape_str(details, fc::escape_control_chars::on, 1024); // this format is parsed by external tools return "action: " + contract_name + ":" + act_name + ", " + details; From 0cee2647742f80ad2578211ca05922400f6ff34c Mon Sep 17 00:00:00 2001 From: Kevin Heifner Date: Fri, 28 Apr 2023 12:10:12 -0500 Subject: [PATCH 57/58] GH-641 std::string_view instead of const char* --- libraries/libfc/include/fc/string.hpp | 6 +++--- libraries/libfc/src/string.cpp | 4 ++-- libraries/libfc/test/test_escape_str.cpp | 6 +++--- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/libraries/libfc/include/fc/string.hpp b/libraries/libfc/include/fc/string.hpp index 9deaa430ad..1e86fc4ffb 100644 --- a/libraries/libfc/include/fc/string.hpp +++ b/libraries/libfc/include/fc/string.hpp @@ -23,7 +23,7 @@ namespace fc std::string trim( const std::string& ); /** - * Convert '\t', '\r', '\n', '\\' and '"' to "\t\r\n\\\"" if escape_control_chars == true + * Convert '\t', '\r', '\n', '\\' and '"' to "\t\r\n\\\"" if escape_ctrl == on * Convert all other < 32 & 127 ascii to escaped unicode "\u00xx" * Removes invalid utf8 characters * Escapes Control sequence Introducer 0x9b to \u009b @@ -33,11 +33,11 @@ namespace fc * @param escape_ctrl if on escapes control chars in str * @param max_len truncate string to max_len * @param add_truncate_str if truncated by max_len, add add_truncate_str to end of any truncated string, - * new length with be max_len + strlen(add_truncate_str), nullptr allowed if no append wanted + * new length with be max_len + add_truncate_str.size() * @return pair */ enum class escape_control_chars { off, on }; std::pair escape_str( std::string& str, escape_control_chars escape_ctrl = escape_control_chars::on, std::size_t max_len = std::numeric_limits::max(), - const char* add_truncate_str = "..." ); + std::string_view add_truncate_str = "..." ); } diff --git a/libraries/libfc/src/string.cpp b/libraries/libfc/src/string.cpp index 452040e4f1..fc31b30953 100644 --- a/libraries/libfc/src/string.cpp +++ b/libraries/libfc/src/string.cpp @@ -88,7 +88,7 @@ namespace fc { } std::pair escape_str( std::string& str, escape_control_chars escape_ctrl, - std::size_t max_len, const char* add_truncate_str ) + std::size_t max_len, std::string_view add_truncate_str ) { bool modified = false, truncated = false; // truncate early to speed up escape @@ -113,7 +113,7 @@ namespace fc { } } - if (truncated && add_truncate_str != nullptr ) { + if (truncated && !add_truncate_str.empty()) { str += add_truncate_str; } diff --git a/libraries/libfc/test/test_escape_str.cpp b/libraries/libfc/test/test_escape_str.cpp index d4595c43b8..87246003a2 100644 --- a/libraries/libfc/test/test_escape_str.cpp +++ b/libraries/libfc/test/test_escape_str.cpp @@ -30,7 +30,7 @@ BOOST_AUTO_TEST_CASE(empty) try { BOOST_CHECK_EQUAL(escape_str(input, fc::escape_control_chars::on, 256, "").first, ""); input = ""; - BOOST_CHECK_EQUAL(escape_str(input, fc::escape_control_chars::off, 512, nullptr).first, ""); + BOOST_CHECK_EQUAL(escape_str(input, fc::escape_control_chars::off, 512, {}).first, ""); } FC_LOG_AND_RETHROW(); BOOST_AUTO_TEST_CASE(truncate) try { @@ -41,7 +41,7 @@ BOOST_AUTO_TEST_CASE(truncate) try { BOOST_CHECK_EQUAL(escape_str(input, fc::escape_control_chars::on, 256, "").first, repeat_256_chars); input = repeat_512_chars; - BOOST_CHECK_EQUAL(escape_str(input, fc::escape_control_chars::on, 256, nullptr).first, repeat_256_chars); + BOOST_CHECK_EQUAL(escape_str(input, fc::escape_control_chars::on, 256, {}).first, repeat_256_chars); input = repeat_512_chars; BOOST_CHECK_EQUAL(escape_str(input, fc::escape_control_chars::on, 256).first, repeat_256_chars + "..."); @@ -58,7 +58,7 @@ BOOST_AUTO_TEST_CASE(modify) try { BOOST_CHECK(escape_str(input, fc::escape_control_chars::on, 256, "").second); input = repeat_512_chars; - BOOST_CHECK(escape_str(input, fc::escape_control_chars::on, 256, nullptr).second); + BOOST_CHECK(escape_str(input, fc::escape_control_chars::on, 256, {}).second); input = repeat_512_chars; BOOST_CHECK(escape_str(input, fc::escape_control_chars::on, 256).second); From 44355def2a6c03942c7457201d5d9639e04151e2 Mon Sep 17 00:00:00 2001 From: Kevin Heifner Date: Fri, 28 Apr 2023 12:37:45 -0500 Subject: [PATCH 58/58] GH-641 Remove unneeded if check --- libraries/libfc/src/string.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libraries/libfc/src/string.cpp b/libraries/libfc/src/string.cpp index fc31b30953..cb0a79fde6 100644 --- a/libraries/libfc/src/string.cpp +++ b/libraries/libfc/src/string.cpp @@ -113,7 +113,7 @@ namespace fc { } } - if (truncated && !add_truncate_str.empty()) { + if (truncated) { str += add_truncate_str; }