Merge branch 'main' into prometheus-peer-statistics

libraries/chain/controller.cpp

@@ -1014,8 +1014,8 @@ struct controller_impl {
       );
    }
 
-   sha256 calculate_integrity_hash() {
-      sha256::encoder enc;
+   fc::sha256 calculate_integrity_hash() {
+      fc::sha256::encoder enc;
       auto hash_writer = std::make_shared<integrity_hash_snapshot_writer>(enc);
       add_to_snapshot(hash_writer);
       hash_writer->finalize();
@@ -2091,7 +2091,7 @@ struct controller_impl {
                   } else {
                      packed_transaction_ptr ptrx( b, &pt ); // alias signed_block_ptr
                      auto fut = transaction_metadata::start_recover_keys(
-                           std::move( ptrx ), thread_pool.get_executor(), chain_id, microseconds::maximum(), transaction_metadata::trx_type::input  );
+                           std::move( ptrx ), thread_pool.get_executor(), chain_id, fc::microseconds::maximum(), transaction_metadata::trx_type::input  );
                      trx_metas.emplace_back( transaction_metadata_ptr{}, std::move( fut ) );
                   }
                }
@@ -3218,7 +3218,7 @@ block_id_type controller::get_block_id_for_num( uint32_t block_num )const { try
    return id;
 } FC_CAPTURE_AND_RETHROW( (block_num) ) }
 
-sha256 controller::calculate_integrity_hash() { try {
+fc::sha256 controller::calculate_integrity_hash() { try {
    return my->calculate_integrity_hash();
 } FC_LOG_AND_RETHROW() }
 

libraries/chain/include/eosio/chain/apply_context.hpp

@@ -14,6 +14,7 @@ namespace chainbase { class database; }
 namespace eosio { namespace chain {
 
 class controller;
+class account_metadata_object;
 
 class apply_context {
    private:

libraries/chain/include/eosio/chain/controller.hpp

@@ -6,8 +6,6 @@
 #include <chainbase/pinnable_mapped_file.hpp>
 #include <boost/signals2/signal.hpp>
 
-#include <eosio/chain/abi_serializer.hpp>
-#include <eosio/chain/account_object.hpp>
 #include <eosio/chain/snapshot.hpp>
 #include <eosio/chain/protocol_feature_manager.hpp>
 #include <eosio/chain/webassembly/eos-vm-oc/config.hpp>
@@ -259,7 +257,7 @@ namespace eosio { namespace chain {
          // thread-safe
          block_id_type get_block_id_for_num( uint32_t block_num )const;
 
-         sha256 calculate_integrity_hash();
+         fc::sha256 calculate_integrity_hash();
          void write_snapshot( const snapshot_writer_ptr& snapshot );
 
          bool sender_avoids_whitelist_blacklist_enforcement( account_name sender )const;

libraries/chain/include/eosio/chain/webassembly/eos-vm-oc/memory.hpp

@@ -24,11 +24,10 @@ class memory {
       static constexpr uint64_t total_memory_per_slice = memory_prologue_size + UINT64_C(0x200000000) + UINT64_C(4096);
 
    public:
-      explicit memory(uint64_t max_pages);
+      explicit memory(uint64_t sliced_pages);
       ~memory();
       memory(const memory&) = delete;
       memory& operator=(const memory&) = delete;
-      void reset(uint64_t max_pages);
 
       uint8_t* const zero_page_memory_base() const { return zeropage_base; }
       uint8_t* const full_page_memory_base() const { return fullpage_base; }

libraries/chain/transaction_context.cpp

@@ -1,4 +1,5 @@
 #include <eosio/chain/apply_context.hpp>
+#include <eosio/chain/account_object.hpp>
 #include <eosio/chain/transaction_context.hpp>
 #include <eosio/chain/authorization_manager.hpp>
 #include <eosio/chain/exceptions.hpp>
@@ -8,16 +9,6 @@
 #include <eosio/chain/global_property_object.hpp>
 #include <eosio/chain/deep_mind.hpp>
 
-#pragma push_macro("N")
-#undef N
-#include <boost/accumulators/accumulators.hpp>
-#include <boost/accumulators/statistics/stats.hpp>
-#include <boost/accumulators/statistics/min.hpp>
-#include <boost/accumulators/statistics/max.hpp>
-#include <boost/accumulators/statistics/weighted_mean.hpp>
-#include <boost/accumulators/statistics/weighted_variance.hpp>
-#pragma pop_macro("N")
-
 #include <chrono>
 
 namespace eosio { namespace chain {
@@ -828,7 +819,7 @@ namespace eosio { namespace chain {
                actors.insert( auth.actor );
          }
       }
-      EOS_ASSERT( one_auth || is_transient(), tx_no_auths, "transaction must have at least one authorization" );
+      EOS_ASSERT( one_auth || is_read_only(), tx_no_auths, "transaction must have at least one authorization" );
 
       if( enforce_actor_whitelist_blacklist ) {
          control.check_actor_list( actors );

libraries/chain/webassembly/authorization.cpp

@@ -1,6 +1,8 @@
 #include <eosio/chain/webassembly/interface.hpp>
 #include <eosio/chain/apply_context.hpp>
 
+#include <fc/io/datastream.hpp>
+
 namespace eosio { namespace chain { namespace webassembly {
    void interface::require_auth( account_name account ) const {
       context.require_authorization( account );
@@ -42,7 +44,7 @@ namespace eosio { namespace chain { namespace webassembly {
 
       auto s = fc::raw::pack_size(result);
       if (s <= packed_result.size()) {
-         datastream<char*> ds(packed_result.data(), s);
+         fc::datastream<char*> ds(packed_result.data(), s);
          fc::raw::pack(ds, result);
       }
       return s;

libraries/chain/webassembly/crypto.cpp

@@ -2,6 +2,7 @@
 #include <eosio/chain/protocol_state_object.hpp>
 #include <eosio/chain/transaction_context.hpp>
 #include <eosio/chain/apply_context.hpp>
+#include <fc/io/datastream.hpp>
 #include <fc/crypto/modular_arithmetic.hpp>
 #include <fc/crypto/blake2.hpp>
 #include <fc/crypto/sha3.hpp>
@@ -25,8 +26,8 @@ namespace eosio { namespace chain { namespace webassembly {
                                        legacy_span<const char> pub ) const {
       fc::crypto::signature s;
       fc::crypto::public_key p;
-      datastream<const char*> ds( sig.data(), sig.size() );
-      datastream<const char*> pubds ( pub.data(), pub.size() );
+      fc::datastream<const char*> ds( sig.data(), sig.size() );
+      fc::datastream<const char*> pubds ( pub.data(), pub.size() );
 
       fc::raw::unpack( ds, s );
       fc::raw::unpack( pubds, p );
@@ -48,7 +49,7 @@ namespace eosio { namespace chain { namespace webassembly {
                                    legacy_span<const char> sig,
                                    legacy_span<char> pub ) const {
       fc::crypto::signature s;
-      datastream<const char*> ds( sig.data(), sig.size() );
+      fc::datastream<const char*> ds( sig.data(), sig.size() );
       fc::raw::unpack(ds, s);
 
       EOS_ASSERT(s.which() < context.db.get<protocol_state_object>().num_supported_key_types, unactivated_signature_type,
@@ -74,7 +75,7 @@ namespace eosio { namespace chain { namespace webassembly {
          // this will do one less copy for those keys while maintaining the rules of
          //    [0..33) dest sizes: assert (asserts in fc::raw::pack)
          //    [33..inf) dest sizes: return packed size (always 33)
-         datastream<char*> out_ds( pub.data(), pub.size() );
+         fc::datastream<char*> out_ds( pub.data(), pub.size() );
          fc::raw::pack(out_ds, recovered);
          return out_ds.tellp();
       }

libraries/chain/webassembly/permission.cpp

@@ -1,4 +1,5 @@
 #include <eosio/chain/webassembly/interface.hpp>
+#include <eosio/chain/account_object.hpp>
 #include <eosio/chain/authorization_manager.hpp>
 #include <eosio/chain/transaction_context.hpp>
 #include <eosio/chain/apply_context.hpp>

libraries/chain/webassembly/privileged.cpp

@@ -1,10 +1,13 @@
+#include <eosio/chain/account_object.hpp>
 #include <eosio/chain/webassembly/interface.hpp>
 #include <eosio/chain/global_property_object.hpp>
 #include <eosio/chain/protocol_state_object.hpp>
 #include <eosio/chain/transaction_context.hpp>
 #include <eosio/chain/resource_limits.hpp>
 #include <eosio/chain/apply_context.hpp>
 
+#include <fc/io/datastream.hpp>
+
 #include <vector>
 #include <set>
 
@@ -94,15 +97,15 @@ namespace eosio { namespace chain { namespace webassembly {
          return s;
 
       if ( s <= packed_parameters.size() ) {
-         datastream<char*> ds( packed_parameters.data(), s );
+         fc::datastream<char*> ds( packed_parameters.data(), s );
          fc::raw::pack(ds, version);
          fc::raw::pack(ds, params);
       }
       return s;
    }
    void interface::set_wasm_parameters_packed( span<const char> packed_parameters ) {
       EOS_ASSERT(!context.trx_context.is_read_only(), wasm_execution_error, "set_wasm_parameters_packed not allowed in a readonly transaction");
-      datastream<const char*> ds( packed_parameters.data(), packed_parameters.size() );
+      fc::datastream<const char*> ds( packed_parameters.data(), packed_parameters.size() );
       uint32_t version;
       chain::wasm_config cfg;
       fc::raw::unpack(ds, version);
@@ -117,7 +120,7 @@ namespace eosio { namespace chain { namespace webassembly {
    }
    int64_t interface::set_proposed_producers( legacy_span<const char> packed_producer_schedule) {
       EOS_ASSERT(!context.trx_context.is_read_only(), wasm_execution_error, "set_proposed_producers not allowed in a readonly transaction");
-      datastream<const char*> ds( packed_producer_schedule.data(), packed_producer_schedule.size() );
+      fc::datastream<const char*> ds( packed_producer_schedule.data(), packed_producer_schedule.size() );
       std::vector<producer_authority> producers;
       std::vector<legacy::producer_key> old_version;
       fc::raw::unpack(ds, old_version);
@@ -137,7 +140,7 @@ namespace eosio { namespace chain { namespace webassembly {
       if (packed_producer_format == 0) {
          return set_proposed_producers(std::move(packed_producer_schedule));
       } else if (packed_producer_format == 1) {
-         datastream<const char*> ds( packed_producer_schedule.data(), packed_producer_schedule.size() );
+         fc::datastream<const char*> ds( packed_producer_schedule.data(), packed_producer_schedule.size() );
          vector<producer_authority> producers;
 
          fc::raw::unpack(ds, producers);
@@ -154,7 +157,7 @@ namespace eosio { namespace chain { namespace webassembly {
       if( packed_blockchain_parameters.size() == 0 ) return s;
 
       if ( s <= packed_blockchain_parameters.size() ) {
-         datastream<char*> ds( packed_blockchain_parameters.data(), s );
+         fc::datastream<char*> ds( packed_blockchain_parameters.data(), s );
          fc::raw::pack(ds, gpo.configuration.v0());
          return s;
       }
@@ -163,7 +166,7 @@ namespace eosio { namespace chain { namespace webassembly {
 
    void interface::set_blockchain_parameters_packed( legacy_span<const char> packed_blockchain_parameters ) {
       EOS_ASSERT(!context.trx_context.is_read_only(), wasm_execution_error, "set_blockchain_parameters_packed not allowed in a readonly transaction");
-      datastream<const char*> ds( packed_blockchain_parameters.data(), packed_blockchain_parameters.size() );
+      fc::datastream<const char*> ds( packed_blockchain_parameters.data(), packed_blockchain_parameters.size() );
       chain::chain_config_v0 cfg;
       fc::raw::unpack(ds, cfg);
       cfg.validate();
@@ -174,7 +177,7 @@ namespace eosio { namespace chain { namespace webassembly {
    }
 
    uint32_t interface::get_parameters_packed( span<const char> packed_parameter_ids, span<char> packed_parameters) const{
-      datastream<const char*> ds_ids( packed_parameter_ids.data(), packed_parameter_ids.size() );
+      fc::datastream<const char*> ds_ids( packed_parameter_ids.data(), packed_parameter_ids.size() );
 
       chain::chain_config cfg = context.control.get_global_properties().configuration;
       std::vector<fc::unsigned_int> ids;
@@ -188,14 +191,14 @@ namespace eosio { namespace chain { namespace webassembly {
                  chain::config_parse_error,
                  "get_parameters_packed: buffer size is smaller than ${size}", ("size", size));
 
-      datastream<char*> ds( packed_parameters.data(), size );
+      fc::datastream<char*> ds( packed_parameters.data(), size );
       fc::raw::pack( ds, config_range );
       return size;
    }
 
    void interface::set_parameters_packed( span<const char> packed_parameters ){
       EOS_ASSERT(!context.trx_context.is_read_only(), wasm_execution_error, "set_parameters_packed not allowed in a readonly transaction");
-      datastream<const char*> ds( packed_parameters.data(), packed_parameters.size() );
+      fc::datastream<const char*> ds( packed_parameters.data(), packed_parameters.size() );
 
       chain::chain_config cfg = context.control.get_global_properties().configuration;
       config_range config_range(cfg, {context.control});

libraries/chain/webassembly/runtimes/eos-vm-oc.cpp

@@ -60,6 +60,9 @@ void eosvmoc_runtime::init_thread_local_data() {
 }
 
 thread_local std::unique_ptr<eosvmoc::executor> eosvmoc_runtime::exec_thread_local {};
-thread_local eosvmoc::memory eosvmoc_runtime::mem_thread_local{ wasm_constraints::maximum_linear_memory/wasm_constraints::wasm_page_size };
+// Set sliced_pages_for_ro_thread to a small number to save upfront virtual memory
+// consumption. Usage beyond this limit will be handled by mprotect.
+constexpr uint32_t sliced_pages_for_ro_thread = 10;
+thread_local eosvmoc::memory eosvmoc_runtime::mem_thread_local{sliced_pages_for_ro_thread};
 
 }}}}

libraries/chain/webassembly/runtimes/eos-vm-oc/memory.cpp

@@ -11,9 +11,9 @@
 
 namespace eosio { namespace chain { namespace eosvmoc {
 
-memory::memory(uint64_t max_pages) {
-   uint64_t number_slices = max_pages + 1;
-   uint64_t wasm_memory_size = max_pages * wasm_constraints::wasm_page_size;
+memory::memory(uint64_t sliced_pages) {
+   uint64_t number_slices = sliced_pages + 1;
+   uint64_t wasm_memory_size = sliced_pages * wasm_constraints::wasm_page_size;
    int fd = syscall(SYS_memfd_create, "eosvmoc_mem", MFD_CLOEXEC);
    FC_ASSERT(fd >= 0, "Failed to create memory memfd");
    auto cleanup_fd = fc::make_scoped_exit([&fd](){close(fd);});
@@ -44,16 +44,6 @@ memory::memory(uint64_t max_pages) {
       intrinsic_jump_table[-intrinsic.second.ordinal] = (uintptr_t)intrinsic.second.function_ptr;
 }
 
-void memory::reset(uint64_t max_pages) {
-   uint64_t old_max_pages = mapsize / memory::total_memory_per_slice - 1;
-   if(max_pages == old_max_pages) return;
-   memory new_memory{max_pages};
-   std::swap(mapbase, new_memory.mapbase);
-   std::swap(mapsize, new_memory.mapsize);
-   std::swap(zeropage_base, new_memory.zeropage_base);
-   std::swap(fullpage_base, new_memory.fullpage_base);
-}
-
 memory::~memory() {
    munmap(mapbase, mapsize);
 }

libraries/chain/webassembly/runtimes/eos-vm.cpp

@@ -1,5 +1,6 @@
 #include <eosio/chain/webassembly/eos-vm.hpp>
 #include <eosio/chain/webassembly/interface.hpp>
+#include <eosio/chain/account_object.hpp>
 #include <eosio/chain/apply_context.hpp>
 #include <eosio/chain/transaction_context.hpp>
 #include <eosio/chain/global_property_object.hpp>

plugins/producer_plugin/producer_plugin.cpp

@@ -1256,48 +1256,6 @@ void producer_plugin_impl::plugin_initialize(const boost::program_options::varia
 
    // only initialize other read-only options when read-only thread pool is enabled
    if (_ro_thread_pool_size > 0) {
-#ifdef EOSIO_EOS_VM_OC_RUNTIME_ENABLED
-      if (chain.is_eos_vm_oc_enabled()) {
-         // EOS VM OC requires 4.2TB Virtual for each executing thread. Make sure the memory
-         // required by configured read-only threads does not exceed the total system virtual memory.
-         std::string   attr_name;
-         size_t        vm_total_kb{0};
-         size_t        vm_used_kb{0};
-         std::ifstream meminfo_file("/proc/meminfo");
-         while (meminfo_file >> attr_name) {
-            if (attr_name == "VmallocTotal:") {
-               if (!(meminfo_file >> vm_total_kb))
-                  break;
-            } else if (attr_name == "VmallocUsed:") {
-               if (!(meminfo_file >> vm_used_kb))
-                  break;
-            }
-            meminfo_file.ignore(std::numeric_limits<std::streamsize>::max(), '\n');
-         }
-
-         EOS_ASSERT(vm_total_kb > 0, plugin_config_exception,
-                    "Unable to get system virtual memory size (not a Linux?), therefore cannot determine if the system has enough "
-                    "virtual memory for multi-threaded read-only transactions on EOS VM OC");
-         EOS_ASSERT(vm_total_kb > vm_used_kb, plugin_config_exception,
-                    "vm total (${t}) must be greater than vm used (${u})",
-                    ("t", vm_total_kb)("u", vm_used_kb));
-         uint32_t num_threads_supported = (vm_total_kb - vm_used_kb) / 4200000000;
-         // reserve 1 for the app thread, 1 for anything else which might use VM
-         EOS_ASSERT(num_threads_supported > 2, plugin_config_exception,
-                    "With the EOS VM OC configured, there is not enough system virtual memory to support the required minimum of "
-                    "3 threads (1 for main thread, 1 for read-only, and 1 for anything else), vm total: ${t}, vm used: ${u}",
-                    ("t", vm_total_kb)("u", vm_used_kb));
-         num_threads_supported -= 2;
-         auto actual_threads_allowed = std::min(_ro_max_threads_allowed, num_threads_supported);
-         ilog("vm total in kb: ${total}, vm used in kb: ${used}, number of EOS VM OC threads supported "
-              "((vm total - vm used)/4.2 TB - 2): ${supp}, max allowed: ${max}, actual allowed: ${actual}",
-              ("total", vm_total_kb)("used", vm_used_kb)("supp", num_threads_supported)("max", _ro_max_threads_allowed)
-              ("actual", actual_threads_allowed));
-         EOS_ASSERT(_ro_thread_pool_size <= actual_threads_allowed, plugin_config_exception,
-                    "read-only-threads (${th}) greater than number of threads allowed for EOS VM OC (${allowed})",
-                    ("th", _ro_thread_pool_size)("allowed", actual_threads_allowed));
-      }
-#endif
       EOS_ASSERT(_ro_thread_pool_size <= _ro_max_threads_allowed,
                  plugin_config_exception,
                  "read-only-threads (${th}) greater than the number of threads allowed (${allowed})",

unittests/delay_tests.cpp

@@ -1901,8 +1901,11 @@ BOOST_AUTO_TEST_CASE( canceldelay_test ) { try {
 
    chain.set_transaction_headers(trx);
    trx.sign(chain.get_private_key("tester"_n, "active"), chain.control->get_chain_id());
+   // first push as a dry_run trx
+   trace = chain.push_transaction(trx, fc::time_point::maximum(), base_tester::DEFAULT_BILLED_CPU_TIME_US, false, transaction_metadata::trx_type::dry_run);
+   BOOST_REQUIRE_EQUAL(transaction_receipt::executed, trace->receipt->status);
+   // now push for real
    trace = chain.push_transaction(trx);
-   //wdump((fc::json::to_pretty_string(trace)));
    BOOST_REQUIRE_EQUAL(transaction_receipt::executed, trace->receipt->status);
    gen_size = chain.control->db().get_index<generated_transaction_multi_index,by_trx_id>().size();
    BOOST_CHECK_EQUAL(2u, gen_size);