Merge pull request #1842 from AntelopeIO/merge-main-10-30-23

IF: Merge main into hotstuff_integration (10/30/2023)

.github/workflows/build.yaml

@@ -39,7 +39,9 @@ jobs:
       contents: read
     with:
       runs-on: '["self-hosted", "enf-x86-beefy"]'
-      platform-files: .cicd/platforms
+      platform-files: |
+        .cicd/platforms
+        tools/reproducible.Dockerfile:builder
 
   build-base:
     name: Run Build Workflow
@@ -77,13 +79,13 @@ jobs:
             echo eos-system-contracts-ref=${{inputs.override-eos-system-contracts}} >> $GITHUB_OUTPUT
           fi
 
-  dev-package:
-    name: Build leap-dev package
+  package:
+    name: Build deb packages
     needs: [platform-cache, build-base]
     strategy:
       fail-fast: false
       matrix:
-        platform: [ubuntu20, ubuntu22]
+        platform: [ubuntu20, ubuntu22, reproducible]
     runs-on: ubuntu-latest
     container: ${{fromJSON(needs.platform-cache.outputs.platforms)[matrix.platform].image}}
     steps:
@@ -94,41 +96,55 @@ jobs:
         uses: actions/download-artifact@v3
         with:
           name: ${{matrix.platform}}-build
-      - name: Build dev package
+      - name: Build packages
         run: |
           zstdcat build.tar.zst | tar x
           cd build
           cpack
+          ../tools/tweak-deb.sh leap_*.deb
       - name: Install dev package
+        if: matrix.platform != 'reproducible'
         run: |
           apt-get update && apt-get upgrade -y
           apt-get install -y ./build/leap_*.deb ./build/leap-dev*.deb
       - name: Test using TestHarness
+        if: matrix.platform != 'reproducible'
         run: |
           python3 -c "from TestHarness import Cluster"
       - name: Upload dev package
         uses: actions/upload-artifact@v3
+        if: matrix.platform != 'reproducible'
         with:
           name: leap-dev-${{matrix.platform}}-amd64
           path: build/leap-dev*.deb
+      - name: Upload leap package
+        uses: actions/upload-artifact@v3
+        if: matrix.platform == 'reproducible'
+        with:
+          name: leap-deb-amd64
+          path: build/leap_*.deb
 
   tests:
-    name: Tests
+    name: Tests (${{matrix.cfg.name}})
     needs: [platform-cache, build-base]
     strategy:
       fail-fast: false
       matrix:
-        platform: [ubuntu20, ubuntu22]
+        include:
+          - cfg: {name: 'ubuntu20',      base: 'ubuntu20', builddir: 'ubuntu20'}
+          - cfg: {name: 'ubuntu22',      base: 'ubuntu22', builddir: 'ubuntu22'}
+          - cfg: {name: 'ubuntu20repro', base: 'ubuntu20', builddir: 'reproducible'}
+          - cfg: {name: 'ubuntu22repro', base: 'ubuntu22', builddir: 'reproducible'}
     runs-on: ["self-hosted", "enf-x86-hightier"]
     container:
-      image: ${{fromJSON(needs.platform-cache.outputs.platforms)[matrix.platform].image}}
+      image: ${{fromJSON(needs.platform-cache.outputs.platforms)[matrix.cfg.base].image}}
       options: --security-opt seccomp=unconfined
     steps:
       - uses: actions/checkout@v3
       - name: Download builddir
         uses: actions/download-artifact@v3
         with:
-          name: ${{matrix.platform}}-build
+          name: ${{matrix.cfg.builddir}}-build
       - name: Run Parallel Tests
         run: |
           # https://github.com/actions/runner/issues/2033  -- need this because of full version label test looking at git revs
@@ -140,66 +156,74 @@ jobs:
         run: awk 'BEGIN {err = 1} /bmi2/ && /adx/ {err = 0} END {exit err}' /proc/cpuinfo
 
   np-tests:
-    name: NP Tests
+    name: NP Tests (${{matrix.cfg.name}})
     needs: [platform-cache, build-base]
     strategy:
       fail-fast: false
       matrix:
-        platform: [ubuntu20, ubuntu22]
+        include:
+          - cfg: {name: 'ubuntu20',      base: 'ubuntu20', builddir: 'ubuntu20'}
+          - cfg: {name: 'ubuntu22',      base: 'ubuntu22', builddir: 'ubuntu22'}
+          - cfg: {name: 'ubuntu20repro', base: 'ubuntu20', builddir: 'reproducible'}
+          - cfg: {name: 'ubuntu22repro', base: 'ubuntu22', builddir: 'reproducible'}
     runs-on: ["self-hosted", "enf-x86-midtier"]
     steps:
       - uses: actions/checkout@v3
       - name: Download builddir
         uses: actions/download-artifact@v3
         with:
-          name: ${{matrix.platform}}-build
+          name: ${{matrix.cfg.builddir}}-build
       - name: Run tests in parallel containers
         uses: ./.github/actions/parallel-ctest-containers
         with:
-          container: ${{fromJSON(needs.platform-cache.outputs.platforms)[matrix.platform].image}}
+          container: ${{fromJSON(needs.platform-cache.outputs.platforms)[matrix.cfg.base].image}}
           error-log-paths: '["build/etc", "build/var", "build/leap-ignition-wd", "build/TestLogs"]'
-          log-tarball-prefix: ${{matrix.platform}}
+          log-tarball-prefix: ${{matrix.cfg.name}}
           tests-label: nonparallelizable_tests
           test-timeout: 420
       - name: Upload logs from failed tests
         uses: actions/upload-artifact@v3
         if: failure()
         with:
-          name: ${{matrix.platform}}-np-logs
+          name: ${{matrix.cfg.name}}-np-logs
           path: '*-logs.tar.gz'
 
   lr-tests:
-    name: LR Tests
+    name: LR Tests (${{matrix.cfg.name}})
     needs: [platform-cache, build-base]
     strategy:
       fail-fast: false
       matrix:
-        platform: [ubuntu20, ubuntu22]
+        include:
+          - cfg: {name: 'ubuntu20',      base: 'ubuntu20', builddir: 'ubuntu20'}
+          - cfg: {name: 'ubuntu22',      base: 'ubuntu22', builddir: 'ubuntu22'}
+          - cfg: {name: 'ubuntu20repro', base: 'ubuntu20', builddir: 'reproducible'}
+          - cfg: {name: 'ubuntu22repro', base: 'ubuntu22', builddir: 'reproducible'}
     runs-on: ["self-hosted", "enf-x86-lowtier"]
     steps:
       - uses: actions/checkout@v3
       - name: Download builddir
         uses: actions/download-artifact@v3
         with:
-          name: ${{matrix.platform}}-build
+          name: ${{matrix.cfg.builddir}}-build
       - name: Run tests in parallel containers
         uses: ./.github/actions/parallel-ctest-containers
         with:
-          container: ${{fromJSON(needs.platform-cache.outputs.platforms)[matrix.platform].image}}
+          container: ${{fromJSON(needs.platform-cache.outputs.platforms)[matrix.cfg.base].image}}
           error-log-paths: '["build/etc", "build/var", "build/leap-ignition-wd", "build/TestLogs"]'
-          log-tarball-prefix: ${{matrix.platform}}
+          log-tarball-prefix: ${{matrix.cfg.name}}
           tests-label: long_running_tests
           test-timeout: 1800
       - name: Upload logs from failed tests
         uses: actions/upload-artifact@v3
         if: failure()
         with:
-          name: ${{matrix.platform}}-lr-logs
+          name: ${{matrix.cfg.name}}-lr-logs
           path: '*-logs.tar.gz'
 
   libtester-tests:
     name: libtester tests
-    needs: [platform-cache, build-base, v, dev-package]
+    needs: [platform-cache, build-base, v, package]
     strategy:
       fail-fast: false
       matrix:
@@ -290,9 +314,9 @@ jobs:
 
   all-passing:
     name: All Required Tests Passed
-    needs: [dev-package, tests, np-tests, libtester-tests]
+    needs: [tests, np-tests, libtester-tests]
     if: always()
     runs-on: ubuntu-latest
     steps:
-      - if: needs.dev-package.result != 'success' || needs.tests.result != 'success' || needs.np-tests.result != 'success' || needs.libtester-tests.result != 'success'
+      - if: needs.tests.result != 'success' || needs.np-tests.result != 'success' || needs.libtester-tests.result != 'success'
         run: false

.github/workflows/release.yaml

@@ -0,0 +1,52 @@
+name: Release Actions
+
+on:
+  release:
+    types: [published]
+
+jobs:
+  eb:
+    name: experimental-binaries
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      packages: write
+      actions: read
+    steps:
+      - name: Get ubuntu20 leap-dev.deb
+        uses: AntelopeIO/asset-artifact-download-action@v3
+        with:
+          owner: ${{github.repository_owner}}
+          repo: ${{github.event.repository.name}}
+          file: 'leap-dev.*amd64.deb'
+          target: ${{github.sha}}
+          artifact-name: leap-dev-ubuntu20-amd64
+          wait-for-exact-target-workflow: true
+      - name: Get ubuntu22 leap-dev.deb
+        uses: AntelopeIO/asset-artifact-download-action@v3
+        with:
+          owner: ${{github.repository_owner}}
+          repo: ${{github.event.repository.name}}
+          file: 'leap-dev.*amd64.deb'
+          target: ${{github.sha}}
+          artifact-name: leap-dev-ubuntu22-amd64
+          wait-for-exact-target-workflow: true
+      - name: Create Dockerfile
+        run: |
+          cat <<EOF > Dockerfile
+          FROM scratch
+          LABEL org.opencontainers.image.description="A collection of experimental Leap binary packages"
+          COPY *.deb /
+          EOF
+      - name: Login to ghcr
+        uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{github.repository_owner}}
+          password: ${{github.token}}
+      - name: Build and push experimental-binaries
+        uses: docker/build-push-action@v3
+        with:
+          push: true
+          tags: ghcr.io/${{github.repository_owner}}/experimental-binaries:${{github.ref_name}}
+          context: .

libraries/chain/controller.cpp

@@ -614,8 +614,6 @@ struct controller_impl {
          ilog( "Snapshot loaded, lib: ${lib}", ("lib", head->block_num) );
 
          init(std::move(check_shutdown));
-         if (conf.revert_to_private_mode)
-            db.revert_to_private_mode();
          auto snapshot_load_time = (fc::time_point::now() - snapshot_load_start_time).to_seconds();
          ilog( "Finished initialization from snapshot (snapshot load time was ${t}s)", ("t", snapshot_load_time) );
       } catch (boost::interprocess::bad_alloc& e) {

libraries/chain/include/eosio/chain/controller.hpp

@@ -81,7 +81,6 @@ namespace eosio { namespace chain {
             bool                     disable_replay_opts    =  false;
             bool                     contracts_console      =  false;
             bool                     allow_ram_billing_in_notify = false;
-            bool                     revert_to_private_mode =  false;
             uint32_t                 maximum_variable_signature_length = chain::config::default_max_variable_signature_length;
             bool                     disable_all_subjective_mitigations = false; //< for developer & testing purposes, can be configured using `disable-all-subjective-mitigations` when `EOSIO_DEVELOPER` build option is provided
             uint32_t                 terminate_at_block     = 0;

libraries/chain/include/eosio/chain/webassembly/eos-vm-oc/eos-vm-oc.h

@@ -7,7 +7,9 @@
 #ifdef __cplusplus
 #include <vector>
 #include <list>
-namespace eosio { namespace chain {class apply_context;}}
+namespace eosio::chain {
+   class apply_context;
+}
 #endif
 
 struct eos_vm_oc_control_block {
@@ -38,3 +40,9 @@ struct eos_vm_oc_control_block {
    int64_t max_linear_memory_pages;
    void* globals;
 };
+
+#ifdef __cplusplus
+namespace eosio::chain::eosvmoc {
+   using control_block = eos_vm_oc_control_block;
+}
+#endif

libraries/chain/include/eosio/chain/webassembly/eos-vm-oc/eos-vm-oc.hpp

@@ -12,13 +12,7 @@
 #include <vector>
 #include <list>
 
-namespace eosio { namespace chain {
-
-class apply_context;
-
-namespace eosvmoc {
-
-using control_block = eos_vm_oc_control_block;
+namespace eosio::chain::eosvmoc {
 
 struct no_offset{};
 struct code_offset {
@@ -52,7 +46,7 @@ enum eosvmoc_exitcode : int {
 
 static constexpr uint8_t current_codegen_version = 1;
 
-}}}
+}
 
 FC_REFLECT(eosio::chain::eosvmoc::no_offset, );
 FC_REFLECT(eosio::chain::eosvmoc::code_offset, (offset));

libraries/chain/include/eosio/chain/webassembly/eos-vm-oc/memory.hpp

@@ -1,14 +1,15 @@
 #pragma once
+#include <fc/exception/exception.hpp>
 
 #include <eosio/chain/wasm_eosio_constraints.hpp>
-#include <eosio/chain/webassembly/eos-vm-oc/eos-vm-oc.hpp>
+#include <eosio/chain/webassembly/eos-vm-oc/eos-vm-oc.h>
 #include <eosio/chain/webassembly/eos-vm-oc/intrinsic_mapping.hpp>
 #include <eosio/chain/webassembly/eos-vm-oc/gs_seg_helpers.h>
 
 #include <stdint.h>
 #include <stddef.h>
 
-namespace eosio { namespace chain { namespace eosvmoc {
+namespace eosio::chain::eosvmoc {
 
 class memory {
       static constexpr uint64_t intrinsic_count                   = intrinsic_table_size();
@@ -66,7 +67,7 @@ class memory {
       uint8_t* fullpage_base;
 };
 
-}}}
+}
 
 #define OFFSET_OF_CONTROL_BLOCK_MEMBER(M) (-(int)eosio::chain::eosvmoc::memory::cb_offset + (int)offsetof(eosio::chain::eosvmoc::control_block, M))
 #define OFFSET_OF_FIRST_INTRINSIC ((int)-eosio::chain::eosvmoc::memory::first_intrinsic_offset)

libraries/chain/webassembly/runtimes/eos-vm-oc/compile_monitor.cpp

@@ -114,7 +114,13 @@ struct compile_monitor_session {
    void read_message_from_compile_task(std::list<std::tuple<code_tuple, local::datagram_protocol::socket>>::iterator current_compile_it) {
       auto& [code, socket] = *current_compile_it;
       socket.async_wait(local::datagram_protocol::socket::wait_read, [this, current_compile_it](auto ec) {
-         //at this point we only expect 1 of 2 things to happen: we either get a reply (success), or we get no reply (failure)
+         //at this point we generally expect 1 of 2 things to happen: we either get a reply (success), or we get an error reading from the
+         // socket (failure). But there is also a third possibility that this compile_monitor_session is being destroyed and thus the
+         // socket is being destroyed by way of current_compiles being destroyed. Since this is an async_wait() and not an async_read(),
+         // for now just consider any error as being due to cancellation at dtor time and completely bail out (there aren't many other
+         // potential errors for an asnyc_wait)
+         if(ec)
+            return;
          auto& [code, socket] = *current_compile_it;
          auto [success, message, fds] = read_message_with_fds(socket);
 

plugins/chain_plugin/chain_plugin.cpp

@@ -931,13 +931,6 @@ void chain_plugin_impl::plugin_initialize(const variables_map& options) {
 
       chain_config->db_map_mode = options.at("database-map-mode").as<pinnable_mapped_file::map_mode>();
 
-      // when loading a snapshot, all the state will be modified, so temporarily use the `mapped` mode instead
-      // of `mapped_private` to lower memory requirements.
-      if (snapshot_path && chain_config->db_map_mode == pinnable_mapped_file::mapped_private) {
-        chain_config->db_map_mode = pinnable_mapped_file::mapped;
-        chain_config->revert_to_private_mode = true; // revert to `mapped_private` mode after loading snapshot.
-      }
-
 #ifdef EOSIO_EOS_VM_OC_RUNTIME_ENABLED
       if( options.count("eos-vm-oc-cache-size-mb") )
          chain_config->eosvmoc_config.cache_size = options.at( "eos-vm-oc-cache-size-mb" ).as<uint64_t>() * 1024u * 1024u;