Merge remote-tracking branch 'origin/hotstuff_integration' into GH-1523…

…-hotstuff-activate
AntelopeIO · Nov 7, 2023 · e10b007 · e10b007
2 parents 0b09468 + 52446c1
commit e10b007
Show file tree

Hide file tree

Showing 182 changed files with 5,436 additions and 3,696 deletions.
diff --git a/.cicd/defaults.json b/.cicd/defaults.json
@@ -4,6 +4,6 @@
        "prerelease":false
     },
     "eossystemcontracts":{
-       "ref":"release/3.1"
+       "ref":"release/3.2"
     }
  }
diff --git a/.cicd/platforms.json b/.cicd/platforms.json
diff --git a/.cicd/platforms/ubuntu20.Dockerfile b/.cicd/platforms/ubuntu20.Dockerfile
@@ -9,7 +9,6 @@ RUN apt-get update && apt-get upgrade -y && \
                        jq                   \
                        libcurl4-openssl-dev \
                        libgmp-dev           \
-                       libssl-dev           \
                        llvm-11-dev          \
                        ninja-build          \
                        python3-numpy        \

diff --git a/.cicd/platforms/ubuntu22.Dockerfile b/.cicd/platforms/ubuntu22.Dockerfile
@@ -8,7 +8,6 @@ RUN apt-get update && apt-get upgrade -y && \
                        jq                   \
                        libcurl4-openssl-dev \
                        libgmp-dev           \
-                       libssl-dev           \
                        llvm-11-dev          \
                        ninja-build          \
                        python3-numpy        \

diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml
@@ -31,23 +31,25 @@ defaults:
     shell: bash
 
 jobs:
-  platforms:
-    name: Run Platforms Workflow
-    uses: ./.github/workflows/platforms.yaml
+  platform-cache:
+    name: Platform Cache
+    uses: AntelopeIO/platform-cache-workflow/.github/workflows/platformcache.yaml@v1
     permissions:
       packages: write
       contents: read
+    with:
+      runs-on: '["self-hosted", "enf-x86-beefy"]'
+      platform-files: |
+        .cicd/platforms
+        tools/reproducible.Dockerfile:builder
 
   build-base:
     name: Run Build Workflow
     uses: ./.github/workflows/build_base.yaml
-    needs: [platforms]
+    needs: [platform-cache]
     with:
-      p: ${{needs.platforms.outputs.p}}
-      platform-matrix: ${{needs.platforms.outputs.platform-matrix}}
-    permissions:
-      packages: write
-      contents: read
+      platforms: ${{needs.platform-cache.outputs.platforms}}
+      platform-list: ${{needs.platform-cache.outputs.platform-list}}
 
   v:
     name: Discover Versions
@@ -77,16 +79,15 @@ jobs:
             echo eos-system-contracts-ref=${{inputs.override-eos-system-contracts}} >> $GITHUB_OUTPUT
           fi
 
-  dev-package:
-    name: Build leap-dev package
-    needs: [platforms, build-base]
-    if: always() && needs.platforms.result == 'success' && needs.build-base.result == 'success'
+  package:
+    name: Build deb packages
+    needs: [platform-cache, build-base]
     strategy:
       fail-fast: false
       matrix:
-        platform: [ubuntu20, ubuntu22]
+        platform: [ubuntu20, ubuntu22, reproducible]
     runs-on: ubuntu-latest
-    container: ${{fromJSON(needs.platforms.outputs.p)[matrix.platform].image}}
+    container: ${{fromJSON(needs.platform-cache.outputs.platforms)[matrix.platform].image}}
     steps:
       - uses: actions/checkout@v3
         with:
@@ -95,42 +96,55 @@ jobs:
         uses: actions/download-artifact@v3
         with:
           name: ${{matrix.platform}}-build
-      - name: Build dev package
+      - name: Build packages
         run: |
           zstdcat build.tar.zst | tar x
           cd build
           cpack
+          ../tools/tweak-deb.sh leap_*.deb
       - name: Install dev package
+        if: matrix.platform != 'reproducible'
         run: |
           apt-get update && apt-get upgrade -y
           apt-get install -y ./build/leap_*.deb ./build/leap-dev*.deb
       - name: Test using TestHarness
+        if: matrix.platform != 'reproducible'
         run: |
           python3 -c "from TestHarness import Cluster"
       - name: Upload dev package
         uses: actions/upload-artifact@v3
+        if: matrix.platform != 'reproducible'
         with:
           name: leap-dev-${{matrix.platform}}-amd64
           path: build/leap-dev*.deb
+      - name: Upload leap package
+        uses: actions/upload-artifact@v3
+        if: matrix.platform == 'reproducible'
+        with:
+          name: leap-deb-amd64
+          path: build/leap_*.deb
 
   tests:
-    name: Tests
-    needs: [platforms, build-base]
-    if: always() && needs.platforms.result == 'success' && needs.build-base.result == 'success'
+    name: Tests (${{matrix.cfg.name}})
+    needs: [platform-cache, build-base]
     strategy:
       fail-fast: false
       matrix:
-        platform: [ubuntu20, ubuntu22]
+        include:
+          - cfg: {name: 'ubuntu20',      base: 'ubuntu20', builddir: 'ubuntu20'}
+          - cfg: {name: 'ubuntu22',      base: 'ubuntu22', builddir: 'ubuntu22'}
+          - cfg: {name: 'ubuntu20repro', base: 'ubuntu20', builddir: 'reproducible'}
+          - cfg: {name: 'ubuntu22repro', base: 'ubuntu22', builddir: 'reproducible'}
     runs-on: ["self-hosted", "enf-x86-hightier"]
     container:
-      image: ${{fromJSON(needs.platforms.outputs.p)[matrix.platform].image}}
+      image: ${{fromJSON(needs.platform-cache.outputs.platforms)[matrix.cfg.base].image}}
       options: --security-opt seccomp=unconfined
     steps:
       - uses: actions/checkout@v3
       - name: Download builddir
         uses: actions/download-artifact@v3
         with:
-          name: ${{matrix.platform}}-build
+          name: ${{matrix.cfg.builddir}}-build
       - name: Run Parallel Tests
         run: |
           # https://github.com/actions/runner/issues/2033  -- need this because of full version label test looking at git revs
@@ -142,76 +156,81 @@ jobs:
         run: awk 'BEGIN {err = 1} /bmi2/ && /adx/ {err = 0} END {exit err}' /proc/cpuinfo
 
   np-tests:
-    name: NP Tests
-    needs: [platforms, build-base]
-    if: always() && needs.platforms.result == 'success' && needs.build-base.result == 'success'
+    name: NP Tests (${{matrix.cfg.name}})
+    needs: [platform-cache, build-base]
     strategy:
       fail-fast: false
       matrix:
-        platform: [ubuntu20, ubuntu22]
+        include:
+          - cfg: {name: 'ubuntu20',      base: 'ubuntu20', builddir: 'ubuntu20'}
+          - cfg: {name: 'ubuntu22',      base: 'ubuntu22', builddir: 'ubuntu22'}
+          - cfg: {name: 'ubuntu20repro', base: 'ubuntu20', builddir: 'reproducible'}
+          - cfg: {name: 'ubuntu22repro', base: 'ubuntu22', builddir: 'reproducible'}
     runs-on: ["self-hosted", "enf-x86-midtier"]
     steps:
       - uses: actions/checkout@v3
       - name: Download builddir
         uses: actions/download-artifact@v3
         with:
-          name: ${{matrix.platform}}-build
+          name: ${{matrix.cfg.builddir}}-build
       - name: Run tests in parallel containers
         uses: ./.github/actions/parallel-ctest-containers
         with:
-          container: ${{fromJSON(needs.platforms.outputs.p)[matrix.platform].image}}
+          container: ${{fromJSON(needs.platform-cache.outputs.platforms)[matrix.cfg.base].image}}
           error-log-paths: '["build/etc", "build/var", "build/leap-ignition-wd", "build/TestLogs"]'
-          log-tarball-prefix: ${{matrix.platform}}
+          log-tarball-prefix: ${{matrix.cfg.name}}
           tests-label: nonparallelizable_tests
           test-timeout: 420
       - name: Upload logs from failed tests
         uses: actions/upload-artifact@v3
         if: failure()
         with:
-          name: ${{matrix.platform}}-np-logs
+          name: ${{matrix.cfg.name}}-np-logs
           path: '*-logs.tar.gz'
 
   lr-tests:
-    name: LR Tests
-    needs: [platforms, build-base]
-    if: always() && needs.platforms.result == 'success' && needs.build-base.result == 'success'
+    name: LR Tests (${{matrix.cfg.name}})
+    needs: [platform-cache, build-base]
     strategy:
       fail-fast: false
       matrix:
-        platform: [ubuntu20, ubuntu22]
+        include:
+          - cfg: {name: 'ubuntu20',      base: 'ubuntu20', builddir: 'ubuntu20'}
+          - cfg: {name: 'ubuntu22',      base: 'ubuntu22', builddir: 'ubuntu22'}
+          - cfg: {name: 'ubuntu20repro', base: 'ubuntu20', builddir: 'reproducible'}
+          - cfg: {name: 'ubuntu22repro', base: 'ubuntu22', builddir: 'reproducible'}
     runs-on: ["self-hosted", "enf-x86-lowtier"]
     steps:
       - uses: actions/checkout@v3
       - name: Download builddir
         uses: actions/download-artifact@v3
         with:
-          name: ${{matrix.platform}}-build
+          name: ${{matrix.cfg.builddir}}-build
       - name: Run tests in parallel containers
         uses: ./.github/actions/parallel-ctest-containers
         with:
-          container: ${{fromJSON(needs.platforms.outputs.p)[matrix.platform].image}}
+          container: ${{fromJSON(needs.platform-cache.outputs.platforms)[matrix.cfg.base].image}}
           error-log-paths: '["build/etc", "build/var", "build/leap-ignition-wd", "build/TestLogs"]'
-          log-tarball-prefix: ${{matrix.platform}}
+          log-tarball-prefix: ${{matrix.cfg.name}}
           tests-label: long_running_tests
           test-timeout: 1800
       - name: Upload logs from failed tests
         uses: actions/upload-artifact@v3
         if: failure()
         with:
-          name: ${{matrix.platform}}-lr-logs
+          name: ${{matrix.cfg.name}}-lr-logs
           path: '*-logs.tar.gz'
 
   libtester-tests:
     name: libtester tests
-    needs: [platforms, build-base, v, dev-package]
-    if: always() && needs.platforms.result == 'success' && needs.v.result == 'success' && needs.dev-package.result == 'success'
+    needs: [platform-cache, build-base, v, package]
     strategy:
       fail-fast: false
       matrix:
         platform: [ubuntu20, ubuntu22]
         test: [build-tree, make-dev-install, deb-install]
     runs-on:  ["self-hosted", "enf-x86-midtier"]
-    container: ${{ matrix.test != 'deb-install' && fromJSON(needs.platforms.outputs.p)[matrix.platform].image || matrix.platform == 'ubuntu20' && 'ubuntu:focal' || 'ubuntu:jammy' }}
+    container: ${{ matrix.test != 'deb-install' && fromJSON(needs.platform-cache.outputs.platforms)[matrix.platform].image || matrix.platform == 'ubuntu20' && 'ubuntu:focal' || 'ubuntu:jammy' }}
     env:
       DEBIAN_FRONTEND: noninteractive
       TZ: Etc/UTC
@@ -295,9 +314,9 @@ jobs:
 
   all-passing:
     name: All Required Tests Passed
-    needs: [dev-package, tests, np-tests, libtester-tests]
+    needs: [tests, np-tests, libtester-tests]
     if: always()
     runs-on: ubuntu-latest
     steps:
-      - if: needs.dev-package.result != 'success' || needs.tests.result != 'success' || needs.np-tests.result != 'success' || needs.libtester-tests.result != 'success'
+      - if: needs.tests.result != 'success' || needs.np-tests.result != 'success' || needs.libtester-tests.result != 'success'
         run: false
diff --git a/.github/workflows/build_base.yaml b/.github/workflows/build_base.yaml
@@ -3,12 +3,12 @@ name: "Build leap"
 on:
   workflow_call:
     inputs:
-      p:
-        description: "Discovered Build Platforms"
+      platforms:
+        description: "Platforms definitions"
         type: string
         required: true
-      platform-matrix:
-        description: "Platform Matrix"
+      platform-list:
+        description: "Array of platforms"
         type: string
         required: true
 
@@ -26,9 +26,9 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        platform: ${{fromJSON(inputs.platform-matrix)}}
+        platform: ${{fromJSON(inputs.platform-list)}}
     runs-on: ["self-hosted", "enf-x86-beefy"]
-    container: ${{fromJSON(inputs.p)[matrix.platform].image}}
+    container: ${{fromJSON(inputs.platforms)[matrix.platform].image}}
     steps:
         - uses: actions/checkout@v3
           with: