Merge branch 'master' into dependabot/cargo/arbitrator/prover/fuzz/ev…

…m-0.41.1

.dockerignore

@@ -9,6 +9,7 @@ go-ethereum/tests
 **/*.yml
 contracts/build
 contracts/cache/
+safe-smart-account/build/
 solgen/go
 **/node_modules
 
@@ -22,10 +23,21 @@ brotli/buildfiles/**/*
 nitro-testnode/**/*
 
 # Arbitrator ignores
+arbitrator/tools/module_roots
+arbitrator/tools/pricer
 
 # Rust outputs
 arbitrator/target/**/*
 arbitrator/target
+arbitrator/stylus/tests/*/target/
+arbitrator/wasm-testsuite/target/
+arbitrator/wasm-libraries/target/
+arbitrator/tools/wasmer/target/
+arbitrator/tools/wasm-tools/
+arbitrator/tools/pricers/
+arbitrator/tools/module_roots/
+arbitrator/langs/rust/target/
+arbitrator/langs/bf/target/
 
 # Compiled files
 **/*.o

.github/workflows/arbitrator-ci.yml

@@ -3,6 +3,12 @@ run-name: Arbitrator CI triggered from @${{ github.actor }} of ${{ github.head_r
 
 on:
   workflow_dispatch:
+    inputs:
+      enable_tmate:
+        type: boolean
+        description: 'Enable tmate'
+        required: false
+        default: false
   merge_group:
   pull_request:
     paths:
@@ -16,51 +22,65 @@ on:
 
 env:
   RUST_BACKTRACE: 1
-  RUSTFLAGS: -Dwarnings
+# RUSTFLAGS: -Dwarnings            # TODO: re-enable after wasmer upgrade
   WABT_VERSION: 1.0.32
 
 jobs:
-  coverage:
+  arbitrator:
     name: Run Arbitrator tests
     runs-on: ubuntu-8
     steps:
+      - name: Setup tmate session
+        uses: mxschmitt/action-tmate@v3
+        if: ${{ github.event_name == 'workflow_dispatch' && inputs.enable_tmate }}
+        with:
+          detached: true
+
       - name: Checkout
         uses: actions/checkout@v4
         with:
           submodules: recursive
 
       - name: Install Ubuntu dependencies
         run: |
-          sudo add-apt-repository -y ppa:ethereum/ethereum
           sudo apt-get update && sudo apt-get install -y \
-            build-essential cmake ethereum lld-14 libudev-dev
+            build-essential cmake lld-14 libudev-dev
           sudo ln -s /usr/bin/wasm-ld-14 /usr/local/bin/wasm-ld
 
       - name: Install go
         uses: actions/setup-go@v4
         with:
-          go-version: 1.20.x
+          go-version: 1.21.x
+
+      - name: Install custom go-ethereum
+        run: |
+          cd /tmp
+          git clone --branch v1.13.8 --depth 1 https://github.com/ethereum/go-ethereum.git
+          cd go-ethereum
+          # Enable KZG point evaluation precompile early
+          sed -i 's#var PrecompiledContractsBerlin = map\[common.Address\]PrecompiledContract{#\0 common.BytesToAddress([]byte{0x0a}): \&kzgPointEvaluation{},#g' core/vm/contracts.go
+          go build -o /usr/local/bin/geth ./cmd/geth
 
       - name: Setup nodejs
         uses: actions/setup-node@v3
         with:
-          node-version: '16'
+          node-version: '18'
           cache: 'yarn'
           cache-dependency-path: '**/yarn.lock'
 
       - name: Install rust stable
         uses: dtolnay/rust-toolchain@stable
         with:
+          toolchain: "1.76"
           components: 'llvm-tools-preview, rustfmt, clippy'
-          targets: 'wasm32-wasi, wasm32-unknown-unknown'
 
-      - name: Install grcov
-        uses: jaxxstorm/[email protected]
+      - name: Install rust nightly
+        uses: dtolnay/rust-toolchain@nightly
+        id: install-rust-nightly
         with:
-          repo: mozilla/grcov
-          tag: v0.8.18
-          extension: "\\.bz2"
-          cache: enable
+          toolchain: "nightly-2024-02-04"
+          targets: 'wasm32-wasi, wasm32-unknown-unknown'
+          components: 'rust-src, rustfmt, clippy'
 
       - name: Cache Rust intermediate build products
         uses: actions/cache@v3
@@ -96,6 +116,9 @@ jobs:
           make -j
           make install
 
+      - name: Install Foundry
+        uses: foundry-rs/foundry-toolchain@v1
+
       - name: Cache cbrotli
         uses: actions/cache@v3
         id: cache-cbrotli
@@ -133,39 +156,32 @@ jobs:
         run: echo "$HOME/wabt-prefix/bin" >> "$GITHUB_PATH"
 
       - name: Make arbitrator libraries
-        run: make -j wasm-ci-build
-
-      - name: Enable rust instrumentation
-        run: |
-          echo LLVM_PROFILE_FILE="your_name-%p-%m.profraw" >> $GITHUB_ENV
-          echo "CARGO_INCREMENTAL=0" >> $GITHUB_ENV
-          echo RUSTFLAGS="-Cinstrument-coverage" >> $GITHUB_ENV
-          echo RUSTDOCFLAGS="-Cpanic=abort" >> $GITHUB_ENV
+        run: make -j wasm-ci-build STYLUS_NIGHTLY_VER="+nightly-2024-02-04"
 
       - name: Clippy check
         run: cargo clippy --all --manifest-path arbitrator/Cargo.toml -- -D warnings
 
       - name: Run rust tests
-        run: cargo test --all --manifest-path arbitrator/Cargo.toml
+        uses: actions-rs/cargo@v1
+        with:
+          command: test
+          args: -p arbutil -p prover -p jit -p stylus --release --manifest-path arbitrator/prover/Cargo.toml
 
       - name: Rustfmt
-        run: cargo fmt --all --manifest-path arbitrator/Cargo.toml -- --check
+        uses: actions-rs/cargo@v1
+        with:
+          command: fmt
+          args: -p arbutil -p prover -p jit -p stylus --manifest-path arbitrator/Cargo.toml -- --check
+
+      - name: Rustfmt - langs/rust
+        uses: actions-rs/cargo@v1
+        with:
+          command: fmt
+          args: --all --manifest-path arbitrator/langs/rust/Cargo.toml -- --check
 
       - name: Make proofs from test cases
         run: make -j test-gen-proofs
 
-      - name: Create code-coverage files
-        run: |
-          grcov . --binary-path arbitrator/target/release/ -s . -t lcov --branch --ignore-not-existing --ignore "/*" -o lcov.info
-
-      - name: Upload to codecov.io
-        uses: codecov/codecov-action@v1
-        with:
-          token: ${{ secrets.CODECOV_TOKEN }}
-          files: ./lcov.info
-          fail_ci_if_error: true
-          verbose: false
-
       - name: Start geth server
         run: |
           geth --dev --http --http.port 8545 &
@@ -177,4 +193,5 @@ jobs:
           cd contracts
           yarn install
           yarn build
+          yarn build:forge:yul
           yarn hardhat --network localhost test test/prover/*.ts

.github/workflows/arbitrator-skip-ci.yml

@@ -13,7 +13,7 @@ on:
       - 'Makefile'
 
 jobs:
-  coverage:
+  arbitrator:
     name: Run Arbitrator tests
     runs-on: ubuntu-latest
     steps:

.github/workflows/ci.yml

@@ -25,7 +25,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        test-mode: [defaults, race, challenge]
+        test-mode: [defaults, race, challenge, stylus, long]
 
     steps:
       - name: Checkout
@@ -39,14 +39,14 @@ jobs:
       - name: Setup nodejs
         uses: actions/setup-node@v3
         with:
-          node-version: '16'
+          node-version: '18'
           cache: 'yarn'
           cache-dependency-path: '**/yarn.lock'
 
       - name: Install go
         uses: actions/setup-go@v4
         with:
-          go-version: 1.20.x
+          go-version: 1.21.x
 
       - name: Install wasm-ld
         run: |
@@ -58,6 +58,24 @@ jobs:
         with:
           targets: 'wasm32-unknown-unknown, wasm32-wasi'
 
+      - name: Install Foundry
+        uses: foundry-rs/foundry-toolchain@v1
+
+      - name: Install rust nightly
+        uses: actions-rs/toolchain@v1
+        id: install-rust-nightly
+        with:
+          profile: minimal
+          toolchain: "nightly"
+
+      - name: Install rust wasm targets
+        run: rustup target add wasm32-wasi wasm32-unknown-unknown
+
+      - name: Install nightly wasm targets
+        run: |
+          rustup component add rust-src --toolchain nightly
+          rustup target add wasm32-unknown-unknown --toolchain nightly
+
       - name: Cache Build Products
         uses: actions/cache@v3
         with:
@@ -114,8 +132,7 @@ jobs:
           skip-pkg-cache: true
       - name: Custom Lint
         run: |
-          go run ./linter/koanf ./...
-          go run ./linter/pointercheck ./...
+          go run ./linters ./...
 
       - name: Set environment variables
         run: |
@@ -124,27 +141,100 @@ jobs:
           echo "GOMEMLIMIT=6GiB" >> "$GITHUB_ENV"
           echo "GOGC=80" >> "$GITHUB_ENV"
 
-      - name: run tests without race detection
+      - name: run tests without race detection and path state scheme
         if: matrix.test-mode == 'defaults'
+        env:
+          TEST_STATE_SCHEME: path
         run: |
           packages=`go list ./...`
-          gotestsum --format short-verbose --packages="$packages" --rerun-fails=1 -- -coverprofile=coverage.txt -covermode=atomic -coverpkg=./...,./go-ethereum/...
-
-      - name: run tests with race detection
+          for package in $packages; do
+            echo running tests for $package
+            if ! stdbuf -oL gotestsum --format short-verbose --packages="$package" --rerun-fails=2 --no-color=false -- -coverprofile=coverage.txt -covermode=atomic -coverpkg=./...,./go-ethereum/... -timeout 20m -tags=cionly > >(stdbuf -oL tee -a full.log | grep -vE "INFO|seal"); then
+              exit 1
+            fi
+          done
+
+      - name: run tests without race detection and hash state scheme
+        if: matrix.test-mode == 'defaults'
+        env:
+          TEST_STATE_SCHEME: hash
+        run: |
+          packages=`go list ./...`
+          for package in $packages; do
+            echo running tests for $package
+            if ! stdbuf -oL gotestsum --format short-verbose --packages="$package" --rerun-fails=2 --no-color=false -- -timeout 20m -tags=cionly; then
+              exit 1
+            fi
+          done
+
+      - name: run tests with race detection and path state scheme
         if: matrix.test-mode == 'race'
-        run:  |
+        env:
+          TEST_STATE_SCHEME: path
+        run: |
           packages=`go list ./...`
-          gotestsum --format short-verbose --packages="$packages" --rerun-fails=1 -- -race
+          for package in $packages; do
+            echo running tests for $package
+            if ! stdbuf -oL gotestsum --format short-verbose --packages="$package" --rerun-fails=2 --no-color=false -- -race -timeout 30m > >(stdbuf -oL tee -a full.log | grep -vE "INFO|seal"); then
+              exit 1
+            fi
+          done
+
+      - name: run tests with race detection and hash state scheme
+        if: matrix.test-mode == 'race'
+        env:
+          TEST_STATE_SCHEME: hash
+        run: |
+          packages=`go list ./...`
+          for package in $packages; do
+            echo running tests for $package
+            if ! stdbuf -oL gotestsum --format short-verbose --packages="$package" --rerun-fails=2 --no-color=false -- -race -timeout 30m; then
+              exit 1
+            fi
+          done
 
       - name: run redis tests
         if: matrix.test-mode == 'defaults'
         run: TEST_REDIS=redis://localhost:6379/0 gotestsum --format short-verbose -- -p 1 -run TestRedis ./arbnode/... ./system_tests/... -coverprofile=coverage-redis.txt -covermode=atomic -coverpkg=./...
 
       - name: run challenge tests
         if: matrix.test-mode == 'challenge'
-        run:  |
+        run: |
+          packages=`go list ./...`
+          for package in $packages; do
+            echo running tests for $package
+            if ! stdbuf -oL gotestsum --format short-verbose --packages="$package" --rerun-fails=2 --no-color=false -- -coverprofile=coverage.txt -covermode=atomic -coverpkg=./...,./go-ethereum/... -tags=challengetest -run=TestChallenge > >(stdbuf -oL tee -a full.log | grep -vE "INFO|seal"); then
+              exit 1
+            fi
+          done
+
+      - name: run stylus tests
+        if: matrix.test-mode == 'stylus'
+        run: |
+          packages=`go list ./...`
+          for package in $packages; do
+            echo running tests for $package
+            if ! stdbuf -oL gotestsum --format short-verbose --packages="$package" --rerun-fails=2 --no-color=false -- -timeout 60m -coverprofile=coverage.txt -covermode=atomic -coverpkg=./...,./go-ethereum/... -tags=stylustest -run="TestProgramArbitrator" > >(stdbuf -oL tee -a full.log | grep -vE "INFO|seal"); then
+              exit 1
+            fi
+          done
+
+      - name: run long stylus tests
+        if: matrix.test-mode == 'long'
+        run: |
           packages=`go list ./...`
-          gotestsum --format short-verbose --packages="$packages" --rerun-fails=1 -- ./... -coverprofile=coverage.txt -covermode=atomic -coverpkg=./...,./go-ethereum/... -tags=challengetest -run=TestChallenge
+          for package in $packages; do
+            echo running tests for $package
+            if ! stdbuf -oL gotestsum --format short-verbose --packages="$package" --rerun-fails=2 --no-color=false -- -timeout 60m -coverprofile=coverage.txt -covermode=atomic -coverpkg=./...,./go-ethereum/... -tags=stylustest -run="TestProgramLong" > >(stdbuf -oL tee -a full.log | grep -vE "INFO|seal"); then
+              exit 1
+            fi
+          done
+
+      - name: Archive detailed run log
+        uses: actions/upload-artifact@v3
+        with:
+          name: ${{ matrix.test-mode }}-full.log
+          path: full.log
 
       - name: Upload coverage to Codecov
         uses: codecov/codecov-action@v2
@@ -154,4 +244,3 @@ jobs:
           files: ./coverage.txt,./coverage-redis.txt
           verbose: false
           token: ${{ secrets.CODECOV_TOKEN }}
-