Clean up github workflows

- Remove bitrot task which was building the benchmarks and examples but we don't have any - Merge cargo fmt, clippy and doc links into a single job to reduce noise - Split heavy tests and light tests into two jobs to parallelize the work - Merge all feature tests into a single job to reduce noise. - Disable integration tests on pull requests because they are taking a long time currently. Enable them only when the trigger-integration-test label is set, when they are scheduled via cron, or when pushing to main.
privacy-scaling-explorations · Jul 21, 2023 · e560228 · e560228
1 parent 350a7a2
commit e560228
Show file tree

Hide file tree

Showing 5 changed files with 70 additions and 159 deletions.
diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml
@@ -28,7 +28,10 @@ concurrency:
 
 jobs:
   set-outputs:
-    if: github.event.pull_request.draft == false
+    if: github.event_name == 'schedule' ||
+        github.event_name == 'push' ||
+        github.event_name == 'workflow_dispatch' ||
+        contains(github.event.pull_request.labels.*.name, 'trigger-integration-tests')
 
     runs-on: ubuntu-latest
     outputs:

diff --git a/.github/workflows/labeler.yml b/.github/workflows/labeler.yml
@@ -1,11 +1,14 @@
-name: "Pull Request Labeler"
+name: Pull Request Labeler
 on:
   - pull_request_target
 
 jobs:
   triage:
+    name: Triage
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/labeler@v3
+      - uses: actions/checkout@v2
+      - name: Labeler
+        uses: actions/labeler@v4
         with:
           repo-token: "${{ secrets.GITHUB_TOKEN }}"
diff --git a/.github/workflows/lints.yml b/.github/workflows/lints.yml
@@ -22,21 +22,21 @@ jobs:
           concurrent_skipping: 'same_content_newer'
           paths_ignore: '["**/README.md"]'
 
-  clippy:
+  lints:
     needs: [skip_check]
     if: |
       github.event.pull_request.draft == false &&
       (github.event.action == 'ready_for_review' || needs.skip_check.outputs.should_skip != 'true')
 
-    name: Clippy
+    name: Various lints
     timeout-minutes: 30
     runs-on: ubuntu-latest
 
     steps:
       - uses: actions/checkout@v2
       - uses: actions-rs/toolchain@v1
         with:
-          components: clippy
+          components: rustfmt, clippy
           override: false
       # Go cache for building geth-utils
       - name: Go cache
@@ -45,7 +45,7 @@ jobs:
           path: |
             ~/.cache/go-build
             ~/go/pkg/mod
-          key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
+          key: lint-${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
           restore-keys: |
             ${{ runner.os }}-go-
       - name: Cargo cache
@@ -57,10 +57,24 @@ jobs:
             ~/.cargo/registry/cache/
             ~/.cargo/git/db/
             target/
-          key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }}
+          key: lint-${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }}
+
+      - name: Check code format
+        uses: actions-rs/cargo@v1
+        with:
+          command: fmt
+          args: --all -- --check
+
       - name: Run clippy
-        uses: actions-rs/clippy-check@v1
+        uses: actions-rs/cargo@v1
         with:
-          name: Clippy
-          token: ${{ secrets.GITHUB_TOKEN }}
+          command: clippy
           args: --all-features --all-targets -- -D warnings
+
+      # Ensure intra-documentation links all resolve correctly
+      # Requires #![deny(intra_doc_link_resolution_failure)] in crates.
+      - name: Check intra-doc links
+        uses: actions-rs/cargo@v1
+        with:
+          command: doc
+          args: --no-deps --all --document-private-items
diff --git a/.github/workflows/ci.yml → .github/workflows/main-tests.yml b/.github/workflows/ci.yml → .github/workflows/main-tests.yml
@@ -1,4 +1,4 @@
-name: CI checks
+name: Main tests
 
 on:
   merge_group:
@@ -28,13 +28,13 @@ jobs:
           concurrent_skipping: 'same_content_newer'
           paths_ignore: '["**/README.md"]'
 
-  test:
+  lighttest:
     needs: [skip_check]
     if: |
       github.event.pull_request.draft == false &&
       (github.event.action == 'ready_for_review' || needs.skip_check.outputs.should_skip != 'true')
 
-    name: Test
+    name: Light unit tests
     runs-on: ["${{github.run_id}}", self-hosted, c5.9xlarge]
 
     steps:
@@ -71,124 +71,30 @@ jobs:
         with:
           command: test
           args: --verbose --release --all --all-features --exclude integration-tests --exclude circuit-benchmarks
-      - name: Run heavy tests # heavy tests are run serially to avoid OOM
-        uses: actions-rs/cargo@v1
-        with:
-          command: test
-          args: --verbose --release --all --all-features --exclude integration-tests --exclude circuit-benchmarks serial_ -- --ignored --test-threads 1
       - name: Run testool internal tests
         uses: actions-rs/cargo@v1
         with:
           command: test
           args: --release --manifest-path testool/Cargo.toml
-  build:
-    needs: [skip_check]
-    if: |
-      github.event.pull_request.draft == false &&
-      (github.event.action == 'ready_for_review' || needs.skip_check.outputs.should_skip != 'true')
-
-    name: Build target ${{ matrix.target }}
-    runs-on: ubuntu-latest
-    strategy:
-      matrix:
-        target:
-          - wasm32-unknown-unknown
-          - wasm32-wasi
-
-    steps:
-      - uses: actions/checkout@v2
-      - uses: actions-rs/toolchain@v1
-        with:
-          override: false
-      - name: Add target
-        run: rustup target add ${{ matrix.target }}
-      # Go cache for building geth-utils
-      - name: Go cache
-        uses: actions/cache@v3
-        with:
-          path: |
-            ~/.cache/go-build
-            ~/go/pkg/mod
-          key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
-          restore-keys: |
-            ${{ runner.os }}-go-
-      - name: Cargo cache
-        uses: actions/cache@v3
-        with:
-          path: |
-            ~/.cargo/bin/
-            ~/.cargo/registry/index/
-            ~/.cargo/registry/cache/
-            ~/.cargo/git/db/
-            target/
-          key: ${{ runner.os }}-${{ matrix.target }}-cargo-${{ hashFiles('**/Cargo.lock') }}
-      - name: cargo build
-        uses: actions-rs/cargo@v1
-        with:
-          command: build
-          args: --all-features
-      # Make sure benchmarks compile.
-      - name: cargo build benchmarks no-run
-        uses: actions-rs/cargo@v1
-        with:
-          command: test
-          args: --verbose --release --all-features -p circuit-benchmarks --no-run
 
-  bitrot:
+  heavytests:
     needs: [skip_check]
     if: |
       github.event.pull_request.draft == false &&
       (github.event.action == 'ready_for_review' || needs.skip_check.outputs.should_skip != 'true')
 
-    name: Bitrot check
-    runs-on: ubuntu-latest
+    name: Heavy unit tests
+    runs-on: ["${{github.run_id}}", self-hosted, c5.9xlarge]
 
     steps:
       - uses: actions/checkout@v2
       - uses: actions-rs/toolchain@v1
         with:
           override: false
-      # Go cache for building geth-utils
-      - name: Go cache
-        uses: actions/cache@v3
-        with:
-          path: |
-            ~/.cache/go-build
-            ~/go/pkg/mod
-          key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
-          restore-keys: |
-            ${{ runner.os }}-go-
-      - name: Cargo cache
-        uses: actions/cache@v3
-        with:
-          path: |
-            ~/.cargo/bin/
-            ~/.cargo/registry/index/
-            ~/.cargo/registry/cache/
-            ~/.cargo/git/db/
-            target/
-          key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }}
-      # Build benchmarks to prevent bitrot
-      - name: Build benchmarks
-        uses: actions-rs/cargo@v1
-        with:
-          command: build
-          args: --benches --examples --all-features
-
-  doc-links:
-    needs: [skip_check]
-    if: |
-      github.event.pull_request.draft == false &&
-      (github.event.action == 'ready_for_review' || needs.skip_check.outputs.should_skip != 'true')
-
-    name: Intra-doc links
-    runs-on: ubuntu-latest
-
-    steps:
-      - uses: actions/checkout@v2
-      - uses: actions-rs/toolchain@v1
+      - name: Setup golang
+        uses: actions/setup-go@v3
         with:
-          override: false
+          go-version: ~1.19
       # Go cache for building geth-utils
       - name: Go cache
         uses: actions/cache@v3
@@ -209,33 +115,32 @@ jobs:
             ~/.cargo/git/db/
             target/
           key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }}
-      - name: cargo fetch
-        uses: actions-rs/cargo@v1
-        with:
-          command: fetch
-
-      # Ensure intra-documentation links all resolve correctly
-      # Requires #![deny(intra_doc_link_resolution_failure)] in crates.
-      - name: Check intra-doc links
+      - name: Run heavy tests # heavy tests are run serially to avoid OOM
         uses: actions-rs/cargo@v1
         with:
-          command: doc
-          args: --no-deps --all --document-private-items
+          command: test
+          args: --verbose --release --all --all-features --exclude integration-tests --exclude circuit-benchmarks serial_ -- --ignored --test-threads 1
 
-  fmt:
+  build:
     needs: [skip_check]
     if: |
       github.event.pull_request.draft == false &&
       (github.event.action == 'ready_for_review' || needs.skip_check.outputs.should_skip != 'true')
 
-    name: Rustfmt
-    timeout-minutes: 30
+    name: Build target ${{ matrix.target }}
     runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        target:
+          - x86_64-unknown-linux-gnu
+
     steps:
       - uses: actions/checkout@v2
       - uses: actions-rs/toolchain@v1
         with:
           override: false
+      - name: Add target
+        run: rustup target add ${{ matrix.target }}
       # Go cache for building geth-utils
       - name: Go cache
         uses: actions/cache@v3
@@ -255,14 +160,9 @@ jobs:
             ~/.cargo/registry/cache/
             ~/.cargo/git/db/
             target/
-          key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }}
-      - name: cargo check
+          key: ${{ runner.os }}-${{ matrix.target }}-cargo-${{ hashFiles('**/Cargo.lock') }}
+      - name: cargo build
         uses: actions-rs/cargo@v1
         with:
-          command: check
+          command: build
           args: --all-features
-      - run: rustup component add rustfmt
-      - uses: actions-rs/cargo@v1
-        with:
-          command: fmt
-          args: --all -- --check
diff --git a/.github/workflows/test-features.yml b/.github/workflows/test-features.yml
@@ -1,4 +1,4 @@
-name: Test feature flags
+name: Feature flags
 
 on:
   merge_group:
@@ -26,18 +26,11 @@ jobs:
     if: |
       github.event.pull_request.draft == false &&
       (github.event.action == 'ready_for_review' || needs.skip_check.outputs.should_skip != 'true')
+    name: Validate features exist
     timeout-minutes: 30
     runs-on: ubuntu-latest
     strategy:
       fail-fast: false
-      matrix:
-        crate:
-          - zkevm-circuits
-        feature:
-          - default
-          - test-circuits
-          - test-util
-          - warn-unimplemented
     steps:
       - uses: actions/checkout@v2
       - uses: actions-rs/toolchain@v1
@@ -51,7 +44,7 @@ jobs:
           path: |
             ~/.cache/go-build
             ~/go/pkg/mod
-          key: ${{ github.workflow }}-${{ matrix.crate }}-${{ matrix.feature }}-${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
+          key: ${{ github.workflow }}-${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
 
       - name: Cargo cache
         uses: actions/cache@v3
@@ -62,21 +55,19 @@ jobs:
             ~/.cargo/registry/cache/
             ~/.cargo/git/db/
             target/
-          key: ${{ github.workflow }}-${{ matrix.crate }}-${{ matrix.feature }}-${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }}
+          key: ${{ github.workflow }}-${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }}
 
-      - name: '${{ matrix.crate }}: ${{ matrix.feature }}'
+      - name: Import with features
         run: |
           GIT_ROOT=$(pwd)
           cd /tmp
-          cargo new foobar
-          cd foobar
-          cp "${GIT_ROOT}/rust-toolchain" . || true
-          cargo add --path "${GIT_ROOT}/${{ matrix.crate }}" --features '${{ matrix.feature }}'
-          cd ../
-          rm -rf foobar
-
-  test_features_complete:
-    needs: [test_features]
-    runs-on: ubuntu-latest
-    steps:
-      - run: echo dummy
+          for crate in zkevm-circuits; do
+            for feature in default test-circuits test-util warn-unimplemented; do
+              cargo new foobar
+              cd foobar
+              cp "${GIT_ROOT}/rust-toolchain" . || true
+              cargo add --path "${GIT_ROOT}/${crate}" --features "${feature}"
+              cd ../
+              rm -rf foobar
+            done
+          done