Enable GHA short benchmarks with frame-omni-bencher
#1354
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: tests misc | |
| on: | |
| push: | |
| branches: | |
| - master | |
| pull_request: | |
| types: [opened, synchronize, reopened, ready_for_review] | |
| merge_group: | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} | |
| cancel-in-progress: true | |
| # Jobs in this workflow depend on each other, only for limiting peak amount of spawned workers | |
| jobs: | |
| #changes: | |
| # permissions: | |
| # pull-requests: read | |
| # uses: ./.github/workflows/reusable-check-changed-files.yml | |
| set-image: | |
| # needs: [ changes ] | |
| # if: needs.changes.outputs.rust || needs.changes.outputs.current-workflow | |
| # GitHub Actions allows using 'env' in a container context. | |
| # However, env variables don't work for forks: https://github.com/orgs/community/discussions/44322 | |
| # This workaround sets the container image for each job using 'set-image' job output. | |
| runs-on: ubuntu-latest | |
| outputs: | |
| IMAGE: ${{ steps.set_image.outputs.IMAGE }} | |
| RUNNER: ${{ steps.set_runner.outputs.RUNNER }} | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - id: set_image | |
| run: cat .github/env >> $GITHUB_OUTPUT | |
| # By default, we use spot machines that can be terminated at any time. | |
| # Merge queues use persistent runners to avoid kicking off from queue when the runner is terminated. | |
| - id: set_runner | |
| run: | | |
| # Run merge queues on persistent runners | |
| if [[ $GITHUB_REF_NAME == *"gh-readonly-queue"* ]]; then | |
| echo "RUNNER=arc-runners-polkadot-sdk-beefy-persistent" >> $GITHUB_OUTPUT | |
| else | |
| echo "RUNNER=arc-runners-polkadot-sdk-beefy" >> $GITHUB_OUTPUT | |
| fi | |
| # more information about this job can be found here: | |
| # https://github.com/paritytech/substrate/pull/3778 | |
| test-full-crypto-feature: | |
| needs: [set-image] | |
| runs-on: ${{ needs.set-image.outputs.RUNNER }} | |
| timeout-minutes: 60 | |
| container: | |
| image: ${{ needs.set-image.outputs.IMAGE }} | |
| env: | |
| # Enable debug assertions since we are running optimized builds for testing | |
| # but still want to have debug assertions. | |
| RUSTFLAGS: "-C debug-assertions" | |
| RUST_BACKTRACE: 1 | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: script | |
| run: | | |
| cd substrate/primitives/core/ | |
| forklift cargo build --locked --no-default-features --features full_crypto | |
| cd ../application-crypto | |
| forklift cargo build --locked --no-default-features --features full_crypto | |
| test-frame-examples-compile-to-wasm: | |
| timeout-minutes: 20 | |
| # into one job | |
| needs: [set-image, test-full-crypto-feature] | |
| runs-on: ${{ needs.set-image.outputs.RUNNER }} | |
| container: | |
| image: ${{ needs.set-image.outputs.IMAGE }} | |
| env: | |
| # Enable debug assertions since we are running optimized builds for testing | |
| # but still want to have debug assertions. | |
| RUSTFLAGS: "-C debug-assertions" | |
| RUST_BACKTRACE: 1 | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: script | |
| run: | | |
| cd substrate/frame/examples/offchain-worker/ | |
| forklift cargo build --locked --target=wasm32-unknown-unknown --no-default-features | |
| cd ../basic | |
| forklift cargo build --locked --target=wasm32-unknown-unknown --no-default-features | |
| test-frame-ui: | |
| timeout-minutes: 60 | |
| needs: [set-image] | |
| runs-on: ${{ needs.set-image.outputs.RUNNER }} | |
| container: | |
| image: ${{ needs.set-image.outputs.IMAGE }} | |
| env: | |
| # Enable debug assertions since we are running optimized builds for testing | |
| # but still want to have debug assertions. | |
| RUSTFLAGS: "-C debug-assertions -D warnings" | |
| RUST_BACKTRACE: 1 | |
| WASM_BUILD_NO_COLOR: 1 | |
| WASM_BUILD_RUSTFLAGS: "-C debug-assertions -D warnings" | |
| # Ensure we run the UI tests. | |
| RUN_UI_TESTS: 1 | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: script | |
| run: | | |
| forklift cargo test --locked -q --profile testnet -p frame-support-test --features=frame-feature-testing,no-metadata-docs,try-runtime,experimental | |
| forklift cargo test --locked -q --profile testnet -p frame-support-test --features=frame-feature-testing,frame-feature-testing-2,no-metadata-docs,try-runtime,experimental | |
| forklift cargo test --locked -q --profile testnet -p xcm-procedural | |
| forklift cargo test --locked -q --profile testnet -p frame-election-provider-solution-type | |
| forklift cargo test --locked -q --profile testnet -p sp-api-test | |
| # There is multiple version of sp-runtime-interface in the repo. So we point to the manifest. | |
| forklift cargo test --locked -q --profile testnet --manifest-path substrate/primitives/runtime-interface/Cargo.toml | |
| test-deterministic-wasm: | |
| timeout-minutes: 20 | |
| needs: [set-image, test-frame-examples-compile-to-wasm] | |
| runs-on: ${{ needs.set-image.outputs.RUNNER }} | |
| container: | |
| image: ${{ needs.set-image.outputs.IMAGE }} | |
| env: | |
| WASM_BUILD_NO_COLOR: 1 | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: script | |
| run: | | |
| # build runtime | |
| forklift cargo build -q --locked --release -p westend-runtime -p rococo-runtime | |
| # make checksum | |
| sha256sum target/release/wbuild/*-runtime/target/wasm32-unknown-unknown/release/*.wasm > checksum.sha256 | |
| cargo clean | |
| # build again | |
| forklift cargo build -q --locked --release -p westend-runtime -p rococo-runtime | |
| # confirm checksum | |
| sha256sum -c checksum.sha256 | |
| cargo-check-benches: | |
| needs: [set-image] | |
| if: ${{ github.event_name == 'pull_request' || github.event_name == 'merge_group' }} | |
| timeout-minutes: 60 | |
| strategy: | |
| matrix: | |
| branch: [master, current] | |
| runs-on: ${{ needs.set-image.outputs.RUNNER }} | |
| container: | |
| image: ${{ needs.set-image.outputs.IMAGE }} | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| with: | |
| # if branch is master, use the branch, otherwise set empty string, so it uses the current context | |
| # either PR (including forks) or merge group (main repo) | |
| ref: ${{ matrix.branch == 'master' && matrix.branch || '' }} | |
| - name: script | |
| run: | | |
| ARTIFACTS_DIR=./artifacts | |
| BENCH_TRIE_READ=::trie::read::small | |
| BENCH_NODE_IMPORT=::node::import::sr25519::transfer_keep_alive::paritydb::small | |
| mkdir -p $ARTIFACTS_DIR | |
| SKIP_WASM_BUILD=1 forklift cargo check --locked --benches --all; | |
| forklift cargo run --locked --release -p node-bench -- $BENCH_TRIE_READ --json | tee $ARTIFACTS_DIR/bench_trie_read_small.json; | |
| forklift cargo run --locked --release -p node-bench -- $BENCH_NODE_IMPORT --json | tee $ARTIFACTS_DIR/bench_transfer_keep_alive.json | |
| - name: Upload artifacts | |
| uses: actions/[email protected] | |
| with: | |
| path: ./artifacts | |
| name: cargo-check-benches-${{ matrix.branch }}-${{ github.sha }} | |
| retention-days: 1 | |
| node-bench-regression-guard: | |
| timeout-minutes: 20 | |
| if: always() && !cancelled() | |
| # runs-on: arc-runners-polkadot-sdk | |
| runs-on: ubuntu-latest | |
| needs: [set-image, cargo-check-benches] | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: Download artifact (master run) | |
| uses: actions/[email protected] | |
| with: | |
| name: cargo-check-benches-master-${{ github.sha }} | |
| path: ./artifacts/master | |
| - name: Download artifact (current run) | |
| uses: actions/[email protected] | |
| with: | |
| name: cargo-check-benches-current-${{ github.sha }} | |
| path: ./artifacts/current | |
| - name: script | |
| id: compare | |
| run: | | |
| docker run --rm \ | |
| -v $PWD/artifacts/master:/artifacts/master \ | |
| -v $PWD/artifacts/current:/artifacts/current \ | |
| paritytech/node-bench-regression-guard:latest \ | |
| node-bench-regression-guard --reference /artifacts/master --compare-with /artifacts/current | |
| if [ $? -ne 0 ]; then | |
| FAILED_MSG='### node-bench-regression-guard failed ❌, check the regression in *cargo-check-benches* job' | |
| echo $FAILED_MSG | |
| echo $FAILED_MSG >> $GITHUB_STEP_SUMMARY | |
| exit 1 | |
| else | |
| echo "### node-bench-regression-guard passed ✅" >> $GITHUB_STEP_SUMMARY | |
| fi | |
| test-node-metrics: | |
| needs: [set-image] | |
| timeout-minutes: 30 | |
| runs-on: ${{ needs.set-image.outputs.RUNNER }} | |
| container: | |
| image: ${{ needs.set-image.outputs.IMAGE }} | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: Run tests | |
| id: tests | |
| env: | |
| RUST_TOOLCHAIN: stable | |
| # Enable debug assertions since we are running optimized builds for testing | |
| # but still want to have debug assertions. | |
| RUSTFLAGS: "-Cdebug-assertions=y -Dwarnings" | |
| run: | | |
| forklift cargo build --bin polkadot-execute-worker --bin polkadot-prepare-worker --profile testnet --verbose --locked | |
| mkdir -p ./artifacts | |
| forklift cargo test --profile testnet --locked --features=runtime-metrics -p polkadot-node-metrics > ./artifacts/log.txt | |
| echo "Metrics test passed" | |
| - name: Upload artifacts if failed | |
| if: ${{ steps.tests.outcome != 'success' }} | |
| uses: actions/[email protected] | |
| with: | |
| name: node-metrics-failed | |
| path: ./artifacts | |
| # more information about this job can be found here: | |
| # https://github.com/paritytech/substrate/pull/6916 | |
| check-tracing: | |
| timeout-minutes: 20 | |
| needs: [set-image, test-node-metrics] | |
| runs-on: ${{ needs.set-image.outputs.RUNNER }} | |
| container: | |
| image: ${{ needs.set-image.outputs.IMAGE }} | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: script | |
| run: | | |
| forklift cargo test --locked --manifest-path ./substrate/primitives/tracing/Cargo.toml --no-default-features | |
| forklift cargo test --locked --manifest-path ./substrate/primitives/tracing/Cargo.toml --no-default-features --features=with-tracing | |
| check-metadata-hash: | |
| timeout-minutes: 20 | |
| needs: [set-image, check-tracing] | |
| runs-on: ${{ needs.set-image.outputs.RUNNER }} | |
| container: | |
| image: ${{ needs.set-image.outputs.IMAGE }} | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: script | |
| run: | | |
| forklift cargo build --locked -p westend-runtime --features metadata-hash | |
| cargo-hfuzz: | |
| timeout-minutes: 20 | |
| needs: [set-image, check-metadata-hash] | |
| runs-on: ${{ needs.set-image.outputs.RUNNER }} | |
| container: | |
| image: ${{ needs.set-image.outputs.IMAGE }} | |
| env: | |
| # max 10s per iteration, 60s per file | |
| HFUZZ_RUN_ARGS: | | |
| --exit_upon_crash | |
| --exit_code_upon_crash 1 | |
| --timeout 10 | |
| --run_time 60 | |
| # use git version of honggfuzz-rs until v0.5.56 is out, we need a few recent changes: | |
| # https://github.com/rust-fuzz/honggfuzz-rs/pull/75 to avoid breakage on debian | |
| # https://github.com/rust-fuzz/honggfuzz-rs/pull/81 fix to the above pr | |
| # https://github.com/rust-fuzz/honggfuzz-rs/pull/82 fix for handling absolute CARGO_TARGET_DIR | |
| HFUZZ_BUILD_ARGS: | | |
| --config=patch.crates-io.honggfuzz.git="https://github.com/altaua/honggfuzz-rs" | |
| --config=patch.crates-io.honggfuzz.rev="205f7c8c059a0d98fe1cb912cdac84f324cb6981" | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: Run honggfuzz | |
| run: | | |
| cd substrate/primitives/arithmetic/fuzzer | |
| forklift cargo hfuzz build | |
| for target in $(cargo read-manifest | jq -r '.targets | .[] | .name'); | |
| do | |
| forklift cargo hfuzz run "$target" || { printf "fuzzing failure for %s\n" "$target"; exit 1; }; | |
| done | |
| - name: Upload artifacts | |
| uses: actions/[email protected] | |
| with: | |
| path: substrate/primitives/arithmetic/fuzzer/hfuzz_workspace/ | |
| name: hfuzz-${{ github.sha }} | |
| cargo-check-each-crate: | |
| timeout-minutes: 140 | |
| needs: [set-image] | |
| runs-on: ${{ needs.set-image.outputs.RUNNER }} | |
| container: | |
| image: ${{ needs.set-image.outputs.IMAGE }} | |
| env: | |
| RUSTFLAGS: "-D warnings" | |
| CI_JOB_NAME: cargo-check-each-crate | |
| strategy: | |
| matrix: | |
| index: [1, 2, 3, 4, 5, 6, 7] # 7 parallel jobs | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: Check Rust | |
| run: | | |
| rustup show | |
| rustup +nightly show | |
| - name: script | |
| run: | | |
| mkdir -p /github/home/.forklift | |
| cp .forklift/config.toml /github/home/.forklift/config.toml | |
| PYTHONUNBUFFERED=x .github/scripts/check-each-crate.py ${{ matrix.index }} ${{ strategy.job-total }} | |
| cargo-check-all-crate-macos: | |
| timeout-minutes: 30 | |
| runs-on: parity-macos | |
| env: | |
| SKIP_WASM_BUILD: 1 | |
| steps: | |
| - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 | |
| - name: Set rust version from env file | |
| run: | | |
| RUST_VERSION=$(cat .github/env | sed -E 's/.*ci-unified:([^-]+)-([^-]+).*/\2/') | |
| echo $RUST_VERSION | |
| echo "RUST_VERSION=${RUST_VERSION}" >> $GITHUB_ENV | |
| - name: Set up Homebrew | |
| uses: Homebrew/actions/setup-homebrew@1ccc07ccd54b6048295516a3eb89b192c35057dc # master from 12.09.2024 | |
| - name: Install rust ${{ env.RUST_VERSION }} | |
| uses: actions-rust-lang/setup-rust-toolchain@1fbea72663f6d4c03efaab13560c8a24cfd2a7cc # v1.9.0 | |
| with: | |
| cache: false | |
| toolchain: ${{ env.RUST_VERSION }} | |
| target: wasm32-unknown-unknown | |
| components: cargo, clippy, rust-docs, rust-src, rustfmt, rustc, rust-std | |
| - name: Install protobuf | |
| run: brew install protobuf | |
| - name: cargo info | |
| run: | | |
| echo "######## rustup show ########" | |
| rustup show | |
| echo "######## cargo --version ########" | |
| cargo --version | |
| - name: Run cargo check | |
| run: cargo check --workspace --locked | |
| confirm-required-test-misc-jobs-passed: | |
| runs-on: ubuntu-latest | |
| name: All test misc tests passed | |
| # If any new job gets added, be sure to add it to this array | |
| needs: | |
| - test-full-crypto-feature | |
| - test-frame-examples-compile-to-wasm | |
| - test-frame-ui | |
| - cargo-check-benches | |
| - node-bench-regression-guard | |
| - test-node-metrics | |
| - check-tracing | |
| - cargo-check-each-crate | |
| - test-deterministic-wasm | |
| # - cargo-hfuzz remove from required for now, as it's flaky | |
| if: always() && !cancelled() | |
| steps: | |
| - run: | | |
| tee resultfile <<< '${{ toJSON(needs) }}' | |
| FAILURES=$(cat resultfile | grep '"result": "failure"' | wc -l) | |
| if [ $FAILURES -gt 0 ]; then | |
| echo "### At least one required job failed ❌" >> $GITHUB_STEP_SUMMARY | |
| exit 1 | |
| else | |
| echo '### Good job! All the required jobs passed 🚀' >> $GITHUB_STEP_SUMMARY | |
| fi |