ci-core-reusable.yml

name: Workflow template for CI jobs for Core Components
on:
  workflow_call:
    inputs:
      compilers:
        description: "JSON of required compilers and their versions"
        type: string
        required: false
        default: '[{ "zksolc": ["1.3.14", "1.3.16", "1.3.17", "1.3.1", "1.3.7", "1.3.18", "1.3.19", "1.3.21"] } , { "zkvyper": ["1.3.13"] }]'

env:
  RUST_BACKTRACE: 1
  PASSED_ENV_VARS: RUST_BACKTRACE

jobs:
  lint:
    name: lint
    uses: ./.github/workflows/ci-core-lint-reusable.yml
  unit-tests:
    runs-on: [ matterlabs-ci-runner-highmem-long ]

    steps:
      - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4
        with:
          submodules: "recursive"
          fetch-depth: 0

      - name: Setup environment
        run: |
          echo ZKSYNC_HOME=$(pwd) >> $GITHUB_ENV
          echo $(pwd)/bin >> $GITHUB_PATH
          echo IN_DOCKER=1 >> .env
          echo "SCCACHE_GCS_BUCKET=matterlabs-infra-sccache-storage" >> .env
          echo "SCCACHE_GCS_SERVICE_ACCOUNT=gha-ci-runners@matterlabs-infra.iam.gserviceaccount.com" >> .env
          echo "SCCACHE_GCS_RW_MODE=READ_WRITE" >> .env
          echo "RUSTC_WRAPPER=sccache" >> .env
          echo RUN_CONTRACT_VERIFICATION_TEST=true >> .env

      # TODO: Remove when we after upgrade of hardhat-plugins
      - name: pre-download compilers
        run: |
          # Download needed versions of vyper compiler
          # Not sanitized due to unconventional path and tags
          mkdir -p ./hardhat-nodejs/compilers-v2/vyper/linux
          wget -nv -O ./hardhat-nodejs/compilers-v2/vyper/linux/0.3.10 https://github.com/vyperlang/vyper/releases/download/v0.3.10/vyper.0.3.10+commit.91361694.linux
          wget -nv -O ./hardhat-nodejs/compilers-v2/vyper/linux/0.3.3 https://github.com/vyperlang/vyper/releases/download/v0.3.3/vyper.0.3.3+commit.48e326f0.linux
          chmod +x  ./hardhat-nodejs/compilers-v2/vyper/linux/0.3.10
          chmod +x  ./hardhat-nodejs/compilers-v2/vyper/linux/0.3.3

          COMPILERS_JSON='${{ inputs.compilers }}'
          echo "$COMPILERS_JSON" | jq -r '.[] | to_entries[] | .key as $compiler | .value[] | "\(.),\($compiler)"' | while IFS=, read -r version compiler; do
            mkdir -p "./hardhat-nodejs/compilers-v2/$compiler"
            wget -nv -O "./hardhat-nodejs/compilers-v2/$compiler/${compiler}-v${version}" "https://github.com/matter-labs/${compiler}-bin/releases/download/v${version}/${compiler}-linux-amd64-musl-v${version}"
            chmod +x "./hardhat-nodejs/compilers-v2/$compiler/${compiler}-v${version}"
          done

      - name: Start services
        run: |
          ci_localnet_up

      - name: Init
        run: |
          ci_run run_retried rustup show
      
      - name: Install zkstack
        run: |
          ci_run ./zkstack_cli/zkstackup/install -g --path ./zkstack_cli/zkstackup/zkstackup
          ci_run zkstackup -g --local

      - name: Build contracts
        run: |
          ci_run zkstack dev contracts

      - name: Contracts unit tests
        run: ci_run yarn l1-contracts test

      - name: Download compilers for contract verifier tests
        run: ci_run zkstack contract-verifier init --zksolc-version=v1.5.3 --zkvyper-version=v1.5.4 --solc-version=0.8.26 --vyper-version=v0.3.10 --era-vm-solc-version=0.8.26-1.0.1 --only --chain era

      - name: Rust unit tests
        run: |
          ci_run zkstack dev test rust
          # Benchmarks are not tested by `cargo nextest` unless specified explicitly, and even then `criterion` harness is incompatible
          # with how `cargo nextest` runs tests. Thus, we run criterion-based benchmark tests manually.
          ci_run cargo test --release -p vm-benchmark --bench oneshot --bench batch

  loadtest:
    runs-on: [ matterlabs-ci-runner-high-performance ]
    strategy:
      fail-fast: false
      matrix:
        vm_mode: [ "OLD", "NEW" ]

    steps:
      - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4
        with:
          submodules: "recursive"
          fetch-depth: 0

      - name: Setup environment
        run: |
          echo ZKSYNC_HOME=$(pwd) >> $GITHUB_ENV
          echo $(pwd)/bin >> $GITHUB_PATH
          echo IN_DOCKER=1 >> .env
          echo "SCCACHE_GCS_BUCKET=matterlabs-infra-sccache-storage" >> .env
          echo "SCCACHE_GCS_SERVICE_ACCOUNT=gha-ci-runners@matterlabs-infra.iam.gserviceaccount.com" >> .env
          echo "SCCACHE_GCS_RW_MODE=READ_WRITE" >> .env
          echo "RUSTC_WRAPPER=sccache" >> .env

      - name: Loadtest configuration
        run: |
          echo EXPECTED_TX_COUNT=${{ matrix.vm_mode == 'NEW' && 30000 || 16000 }} >> .env
          echo ACCOUNTS_AMOUNT="100" >> .env
          echo MAX_INFLIGHT_TXS="10" >> .env
          echo SYNC_API_REQUESTS_LIMIT="15" >> .env
          echo FAIL_FAST=true >> .env
          echo IN_DOCKER=1 >> .env

      - name: Start services
        run: |
          ci_localnet_up
          ci_run sccache --start-server

      - name: Init
        run: |
          ci_run git config --global --add safe.directory /usr/src/zksync
          ci_run git config --global --add safe.directory /usr/src/zksync/sdk/binaryen
          ci_run git config --global --add safe.directory /usr/src/zksync/contracts/system-contracts
          ci_run git config --global --add safe.directory /usr/src/zksync/contracts

      - name: Install zkstack
        run: |
          ci_run ./zkstack_cli/zkstackup/install -g --path ./zkstack_cli/zkstackup/zkstackup || true
          ci_run zkstackup -g --local
  

      - name: Create and initialize legacy chain
        run: |
          ci_run zkstack chain create \
            --chain-name legacy \
            --chain-id sequential \
            --prover-mode no-proofs \
            --wallet-creation localhost \
            --l1-batch-commit-data-generator-mode rollup \
            --base-token-address 0x0000000000000000000000000000000000000001 \
            --base-token-price-nominator 1 \
            --base-token-price-denominator 1 \
            --set-as-default false \
            --ignore-prerequisites \
            --legacy-bridge \
            --evm-emulator false

          ci_run zkstack ecosystem init --dev --verbose
          ci_run zkstack dev contracts --test-contracts

      # `sleep 60` because we need to wait until server added all the tokens
      - name: Run server
        run: |
          ci_run zkstack dev config-writer --path ${{ matrix.vm_mode == 'NEW' && 'etc/env/file_based/overrides/tests/loadtest-new.yaml' || 'etc/env/file_based/overrides/tests/loadtest-old.yaml' }} --chain legacy
          ci_run zkstack server --uring --chain=legacy --components api,tree,eth,state_keeper,housekeeper,commitment_generator,vm_runner_protective_reads &>server.log &
          ci_run sleep 60

      - name: Perform loadtest
        run: ci_run zkstack dev t loadtest -v --chain=legacy

      - name: Show server.log logs
        if: always()
        run: ci_run cat server.log || true

      - name: Show sccache logs
        if: always()
        run: |
          ci_run sccache --show-stats || true
          ci_run cat /tmp/sccache_log.txt || true

  integration-tests:
    runs-on: [ matterlabs-ci-runner-ultra-performance ]
    steps:
      - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4
        with:
          submodules: "recursive"
          fetch-depth: 0


      - name: Setup environment
        run: |
          echo ZKSYNC_HOME=$(pwd) >> $GITHUB_ENV
          echo $(pwd)/bin >> $GITHUB_PATH
          echo IN_DOCKER=1 >> .env
          echo "SCCACHE_GCS_BUCKET=matterlabs-infra-sccache-storage" >> .env
          echo "SCCACHE_GCS_SERVICE_ACCOUNT=gha-ci-runners@matterlabs-infra.iam.gserviceaccount.com" >> .env
          echo "SCCACHE_GCS_RW_MODE=READ_WRITE" >> .env
          echo "RUSTC_WRAPPER=sccache" >> .env
          echo "GITHUB_TOKEN=${{ secrets.GITHUB_TOKEN }}" >> .env
          echo RUN_CONTRACT_VERIFICATION_TEST=true >> $GITHUB_ENV

      - name: Start services
        run: |
          ci_localnet_up

      - name: Install zkstack
        run: |
          ci_run ./zkstack_cli/zkstackup/install -g --path ./zkstack_cli/zkstackup/zkstackup || true
          ci_run zkstackup -g --local
  
      - name: Create log directories
        run: |
          SERVER_LOGS_DIR=logs/server
          INTEGRATION_TESTS_LOGS_DIR=logs/integration_tests
          INTEGRATION_TESTS_EN_LOGS_DIR=logs/integration_tests/en
          SNAPSHOT_RECOVERY_LOGS_DIR=logs/snapshot_recovery/
          GENESIS_RECOVERY_LOGS_DIR=logs/genesis_recovery/
          EXTERNAL_NODE_LOGS_DIR=logs/external_node
          FEES_LOGS_DIR=logs/fees
          REVERT_LOGS_DIR=logs/revert

          mkdir -p $SERVER_LOGS_DIR
          mkdir -p $INTEGRATION_TESTS_LOGS_DIR
          mkdir -p $INTEGRATION_TESTS_EN_LOGS_DIR
          mkdir -p $SNAPSHOT_RECOVERY_LOGS_DIR
          mkdir -p $GENESIS_RECOVERY_LOGS_DIR
          mkdir -p $EXTERNAL_NODE_LOGS_DIR
          mkdir -p $FEES_LOGS_DIR
          mkdir -p $REVERT_LOGS_DIR

          echo "SERVER_LOGS_DIR=$SERVER_LOGS_DIR" >> $GITHUB_ENV
          echo "INTEGRATION_TESTS_LOGS_DIR=$INTEGRATION_TESTS_LOGS_DIR" >> $GITHUB_ENV
          echo "INTEGRATION_TESTS_EN_LOGS_DIR=$INTEGRATION_TESTS_EN_LOGS_DIR" >> $GITHUB_ENV
          echo "SNAPSHOT_RECOVERY_LOGS_DIR=$SNAPSHOT_RECOVERY_LOGS_DIR" >> $GITHUB_ENV
          echo "GENESIS_RECOVERY_LOGS_DIR=$GENESIS_RECOVERY_LOGS_DIR" >> $GITHUB_ENV
          echo "EXTERNAL_NODE_LOGS_DIR=$EXTERNAL_NODE_LOGS_DIR" >> $GITHUB_ENV
          echo "FEES_LOGS_DIR=$FEES_LOGS_DIR" >> $GITHUB_ENV
          echo "REVERT_LOGS_DIR=$REVERT_LOGS_DIR" >> $GITHUB_ENV

      - name: Initialize ecosystem
        run: |
          ci_run git config --global --add safe.directory /usr/src/zksync
          ci_run git config --global --add safe.directory /usr/src/zksync/contracts/system-contracts
          ci_run git config --global --add safe.directory /usr/src/zksync/contracts

          ci_run zkstack ecosystem init --deploy-paymaster --deploy-erc20 \
          --deploy-ecosystem --l1-rpc-url=http://localhost:8545 \
          --server-db-url=postgres://postgres:notsecurepassword@localhost:5432 \
          --server-db-name=zksync_server_localhost_era \
          --ignore-prerequisites --verbose \
          --observability=false

      - name: Read Custom Token address and set as environment variable
        run: |
          CUSTOM_TOKEN_ADDRESS=$(awk -F": " '/tokens:/ {found_tokens=1} found_tokens && /DAI:/ {found_dai=1} found_dai && /address:/ {print $2; exit}' ./configs/erc20.yaml)
          echo "CUSTOM_TOKEN_ADDRESS=$CUSTOM_TOKEN_ADDRESS"
          echo "CUSTOM_TOKEN_ADDRESS=$CUSTOM_TOKEN_ADDRESS" >> $GITHUB_ENV

      - name: Create and initialize Validium chain
        run: |
          ci_run zkstack chain create \
          --chain-name validium \
          --chain-id sequential \
          --prover-mode no-proofs \
          --wallet-creation localhost \
          --l1-batch-commit-data-generator-mode validium \
          --base-token-address 0x0000000000000000000000000000000000000001 \
          --base-token-price-nominator 1 \
          --base-token-price-denominator 1 \
          --set-as-default false \
          --ignore-prerequisites \
          --evm-emulator false

          ci_run zkstack chain init \
          --deploy-paymaster \
          --l1-rpc-url=http://localhost:8545 \
          --server-db-url=postgres://postgres:notsecurepassword@localhost:5432 \
          --server-db-name=zksync_server_localhost_validium \
          --chain validium

      - name: Create and initialize chain with Custom Token
        run: |
          ci_run zkstack chain create \
          --chain-name custom_token \
          --chain-id sequential \
          --prover-mode no-proofs \
          --wallet-creation localhost \
          --l1-batch-commit-data-generator-mode rollup \
          --base-token-address ${{ env.CUSTOM_TOKEN_ADDRESS }} \
          --base-token-price-nominator 314 \
          --base-token-price-denominator 1000 \
          --set-as-default false \
          --ignore-prerequisites \
          --evm-emulator false

          ci_run zkstack chain init \
          --deploy-paymaster \
          --l1-rpc-url=http://localhost:8545 \
          --server-db-url=postgres://postgres:notsecurepassword@localhost:5432 \
          --server-db-name=zksync_server_localhost_custom_token \
          --chain custom_token

      - name: Create and register chain with transactions signed "offline"
        run: |
          ci_run zkstack chain create \
          --chain-name offline_chain \
          --chain-id sequential \
          --prover-mode no-proofs \
          --wallet-creation localhost \
          --l1-batch-commit-data-generator-mode rollup \
          --base-token-address 0x0000000000000000000000000000000000000001 \
          --base-token-price-nominator 1 \
          --base-token-price-denominator 1 \
          --set-as-default false \
          --ignore-prerequisites \
          --evm-emulator false

          ci_run zkstack chain build-transactions --chain offline_chain --l1-rpc-url http://127.0.0.1:8545

          governor_pk=$(awk '/governor:/ {flag=1} flag && /private_key:/ {print $2; exit}' ./configs/wallets.yaml)

          ci_run zkstack dev send-transactions \
          --file ./transactions/chain/offline_chain/register-hyperchain-txns.json \
          --l1-rpc-url http://127.0.0.1:8545 \
          --private-key $governor_pk

          bridge_hub=$(awk '/bridgehub_proxy_addr/ {print $2}' ./configs/contracts.yaml)
          chain_id=$(awk '/chain_id:/ {print $2}' ./chains/offline_chain/ZkStack.yaml)

          hyperchain_output=$(ci_run cast call $bridge_hub "getHyperchain(uint256)" $chain_id)

          if [[ $hyperchain_output == 0x* && ${#hyperchain_output} -eq 66 ]]; then
              echo "Chain successfully registered: $hyperchain_output"
          else
              echo "Failed to register chain: $hyperchain_output"
              exit 1
          fi

      - name: Create and initialize Consensus chain
        run: |
          ci_run zkstack chain create \
          --chain-name consensus \
          --chain-id sequential \
          --prover-mode no-proofs \
          --wallet-creation localhost \
          --l1-batch-commit-data-generator-mode validium \
          --base-token-address ${{ env.CUSTOM_TOKEN_ADDRESS }} \
          --base-token-price-nominator 314 \
          --base-token-price-denominator 1000 \
          --set-as-default false \
          --ignore-prerequisites \
          --evm-emulator false

          ci_run zkstack chain init \
          --deploy-paymaster \
          --l1-rpc-url=http://localhost:8545 \
          --server-db-url=postgres://postgres:notsecurepassword@localhost:5432 \
          --server-db-name=zksync_server_localhost_consensus \
          --chain consensus

      - name: Export chain list to environment variable
        run: |
          CHAINS="era,validium,custom_token,consensus"
          echo "CHAINS=$CHAINS" >> $GITHUB_ENV

      - name: Build test dependencies
        run: |
          ci_run zkstack dev test build

      - name: Build tested binaries
        run: |
          ci_run zkstack server build
          ci_run zkstack external-node build
          ci_run zkstack contract-verifier build

      - name: Initialize Contract verifier
        run: |
          ci_run zkstack contract-verifier init --zksolc-version=v1.5.3 --zkvyper-version=v1.5.4 --solc-version=0.8.26 --vyper-version=v0.3.10 --era-vm-solc-version=0.8.26-1.0.1 --only --chain era
          ci_run zkstack contract-verifier run --chain era &> ${{ env.SERVER_LOGS_DIR }}/contract-verifier-rollup.log &
          ci_run zkstack contract-verifier wait --chain era --verbose

      - name: Run servers
        run: |
          # Override config for part of chains to test the default config as well
          ci_run zkstack dev config-writer --path etc/env/file_based/overrides/tests/integration.yaml --chain era
          ci_run zkstack dev config-writer --path etc/env/file_based/overrides/tests/integration.yaml --chain validium
          
          ci_run zkstack server --ignore-prerequisites --chain era &> ${{ env.SERVER_LOGS_DIR }}/rollup.log &
          ci_run zkstack server --ignore-prerequisites --chain validium &> ${{ env.SERVER_LOGS_DIR }}/validium.log &
          ci_run zkstack server --ignore-prerequisites --chain custom_token &> ${{ env.SERVER_LOGS_DIR }}/custom_token.log &
          ci_run zkstack server --ignore-prerequisites --chain consensus \
            --components=api,tree,eth,state_keeper,housekeeper,commitment_generator,vm_runner_protective_reads,vm_runner_bwip,vm_playground,da_dispatcher,consensus \
            &> ${{ env.SERVER_LOGS_DIR }}/consensus.log &

          ci_run zkstack server wait --ignore-prerequisites --verbose --chain era
          ci_run zkstack server wait --ignore-prerequisites --verbose --chain validium
          ci_run zkstack server wait --ignore-prerequisites --verbose --chain custom_token
          ci_run zkstack server wait --ignore-prerequisites --verbose --chain consensus

      - name: Set up attester committee for the consensus chain
        run: |
          ci_run zkstack consensus wait-for-registry --ignore-prerequisites --verbose --chain consensus
          ci_run zkstack consensus set-attester-committee --chain consensus --from-genesis &> ${{ env.INTEGRATION_TESTS_LOGS_DIR }}/consensus.log 

      - name: Run integration tests
        run: |
          ci_run ./bin/run_on_all_chains.sh "zkstack dev test integration --no-deps --ignore-prerequisites" ${{ env.CHAINS }} ${{ env.INTEGRATION_TESTS_LOGS_DIR }}

      - name: Init external nodes
        run: |
          ci_run zkstack external-node configs --db-url=postgres://postgres:notsecurepassword@localhost:5432 \
          --db-name=zksync_en_localhost_era_rollup --l1-rpc-url=http://localhost:8545 --chain era
          ci_run zkstack external-node init --ignore-prerequisites --chain era

          ci_run zkstack external-node configs --db-url=postgres://postgres:notsecurepassword@localhost:5432 \
          --db-name=zksync_en_localhost_era_validium1 --l1-rpc-url=http://localhost:8545 --chain validium
          ci_run zkstack external-node init --ignore-prerequisites --chain validium

          ci_run zkstack external-node configs --db-url=postgres://postgres:notsecurepassword@localhost:5432 \
          --db-name=zksync_en_localhost_era_custom_token --l1-rpc-url=http://localhost:8545 --chain custom_token
          ci_run zkstack external-node init --ignore-prerequisites --chain custom_token

          ci_run zkstack external-node configs --db-url=postgres://postgres:notsecurepassword@localhost:5432 \
          --db-name=zksync_en_localhost_era_consensus --l1-rpc-url=http://localhost:8545 --chain consensus
          ci_run zkstack external-node init --ignore-prerequisites --chain consensus

      - name: Run recovery tests (from snapshot)
        run: |
          ci_run ./bin/run_on_all_chains.sh "zkstack dev test recovery --snapshot --no-deps --ignore-prerequisites --verbose" ${{ env.CHAINS }} ${{ env.INTEGRATION_TESTS_LOGS_DIR }}

      - name: Run recovery tests (from genesis)
        run: |
          ci_run ./bin/run_on_all_chains.sh "zkstack dev test recovery --no-deps --no-kill --ignore-prerequisites --verbose" ${{ env.CHAINS }} ${{ env.INTEGRATION_TESTS_LOGS_DIR }}

      - name: Run external node server
        run: |
          ci_run zkstack external-node run --ignore-prerequisites --chain era &> ${{ env.EXTERNAL_NODE_LOGS_DIR }}/rollup.log &
          ci_run zkstack external-node run --ignore-prerequisites --chain validium &> ${{ env.EXTERNAL_NODE_LOGS_DIR }}/validium.log &
          ci_run zkstack external-node run --ignore-prerequisites --chain custom_token &> ${{ env.EXTERNAL_NODE_LOGS_DIR }}/custom_token.log &
          ci_run zkstack external-node run --ignore-prerequisites --chain consensus --enable-consensus &> ${{ env.EXTERNAL_NODE_LOGS_DIR }}/consensus.log &
          
          ci_run zkstack external-node wait --ignore-prerequisites --verbose --chain era
          ci_run zkstack external-node wait --ignore-prerequisites --verbose --chain validium
          ci_run zkstack external-node wait --ignore-prerequisites --verbose --chain custom_token
          ci_run zkstack external-node wait --ignore-prerequisites --verbose --chain consensus

      - name: Run integration tests en
        run: |
          ci_run ./bin/run_on_all_chains.sh "zkstack dev test integration --no-deps --ignore-prerequisites --external-node" ${{ env.CHAINS }} ${{ env.INTEGRATION_TESTS_LOGS_DIR }}

      - name: Fee projection tests
        run: |
          ci_run killall -INT zksync_server || true
          ci_run ./bin/run_on_all_chains.sh "zkstack dev test fees --no-deps --no-kill" ${{ env.CHAINS }} ${{ env.FEES_LOGS_DIR }}

      - name: Run revert tests
        run: |
          ci_run killall -INT zksync_server || true
          ci_run killall -INT zksync_external_node || true

          ci_run ./bin/run_on_all_chains.sh "zkstack dev test revert --no-deps --external-node --no-kill --ignore-prerequisites" ${{ env.CHAINS }} ${{ env.INTEGRATION_TESTS_LOGS_DIR }}

      # Upgrade tests should run last, because as soon as they
      # finish the bootloader will be different
      # TODO make upgrade tests safe to run multiple times
      - name: Run upgrade test
        run: |
          ci_run zkstack dev test upgrade --no-deps --chain era


      - name: Upload logs
        uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874 # v4.4.0
        if: always()
        with:
          name: logs
          path: logs