test.yml

name: Test

on:
  # We want to run the CI when anything is pushed to master.
  # Since master is a protected branch this only happens when a PR is merged.
  # This is a double check in case the PR was stale and had some issues.
  push:
    branches:
      - master
    paths-ignore: # Prevents from running if only docs are updated
      - 'doc/**'
      - '**/*README*'
      - '**.md'
      - '**.rst'
  pull_request:
    paths-ignore: # Prevents from running if only docs are updated
      - 'doc/**'
      - '**/*README*'
      - '**.md'
      - '**.rst'
  workflow_dispatch:
  schedule:
  - cron: '0 0 * * *' # daily

# We want to cancel previous runs for a given PR or branch / ref if another CI
# run is requested.
# See: https://docs.github.com/en/actions/using-jobs/using-concurrency
concurrency:
  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
  cancel-in-progress: true

env:
  # default compiler for all non-compatibility tests
  MATRIX_EVAL: "CC=gcc-11 && CXX=g++-11"

jobs:
  Build:
    name: 'B: Building VtR'
    runs-on: ubuntu-22.04
    strategy:
      fail-fast: false
      matrix:
        include:
        - { build_type: 'release', verbose: '0' }
        - { build_type: 'debug',   verbose: '0' }
        - { build_type: 'debug',   verbose: '1' }
    steps:

    - uses: actions/setup-python@v5
      with:
        python-version: 3.10.10
    - uses: actions/checkout@v4
      with:
        submodules: 'true'
    - run: ./.github/scripts/install_dependencies.sh

    - uses: hendrikmuhs/ccache-action@v1.2

    - name: Test
      env:
        BUILD_TYPE: ${{ matrix.build_type }}
      run: |
        export PATH="/usr/lib/ccache:/usr/local/opt/ccache/libexec:$PATH"
        ./.github/scripts/build.sh VERBOSE=${{ matrix.verbose }}


  Format:
    runs-on: ubuntu-22.04
    strategy:
      fail-fast: false
      matrix:
        include:
        - { name: 'C/C++',       script: 'check-format.sh'    }
        - { name: 'Python',      script: 'check-format-py.sh' }
        - { name: 'Python Lint', script: 'pylint_check.py'    }
    name: 'F: ${{ matrix.name }}'
    steps:

    - uses: actions/setup-python@v5
      with:
        python-version: 3.10.10
    - uses: actions/checkout@v4
    - run: ./.github/scripts/install_dependencies.sh

    - name: Test
      run: ./dev/${{ matrix.script }}


  UniTests:
    name: 'U: C++ Unit Tests'
    runs-on: ubuntu-22.04
    steps:

    - uses: actions/setup-python@v5
      with:
        python-version: 3.10.10
    - uses: actions/checkout@v4
      with:
        submodules: 'true'
    - run: ./.github/scripts/install_dependencies.sh

    - name: Test
      env:
        CMAKE_PARAMS: "-DVTR_ASSERT_LEVEL=3 -DWITH_BLIFEXPLORER=on"
      run: ./.github/scripts/unittest.sh


  Warnings:
    name: 'W: Check Compilation Warnings'
    runs-on: ubuntu-22.04
    steps:

    - uses: actions/setup-python@v5
      with:
        python-version: 3.10.10
    - uses: actions/checkout@v4
      with:
        submodules: 'true'
    - run: ./.github/scripts/install_dependencies.sh

    - uses: hendrikmuhs/ccache-action@v1.2

    - name: Test
      env:
        #In order to get compilation warnings produced per source file, we must do a non-IPO build
        #We also turn warnings into errors for this target by doing a strict compile
        CMAKE_PARAMS: "-DVTR_ASSERT_LEVEL=3 -DWITH_BLIFEXPLORER=on -DVTR_ENABLE_STRICT_COMPILE=on -DVTR_IPO_BUILD=off"
      run: |
        export PATH="/usr/lib/ccache:/usr/local/opt/ccache/libexec:$PATH"
        ./.github/scripts/build.sh


  Regression:
    runs-on: ubuntu-22.04
    strategy:
      fail-fast: false
      matrix:
        include: [
          {
            name: 'Basic',
            params: '-DCMAKE_COMPILE_WARNING_AS_ERROR=on -DVTR_ASSERT_LEVEL=3 -DWITH_BLIFEXPLORER=on',
            suite: 'vtr_reg_basic'
          },
          {
            name: 'Basic with highest assertion level',
            params: '-DCMAKE_COMPILE_WARNING_AS_ERROR=on -DVTR_ASSERT_LEVEL=4 -DWITH_BLIFEXPLORER=on',
            suite: 'vtr_reg_basic'
          },
          {
            name: 'Basic_odin',
            params: '-DCMAKE_COMPILE_WARNING_AS_ERROR=on -DVTR_ASSERT_LEVEL=3 -DWITH_BLIFEXPLORER=on -DWITH_PARMYS=OFF -DWITH_ODIN=on',
            suite: 'vtr_reg_basic_odin'
          },
          {
            name: 'Basic with NO_GRAPHICS',
            params: '-DCMAKE_COMPILE_WARNING_AS_ERROR=on -DVTR_ASSERT_LEVEL=3 -DWITH_BLIFEXPLORER=on -DVPR_USE_EZGL=off',
            suite: 'vtr_reg_basic'
          },
          {
            name: 'Basic with NO_SERVER',
            params: '-DVTR_ASSERT_LEVEL=3 -DWITH_BLIFEXPLORER=on -DVPR_USE_EZGL=on -DVPR_USE_SERVER=off',
            suite: 'vtr_reg_basic'
          },
          {
            name: 'Basic with CAPNPROTO disabled',
            params: '-DCMAKE_COMPILE_WARNING_AS_ERROR=on -DVTR_ASSERT_LEVEL=3 -DWITH_BLIFEXPLORER=on -DVTR_ENABLE_CAPNPROTO=off',
            suite: 'vtr_reg_basic'
          },
          {
            name: 'Basic with VTR_ENABLE_DEBUG_LOGGING',
            params: '-DCMAKE_COMPILE_WARNING_AS_ERROR=on -DVTR_ASSERT_LEVEL=3 -DWITH_BLIFEXPLORER=on -DVTR_ENABLE_DEBUG_LOGGING=on',
            suite: 'vtr_reg_basic'
          },
          {
            name: 'Basic_odin with VTR_ENABLE_DEBUG_LOGGING',
            params: '-DCMAKE_COMPILE_WARNING_AS_ERROR=on -DVTR_ASSERT_LEVEL=3 -DWITH_BLIFEXPLORER=on -DVTR_ENABLE_DEBUG_LOGGING=on -DWITH_PARMYS=OFF -DWITH_ODIN=on',
            suite: 'vtr_reg_basic_odin'
          },
          {
            name: 'Strong',
            params: '-DCMAKE_COMPILE_WARNING_AS_ERROR=on -DVTR_ASSERT_LEVEL=3 -DWITH_BLIFEXPLORER=on',
            suite: 'vtr_reg_strong'
          },
          {
            name: 'Strong_odin',
            params: '-DCMAKE_COMPILE_WARNING_AS_ERROR=on -DVTR_ASSERT_LEVEL=3 -DWITH_BLIFEXPLORER=on -DWITH_PARMYS=OFF -DWITH_ODIN=on',
            suite: 'vtr_reg_strong_odin'
          },
          {
            name: 'Valgrind Memory',
            params: '-DCMAKE_COMPILE_WARNING_AS_ERROR=on -DVTR_ASSERT_LEVEL=3 -DWITH_BLIFEXPLORER=on -DWITH_ODIN=on',
            suite: 'vtr_reg_valgrind_small'
          }
        ]
    name: 'R: ${{ matrix.name }}'
    steps:

    - uses: actions/setup-python@v5
      with:
        python-version: 3.10.10
    - uses: actions/checkout@v4
      with:
        submodules: 'true'
    - run: ./.github/scripts/install_dependencies.sh

    - uses: hendrikmuhs/ccache-action@v1.2

    - name: Test
      env:
        CMAKE_PARAMS: ${{ matrix.params }}
      run: |
        export PATH="/usr/lib/ccache:/usr/local/opt/ccache/libexec:$PATH"
        ./.github/scripts/build.sh
        ./run_reg_test.py ${{ matrix.suite }} -show_failures -j2

    - name: Upload regression run files
      if: ${{ !cancelled() }}
      uses: actions/upload-artifact@v4
      with:
        name: ${{matrix.name}}_run_files
        path: |
          vtr_flow/**/*.out
          # vtr_flow/**/*.blif  # Removed since it was taking too much space and was hardly used.
          vtr_flow/**/*.p
          vtr_flow/**/*.net
          vtr_flow/**/*.r

    - name: Upload regression results
      if: ${{ !cancelled() }}
      uses: actions/upload-artifact@v4
      with:
        name: ${{matrix.name}}_results
        path: |
          vtr_flow/**/*.log
          vtr_flow/**/parse_results*.txt

  Sanitized:
    runs-on: ubuntu-22.04
    strategy:
      fail-fast: false
      matrix:
        include: [
          {
            name: 'Basic',
            params: '-DVTR_ASSERT_LEVEL=3 -DVTR_ENABLE_SANITIZE=on -DVTR_IPO_BUILD=off -DWITH_BLIFEXPLORER=on -DWITH_PARMYS=OFF -DWITH_ODIN=on',
            suite: 'vtr_reg_basic_odin'
          }
          #- { name: 'Strong', suite: 'vtr_reg_strong' } # SKIP Too long to run on GitHub Actions (max 6h)
        ]


    name: 'S: ${{ matrix.name }}'
    steps:

    - uses: actions/setup-python@v5
      with:
        python-version: 3.10.10
    - uses: actions/checkout@v4
      with:
        submodules: 'true'

    - run: ./.github/scripts/install_dependencies.sh

    - uses: hendrikmuhs/ccache-action@v1.2

    - name: Test
      env:
        CMAKE_PARAMS: ${{ matrix.params }}
        BUILD_TYPE: debug
        LSAN_OPTIONS: 'exitcode=42' #Use a non-standard exit code to ensure LSAN errors are detected
        # In Ubuntu 20240310.1.0, the entropy of ASLR has increased (28 -> 32). LLVM 14 in this
        # image is not compatible with this increased ASLR entropy. Apparently, memory sanitizer
        # depends on LLVM and all CI tests where VTR_ENABLE_SANITIZE is enabled fail. For a temporary
        # fix, we manually reduce the entropy. This quick fix should be removed in the future
        # when github deploys a more stable Ubuntu image.
      run: |
        sudo sysctl -w vm.mmap_rnd_bits=28
        export PATH="/usr/lib/ccache:/usr/local/opt/ccache/libexec:$PATH"
        ./.github/scripts/build.sh
        # We skip QoR since we are only checking for errors in sanitizer runs
        ./run_reg_test.py ${{ matrix.suite }} -show_failures -j2 -skip_qor


  Parmys:
    name: 'Parmys Basic Test'
    runs-on: ubuntu-22.04
    steps:

      - uses: actions/setup-python@v5
        with:
          python-version: 3.10.10
      - uses: actions/checkout@v4
        with:
          submodules: 'true'
      - run: ./.github/scripts/install_dependencies.sh

      - uses: hendrikmuhs/ccache-action@v1.2

      - name: Test
        env:
          CMAKE_PARAMS: '-DVTR_IPO_BUILD=off'
          BUILD_TYPE: debug
        run: |
          export PATH="/usr/lib/ccache:/usr/local/opt/ccache/libexec:$PATH"
          ./.github/scripts/build.sh
          ./run_reg_test.py parmys_reg_basic -show_failures -j2


  ODINII:
    name: 'ODIN-II Basic Test'
    runs-on: ubuntu-22.04
    steps:

    - uses: actions/setup-python@v5
      with:
        python-version: 3.10.10
    - uses: actions/checkout@v4
      with:
        submodules: 'true'
    - run: ./.github/scripts/install_dependencies.sh

    - uses: hendrikmuhs/ccache-action@v1.2

    - name: Test
      env:
        CMAKE_PARAMS: '-DVTR_ASSERT_LEVEL=3 -DVTR_ENABLE_SANITIZE=on -DVTR_IPO_BUILD=off -DWITH_BLIFEXPLORER=on -DWITH_PARMYS=OFF -DWITH_ODIN=on'
        BUILD_TYPE: debug
      run: |
        sudo sysctl -w vm.mmap_rnd_bits=28
        export PATH="/usr/lib/ccache:/usr/local/opt/ccache/libexec:$PATH"
        ./.github/scripts/build.sh
        ./run_reg_test.py odin_reg_basic -show_failures -j2


  VQM2BLIF:
    name: 'VQM2BLIF Basic Tests'
    runs-on: ubuntu-22.04
    steps:

    - uses: actions/setup-python@v5
      with:
        python-version: 3.10.10
    - uses: actions/checkout@v4
      with:
        submodules: 'true'
    - run: ./.github/scripts/install_dependencies.sh

    - uses: hendrikmuhs/ccache-action@v1.2

    - name: Test
      env:
        BUILD_TYPE: release
      run: |
        export PATH="/usr/lib/ccache:/usr/local/opt/ccache/libexec:$PATH"
        ./.github/scripts/build.sh
        ./utils/vqm2blif/test/scripts/test_vqm2blif.sh

  Compatibility:
    runs-on: ubuntu-22.04
    strategy:
      fail-fast: false
      matrix:
        include:
        - { name: 'GCC 9 (Ubuntu Jammy - 22.04)',     eval: 'CC=gcc-9 && CXX=g++-9',           }
        - { name: 'GCC 10 (Ubuntu Jammy - 22.04)',    eval: 'CC=gcc-10 && CXX=g++-10',         }
        - { name: 'GCC 11 (Ubuntu Jammy - 22.04)',    eval: 'CC=gcc-11 && CXX=g++-11',         }
        - { name: 'GCC 12 (Ubuntu Jammy - 22.04)',    eval: 'CC=gcc-12 && CXX=g++-12',         }
        - { name: 'Clang 11 (Ubuntu Jammy - 22.04)',  eval: 'CC=clang-11 && CXX=clang++-11',   }
        - { name: 'Clang 12 (Ubuntu Jammy - 22.04)',  eval: 'CC=clang-12 && CXX=clang++-12',   }
        - { name: 'Clang 13 (Ubuntu Jammy - 22.04)',  eval: 'CC=clang-13 && CXX=clang++-13',   }
        - { name: 'Clang 14 (Ubuntu Jammy - 22.04)',  eval: 'CC=clang-14 && CXX=clang++-14',   }
    name: 'B: ${{ matrix.name }}'
    steps:

    - uses: actions/setup-python@v5
      with:
        python-version: 3.10.10
    - uses: actions/checkout@v4
      with:
        submodules: 'true'
    - run: ./.github/scripts/install_dependencies.sh

    - uses: hendrikmuhs/ccache-action@v1.2

    - name: Test
      env:
        CMAKE_PARAMS: "-DVTR_ASSERT_LEVEL=3 -DWITH_BLIFEXPLORER=on -DWITH_ODIN=on"
        MATRIX_EVAL: ${{ matrix.eval }}
        BUILD_TYPE: release
      run: |
        export PATH="/usr/lib/ccache:/usr/local/opt/ccache/libexec:$PATH"
        ./.github/scripts/build.sh

  Coverity:
    name: 'Coverity Scan'
    needs:
      - Build
      - Format
      - UniTests
      - Warnings
      - Regression
      - Sanitized
      - Parmys
      - ODINII
      - VQM2BLIF
      - Compatibility
    runs-on: ubuntu-22.04
    steps:

    - uses: actions/setup-python@v5
      with:
        python-version: 3.10.10
    - uses: actions/checkout@v4
      with:
        submodules: 'true'
    - run: ./.github/scripts/install_dependencies.sh

    - uses: hendrikmuhs/ccache-action@v1.2

    - name: Test
      env:
        CMAKE_PARAMS: '-DVTR_ASSERT_LEVEL=3 -DWITH_BLIFEXPLORER=on'
        _COVERITY_URL: 'https://scan.coverity.com/download/linux64'
        _COVERITY_MD5: 'd0d7d7df9d6609e578f85096a755fb8f'
      run: |
        export PATH="/usr/lib/ccache:/usr/local/opt/ccache/libexec:$PATH"
        ./.github/scripts/build.sh