main.yml

name: CI

on:
  push:
    branches:
      - trunk
  pull_request:
    branches:
      - trunk
  schedule:
    - cron: '0 1 * * *'

permissions:
  actions: write  # Needed for skip-duplicate-jobs job
  contents: read

jobs:
  # Special job which skips duplicate jobs
  pre_job:
    name: Skip Duplicate Jobs Pre Job
    runs-on: ubuntu-latest
    # Map a step output to a job output
    outputs:
      should_skip: ${{ steps.skip_check.outputs.should_skip }}
    steps:
      - name: Checkout code
        uses: actions/checkout@master
        with:
          persist-credentials: false
          submodules: recursive

      - id: skip_check
        # NOTE: We store action as submodule since ASF doesn't allow directly referencing external
        # actions
        uses: ./.github/actions/skip-duplicate-actions # v4.0.0
        with:
          cancel_others: 'true'
          github_token: ${{ github.token }}

  unit_tests:
    name: Unit Tests (Python ${{ matrix.python_version }})
    runs-on: ${{ matrix.os }}
    timeout-minutes: 8

    needs: pre_job
    if: ${{ needs.pre_job.outputs.should_skip == 'false' || github.ref == 'refs/heads/trunk' }}

    strategy:
      fail-fast: false
      matrix:
        python_version:
          - 3.7
          - 3.8
          - 3.9
          - "3.10"
          - "3.11"
          - "pypy-3.7"
        os:
          - ubuntu-latest
        include:
          - python_version: pyjion
            os: ubuntu-20.04

    steps:
      - uses: actions/checkout@master
        with:
          fetch-depth: 1

      - name: Use Python ${{ matrix.python_version }}
        if: ${{ matrix.python_version != 'pyjion' }}
        uses: actions/setup-python@v4
        with:
          python-version: ${{ matrix.python_version }}

      - name: Install OS / deb dependencies
        run: |
          sudo DEBIAN_FRONTEND=noninteractive apt-get update
          sudo DEBIAN_FRONTEND=noninteractive apt-get install -yq gcc libvirt-dev

      - name: Use Python 3.10
        if: ${{ matrix.python_version == 'pyjion' }}
        uses: actions/setup-python@v4
        with:
          python-version: "3.10"

      # From https://github.com/tonybaloney/Pyjion/blob/develop/main/.github/workflows/benchmark.yml#L26 (MIT)
      - name: Install OS / deb dependencies
        if: ${{ matrix.python_version == 'pyjion' }}
        run: |
          sudo DEBIAN_FRONTEND=noninteractive apt-get update
          sudo DEBIAN_FRONTEND=noninteractive apt-get install -yq cmake llvm-9 clang-9 autoconf automake \
          libtool build-essential python curl git lldb-6.0 liblldb-6.0-dev \
          libunwind8 libunwind8-dev gettext libicu-dev liblttng-ust-dev \
          libssl-dev libnuma-dev libkrb5-dev zlib1g-dev

      - name: Setup Dotnet 6
        if: ${{ matrix.python_version == 'pyjion' }}
        uses: actions/setup-dotnet@v3.0.3
        with:
          dotnet-version: "6.0.100"

      - name: Cache Python Dependencies
        uses: actions/cache@v3
        with:
          path: ~/.cache/pip
          key: ${{ runner.os }}-pip-${{ hashFiles('requirements-tests.txt', '') }}
          restore-keys: |
            ${{ runner.os }}-pip-

      - name: Install Pyjion
        if: ${{ matrix.python_version == 'pyjion' }}
        run: |
          pip install pyjion

      - name: Install Python Dependencies
        run: |
          pip install "tox==4.4.2"

      - name: Run unit tests tox target
        run: |
         script -e -c "tox -e py${{ matrix.python_version }}"

      - name: Run dist install checks tox target
        if: ${{ matrix.python_version != 'pypy-3.7' && matrix.python_version != 'pyjion' }}
        run: |
         script -e -c "tox -e py${{ matrix.python_version }}-dist,py${{ matrix.python_version }}-dist-wheel"

  code_coverage:
    name: Generate Code Coverage
    runs-on: ubuntu-latest

    needs: pre_job
    if: ${{ needs.pre_job.outputs.should_skip == 'false' || github.ref == 'refs/heads/trunk' }}

    strategy:
      matrix:
        python_version: [3.8]

    steps:
      - uses: actions/checkout@master
        with:
          fetch-depth: 1

      - name: Use Python ${{ matrix.python_version }}
        uses: actions/setup-python@v4
        with:
          python-version: ${{ matrix.python_version }}

      - name: Install OS / deb dependencies
        run: |
          sudo DEBIAN_FRONTEND=noninteractive apt-get update
          sudo DEBIAN_FRONTEND=noninteractive apt-get install -yq graphviz gcc libvirt-dev

      - name: Cache Python Dependencies
        uses: actions/cache@v3
        with:
          path: ~/.cache/pip
          key: ${{ runner.os }}-pip-${{ hashFiles('requirements-tests.txt') }}
          restore-keys: |
            ${{ runner.os }}-pip-

      - name: Install Python Dependencies
        run: |
          pip install "tox==4.4.2"

      - name: Run Checks
        run: |
          script -e -c "tox -e coverage-ci"

  lint_checks:
    name: Run Various Lint and Other Checks
    runs-on: ubuntu-latest

    needs: pre_job
    if: ${{ needs.pre_job.outputs.should_skip == 'false' || github.ref == 'refs/heads/trunk' }}

    strategy:
      matrix:
        python_version: [3.8]

    steps:
      - uses: actions/checkout@master
        with:
          fetch-depth: 1

      - name: Use Python ${{ matrix.python_version }}
        uses: actions/setup-python@v4
        with:
          python-version: ${{ matrix.python_version }}

      - name: Install OS / deb dependencies
        run: |
          sudo DEBIAN_FRONTEND=noninteractive apt-get update
          sudo DEBIAN_FRONTEND=noninteractive apt-get install -yq graphviz gcc libvirt-dev

      - name: Cache Python Dependencies
        uses: actions/cache@v3
        with:
          path: ~/.cache/pip
          key: ${{ runner.os }}-pip-${{ hashFiles('requirements-lint.txt') }}
          restore-keys: |
            ${{ runner.os }}-pip-

      - name: Install Python Dependencies
        run: |
          pip install "tox==4.4.2"

      - name: Run Checks
        run: |
          script -e -c "tox -e black-check,isort-check,pyupgrade,checks,import-timings,lint,pylint"

  build_test_docker_image:
    name: Build and Verify Docker Image
    runs-on: ubuntu-latest

    needs: pre_job
    if: ${{ needs.pre_job.outputs.should_skip == 'false' || github.ref == 'refs/heads/trunk' }}

    strategy:
      matrix:
        python_version: [3.8]

    steps:
      - uses: actions/checkout@master
        with:
          fetch-depth: 1

      - name: Build Testing Docker Image
        run: |
          docker build -f contrib/Dockerfile -t libcloud_runtest_img .

      - name: Verify Image Works
        # This step runs checks under various Python versions and it's slow so
        # we only run it on nightly basis
        if: ${{ github.event.schedule == '0 1 * * *' }}
        run: |
          docker run libcloud_runtest_img

  security_checks:
    name: Run Security Checks
    runs-on: ubuntu-20.04

    needs: pre_job
    if: ${{ needs.pre_job.outputs.should_skip == 'false' || github.ref == 'refs/heads/trunk' }}

    strategy:
      matrix:
        python_version: [3.8]

    steps:
      - uses: actions/checkout@master
        with:
          fetch-depth: 1

      - name: Install OS / deb dependencies
        run: |
          sudo DEBIAN_FRONTEND=noninteractive apt-get update
          sudo DEBIAN_FRONTEND=noninteractive apt-get install -yq gcc libvirt-dev

      - name: Use Python ${{ matrix.python_version }}
        uses: actions/setup-python@v4
        with:
          python-version: ${{ matrix.python_version }}

      - name: Cache Python Dependencies
        uses: actions/cache@v3
        with:
          path: ~/.cache/pip
          key: ${{ runner.os }}-pip-${{ hashFiles('requirements-lint.txt') }}
          restore-keys: |
            ${{ runner.os }}-pip-

      - name: Install Python Dependencies
        run: |
          pip install "tox==4.4.2"

      - name: Install Library Into Virtualenv
        run: |
          python -m venv venv/
          source venv/bin/activate
          python -m pip install .

      - name: Run Pip Audit Check On Main Library Dependencies
        uses: ./.github/actions/gh-action-pip-audit/  # v1.0.0
        with:
          virtual-environment: venv/
          # setuptools which we don't install or depend on directly
          ignore-vulns: |
            GHSA-r9hx-vwmv-q579

      - name: Cleanup
        run: |
          rm -rf venv/ || true

      - name: Run Pip Audit Check On All Development And Test Dependencies
        uses: ./.github/actions/gh-action-pip-audit/  # v1.0.0
        with:
          inputs: requirements-tests.txt requirements-lint.txt requirements-mypy.txt requirements-docs.txt
          # setuptools which we don't install or depend on directly
          ignore-vulns: |
            GHSA-r9hx-vwmv-q579

      - name: Run Bandit Check
        run: |
          script -e -c "tox -e bandit"

  micro-benchmarks:
    name: Micro Benchmarks
    runs-on: ubuntu-latest

    needs: pre_job
    if: ${{ needs.pre_job.outputs.should_skip == 'false' || github.ref == 'refs/heads/trunk' }}

    strategy:
      matrix:
        python_version: [3.8]

    steps:
      - uses: actions/checkout@master
        with:
          fetch-depth: 1

      - name: Use Python ${{ matrix.python_version }}
        uses: actions/setup-python@v4
        with:
          python-version: ${{ matrix.python_version }}

      - name: Install OS / deb dependencies
        run: |
          sudo DEBIAN_FRONTEND=noninteractive apt-get update
          sudo DEBIAN_FRONTEND=noninteractive apt-get install -yq graphviz gcc libvirt-dev

      - name: Cache Python Dependencies
        uses: actions/cache@v3
        with:
          path: ~/.cache/pip
          key: ${{ runner.os }}-pip-${{ hashFiles('requirements-tests.txt') }}
          restore-keys: |
            ${{ runner.os }}-pip-

      - name: Install Python Dependencies
        run: |
          pip install "tox==4.4.2"

      - name: Run Micro Benchmarks
        run: |
          script -e -c "tox -e micro-benchmarks"

  docs:
    name: Build and upload Documentation
    runs-on: ubuntu-latest

    needs: pre_job
    if: ${{ needs.pre_job.outputs.should_skip == 'false' || github.ref == 'refs/heads/trunk' }}

    strategy:
      matrix:
        python_version: [3.8]

    steps:
      - name: Print Environment Info
        id: printenv
        run: |
          printenv | sort

      - uses: actions/checkout@master
        with:
          fetch-depth: 1

      - name: Use Python ${{ matrix.python_version }}
        uses: actions/setup-python@v4
        with:
          python-version: ${{ matrix.python_version }}

      - name: Install OS / deb dependencies
        run: |
          sudo DEBIAN_FRONTEND=noninteractive apt-get update
          sudo DEBIAN_FRONTEND=noninteractive apt-get install -yq graphviz gcc libvirt-dev

      - name: Cache Python Dependencies
        uses: actions/cache@v3
        with:
          path: ~/.cache/pip
          key: ${{ runner.os }}-pip-${{ hashFiles('requirements-docs.txt') }}
          restore-keys: |
            ${{ runner.os }}-pip-

      - name: Install Python Dependencies
        run: |
          pip install "tox==4.4.2"

      - name: Build Docs
        run: |
          script -e -c "tox -e docs-ci"

      - name: Trigger ReadTheDocs build
        env:
          RTD_TOKEN: ${{ secrets.RTD_TOKEN }}
        run: |
          pip install requests
          python ./contrib/trigger_rtd_build.py