pr-test.yml

name: PR Test

on: pull_request

permissions:
  contents: read

jobs:
  lint:
    uses: bridgecrewio/gha-reusable-workflows/.github/workflows/pre-commit.yaml@main
    with:
      python-version: "3.9"

  danger-check:
    runs-on: [ self-hosted, public, linux, x64 ]
    permissions:
      contents: read
      pull-requests: read
    steps:
      - name: Checkout code
        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11  # v3
      - name: Install Node.js
        uses: actions/setup-node@b39b52d1213e96004bfcb1c61a8a6fa8ab84f3e8  # v4
        with:
          node-version: "16"
      - name: Install and run DangerJS
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
        run: |
          npm install -g danger
          danger ci --verbose --failOnErrors
  cfn-lint:
    runs-on: ubuntu-latest
    env:
      PYTHON_VERSION: "3.8"
    steps:
      - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11  # v3
      - uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c  # v4
        with:
          python-version: ${{ env.PYTHON_VERSION }}
      - name: Get changed CFN test files
        id: changed-files-specific
        uses: tj-actions/changed-files@eaf854ef0c266753e1abec356dcf17d92695b251 # v44
        with:
          files: tests/cloudformation/checks/resource/aws/**/*
      - name: Install cfn-lint
        if: steps.changed-files-specific.outputs.any_changed == 'true'
        run: |
          pip install -U cfn-lint
      - name: Lint Cloudformation templates
        if: steps.changed-files-specific.outputs.any_changed == 'true'
        env:
          ALL_CHANGED_FILES: ${{ steps.changed-files-specific.outputs.all_changed_files }}
        run: |
          for file in $ALL_CHANGED_FILES; do
            cfn-lint "$file" -i W
          done

  mypy:
    uses: bridgecrewio/gha-reusable-workflows/.github/workflows/mypy.yaml@main
    with:
      python-version: "3.8"

  unit-tests:
    strategy:
      fail-fast: true
      matrix:
        python: ["3.8", "3.9", "3.10", "3.11", "3.12"]
    runs-on: ubuntu-latest
    timeout-minutes: 30
    steps:
      - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11  # v3
      - name: Set up Python ${{ matrix.python }}
        uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c  # v4
        with:
          python-version: ${{ matrix.python }}
          allow-prereleases: true
          cache: "pipenv"
          cache-dependency-path: "Pipfile.lock"
      - uses: azure/setup-helm@5119fcb9089d432beecbf79bb2c7915207344b78  # v3
        with:
          token: ${{ secrets.GITHUB_TOKEN }}
      - uses: imranismail/setup-kustomize@a76db1c6419124d51470b1e388c4b29476f495f1  # v2
        with:
          github-token: ${{ secrets.GITHUB_TOKEN }}
      - name: Install pipenv
        run: |
          python -m pip install --no-cache-dir --upgrade pipenv
      - name: Install dependencies
        run: |
          # remove venv, if exists
          pipenv --rm || true
          pipenv --python ${{ matrix.python }}

          if [ '${{ matrix.python }}' == '3.12' ]; then
            # needed for numpy
            pipenv install --skip-lock --dev -v
          else
            pipenv install --dev -v
          fi

          # list all dependencies to get a better view about installed package versions
          pipenv run pip list

      - name: Get venv path
        id: get-venv
        run: |
          echo "venv=$(pipenv --venv)" >> "$GITHUB_OUTPUT"

      - name: Run Redefine.dev
        uses: redefinedev/redefine-action@main
        with:
          auth: ${{ secrets.REDEFINE_AUTH }}
          python-venv-path: ${{ steps.get-venv.outputs.venv }}
          testing-framework: pytest
          mode: fail-fast
          time-limit: 30  # approx 15% of the run time
          config-args: matrix_value=${{ matrix.python }} file_based_prediction=true

      - name: Unit tests
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
        run: pipenv run python -m pytest tests

  integration-tests:
    strategy:
      fail-fast: true
      matrix:
        python: ["3.10", "3.11", "3.12"]
        os: [ubuntu-latest, macos-latest, windows-latest]
    runs-on: ${{ matrix.os }}
    steps:
      - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11  # v3
      - uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c  # v4
        with:
          python-version: ${{ matrix.python }}
          allow-prereleases: true
          cache: "pipenv"
          cache-dependency-path: "Pipfile.lock"
      - uses: actions/setup-node@b39b52d1213e96004bfcb1c61a8a6fa8ab84f3e8  # v4
      - uses: azure/setup-helm@5119fcb9089d432beecbf79bb2c7915207344b78  # v3
        with:
          token: ${{ secrets.GITHUB_TOKEN }}
      - uses: imranismail/setup-kustomize@a76db1c6419124d51470b1e388c4b29476f495f1  # v2
        if: ${{ runner.os != 'windows' }}
        with:
          github-token: ${{ secrets.GITHUB_TOKEN }}
      - name: Install pipenv
        run: |
          python -m pip install --no-cache-dir --upgrade pipenv
      - name: Build & install checkov package
        shell: bash
        run: |
          # remove venv, if exists
          pipenv --rm || true
          pipenv --python ${{ matrix.python }}
          pipenv run pip install pytest pytest-xdist setuptools wheel
          pipenv run python setup.py sdist bdist_wheel
          bash -c 'pipenv run pip install dist/checkov-*.whl'
      - name: Clone Terragoat - vulnerable terraform
        run: git clone https://github.com/bridgecrewio/terragoat
      - name: Clone Cfngoat - vulnerable cloudformation
        run: git clone https://github.com/bridgecrewio/cfngoat
      - name: Clone Kubernetes-goat - vulnerable kubernetes
        run: git clone https://github.com/madhuakula/kubernetes-goat
      - name: Clone kustomize-goat - vulnerable kustomize
        run: git clone https://github.com/bridgecrewio/kustomizegoat
      - name: Create checkov reports
        env:
          LOG_LEVEL: INFO
          BC_KEY: ${{ secrets.PRISMA_KEY_API2 }}
          PRISMA_API_URL: ${{ secrets.PRISMA_API_URL_2 }}
        run: |
          # Just making sure the API key tests don't run on PRs
          bash -c './integration_tests/prepare_data.sh ${{ matrix.os }} 3.8'
      - name: Run integration tests
        run: |
          pipenv run pytest integration_tests -k 'not api_key'

  integration-tests-old-python:
    strategy:
      fail-fast: true
      matrix:
        python: ["3.8", "3.9"]
        os: [ubuntu-latest, macos-12, windows-latest]
    runs-on: ${{ matrix.os }}
    steps:
      - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11  # v3
      - uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c  # v4
        with:
          python-version: ${{ matrix.python }}
          allow-prereleases: true
          cache: "pipenv"
          cache-dependency-path: "Pipfile.lock"
      - uses: actions/setup-node@b39b52d1213e96004bfcb1c61a8a6fa8ab84f3e8  # v4
      - uses: azure/setup-helm@5119fcb9089d432beecbf79bb2c7915207344b78  # v3
        with:
          token: ${{ secrets.GITHUB_TOKEN }}
      - uses: imranismail/setup-kustomize@a76db1c6419124d51470b1e388c4b29476f495f1  # v2
        if: ${{ runner.os != 'windows' }}
        with:
          github-token: ${{ secrets.GITHUB_TOKEN }}
      - name: Install pipenv
        run: |
          python -m pip install --no-cache-dir --upgrade pipenv
      - name: Build & install checkov package
        shell: bash
        run: |
          # remove venv, if exists
          pipenv --rm || true
          pipenv --python ${{ matrix.python }}
          pipenv run pip install pytest pytest-xdist setuptools wheel
          pipenv run python setup.py sdist bdist_wheel
          bash -c 'pipenv run pip install dist/checkov-*.whl'
      - name: Clone Terragoat - vulnerable terraform
        run: git clone https://github.com/bridgecrewio/terragoat
      - name: Clone Cfngoat - vulnerable cloudformation
        run: git clone https://github.com/bridgecrewio/cfngoat
      - name: Clone Kubernetes-goat - vulnerable kubernetes
        run: git clone https://github.com/madhuakula/kubernetes-goat
      - name: Clone kustomize-goat - vulnerable kustomize
        run: git clone https://github.com/bridgecrewio/kustomizegoat
      - name: Create checkov reports
        env:
          LOG_LEVEL: INFO
          BC_KEY: ${{ secrets.PRISMA_KEY_API2 }}
          PRISMA_API_URL: ${{ secrets.PRISMA_API_URL_2 }}
        run: |
          # Just making sure the API key tests don't run on PRs
          bash -c './integration_tests/prepare_data.sh ${{ matrix.os }} 3.8'
      - name: Run integration tests
        run: |
          pipenv run pytest integration_tests -k 'not api_key'

  sast-integration-tests:
    strategy:
      fail-fast: true
      matrix:
        python: ["3.12"]
        os: [ubuntu-latest, macos-latest]
    runs-on: ${{ matrix.os }}
    steps:
      - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11  # v3
      - uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c  # v4
        with:
          python-version: ${{ matrix.python }}
          allow-prereleases: true
          cache: "pipenv"
          cache-dependency-path: "Pipfile.lock"
      - name: Install pipenv
        run: |
          python -m pip install --no-cache-dir --upgrade pipenv
      - name: Build & install checkov package
        run: |
          # remove venv, if exists
          pipenv --rm || true
          pipenv --python ${{ matrix.python }}
          pipenv run pip install pytest pytest-xdist setuptools wheel
          pipenv run python setup.py sdist bdist_wheel
          bash -c 'pipenv run pip install dist/checkov-*.whl'
      - name: Clone flask - Python repo for SAST
        run: git clone https://github.com/pallets/flask
      - name: Clone WebGoat - Java repo for SAST
        run: git clone https://github.com/WebGoat/WebGoat
      - name: Clone axios - JavaScript repo for SAST
        run: git clone https://github.com/axios/axios
      - name: Create checkov reports
        env:
          LOG_LEVEL: INFO
          BC_API_KEY: ${{ secrets.PRISMA_KEY_API2 }}
          PRISMA_API_URL: ${{ secrets.PRISMA_API_URL_2 }}
        if: env.BC_API_KEY != null
        run: bash -c './sast_integration_tests/prepare_data.sh'
      - name: Run integration tests
        env:
          LOG_LEVEL: INFO
          BC_API_KEY: ${{ secrets.PRISMA_KEY_API2 }}
          PRISMA_API_URL: ${{ secrets.PRISMA_API_URL_2 }}
        if: env.BC_API_KEY != null
        run: |
          pipenv run pytest sast_integration_tests

  sast-integration-tests-old-python:
    strategy:
      fail-fast: true
      matrix:
        python: ["3.8"]
        os: [ubuntu-latest, macos-12]
    runs-on: ${{ matrix.os }}
    steps:
      - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11  # v3
      - uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c  # v4
        with:
          python-version: ${{ matrix.python }}
          allow-prereleases: true
          cache: "pipenv"
          cache-dependency-path: "Pipfile.lock"
      - name: Install pipenv
        run: |
          python -m pip install --no-cache-dir --upgrade pipenv
      - name: Build & install checkov package
        run: |
          # remove venv, if exists
          pipenv --rm || true
          pipenv --python ${{ matrix.python }}
          pipenv run pip install pytest pytest-xdist setuptools wheel
          pipenv run python setup.py sdist bdist_wheel
          bash -c 'pipenv run pip install dist/checkov-*.whl'
      - name: Clone flask - Python repo for SAST
        run: git clone https://github.com/pallets/flask
      - name: Clone WebGoat - Java repo for SAST
        run: git clone https://github.com/WebGoat/WebGoat
      - name: Clone axios - JavaScript repo for SAST
        run: git clone https://github.com/axios/axios
      - name: Create checkov reports
        env:
          LOG_LEVEL: INFO
          BC_API_KEY: ${{ secrets.PRISMA_KEY_API2 }}
          PRISMA_API_URL: ${{ secrets.PRISMA_API_URL_2 }}
        if: env.BC_API_KEY != null
        run: bash -c './sast_integration_tests/prepare_data.sh'
      - name: Run integration tests
        env:
          LOG_LEVEL: INFO
          BC_API_KEY: ${{ secrets.PRISMA_KEY_API2 }}
          PRISMA_API_URL: ${{ secrets.PRISMA_API_URL_2 }}
        if: env.BC_API_KEY != null
        run: |
          pipenv run pytest sast_integration_tests

  cdk-integration-tests:
    strategy:
      fail-fast: true
      matrix:
        python: ["3.12"]
        os: [ubuntu-latest, macos-latest]
    runs-on: ${{ matrix.os }}
    steps:
      - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11  # v3
      - uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c  # v4
        with:
          python-version: ${{ matrix.python }}
          allow-prereleases: true
          cache: "pipenv"
          cache-dependency-path: "Pipfile.lock"
      - name: Install pipenv
        run: |
          python -m pip install --no-cache-dir --upgrade pipenv
      - name: Build & install checkov package
        run: |
          # remove venv, if exists
          pipenv --rm || true
          pipenv --python ${{ matrix.python }}
          pipenv run pip install pytest pytest-xdist setuptools wheel
          pipenv run python setup.py sdist bdist_wheel
          bash -c 'pipenv run pip install dist/checkov-*.whl'
      - name: Create checkov reports
        env:
          LOG_LEVEL: INFO
          BC_API_KEY: ${{ secrets.PRISMA_KEY_API2 }}
          PRISMA_API_URL: ${{ secrets.PRISMA_API_URL_2 }}
        if: env.BC_API_KEY != null
        run: bash -c './cdk_integration_tests/prepare_data.sh'
      - name: Run integration tests
        env:
          LOG_LEVEL: INFO
          BC_API_KEY: ${{ secrets.PRISMA_KEY_API2 }}
          PRISMA_API_URL: ${{ secrets.PRISMA_API_URL_2 }}
        if: env.BC_API_KEY != null
        run: |
          pipenv run pytest cdk_integration_tests

  cdk-integration-tests-old-python:
    strategy:
      fail-fast: true
      matrix:
        python: ["3.8"]
        os: [ubuntu-latest, macos-12]
    runs-on: ${{ matrix.os }}
    steps:
      - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11  # v3
      - uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c  # v4
        with:
          python-version: ${{ matrix.python }}
          allow-prereleases: true
          cache: "pipenv"
          cache-dependency-path: "Pipfile.lock"
      - name: Install pipenv
        run: |
          python -m pip install --no-cache-dir --upgrade pipenv
      - name: Build & install checkov package
        run: |
          # remove venv, if exists
          pipenv --rm || true
          pipenv --python ${{ matrix.python }}
          pipenv run pip install pytest pytest-xdist setuptools wheel
          pipenv run python setup.py sdist bdist_wheel
          bash -c 'pipenv run pip install dist/checkov-*.whl'
      - name: Create checkov reports
        env:
          LOG_LEVEL: INFO
          BC_API_KEY: ${{ secrets.PRISMA_KEY_API2 }}
          PRISMA_API_URL: ${{ secrets.PRISMA_API_URL_2 }}
        if: env.BC_API_KEY != null
        run: bash -c './cdk_integration_tests/prepare_data.sh'
      - name: Run integration tests
        env:
          LOG_LEVEL: INFO
          BC_API_KEY: ${{ secrets.PRISMA_KEY_API2 }}
          PRISMA_API_URL: ${{ secrets.PRISMA_API_URL_2 }}
        if: env.BC_API_KEY != null
        run: |
          pipenv run pytest cdk_integration_tests

  performance-tests:
    env:
      PYTHON_VERSION: "3.8"
      working-directory: ./performance_tests
    runs-on: [self-hosted, public, linux, x64]
    steps:
      - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11  # v3
      - uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c  # v4
        with:
          python-version: ${{ env.PYTHON_VERSION }}
          cache: "pipenv"
          cache-dependency-path: "Pipfile.lock"
      - uses: actions/setup-node@b39b52d1213e96004bfcb1c61a8a6fa8ab84f3e8  # v4
      - uses: azure/setup-helm@5119fcb9089d432beecbf79bb2c7915207344b78  # v3
        with:
          token: ${{ secrets.GITHUB_TOKEN }}
      - uses: imranismail/setup-kustomize@a76db1c6419124d51470b1e388c4b29476f495f1  # v2
        with:
          github-token: ${{ secrets.GITHUB_TOKEN }}
      - name: Install pipenv
        run: |
          python -m pip install --no-cache-dir --upgrade pipenv
      - name: Build & install checkov package
        run: |
          # remove venv, if exists
          pipenv --rm || true
          pipenv --python ${{ env.PYTHON_VERSION }}
          # 'py' package is used in 'pytest-benchmark', but 'pytest' removed it in their latest version
          pipenv run pip install pytest pytest-benchmark py
          pipenv run python setup.py sdist bdist_wheel
          bash -c 'pipenv run pip install dist/checkov-*.whl'
      - name: Clone terraform-aws-components
        run: git clone --branch 0.182.0 https://github.com/cloudposse/terraform-aws-components.git
        working-directory: ${{ env.working-directory }}
      - name: Clone aws-cloudformation-templates
        run: git clone --branch 0.0.1 https://github.com/awslabs/aws-cloudformation-templates.git
        working-directory: ${{ env.working-directory }}
      - name: Clone kubernetes-yaml-templates
        run: git clone https://github.com/dennyzhang/kubernetes-yaml-templates.git
        working-directory: ${{ env.working-directory }}
# TODO: migrate to separate performance tests
#      - name: Clone Python-Mini-Projects
#        run: git clone https://github.com/alimoustafa2000/Python-Mini-Projects.git
#        working-directory: ${{ env.working-directory }}
#      - name: Clone NodeJs
#        run: git clone https://github.com/harshitbansal373/NodeJs.git
#        working-directory: ${{ env.working-directory }}
#      - name: Clone Mini-Project-using-Java
#        run: git clone https://github.com/ikanurfitriani/Mini-Project-using-Java.git
#        working-directory: ${{ env.working-directory }}
      - name: Run performance tests
        run: |
          pipenv run pytest
        working-directory: ${{ env.working-directory }}

  dogfood-tests:
    runs-on: ubuntu-latest
    env:
      PYTHON_VERSION: "3.8"
      WORKING_DIRECTORY: ./dogfood_tests
    steps:
      - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11  # v3
      - uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c  # v4
        with:
          python-version: ${{ env.PYTHON_VERSION }}
          cache: "pipenv"
          cache-dependency-path: "Pipfile.lock"
      - uses: azure/setup-helm@5119fcb9089d432beecbf79bb2c7915207344b78  # v3
        with:
          token: ${{ secrets.GITHUB_TOKEN }}
      - uses: imranismail/setup-kustomize@a76db1c6419124d51470b1e388c4b29476f495f1  # v2
        with:
          github-token: ${{ secrets.GITHUB_TOKEN }}
      - name: Install pipenv
        run: |
          python -m pip install --no-cache-dir --upgrade pipenv

      - name: Build & install checkov package
        run: |
          # remove venv, if exists
          pipenv --rm || true
          pipenv --python ${{ env.PYTHON_VERSION }}
          pipenv run pip install pytest pytest-xdist
          pipenv run python setup.py sdist bdist_wheel
          bash -c 'pipenv run pip install dist/checkov-*.whl'
      - name: Run dogfood tests
        run: |
          pipenv run pytest
        working-directory: ${{ env.WORKING_DIRECTORY }}