ci.yml

name: CI

on:
  pull_request:


jobs:
  commit-lint:
    runs-on: ubuntu-latest
    steps:
      - name: find the prev warning if exist
        uses: peter-evans/find-comment@v1
        id: fc
        with:
          issue-number: ${{ github.event.pull_request.number }}
          comment-author: "github-actions[bot]"
          body-includes: "bad commit message"
      - name: Delete comment if exist
        if: ${{ steps.fc.outputs.comment-id != 0 }}
        uses: actions/github-script@v3
        with:
          github-token: ${{ secrets.GITHUB_TOKEN }}
          script: |
            github.issues.deleteComment({
              owner: context.repo.owner,
              repo: context.repo.repo,
              comment_id: ${{ steps.fc.outputs.comment-id }},
            })
      - uses: actions/checkout@v2.5.0
        with:
          fetch-depth: 0
      - run: 'echo "module.exports = {extends: [''@commitlint/config-conventional'']}" > commitlint.config.js'
      - uses: wagoid/commitlint-github-action@v4
        env:
          GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}"
      - name: if lint failed
        if: ${{ failure() }}
        uses: peter-evans/create-or-update-comment@v1
        with:
          issue-number: ${{ github.event.pull_request.number }}
          body: |
            Thanks for your contribution :heart:
            :broken_heart: Unfortunately, this PR has one or more **bad commit messages**, it can not be merged. To fix this problem, please refer to:
            - [Commit Message Guideline for the First Time Contributor](https://github.com/jina-ai/jina/issues/553)
            - [Contributing Guideline](https://github.com/jina-ai/jina/blob/master/CONTRIBUTING.md)
            This message will be deleted automatically when the commit messages get fixed.
          reaction-type: "eyes"

  lint-flake-8:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v2
      - name: Set up Python 3.7
        uses: actions/setup-python@v2
        with:
          python-version: 3.7
      - name: Lint with flake8
        run: |
          pip install flake8
          # stop the build if there are Python syntax errors or undefined names
          flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics --exclude .git,__pycache__,docs/source/conf.py,old,build,dist,tests/,jina/resources/
          # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide
          flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics --exclude .git,__pycache__,docs/source/conf.py,old,build,dist,tests/,jina/resources/

  check-black:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v2
        with:
          fetch-depth: 0
      - name: Set up Python 3.7
        uses: actions/setup-python@v2
        with:
          python-version: 3.7
      - id: file_changes
        uses: Ana06/get-changed-files@v1.2
      - name: check black
        run: ./scripts/black.sh
        env:
          CHANGED_FILES: ${{ steps.file_changes.outputs.added_modified }}

  prep-testbed:
    runs-on: ubuntu-latest
    needs: [lint-flake-8, check-black]
    steps:
      - uses: actions/checkout@v2
      - id: set-matrix
        run: |
          sudo apt-get install jq
          echo "::set-output name=matrix::$(bash scripts/get-all-test-paths.sh)"
    outputs:
      matrix: ${{ steps.set-matrix.outputs.matrix }}

  core-test:
    needs: prep-testbed
    runs-on: ubuntu-latest
    strategy:
      fail-fast: false
      matrix:
        python-version: [3.7]
        test-path: ${{fromJson(needs.prep-testbed.outputs.matrix)}}
    steps:
      - uses: actions/checkout@v2
      - name: Set up Python ${{ matrix.python-version }}
        uses: actions/setup-python@v2
        with:
          python-version: ${{ matrix.python-version }}
      - name: Prepare enviroment
        run: |
          python -m pip install --upgrade pip
          python -m pip install wheel pytest pytest-cov
          pip install --no-cache-dir "client/[test]"
          pip install --no-cache-dir "server/[onnx]"
          pip install --no-cache-dir "server/[transformers]"
          pip install --no-cache-dir "server/[search]"
          pip install --no-cache-dir "server/[cn_clip]"
      - name: Test
        id: test
        run: |
          pytest --suppress-no-test-exit-code --cov=clip_client --cov=clip_server --cov-report=xml \
            -v -s ${{ matrix.test-path }}
          echo "::set-output name=codecov_flag::cas"
        timeout-minutes: 30
      - name: Check codecov file
        id: check_files
        uses: andstor/file-existence-action@v1
        with:
          files: "coverage.xml"
      - name: Upload coverage from test to Codecov
        uses: codecov/codecov-action@v3
        if: steps.check_files.outputs.files_exists == 'true' && ${{ matrix.python-version }} == '3.7'
        with:
          file: coverage.xml
          name: ${{ matrix.test-path }}-codecov
          flags: ${{ steps.test.outputs.codecov_flag }}
          fail_ci_if_error: false
          token: ${{ secrets.CODECOV_TOKEN }} # not required for public repos

  trt-gpu-test:
    needs: prep-testbed
    runs-on: [self-hosted, x64, gpu, linux]
    strategy:
      fail-fast: false
      matrix:
        python-version: [ 3.7 ]
    steps:
      - uses: actions/checkout@v2
        with:
          # For coverage builds fetch the whole history
          fetch-depth: 0
      - name: Set up Python ${{ matrix.python-version }}
        uses: actions/setup-python@v2
        with:
          python-version: ${{ matrix.python-version }}
      - name: Prepare enviroment
        run: |
          python -m pip install pip==23.0.1
          python -m pip install wheel pytest pytest-cov nvidia-pyindex
          pip install -e "client/[test]"
          pip install -e "server/[tensorrt]"
          pip install -e "server/[onnx]"
          pip install -e "server/[transformers]"
          {
            pip install -e "server/[flash-attn]"
          } || {
            echo "flash attention was not installed."
          }
          pip install --no-cache-dir "server/[cn_clip]"
      - name: Test
        id: test
        run: |
          pytest --suppress-no-test-exit-code --cov=clip_client --cov=clip_server --cov-report=xml \
            -v -s -m "gpu" ./tests/test_tensorrt.py
          pytest --suppress-no-test-exit-code --cov=clip_client --cov=clip_server --cov-report=xml \
            -v -s -m "gpu" ./tests/test_simple.py
          echo "::set-output name=codecov_flag::cas"
        timeout-minutes: 30
        env:
          # fix re-initialized torch runtime error on cuda device
          JINA_MP_START_METHOD: spawn
      - name: Check codecov file
        id: check_files
        uses: andstor/file-existence-action@v1
        with:
          files: "coverage.xml"
      - name: Upload coverage from test to Codecov
        uses: codecov/codecov-action@v3
        if: steps.check_files.outputs.files_exists == 'true' && ${{ matrix.python-version }} == '3.7'
        with:
          file: coverage.xml
          name: gpu-related-codecov
          flags: ${{ steps.test.outputs.codecov_flag }}
          fail_ci_if_error: false
          token: ${{ secrets.CODECOV_TOKEN }} # not required for public repos

  gpu-model-test:
    needs: prep-testbed
    runs-on: [ self-hosted, x64, gpu, linux ]
    strategy:
      fail-fast: false
      matrix:
        python-version: [ 3.7 ]
    steps:
      - uses: actions/checkout@v2
        with:
          # For coverage builds fetch the whole history
          fetch-depth: 0
      - name: Set up Python ${{ matrix.python-version }}
        uses: actions/setup-python@v2
        with:
          python-version: ${{ matrix.python-version }}
      - name: Prepare enviroment
        run: |
          python -m pip install pip==23.0.1
          python -m pip install wheel pytest pytest-cov nvidia-pyindex
          pip install -e "client/[test]"
          pip install -e "server/[onnx]"
          pip install -e "server/[transformers]"
          {
            pip install -e "server/[flash-attn]"
          } || {
            echo "flash attention was not installed."
          }
          pip install --no-cache-dir "server/[cn_clip]"
      - name: Test
        id: test
        run: |
          pytest --suppress-no-test-exit-code --cov=clip_client --cov=clip_server --cov-report=xml \
            -v -s -m "gpu" ./tests/test_model.py
          echo "::set-output name=codecov_flag::cas"
        timeout-minutes: 30
        env:
          # fix re-initialized torch runtime error on cuda device
          JINA_MP_START_METHOD: spawn
      - name: Check codecov file
        id: check_files
        uses: andstor/file-existence-action@v1
        with:
          files: "coverage.xml"
      - name: Upload coverage from test to Codecov
        uses: codecov/codecov-action@v3
        if: steps.check_files.outputs.files_exists == 'true' && ${{ matrix.python-version }} == '3.7'
        with:
          file: coverage.xml
          name: gpu-related-codecov
          flags: ${{ steps.test.outputs.codecov_flag }}
          fail_ci_if_error: false
          token: ${{ secrets.CODECOV_TOKEN }} # not required for public repos

  # just for blocking the merge until all parallel core-test are successful
  success-all-test:
    needs: [commit-lint, core-test, trt-gpu-test, gpu-model-test]
    if: always()
    runs-on: ubuntu-latest
    steps:
      - uses: technote-space/workflow-conclusion-action@v2
      - name: Check Failure
        if: env.WORKFLOW_CONCLUSION == 'failure'
        run: exit 1
      - name: Success
        if: ${{ success() }}
        run: echo "All Done"