ci.yml

name: CI

on: [push, pull_request]

env:
  CI: true
  PIPENV_VENV_IN_PROJECT: true
  AUTOTHROTTLE_MAX_DELAY: 30.0
  AUTOTHROTTLE_START_DELAY: 1.5
  AUTOTHROTTLE_TARGET_CONCURRENCY: 3.0

jobs:
  check:
    runs-on: ubuntu-latest
    strategy:
      max-parallel: 4
      matrix:
        python-version: [3.11]

    steps:
      - uses: actions/checkout@v4
        with:
          fetch-depth: 0

      - name: Set up Python ${{ matrix.python-version }}
        uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python-version }}

      - name: Install Pipenv
        run: pip install --user pipenv

      - name: Cache Python dependencies
        uses: actions/cache@v4
        with:
          path: .venv
          key: pip-${{ matrix.python-version }}-${{ hashFiles('**/Pipfile.lock') }}
          restore-keys: |
            pip-${{ matrix.python-version }}-
            pip-

      - name: Install dependencies
        run: pipenv sync --dev
        env:
          PIPENV_DEFAULT_PYTHON_VERSION: ${{ matrix.python-version }}

      - name: Check imports with isort
        run: |
          pipenv run isort . --check-only

      - name: Check style with black
        run: |
          pipenv run black . --check

      - name: Lint with flake8
        run: |
          pipenv run flake8 .

      - name: Test with pytest
        # Ignores exit code 5 (no tests collected)
        run: |
          pipenv run pytest || [ $? -eq 5 ]

      - name: Validate output with scrapy
        if: github.event_name == 'pull_request'
        run: |
          git checkout ${{ github.base_ref }}
          git checkout $(git show-ref | grep pull | awk '{ print $2 }')
          git diff-index --name-only --diff-filter=d $(git merge-base HEAD ${{ github.base_ref }}) | \
          grep -Pio '(?<=/spiders/).*(?=\.py)' | \
          xargs pipenv run scrapy validate