Adapt s3 connector for kafka fix (#499) #315
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: CI | |
| # ensures that main has our basic test and code quality | |
| on: | |
| push: | |
| branches: [main] | |
| jobs: | |
| build-pex: | |
| runs-on: ubuntu-22.04 | |
| strategy: | |
| matrix: | |
| python-version: ["3.10"] | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - name: Set up Python | |
| uses: actions/setup-python@v3 | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| cache: "pip" | |
| - name: Install dependencies | |
| run: | | |
| python -m pip install --upgrade pip | |
| pip install ansible | |
| pip install virtualenv | |
| pip install wheel | |
| pip install pex | |
| - name: Repack confluent-kafka wheel | |
| run: | | |
| rm -rf tmp_pip_cache && | |
| mkdir tmp_pip_cache && | |
| cd tmp_pip_cache && | |
| python -m pip download $(cat ../requirements.txt | grep confluent-kafka) && | |
| unzip * && | |
| rm *.whl && | |
| python -m wheel pack . | |
| - name: Build Pex File | |
| run: | | |
| pex . -r requirements.txt -o ./logprep.pex -c logprep --pex-root=tmp_pip_cache | |
| - name: Upload PEX | |
| uses: actions/upload-artifact@v2 | |
| with: | |
| name: Logprep | |
| path: logprep.pex | |
| test: | |
| runs-on: ubuntu-22.04 | |
| strategy: | |
| matrix: | |
| python-version: ["3.10", "3.11", "3.12"] | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - name: Set up Python | |
| uses: actions/setup-python@v3 | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| cache: "pip" | |
| - name: Install dependencies | |
| run: | | |
| pip install --upgrade pip wheel | |
| pip install -r requirements_dev.txt | |
| - name: Perform unit tests | |
| env: | |
| PYTEST_ADDOPTS: "--color=yes" | |
| run: | | |
| pytest -vv tests/unit | |
| - name: Perform acceptance tests | |
| env: | |
| PYTEST_ADDOPTS: "--color=yes" | |
| run: | | |
| pytest -vv tests/acceptance | |
| code-quality: | |
| runs-on: ubuntu-22.04 | |
| strategy: | |
| matrix: | |
| python-version: ["3.10"] | |
| steps: | |
| - uses: actions/checkout@v3 | |
| with: | |
| fetch-depth: 0 | |
| - uses: actions/checkout@v3 | |
| - name: Set up Python | |
| uses: actions/setup-python@v3 | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| cache: "pip" | |
| - name: Get changed python files | |
| id: changed-files | |
| uses: tj-actions/[email protected] | |
| with: | |
| files: | | |
| **/*.py | |
| - name: Install dependencies | |
| run: | | |
| sudo apt-get update && sudo apt-get -y install libhyperscan-dev librdkafka-dev | |
| pip install --upgrade pip wheel | |
| pip install -r requirements_dev.txt | |
| - name: check black formating | |
| run: | | |
| black --check --diff --config ./pyproject.toml . | |
| - name: lint changed and added files | |
| if: steps.changed-files.outputs.all_changed_files | |
| run: | | |
| pylint --rcfile=.pylintrc --fail-under 9.5 ${{ steps.changed-files.outputs.all_changed_files }} | |
| - name: Run tests and collect coverage | |
| run: pytest tests/unit --cov=logprep --cov-report=xml | |
| - name: Upload coverage reports to Codecov with GitHub Action | |
| uses: codecov/codecov-action@v2 |