[CHORE] Adding more test fixtures for different I/O sources #3497
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # This workflow will install Python dependencies, run tests and lint with a variety of Python versions | |
| # For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions | |
| name: daft | |
| on: | |
| push: | |
| branches: [main] | |
| pull_request: | |
| branches: [main] | |
| env: | |
| DAFT_ANALYTICS_ENABLED: '0' | |
| jobs: | |
| unit-tests-with-coverage: | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 15 | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| python-version: ['3.7', '3.10'] | |
| daft-runner: [py, ray] | |
| pyarrow-version: [6.0.1, 12.0] | |
| exclude: | |
| - daft-runner: ray | |
| pyarrow-version: 6.0.1 | |
| - daft-runner: py | |
| python-version: '3.10' | |
| pyarrow-version: 6.0.1 | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - uses: moonrepo/setup-rust@v0 | |
| - name: Install cargo-llvm-cov | |
| uses: taiki-e/install-action@cargo-llvm-cov | |
| - uses: actions/cache@v3 | |
| env: | |
| cache-name: cache-cargo | |
| with: | |
| path: | | |
| ~/.cargo/registry | |
| ~/.cargo/git | |
| key: ${{ runner.os }}-build-${{ env.cache-name }}-${{ hashFiles('**/Cargo.lock') }} | |
| restore-keys: | | |
| ${{ runner.os }}-build-${{ env.cache-name }}- | |
| ${{ runner.os }}-build- | |
| ${{ runner.os }}- | |
| - name: Set up Python ${{ matrix.python-version }} | |
| uses: actions/setup-python@v4 | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| - name: Setup Virtual Env | |
| run: | | |
| python -m venv venv | |
| echo "$GITHUB_WORKSPACE/venv/bin" >> $GITHUB_PATH | |
| - name: Install dependencies | |
| run: | | |
| pip install --upgrade pip | |
| pip install -r requirements-dev.txt | |
| - name: Override pyarrow | |
| if: ${{ matrix.pyarrow-version }} | |
| run: pip install pyarrow==${{ matrix.pyarrow-version }} | |
| - name: Build library and Test with pytest | |
| run: | | |
| source activate | |
| # source <(cargo llvm-cov show-env --export-prefix) | |
| # export CARGO_TARGET_DIR=$CARGO_LLVM_COV_TARGET_DIR | |
| # export CARGO_INCREMENTAL=1 | |
| # cargo llvm-cov clean --workspace | |
| maturin develop | |
| mkdir -p report-output && pytest --cov=daft --ignore tests/integration --durations=50 | |
| coverage combine -a --data-file='.coverage' || true | |
| coverage xml -o ./report-output/coverage-${{ runner.os }}-${{ matrix.python-version }}-${{ matrix.daft-runner }}.xml | |
| # cargo llvm-cov --no-run --lcov --output-path report-output/rust-coverage-${{ runner.os }}-${{ matrix.python-version }}-${{ matrix.daft-runner }}.lcov | |
| env: | |
| DAFT_RUNNER: ${{ matrix.daft-runner }} | |
| - name: Upload coverage report | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: coverage-reports | |
| path: ./report-output | |
| - name: Send Slack notification on failure | |
| uses: slackapi/[email protected] | |
| if: ${{ failure() && (github.ref == 'refs/heads/main') }} | |
| with: | |
| payload: | | |
| { | |
| "blocks": [ | |
| { | |
| "type": "section", | |
| "text": { | |
| "type": "mrkdwn", | |
| "text": ":rotating_light: [CI] Pytest Unit Tests <${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}|workflow> *FAILED on main* :rotating_light:" | |
| } | |
| } | |
| ] | |
| } | |
| env: | |
| SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} | |
| SLACK_WEBHOOK_TYPE: INCOMING_WEBHOOK | |
| integration-tests: | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 30 | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| python-version: ['3.7'] | |
| daft-runner: [py, ray] | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - uses: moonrepo/setup-rust@v0 | |
| - name: Install cargo-llvm-cov | |
| uses: taiki-e/install-action@cargo-llvm-cov | |
| - uses: actions/cache@v3 | |
| env: | |
| cache-name: cache-cargo | |
| with: | |
| path: | | |
| ~/.cargo/registry | |
| ~/.cargo/git | |
| key: ${{ runner.os }}-build-${{ env.cache-name }}-${{ hashFiles('**/Cargo.lock') }} | |
| restore-keys: | | |
| ${{ runner.os }}-build-${{ env.cache-name }}- | |
| ${{ runner.os }}-build- | |
| ${{ runner.os }}- | |
| - name: Set up Python ${{ matrix.python-version }} | |
| uses: actions/setup-python@v4 | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| - name: Setup Virtual Env | |
| run: | | |
| python -m venv venv | |
| echo "$GITHUB_WORKSPACE/venv/bin" >> $GITHUB_PATH | |
| - name: Install dependencies | |
| run: | | |
| pip install --upgrade pip | |
| pip install -r requirements-dev.txt | |
| - uses: actions/cache@v3 | |
| env: | |
| cache-name: cache-tpch-data | |
| with: | |
| path: data/tpch-dbgen | |
| key: ${{ runner.os }}-build-${{ env.cache-name }}-${{ hashFiles('tests/integration/test_tpch.py', 'benchmarking/tpch/**') }} | |
| - name: Build library in release mode and run TPCH integration tests | |
| run: | | |
| source activate | |
| maturin develop --release | |
| pytest tests/integration/ --durations=50 | |
| env: | |
| DAFT_RUNNER: ${{ matrix.daft-runner }} | |
| - name: Send Slack notification on failure | |
| uses: slackapi/[email protected] | |
| if: ${{ failure() && (github.ref == 'refs/heads/main') }} | |
| with: | |
| payload: | | |
| { | |
| "blocks": [ | |
| { | |
| "type": "section", | |
| "text": { | |
| "type": "mrkdwn", | |
| "text": ":rotating_light: [CI] TPCH Integration Tests <${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}|workflow> *FAILED on main* :rotating_light:" | |
| } | |
| } | |
| ] | |
| } | |
| env: | |
| SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} | |
| SLACK_WEBHOOK_TYPE: INCOMING_WEBHOOK | |
| rust-tests: | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 15 | |
| strategy: | |
| fail-fast: false | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - uses: moonrepo/setup-rust@v0 | |
| - uses: actions/cache@v3 | |
| env: | |
| cache-name: cache-cargo | |
| with: | |
| path: | | |
| ~/.cargo/registry | |
| ~/.cargo/git | |
| key: ${{ runner.os }}-rust-package-${{ env.cache-name }}-${{ hashFiles('**/Cargo.lock') }} | |
| restore-keys: | | |
| ${{ runner.os }}-build-${{ env.cache-name }}- | |
| ${{ runner.os }}-build- | |
| ${{ runner.os }}- | |
| - name: Install cargo-llvm-cov | |
| uses: taiki-e/install-action@cargo-llvm-cov | |
| - name: Generate code coverage | |
| run: mkdir -p report-output && cargo llvm-cov --no-default-features --workspace --lcov --output-path ./report-output/lcov.info | |
| - name: Upload coverage report | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: coverage-reports | |
| path: ./report-output | |
| - name: Send Slack notification on failure | |
| uses: slackapi/[email protected] | |
| if: ${{ failure() && (github.ref == 'refs/heads/main') }} | |
| with: | |
| payload: | | |
| { | |
| "blocks": [ | |
| { | |
| "type": "section", | |
| "text": { | |
| "type": "mrkdwn", | |
| "text": ":rotating_light: [CI] Rust Unit Tests <${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}|workflow> *FAILED on main* :rotating_light:" | |
| } | |
| } | |
| ] | |
| } | |
| env: | |
| SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} | |
| SLACK_WEBHOOK_TYPE: INCOMING_WEBHOOK | |
| publish-coverage-reports: | |
| name: Publish coverage reports to CodeCov | |
| runs-on: ubuntu-latest | |
| needs: | |
| - unit-tests-with-coverage | |
| - rust-tests | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - uses: actions/download-artifact@v3 | |
| with: | |
| name: coverage-reports | |
| path: ./report-output | |
| - name: Upload coverage reports to Codecov with GitHub Action | |
| uses: codecov/codecov-action@v3 | |
| env: | |
| CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} | |
| files: ./report-output/* | |
| - name: Send Slack notification on failure | |
| uses: slackapi/[email protected] | |
| if: ${{ failure() && (github.ref == 'refs/heads/main') }} | |
| with: | |
| payload: | | |
| { | |
| "blocks": [ | |
| { | |
| "type": "section", | |
| "text": { | |
| "type": "mrkdwn", | |
| "text": ":rotating_light: [CI] Codecov Uploads <${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}|workflow> *FAILED on main* :rotating_light:" | |
| } | |
| } | |
| ] | |
| } | |
| env: | |
| SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} | |
| SLACK_WEBHOOK_TYPE: INCOMING_WEBHOOK | |
| test-imports: | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 15 | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| python-version: ['3.7'] | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - uses: moonrepo/setup-rust@v0 | |
| - uses: actions/cache@v3 | |
| env: | |
| cache-name: cache-cargo | |
| with: | |
| path: | | |
| ~/.cargo/registry | |
| ~/.cargo/git | |
| key: ${{ runner.os }}-build-${{ env.cache-name }}-${{ hashFiles('**/Cargo.lock') }} | |
| restore-keys: | | |
| ${{ runner.os }}-build-${{ env.cache-name }}- | |
| ${{ runner.os }}-build- | |
| ${{ runner.os }}- | |
| - name: Set up Python ${{ matrix.python-version }} | |
| uses: actions/setup-python@v4 | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| - name: Setup Virtual Env | |
| run: | | |
| python -m venv venv | |
| source venv/bin/activate | |
| pip install maturin | |
| - name: Build Rust Library | |
| run: | | |
| venv/bin/maturin build --out dist | |
| - name: Test Imports in Clean Env | |
| run: | | |
| rm -rf daft | |
| rm -rf venv | |
| python -m venv venv | |
| source venv/bin/activate | |
| ls -R ./dist | |
| venv/bin/pip install dist/*.whl | |
| venv/bin/python -c 'import daft; from daft import *' | |
| - name: Send Slack notification on failure | |
| uses: slackapi/[email protected] | |
| if: ${{ failure() && (github.ref == 'refs/heads/main') }} | |
| with: | |
| payload: | | |
| { | |
| "blocks": [ | |
| { | |
| "type": "section", | |
| "text": { | |
| "type": "mrkdwn", | |
| "text": ":rotating_light: [CI] Python Import Checks <${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}|workflow> *FAILED on main* :rotating_light:" | |
| } | |
| } | |
| ] | |
| } | |
| env: | |
| SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} | |
| SLACK_WEBHOOK_TYPE: INCOMING_WEBHOOK | |
| style: | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 15 | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| python-version: ['3.8'] | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - name: Set up Python ${{ matrix.python-version }} | |
| uses: actions/setup-python@v4 | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| - name: Install pre-commit | |
| run: | | |
| pip install --upgrade pip | |
| pip install pre-commit | |
| - uses: moonrepo/setup-rust@v0 | |
| - uses: actions/cache@v3 | |
| env: | |
| cache-name: cache-cargo | |
| with: | |
| path: | | |
| ~/.cargo/registry | |
| ~/.cargo/git | |
| key: ${{ runner.os }}-build-${{ env.cache-name }}-${{ hashFiles('**/Cargo.lock') }} | |
| restore-keys: | | |
| ${{ runner.os }}-build-${{ env.cache-name }}- | |
| ${{ runner.os }}-build- | |
| ${{ runner.os }}- | |
| - uses: actions/cache@v3 | |
| id: pre-commit-cache | |
| with: | |
| path: ~/.cache/pre-commit/ | |
| key: ${{ runner.os }}-python-${{ steps.setup-python.outputs.python-version }}-pre-commit-${{ hashFiles('.pre-commit-config.yaml') }} | |
| - name: Python And Rust Style Check | |
| run: | | |
| pre-commit run --all-files | |
| - name: Send Slack notification on failure | |
| uses: slackapi/[email protected] | |
| if: ${{ failure() && (github.ref == 'refs/heads/main') }} | |
| with: | |
| payload: | | |
| { | |
| "blocks": [ | |
| { | |
| "type": "section", | |
| "text": { | |
| "type": "mrkdwn", | |
| "text": ":rotating_light: [CI] Style Checks <${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}|workflow> *FAILED on main* :rotating_light:" | |
| } | |
| } | |
| ] | |
| } | |
| env: | |
| SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} | |
| SLACK_WEBHOOK_TYPE: INCOMING_WEBHOOK |