Tests #68
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # Licensed to the Apache Software Foundation (ASF) under one | |
| # or more contributor license agreements. See the NOTICE file | |
| # distributed with this work for additional information | |
| # regarding copyright ownership. The ASF licenses this file | |
| # to you under the Apache License, Version 2.0 (the | |
| # "License"); you may not use this file except in compliance | |
| # with the License. You may obtain a copy of the License at | |
| # | |
| # http://www.apache.org/licenses/LICENSE-2.0 | |
| # | |
| # Unless required by applicable law or agreed to in writing, | |
| # software distributed under the License is distributed on an | |
| # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY | |
| # KIND, either express or implied. See the License for the | |
| # specific language governing permissions and limitations | |
| # under the License. | |
| # | |
| --- | |
| name: Tests | |
| on: # yamllint disable-line rule:truthy | |
| schedule: | |
| - cron: '28 0 * * *' | |
| push: | |
| branches: ['main', 'v[0-9]+-[0-9]+-test'] | |
| pull_request: | |
| branches: ['main'] | |
| permissions: | |
| # All other permissions are set to none | |
| contents: read | |
| # Technically read access while waiting for images should be more than enough. However, | |
| # there is a bug in GitHub Actions/Packages and in case private repositories are used, you get a permission | |
| # denied error when attempting to just pull private image, changing the token permission to write solves the | |
| # issue. This is not dangerous, because if it is for "apache/airflow", only maintainers can push ci.yml | |
| # changes. If it is for a fork, then the token is read-only anyway. | |
| packages: write | |
| env: | |
| ANSWER: "yes" | |
| DB_RESET: "true" | |
| VERBOSE: "true" | |
| GITHUB_REPOSITORY: ${{ github.repository }} | |
| GITHUB_USERNAME: ${{ github.actor }} | |
| # You can override CONSTRAINTS_GITHUB_REPOSITORY by setting secret in your repo but by default the | |
| # Airflow one is going to be used | |
| CONSTRAINTS_GITHUB_REPOSITORY: >- | |
| ${{ secrets.CONSTRAINTS_GITHUB_REPOSITORY != '' && | |
| secrets.CONSTRAINTS_GITHUB_REPOSITORY || 'apache/airflow' }} | |
| # In builds from forks, this token is read-only. For scheduler/direct push it is WRITE one | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| ENABLE_TEST_COVERAGE: "${{ github.event_name == 'push' }}" | |
| IMAGE_TAG: "${{ github.event.pull_request.head.sha || github.sha }}" | |
| USE_SUDO: "true" | |
| INCLUDE_SUCCESS_OUTPUTS: "true" | |
| AIRFLOW_ENABLE_AIP_44: "true" | |
| concurrency: | |
| group: ci-${{ github.event.pull_request.number || github.ref }} | |
| cancel-in-progress: true | |
| jobs: | |
| build-info: | |
| name: "Build info" | |
| # The runs-on cannot refer to env. or secrets. context, so we have no | |
| # option but to specify a hard-coded list here. This is "safe", as the list | |
| # is checked again by the runner using it's own list, so a PR author cannot | |
| # change this and get access to our self-hosted runners | |
| # | |
| # This list is kept up-to-date from the list of authors found in the | |
| # 'airflow-ci-infra' by the 'sync_authors' Github workflow. It uses a regexp | |
| # to find the list of authors and replace them, so any changes to the | |
| # formatting of the contains(fromJSON()) structure below will need to be | |
| # reflected in that workflow too. | |
| runs-on: "ubuntu-22.04" | |
| env: | |
| GITHUB_CONTEXT: ${{ toJson(github) }} | |
| outputs: | |
| cache-directive: ${{ steps.selective-checks.outputs.cache-directive }} | |
| affected-providers-list-as-string: >- | |
| ${{ steps.selective-checks.outputs.affected-providers-list-as-string }} | |
| upgrade-to-newer-dependencies: ${{ steps.selective-checks.outputs.upgrade-to-newer-dependencies }} | |
| python-versions: ${{ steps.selective-checks.outputs.python-versions }} | |
| python-versions-list-as-string: ${{ steps.selective-checks.outputs.python-versions-list-as-string }} | |
| all-python-versions-list-as-string: >- | |
| ${{ steps.selective-checks.outputs.all-python-versions-list-as-string }} | |
| default-python-version: ${{ steps.selective-checks.outputs.default-python-version }} | |
| kubernetes-versions-list-as-string: >- | |
| ${{ steps.selective-checks.outputs.kubernetes-versions-list-as-string }} | |
| kubernetes-combos-list-as-string: >- | |
| ${{ steps.selective-checks.outputs.kubernetes-combos-list-as-string }} | |
| default-kubernetes-version: ${{ steps.selective-checks.outputs.default-kubernetes-version }} | |
| postgres-versions: ${{ steps.selective-checks.outputs.postgres-versions }} | |
| default-postgres-version: ${{ steps.selective-checks.outputs.default-postgres-version }} | |
| mysql-versions: ${{ steps.selective-checks.outputs.mysql-versions }} | |
| mssql-versions: ${{ steps.selective-checks.outputs.mssql-versions }} | |
| default-mysql-version: ${{ steps.selective-checks.outputs.default-mysql-version }} | |
| default-helm-version: ${{ steps.selective-checks.outputs.default-helm-version }} | |
| default-kind-version: ${{ steps.selective-checks.outputs.default-kind-version }} | |
| full-tests-needed: ${{ steps.selective-checks.outputs.full-tests-needed }} | |
| parallel-test-types-list-as-string: >- | |
| ${{ steps.selective-checks.outputs.parallel-test-types-list-as-string }} | |
| mssql-parallelism: ${{ steps.selective-checks.outputs.mssql-parallelism }} | |
| postgres-exclude: ${{ steps.selective-checks.outputs.postgres-exclude }} | |
| mysql-exclude: ${{ steps.selective-checks.outputs.mysql-exclude }} | |
| mssql-exclude: ${{ steps.selective-checks.outputs.mssql-exclude }} | |
| sqlite-exclude: ${{ steps.selective-checks.outputs.sqlite-exclude }} | |
| skip-provider-tests: ${{ steps.selective-checks.outputs.skip-provider-tests }} | |
| run-tests: ${{ steps.selective-checks.outputs.run-tests }} | |
| run-amazon-tests: ${{ steps.selective-checks.outputs.run-amazon-tests }} | |
| run-www-tests: ${{ steps.selective-checks.outputs.run-www-tests }} | |
| run-kubernetes-tests: ${{ steps.selective-checks.outputs.run-kubernetes-tests }} | |
| basic-checks-only: ${{ steps.selective-checks.outputs.basic-checks-only }} | |
| image-build: ${{ steps.selective-checks.outputs.image-build }} | |
| docs-build: ${{ steps.selective-checks.outputs.docs-build }} | |
| needs-helm-tests: ${{ steps.selective-checks.outputs.needs-helm-tests }} | |
| needs-api-tests: ${{ steps.selective-checks.outputs.needs-api-tests }} | |
| needs-api-codegen: ${{ steps.selective-checks.outputs.needs-api-codegen }} | |
| default-branch: ${{ steps.selective-checks.outputs.default-branch }} | |
| default-constraints-branch: ${{ steps.selective-checks.outputs.default-constraints-branch }} | |
| docs-filter-list-as-string: ${{ steps.selective-checks.outputs.docs-filter-list-as-string }} | |
| skip-pre-commits: ${{ steps.selective-checks.outputs.skip-pre-commits }} | |
| helm-test-packages: ${{ steps.selective-checks.outputs.helm-test-packages }} | |
| debug-resources: ${{ steps.selective-checks.outputs.debug-resources }} | |
| runs-on: ${{ steps.selective-checks.outputs.runs-on }} | |
| source-head-repo: ${{ steps.source-run-info.outputs.source-head-repo }} | |
| pull-request-labels: ${{ steps.source-run-info.outputs.pr-labels }} | |
| in-workflow-build: ${{ steps.source-run-info.outputs.in-workflow-build }} | |
| build-job-description: ${{ steps.source-run-info.outputs.build-job-description }} | |
| canary-run: ${{ steps.source-run-info.outputs.canary-run }} | |
| run-coverage: ${{ steps.source-run-info.outputs.run-coverage }} | |
| steps: | |
| - name: Cleanup repo | |
| run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" | |
| - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" | |
| uses: actions/checkout@v3 | |
| with: | |
| persist-credentials: false | |
| submodules: recursive | |
| - name: Fetch incoming commit ${{ github.sha }} with its parent | |
| uses: actions/checkout@v3 | |
| with: | |
| ref: ${{ github.sha }} | |
| fetch-depth: 2 | |
| persist-credentials: false | |
| - name: "Install Breeze" | |
| uses: ./.github/actions/breeze | |
| - name: "Retrieve defaults from branch_defaults.py" | |
| id: defaults | |
| # We could retrieve it differently here - by just importing the variables and | |
| # printing them from python code, however we want to have the same code as used in | |
| # the build-images.yml (there we cannot import python code coming from the PR - we need to | |
| # treat the python code as text and extract the variables from there. | |
| run: | | |
| python - <<EOF >> ${GITHUB_ENV} | |
| from pathlib import Path | |
| import re | |
| import sys | |
| DEFAULTS_CONTENT = Path('dev/breeze/src/airflow_breeze/branch_defaults.py').read_text() | |
| BRANCH_PATTERN = r'^AIRFLOW_BRANCH = "(.*)"$' | |
| CONSTRAINTS_BRANCH_PATTERN = r'^DEFAULT_AIRFLOW_CONSTRAINTS_BRANCH = "(.*)"$' | |
| branch = re.search(BRANCH_PATTERN, DEFAULTS_CONTENT, re.MULTILINE).group(1) | |
| constraints_branch = re.search(CONSTRAINTS_BRANCH_PATTERN, DEFAULTS_CONTENT, re.MULTILINE).group(1) | |
| output = f""" | |
| DEFAULT_BRANCH={branch} | |
| DEFAULT_CONSTRAINTS_BRANCH={constraints_branch} | |
| """.strip() | |
| print(output) | |
| # Stdout is redirected to GITHUB_ENV but we also print it to stderr to see it in ci log | |
| print(output, file=sys.stderr) | |
| EOF | |
| - name: "Get information about the Workflow" | |
| id: source-run-info | |
| run: breeze ci get-workflow-info 2>> ${GITHUB_OUTPUT} | |
| - name: Selective checks | |
| id: selective-checks | |
| env: | |
| PR_LABELS: "${{ steps.source-run-info.outputs.pr-labels }}" | |
| COMMIT_REF: "${{ github.sha }}" | |
| VERBOSE: "false" | |
| run: breeze ci selective-check 2>> ${GITHUB_OUTPUT} | |
| - name: env | |
| run: printenv | |
| env: | |
| PR_LABELS: ${{ steps.source-run-info.outputs.pr-labels }} | |
| GITHUB_CONTEXT: ${{ toJson(github) }} | |
| # Push early BuildX cache to GitHub Registry in Apache repository, This cache does not wait for all the | |
| # tests to complete - it is run very early in the build process for "main" merges in order to refresh | |
| # cache using the current constraints. This will speed up cache refresh in cases when setup.py | |
| # changes or in case of Dockerfile changes. Failure in this step is not a problem (at most it will | |
| # delay cache refresh. It does not attempt to upgrade to newer dependencies. | |
| # We only push CI cache as PROD cache usually does not gain as much from fresh cache because | |
| # it uses prepared airflow and provider packages that invalidate the cache anyway most of the time | |
| push-early-buildx-cache-to-github-registry: | |
| timeout-minutes: 50 | |
| name: "Push Early Image Cache" | |
| runs-on: "${{needs.build-info.outputs.runs-on}}" | |
| needs: | |
| - build-info | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| platform: ["linux/amd64", "linux/arm64"] | |
| env: | |
| RUNS_ON: "${{needs.build-info.outputs.runs-on}}" | |
| UPGRADE_TO_NEWER_DEPENDENCIES: false | |
| VERSION_SUFFIX_FOR_PYPI: "dev0" | |
| continue-on-error: true | |
| steps: | |
| - name: Cleanup repo | |
| run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" | |
| if: needs.build-info.outputs.canary-run == 'true' && needs.build-info.outputs.default-branch == 'main' | |
| - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" | |
| uses: actions/checkout@v3 | |
| with: | |
| persist-credentials: false | |
| if: needs.build-info.outputs.canary-run == 'true' && needs.build-info.outputs.default-branch == 'main' | |
| - name: "Install Breeze" | |
| uses: ./.github/actions/breeze | |
| if: needs.build-info.outputs.canary-run == 'true' && needs.build-info.outputs.default-branch == 'main' | |
| - name: "Start ARM instance" | |
| run: ./scripts/ci/images/ci_start_arm_instance_and_connect_to_docker.sh | |
| if: > | |
| matrix.platform == 'linux/arm64' && needs.build-info.outputs.canary-run == 'true' | |
| && needs.build-info.outputs.default-branch == 'main' | |
| - name: "Push CI cache ${{ matrix.platform }}" | |
| run: > | |
| breeze ci-image build | |
| --builder airflow_cache | |
| --prepare-buildx-cache | |
| --run-in-parallel | |
| --force-build | |
| --platform ${{ matrix.platform }} | |
| env: | |
| DEBUG_RESOURCES: ${{needs.build-info.outputs.debug-resources}} | |
| COMMIT_SHA: ${{ github.sha }} | |
| if: needs.build-info.outputs.canary-run == 'true' && needs.build-info.outputs.default-branch == 'main' | |
| - name: "Push CI latest image ${{ matrix.platform }}" | |
| run: > | |
| breeze ci-image build | |
| --tag-as-latest --push --run-in-parallel --platform ${{ matrix.platform }} | |
| env: | |
| DEBUG_RESOURCES: ${{needs.build-info.outputs.debug-resources}} | |
| COMMIT_SHA: ${{ github.sha }} | |
| # We only push "amd" image as it is really only needed for any kind of automated builds in CI | |
| # and currently there is not an easy way to make multi-platform image from two separate builds | |
| if: > | |
| matrix.platform == 'linux/amd64' && needs.build-info.outputs.canary-run == 'true' | |
| && needs.build-info.outputs.default-branch == 'main' | |
| - name: "Stop ARM instance" | |
| run: ./scripts/ci/images/ci_stop_arm_instance.sh | |
| if: > | |
| always() && matrix.platform == 'linux/arm64' && needs.build-info.outputs.canary-run == 'true' | |
| && needs.build-info.outputs.default-branch == 'main' | |
| - name: "Clean docker cache for ${{ matrix.platform }}" | |
| run: docker system prune --all --force | |
| if: > | |
| matrix.platform == 'linux/amd64' && needs.build-info.outputs.canary-run == 'true' | |
| && needs.build-info.outputs.default-branch == 'main' | |
| - name: "Fix ownership" | |
| run: breeze ci fix-ownership | |
| if: > | |
| always() && needs.build-info.outputs.canary-run == 'true' | |
| && needs.build-info.outputs.default-branch == 'main' | |
| # Check that after earlier cache push, breeze command will build quickly | |
| check-that-image-builds-quickly: | |
| timeout-minutes: 5 | |
| name: "Check that image builds quickly" | |
| runs-on: "${{needs.build-info.outputs.runs-on}}" | |
| needs: | |
| - build-info | |
| - push-early-buildx-cache-to-github-registry | |
| env: | |
| RUNS_ON: "${{needs.build-info.outputs.runs-on}}" | |
| UPGRADE_TO_NEWER_DEPENDENCIES: false | |
| PLATFORM: "linux/amd64" | |
| if: > | |
| needs.build-info.outputs.canary-run == 'true' | |
| && needs.build-info.outputs.default-branch == 'main' | |
| steps: | |
| - name: Cleanup repo | |
| run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" | |
| - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" | |
| uses: actions/checkout@v3 | |
| with: | |
| persist-credentials: false | |
| - name: "Install Breeze" | |
| uses: ./.github/actions/breeze | |
| - name: "Check that image builds quickly" | |
| run: breeze shell --max-time 120 | |
| - name: "Fix ownership" | |
| run: breeze ci fix-ownership | |
| if: always() | |
| build-ci-images: | |
| timeout-minutes: 80 | |
| name: >- | |
| ${{needs.build-info.outputs.build-job-description}} CI images | |
| ${{needs.build-info.outputs.all-python-versions-list-as-string}} | |
| runs-on: "${{needs.build-info.outputs.runs-on}}" | |
| needs: [build-info] | |
| env: | |
| DEFAULT_BRANCH: ${{ needs.build-info.outputs.default-branch }} | |
| DEFAULT_CONSTRAINTS_BRANCH: ${{ needs.build-info.outputs.default-constraints-branch }} | |
| RUNS_ON: "${{needs.build-info.outputs.runs-on}}" | |
| # Force more parallelism for build even on public images | |
| PARALLELISM: 6 | |
| VERSION_SUFFIX_FOR_PYPI: "dev0" | |
| steps: | |
| - name: Cleanup repo | |
| run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" | |
| if: needs.build-info.outputs.in-workflow-build == 'true' | |
| - uses: actions/checkout@v3 | |
| with: | |
| ref: ${{ needs.build-info.outputs.targetCommitSha }} | |
| persist-credentials: false | |
| submodules: recursive | |
| if: needs.build-info.outputs.in-workflow-build == 'true' | |
| - name: > | |
| Build CI Images | |
| ${{needs.build-info.outputs.all-python-versions-list-as-string}}:${{env.IMAGE_TAG}} | |
| uses: ./.github/actions/build-ci-images | |
| if: needs.build-info.outputs.in-workflow-build == 'true' | |
| env: | |
| UPGRADE_TO_NEWER_DEPENDENCIES: ${{ needs.build-info.outputs.upgrade-to-newer-dependencies }} | |
| DOCKER_CACHE: ${{ needs.build-info.outputs.cache-directive }} | |
| PYTHON_VERSIONS: ${{needs.build-info.outputs.all-python-versions-list-as-string}} | |
| DEBUG_RESOURCES: ${{needs.build-info.outputs.debug-resources}} | |
| build-prod-images: | |
| timeout-minutes: 80 | |
| name: > | |
| ${{needs.build-info.outputs.build-job-description}} PROD images | |
| ${{needs.build-info.outputs.all-python-versions-list-as-string}} | |
| runs-on: "${{needs.build-info.outputs.runs-on}}" | |
| needs: [build-info, build-ci-images] | |
| env: | |
| DEFAULT_BRANCH: ${{ needs.build-info.outputs.default-branch }} | |
| DEFAULT_CONSTRAINTS_BRANCH: ${{ needs.build-info.outputs.default-constraints-branch }} | |
| RUNS_ON: "${{needs.build-info.outputs.runs-on}}" | |
| BACKEND: sqlite | |
| DOCKER_CACHE: ${{ needs.build-info.outputs.cache-directive }} | |
| VERSION_SUFFIX_FOR_PYPI: "dev0" | |
| DEBUG_RESOURCES: ${{needs.build-info.outputs.debug-resources}} | |
| # Force more parallelism for build even on public images | |
| PARALLELISM: 6 | |
| steps: | |
| - name: Cleanup repo | |
| run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" | |
| if: needs.build-info.outputs.in-workflow-build == 'true' | |
| - uses: actions/checkout@v3 | |
| with: | |
| ref: ${{ needs.build-info.outputs.targetCommitSha }} | |
| persist-credentials: false | |
| submodules: recursive | |
| if: needs.build-info.outputs.in-workflow-build == 'true' | |
| - name: > | |
| Build PROD Images | |
| ${{needs.build-info.outputs.all-python-versions-list-as-string}}:${{env.IMAGE_TAG}} | |
| uses: ./.github/actions/build-prod-images | |
| if: needs.build-info.outputs.in-workflow-build == 'true' | |
| with: | |
| build-provider-packages: ${{ needs.build-info.outputs.default-branch == 'main' }} | |
| env: | |
| UPGRADE_TO_NEWER_DEPENDENCIES: ${{ needs.build-info.outputs.upgrade-to-newer-dependencies }} | |
| DOCKER_CACHE: ${{ needs.build-info.outputs.cache-directive }} | |
| PYTHON_VERSIONS: ${{needs.build-info.outputs.all-python-versions-list-as-string}} | |
| DEBUG_RESOURCES: ${{ needs.build-info.outputs.debug-resources }} | |
| run-breeze-tests: | |
| timeout-minutes: 10 | |
| name: Breeze unit tests | |
| runs-on: "${{needs.build-info.outputs.runs-on}}" | |
| needs: [build-info] | |
| steps: | |
| - name: Cleanup repo | |
| run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" | |
| - uses: actions/checkout@v3 | |
| with: | |
| persist-credentials: false | |
| - uses: actions/setup-python@v4 | |
| with: | |
| python-version: "${{needs.build-info.outputs.default-python-version}}" | |
| cache: 'pip' | |
| cache-dependency-path: ./dev/breeze/setup* | |
| - run: python -m pip install --editable ./dev/breeze/ | |
| - run: > | |
| breeze setup version || | |
| (echo "\033[0;31mERROR: Breeze version check failed\033[0m" && | |
| echo "Please run 'breeze setup version' and commit the changes." && false) | |
| - run: python -m pytest -n auto --color=yes | |
| working-directory: ./dev/breeze/ | |
| - run: breeze setup check-all-params-in-groups | |
| tests-www: | |
| timeout-minutes: 10 | |
| name: React WWW tests | |
| runs-on: "${{needs.build-info.outputs.runs-on}}" | |
| needs: [build-info] | |
| if: needs.build-info.outputs.run-www-tests == 'true' | |
| steps: | |
| - name: Cleanup repo | |
| run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" | |
| - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" | |
| uses: actions/checkout@v3 | |
| with: | |
| persist-credentials: false | |
| - name: "Setup node" | |
| uses: actions/setup-node@v3 | |
| with: | |
| node-version: 14 | |
| - name: "Cache eslint" | |
| uses: actions/cache@v3 | |
| with: | |
| path: 'airflow/www/node_modules' | |
| key: ${{ runner.os }}-www-node-modules-${{ hashFiles('airflow/www/**/yarn.lock') }} | |
| - run: yarn --cwd airflow/www/ install --frozen-lockfile --non-interactive | |
| - run: yarn --cwd airflow/www/ run test | |
| env: | |
| FORCE_COLOR: 2 | |
| test-openapi-client-generation: | |
| timeout-minutes: 10 | |
| name: "Test OpenAPI client generation" | |
| runs-on: "${{needs.build-info.outputs.runs-on}}" | |
| needs: [build-info] | |
| if: needs.build-info.outputs.needs-api-codegen == 'true' | |
| steps: | |
| - name: Cleanup repo | |
| run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" | |
| - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" | |
| uses: actions/checkout@v3 | |
| with: | |
| fetch-depth: 2 | |
| persist-credentials: false | |
| - name: "Generate client codegen diff" | |
| run: ./scripts/ci/openapi/client_codegen_diff.sh | |
| test-examples-of-prod-image-building: | |
| timeout-minutes: 60 | |
| name: "Test examples of production image building" | |
| runs-on: "${{needs.build-info.outputs.runs-on}}" | |
| needs: [build-info] | |
| if: needs.build-info.outputs.image-build == 'true' | |
| steps: | |
| - name: Cleanup repo | |
| run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" | |
| - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" | |
| uses: actions/checkout@v3 | |
| with: | |
| fetch-depth: 2 | |
| persist-credentials: false | |
| - name: "Setup python" | |
| uses: actions/setup-python@v4 | |
| with: | |
| python-version: "${{needs.build-info.outputs.default-python-version}}" | |
| cache: 'pip' | |
| cache-dependency-path: ./dev/requirements.txt | |
| - name: "Test examples of PROD image building" | |
| run: > | |
| cd ./docker_tests && | |
| python -m pip install -r requirements.txt && | |
| python -m pytest test_examples_of_prod_image_building.py -n auto --color=yes | |
| test-git-clone-on-windows: | |
| timeout-minutes: 5 | |
| name: "Test git clone on Windows" | |
| runs-on: windows-latest | |
| needs: [build-info] | |
| steps: | |
| - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" | |
| uses: actions/checkout@v3 | |
| with: | |
| fetch-depth: 2 | |
| persist-credentials: false | |
| if: needs.build-info.outputs.runs-on != 'self-hosted' | |
| wait-for-ci-images: | |
| timeout-minutes: 120 | |
| name: "Wait for CI images" | |
| runs-on: "ubuntu-22.04" | |
| needs: [build-info, build-ci-images] | |
| if: needs.build-info.outputs.image-build == 'true' | |
| env: | |
| RUNS_ON: "${{needs.build-info.outputs.runs-on}}" | |
| BACKEND: sqlite | |
| # Force more parallelism for pull even on public images | |
| PARALLELISM: 6 | |
| steps: | |
| - name: Cleanup repo | |
| run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" | |
| - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" | |
| uses: actions/checkout@v3 | |
| with: | |
| persist-credentials: false | |
| - name: "Install Breeze" | |
| uses: ./.github/actions/breeze | |
| - name: Wait for CI images ${{ env.PYTHON_VERSIONS }}:${{ env.IMAGE_TAG }} | |
| id: wait-for-images | |
| run: breeze ci-image pull --run-in-parallel --verify --wait-for-image --tag-as-latest | |
| env: | |
| PYTHON_VERSIONS: ${{ needs.build-info.outputs.python-versions-list-as-string }} | |
| DEBUG_RESOURCES: ${{needs.build-info.outputs.debug-resources}} | |
| - name: "Fix ownership" | |
| run: breeze ci fix-ownership | |
| if: always() | |
| static-checks: | |
| timeout-minutes: 30 | |
| name: "Static checks" | |
| runs-on: "${{needs.build-info.outputs.runs-on}}" | |
| needs: [build-info, wait-for-ci-images] | |
| env: | |
| RUNS_ON: "${{needs.build-info.outputs.runs-on}}" | |
| PYTHON_MAJOR_MINOR_VERSION: "${{needs.build-info.outputs.default-python-version}}" | |
| UPGRADE_TO_NEWER_DEPENDENCIES: "${{ needs.build-info.outputs.upgrade-to-newer-dependencies }}" | |
| if: needs.build-info.outputs.basic-checks-only == 'false' | |
| steps: | |
| - name: Cleanup repo | |
| run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" | |
| - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" | |
| uses: actions/checkout@v3 | |
| with: | |
| persist-credentials: false | |
| - name: > | |
| Prepare breeze & CI image: ${{needs.build-info.outputs.default-python-version}}:${{env.IMAGE_TAG}} | |
| uses: ./.github/actions/prepare_breeze_and_image | |
| id: breeze | |
| - name: Cache pre-commit envs | |
| uses: actions/cache@v3 | |
| with: | |
| path: ~/.cache/pre-commit | |
| # yamllint disable-line rule:line-length | |
| key: "pre-commit-full-${{steps.breeze.outputs.host-python-version}}-${{ hashFiles('.pre-commit-config.yaml') }}" | |
| restore-keys: | | |
| pre-commit-full-${{steps.breeze.outputs.host-python-version}}- | |
| - name: "Static checks" | |
| run: breeze static-checks --all-files --show-diff-on-failure --color always --initialize-environment | |
| env: | |
| VERBOSE: "false" | |
| SKIP: ${{ needs.build-info.outputs.skip-pre-commits }} | |
| COLUMNS: "250" | |
| SKIP_GROUP_OUTPUT: "true" | |
| DEFAULT_BRANCH: ${{ needs.build-info.outputs.default-branch }} | |
| RUFF_FORMAT: "github" | |
| - name: "Fix ownership" | |
| run: breeze ci fix-ownership | |
| if: always() | |
| # Those checks are run if no image needs to be built for checks. This is for simple changes that | |
| # Do not touch any of the python code or any of the important files that might require building | |
| # The CI Docker image and they can be run entirely using the pre-commit virtual environments on host | |
| static-checks-basic-checks-only: | |
| timeout-minutes: 30 | |
| name: "Static checks: basic checks only" | |
| runs-on: "${{needs.build-info.outputs.runs-on}}" | |
| needs: [build-info] | |
| env: | |
| RUNS_ON: "${{needs.build-info.outputs.runs-on}}" | |
| if: needs.build-info.outputs.basic-checks-only == 'true' | |
| steps: | |
| - name: Cleanup repo | |
| run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" | |
| - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" | |
| uses: actions/checkout@v3 | |
| with: | |
| persist-credentials: false | |
| - name: "Setup python" | |
| uses: actions/setup-python@v4 | |
| with: | |
| python-version: "${{needs.build-info.outputs.default-python-version}}" | |
| cache: 'pip' | |
| cache-dependency-path: ./dev/breeze/setup* | |
| - name: "Install Breeze" | |
| uses: ./.github/actions/breeze | |
| id: breeze | |
| - name: Cache pre-commit envs | |
| uses: actions/cache@v3 | |
| with: | |
| path: ~/.cache/pre-commit | |
| # yamllint disable-line rule:line-length | |
| key: "pre-commit-basic-${{steps.breeze.outputs.host-python-version}}-${{ hashFiles('.pre-commit-config.yaml') }}" | |
| restore-keys: "\ | |
| pre-commit-full-${{steps.breeze.outputs.host-python-version}}-\ | |
| ${{ hashFiles('.pre-commit-config.yaml') }}\n | |
| pre-commit-basic-${{steps.breeze.outputs.host-python-version}}-\n | |
| pre-commit-full-${{steps.breeze.outputs.host-python-version}}-" | |
| - name: Fetch incoming commit ${{ github.sha }} with its parent | |
| uses: actions/checkout@v3 | |
| with: | |
| ref: ${{ github.sha }} | |
| fetch-depth: 2 | |
| persist-credentials: false | |
| - name: "Static checks: basic checks only" | |
| run: > | |
| breeze static-checks --show-diff-on-failure --color always --initialize-environment | |
| --skip-image-check --commit-ref "${{ github.sha }}" | |
| env: | |
| VERBOSE: "false" | |
| SKIP_IMAGE_PRE_COMMITS: "true" | |
| SKIP: ${{ needs.build-info.outputs.skip-pre-commits }} | |
| COLUMNS: "250" | |
| - name: "Fix ownership" | |
| run: breeze ci fix-ownership | |
| if: always() | |
| docs: | |
| timeout-minutes: 90 | |
| name: "Build docs" | |
| runs-on: "${{needs.build-info.outputs.runs-on}}" | |
| needs: [build-info, wait-for-ci-images] | |
| if: needs.build-info.outputs.docs-build == 'true' | |
| env: | |
| RUNS_ON: "${{needs.build-info.outputs.runs-on}}" | |
| PYTHON_MAJOR_MINOR_VERSION: "${{needs.build-info.outputs.default-python-version}}" | |
| steps: | |
| - name: Cleanup repo | |
| run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" | |
| - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" | |
| uses: actions/checkout@v3 | |
| with: | |
| persist-credentials: false | |
| submodules: recursive | |
| - name: > | |
| Prepare breeze & CI image: ${{needs.build-info.outputs.default-python-version}}:${{env.IMAGE_TAG}} | |
| uses: ./.github/actions/prepare_breeze_and_image | |
| - uses: actions/cache@v3 | |
| id: cache-doc-inventories | |
| with: | |
| path: ./docs/_inventory_cache/ | |
| key: docs-inventory-${{ hashFiles('setup.py','setup.cfg','pyproject.toml;') }} | |
| restore-keys: | | |
| docs-inventory-${{ hashFiles('setup.py','setup.cfg','pyproject.toml;') }} | |
| docs-inventory- | |
| - name: "Build docs" | |
| run: > | |
| breeze build-docs | |
| ${{ needs.build-info.outputs.docs-filter-list-as-string }} | |
| - name: "Clone airflow-site" | |
| run: > | |
| git clone https://github.com/apache/airflow-site.git ${GITHUB_WORKSPACE}/airflow-site && | |
| echo "AIRFLOW_SITE_DIRECTORY=${GITHUB_WORKSPACE}/airflow-site" >> "$GITHUB_ENV" | |
| - name: "Publish docs" | |
| run: > | |
| breeze release-management publish-docs | |
| --override-versioned | |
| ${{ needs.build-info.outputs.docs-filter-list-as-string }} | |
| - name: Configure AWS credentials | |
| uses: ./.github/actions/configure-aws-credentials | |
| if: > | |
| github.ref == 'refs/heads/main' && github.repository == 'apache/airflow' && | |
| github.event_name == 'push' | |
| with: | |
| aws-access-key-id: ${{ secrets.DOCS_AWS_ACCESS_KEY_ID }} | |
| aws-secret-access-key: ${{ secrets.DOCS_AWS_SECRET_ACCESS_KEY }} | |
| aws-region: eu-central-1 | |
| - name: "Upload documentation to AWS S3" | |
| if: > | |
| github.ref == 'refs/heads/main' && github.repository == 'apache/airflow' && | |
| github.event_name == 'push' | |
| run: aws s3 sync --delete ./files/documentation s3://apache-airflow-docs | |
| - name: "Fix ownership" | |
| run: breeze ci fix-ownership | |
| if: always() | |
| prepare-test-provider-packages-wheel: | |
| timeout-minutes: 80 | |
| name: "Provider packages wheel (verify)" | |
| runs-on: "${{needs.build-info.outputs.runs-on}}" | |
| needs: [build-info, wait-for-ci-images] | |
| env: | |
| RUNS_ON: "${{needs.build-info.outputs.runs-on}}" | |
| PYTHON_MAJOR_MINOR_VERSION: "${{needs.build-info.outputs.default-python-version}}" | |
| if: needs.build-info.outputs.skip-provider-tests != 'true' | |
| steps: | |
| - name: Cleanup repo | |
| run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" | |
| - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" | |
| uses: actions/checkout@v3 | |
| with: | |
| persist-credentials: false | |
| - name: > | |
| Prepare breeze & CI image: ${{needs.build-info.outputs.default-python-version}}:${{env.IMAGE_TAG}} | |
| uses: ./.github/actions/prepare_breeze_and_image | |
| - name: "Cleanup dist files" | |
| run: rm -fv ./dist/* | |
| - name: "Prepare provider documentation" | |
| run: > | |
| breeze release-management prepare-provider-documentation | |
| ${{ needs.build-info.outputs.affected-providers-list-as-string }} | |
| - name: "Prepare provider packages: wheel" | |
| run: > | |
| breeze release-management prepare-provider-packages --version-suffix-for-pypi dev0 | |
| --package-format wheel ${{ needs.build-info.outputs.affected-providers-list-as-string }} | |
| - name: "Prepare airflow package: wheel" | |
| run: breeze release-management prepare-airflow-package --version-suffix-for-pypi dev0 | |
| - name: "Verify wheel packages with twine" | |
| run: pipx install twine --force && twine check dist/*.whl | |
| - name: "Test providers issue generation automatically" | |
| run: > | |
| breeze release-management generate-issue-content-providers | |
| --only-available-in-dist --disable-progress | |
| - name: "Install and verify all provider packages and airflow via wheel files" | |
| run: > | |
| breeze release-management verify-provider-packages --use-packages-from-dist | |
| --package-format wheel --use-airflow-version wheel | |
| if: needs.build-info.outputs.affected-providers-list-as-string == '' | |
| env: | |
| SKIP_CONSTRAINTS: "${{ needs.build-info.outputs.upgrade-to-newer-dependencies }}" | |
| - name: "Install affected provider packages and airflow via wheel files" | |
| run: > | |
| breeze release-management install-provider-packages | |
| --package-format wheel --use-airflow-version wheel --run-in-parallel | |
| if: needs.build-info.outputs.affected-providers-list-as-string != '' | |
| env: | |
| SKIP_CONSTRAINTS: "${{ needs.build-info.outputs.upgrade-to-newer-dependencies }}" | |
| - name: "Fix ownership" | |
| run: breeze ci fix-ownership | |
| if: always() | |
| provider-airflow-compatibility-check: | |
| timeout-minutes: 80 | |
| name: "Providers Airflow 2.4 compatibility check" | |
| runs-on: "${{needs.build-info.outputs.runs-on}}" | |
| needs: [build-info, wait-for-ci-images] | |
| env: | |
| RUNS_ON: "${{needs.build-info.outputs.runs-on}}" | |
| PYTHON_MAJOR_MINOR_VERSION: "${{needs.build-info.outputs.default-python-version}}" | |
| if: needs.build-info.outputs.skip-provider-tests != 'true' | |
| steps: | |
| - name: Cleanup repo | |
| run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" | |
| - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" | |
| uses: actions/checkout@v3 | |
| with: | |
| persist-credentials: false | |
| - name: > | |
| Prepare breeze & CI image: ${{needs.build-info.outputs.default-python-version}}:${{env.IMAGE_TAG}} | |
| uses: ./.github/actions/prepare_breeze_and_image | |
| - name: "Cleanup dist files" | |
| run: rm -fv ./dist/* | |
| - name: "Prepare provider packages: wheel" | |
| run: > | |
| breeze release-management prepare-provider-packages --version-suffix-for-pypi dev0 | |
| --package-format wheel ${{ needs.build-info.outputs.affected-providers-list-as-string }} | |
| - name: "Fix incompatible 2.4 provider packages" | |
| run: | | |
| # This step should remove the provider packages that are not compatible with 2.4 | |
| # or replace them with 2.4 compatible versions. Sometimes we have good reasons to bump | |
| # the min airflow versions for some providers and then we need to add exclusions here. | |
| # | |
| # The Removal can be done with: | |
| # | |
| # rm -vf dist/apache_airflow_providers_<PROVIDER>*.whl | |
| # | |
| # Then it can be followed by downloading a compatible version from PyPI in case other | |
| # providers depend on it and fail with import errors (you need to download compatible version): | |
| # | |
| # pip download --no-deps --dest dist apache-airflow-providers-<PROVIDER>==3.1.0 | |
| # | |
| rm -vf dist/apache_airflow_providers_openlineage*.whl | |
| - name: "Get all provider extras as AIRFLOW_EXTRAS env variable" | |
| # Extras might be different on S3 so rather than relying on "all" we should get the list of | |
| # packages to be installed from the current provider_dependencies.json file | |
| run: | | |
| python -c 'from pathlib import Path; import json | |
| providers = json.loads(Path("generated/provider_dependencies.json").read_text()) | |
| provider_keys = ",".join(providers.keys()) | |
| print("AIRFLOW_EXTRAS={}".format(provider_keys))' >> $GITHUB_ENV | |
| - name: "Install and verify all provider packages and airflow on Airflow 2.4 files" | |
| run: > | |
| breeze release-management verify-provider-packages --use-airflow-version 2.4.0 | |
| --use-packages-from-dist --airflow-constraints-reference constraints-2.4.0 | |
| if: needs.build-info.outputs.affected-providers-list-as-string == '' | |
| - name: "Install affected provider packages and airflow on Airflow 2.4 files" | |
| run: > | |
| breeze release-management install-provider-packages --use-airflow-version 2.4.0 | |
| --airflow-constraints-reference constraints-2.4.0 --run-in-parallel | |
| # Make sure to skip the run if the only provider to be installed has been removed | |
| # in the previous step | |
| if: > | |
| needs.build-info.outputs.affected-providers-list-as-string != '' && | |
| needs.build-info.outputs.affected-providers-list-as-string != 'openlineage' | |
| - name: "Fix ownership" | |
| run: breeze ci fix-ownership | |
| if: always() | |
| prepare-install-provider-packages-sdist: | |
| timeout-minutes: 80 | |
| name: "Provider packages sdist (install)" | |
| runs-on: "${{needs.build-info.outputs.runs-on}}" | |
| needs: [build-info, wait-for-ci-images] | |
| env: | |
| RUNS_ON: "${{needs.build-info.outputs.runs-on}}" | |
| PYTHON_MAJOR_MINOR_VERSION: "${{needs.build-info.outputs.default-python-version}}" | |
| USE_AIRFLOW_VERSION: "sdist" | |
| if: > | |
| needs.build-info.outputs.canary-run == 'true' && | |
| needs.build-info.outputs.skip-provider-tests != 'true' | |
| steps: | |
| - name: Cleanup repo | |
| run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" | |
| - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" | |
| uses: actions/checkout@v3 | |
| with: | |
| persist-credentials: false | |
| - name: > | |
| Prepare breeze & CI image: ${{needs.build-info.outputs.default-python-version}}:${{env.IMAGE_TAG}} | |
| uses: ./.github/actions/prepare_breeze_and_image | |
| - name: "Cleanup dist files" | |
| run: rm -fv ./dist/* | |
| - name: "Prepare provider packages: sdist" | |
| run: > | |
| breeze release-management prepare-provider-packages | |
| --version-suffix-for-pypi dev0 --package-format sdist | |
| ${{ needs.build-info.outputs.affected-providers-list-as-string }} | |
| - name: "Prepare airflow package: sdist" | |
| run: > | |
| breeze release-management prepare-airflow-package | |
| --version-suffix-for-pypi dev0 --package-format sdist | |
| - name: "Verify sdist packages with twine" | |
| run: pipx install twine --force && twine check dist/*.tar.gz | |
| - name: "Install all provider packages and airflow via sdist files" | |
| run: > | |
| breeze release-management install-provider-packages | |
| --package-format sdist --use-airflow-version sdist --run-in-parallel | |
| if: needs.build-info.outputs.affected-providers-list-as-string == '' | |
| - name: "Install affected provider packages and airflow via sdist files" | |
| run: > | |
| breeze release-management install-provider-packages | |
| --package-format sdist --use-airflow-version sdist --run-in-parallel | |
| if: needs.build-info.outputs.affected-providers-list-as-string != '' | |
| - name: "Fix ownership" | |
| run: breeze ci fix-ownership | |
| if: always() | |
| test-airflow-release-commands: | |
| timeout-minutes: 80 | |
| name: "Test Airflow release commands" | |
| runs-on: "${{needs.build-info.outputs.runs-on}}" | |
| needs: [build-info, wait-for-ci-images] | |
| if: needs.build-info.outputs.runs-on == 'self-hosted' | |
| env: | |
| RUNS_ON: "${{needs.build-info.outputs.runs-on}}" | |
| PYTHON_MAJOR_MINOR_VERSION: "${{needs.build-info.outputs.default-python-version}}" | |
| steps: | |
| - name: Cleanup repo | |
| run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" | |
| - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" | |
| uses: actions/checkout@v3 | |
| with: | |
| persist-credentials: false | |
| - name: > | |
| Prepare breeze & CI image: ${{needs.build-info.outputs.default-python-version}}:${{env.IMAGE_TAG}} | |
| uses: ./.github/actions/prepare_breeze_and_image | |
| - name: "Cleanup dist files" | |
| run: rm -fv ./dist/* | |
| - name: "Install required packages" | |
| run: python -m pip install --editable ./dev/breeze/ | |
| - name: "Check Airflow create minor branch command" | |
| run: breeze release-management create-minor-branch --version-branch 2-4 -a y | |
| - name: "Check Airflow RC process command" | |
| run: | | |
| breeze release-management start-rc-process --version 2.4.3rc1 --previous-version 2.4.2 -a y | |
| - name: "Check Airflow release process command" | |
| run: | | |
| breeze release-management start-release --release-candidate 2.4.3rc1 --previous-release 2.4.2 -a y | |
| - name: "Fix ownership" | |
| run: breeze ci fix-ownership | |
| if: always() | |
| tests-helm: | |
| timeout-minutes: 80 | |
| name: "Unit tests Helm: ${{matrix.helm-test-package}}" | |
| runs-on: "${{needs.build-info.outputs.runs-on}}" | |
| needs: [build-info, wait-for-ci-images] | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| helm-test-package: ${{fromJson(needs.build-info.outputs.helm-test-packages)}} | |
| env: | |
| RUNS_ON: "${{needs.build-info.outputs.runs-on}}" | |
| PARALLEL_TEST_TYPES: "Helm" | |
| BACKEND: "" | |
| DB_RESET: "false" | |
| PYTHON_MAJOR_MINOR_VERSION: "${{needs.build-info.outputs.default-python-version}}" | |
| JOB_ID: "helm-tests" | |
| COVERAGE: "${{needs.build-info.outputs.run-coverage}}" | |
| if: > | |
| needs.build-info.outputs.needs-helm-tests == 'true' && | |
| (github.repository == 'apache/airflow' || github.event_name != 'schedule') && | |
| needs.build-info.outputs.default-branch == 'main' | |
| steps: | |
| - name: Cleanup repo | |
| run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" | |
| - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" | |
| uses: actions/checkout@v3 | |
| with: | |
| persist-credentials: false | |
| - name: > | |
| Prepare breeze & CI image: ${{needs.build-info.outputs.default-python-version}}:${{env.IMAGE_TAG}} | |
| uses: ./.github/actions/prepare_breeze_and_image | |
| - name: "Helm Unit Tests: ${{ matrix.helm-test-package }}" | |
| run: breeze testing helm-tests --helm-test-package "${{ matrix.helm-test-package }}" | |
| - name: > | |
| Post Tests success: Helm" | |
| uses: ./.github/actions/post_tests_success | |
| if: success() | |
| - name: > | |
| Post Tests failure: Helm" | |
| uses: ./.github/actions/post_tests_failure | |
| if: failure() | |
| tests-postgres: | |
| timeout-minutes: 130 | |
| name: > | |
| Postgres${{matrix.postgres-version}},Py${{matrix.python-version}}: | |
| ${{needs.build-info.outputs.parallel-test-types-list-as-string}} | |
| runs-on: "${{needs.build-info.outputs.runs-on}}" | |
| needs: [build-info, wait-for-ci-images] | |
| strategy: | |
| matrix: | |
| python-version: "${{fromJson(needs.build-info.outputs.python-versions)}}" | |
| postgres-version: "${{fromJson(needs.build-info.outputs.postgres-versions)}}" | |
| exclude: "${{fromJson(needs.build-info.outputs.postgres-exclude)}}" | |
| fail-fast: false | |
| env: | |
| RUNS_ON: "${{needs.build-info.outputs.runs-on}}" | |
| PARALLEL_TEST_TYPES: "${{needs.build-info.outputs.parallel-test-types-list-as-string}}" | |
| PR_LABELS: "${{needs.build-info.outputs.pull-request-labels}}" | |
| FULL_TESTS_NEEDED: "${{needs.build-info.outputs.full-tests-needed}}" | |
| DEBUG_RESOURCES: "${{needs.build-info.outputs.debug-resources}}" | |
| BACKEND: "postgres" | |
| PYTHON_MAJOR_MINOR_VERSION: "${{matrix.python-version}}" | |
| POSTGRES_VERSION: "${{matrix.postgres-version}}" | |
| BACKEND_VERSION: "${{matrix.postgres-version}}" | |
| JOB_ID: "postgres-${{matrix.postgres-version}}-${{matrix.python-version}}" | |
| COVERAGE: "${{needs.build-info.outputs.run-coverage}}" | |
| if: needs.build-info.outputs.run-tests == 'true' | |
| steps: | |
| - name: Cleanup repo | |
| shell: bash | |
| run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" | |
| - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" | |
| uses: actions/checkout@v3 | |
| with: | |
| persist-credentials: false | |
| - name: "Prepare breeze & CI image: ${{matrix.python-version}}:${{env.IMAGE_TAG}}" | |
| uses: ./.github/actions/prepare_breeze_and_image | |
| - name: > | |
| Migration Tests: | |
| ${{matrix.python-version}}:${{needs.build-info.outputs.parallel-test-types-list-as-string}} | |
| uses: ./.github/actions/migration_tests | |
| - name: > | |
| Tests: ${{matrix.python-version}}:${{needs.build-info.outputs.parallel-test-types-list-as-string}} | |
| run: breeze testing tests --run-in-parallel | |
| - name: "Tests ARM Pytest collection: ${{matrix.python-version}}" | |
| run: breeze testing tests --run-in-parallel --collect-only --remove-arm-packages | |
| if: matrix.postgres-version == needs.build-info.outputs.default-postgres-version | |
| - name: > | |
| Post Tests success: Postgres" | |
| uses: ./.github/actions/post_tests_success | |
| if: success() | |
| - name: > | |
| Post Tests failure: Postgres" | |
| uses: ./.github/actions/post_tests_failure | |
| if: failure() | |
| tests-postgres-boto: | |
| timeout-minutes: 130 | |
| name: > | |
| LatestBotoPostgres${{needs.build-info.outputs.default-postgres-version}}, | |
| Py${{needs.build-info.outputs.default-python-version}}: | |
| ${{needs.build-info.outputs.parallel-test-types-list-as-string}} | |
| runs-on: "${{needs.build-info.outputs.runs-on}}" | |
| needs: [build-info, wait-for-ci-images] | |
| env: | |
| RUNS_ON: "${{needs.build-info.outputs.runs-on}}" | |
| PARALLEL_TEST_TYPES: "${{needs.build-info.outputs.parallel-test-types-list-as-string}}" | |
| PR_LABELS: "${{needs.build-info.outputs.pull-request-labels}}" | |
| FULL_TESTS_NEEDED: "${{needs.build-info.outputs.full-tests-needed}}" | |
| DEBUG_RESOURCES: "${{needs.build-info.outputs.debug-resources}}" | |
| BACKEND: "postgres" | |
| PYTHON_MAJOR_MINOR_VERSION: "${{needs.build-info.outputs.default-python-version}}" | |
| PYTHON_VERSION: "${needs.build-info.outputs.default-python-version}}" | |
| POSTGRES_VERSION: "${{needs.build-info.outputs.default-postgres-version}}" | |
| BACKEND_VERSION: "${{needs.build-info.outputs.default-postgres-version}}" | |
| UPGRADE_BOTO: "true" | |
| JOB_ID: > | |
| postgres-boto-${{needs.build-info.outputs.default-python-version}}- | |
| ${{needs.build-info.outputs.default-postgres-version}} | |
| COVERAGE: "${{needs.build-info.outputs.run-coverage}}" | |
| if: needs.build-info.outputs.run-tests == 'true' && needs.build-info.outputs.run-amazon-tests == 'true' | |
| steps: | |
| - name: Cleanup repo | |
| shell: bash | |
| run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" | |
| - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" | |
| uses: actions/checkout@v3 | |
| with: | |
| persist-credentials: false | |
| - name: > | |
| Prepare breeze & CI image: ${{needs.build-info.outputs.default-python-version}}:${{env.IMAGE_TAG}} | |
| uses: ./.github/actions/prepare_breeze_and_image | |
| - name: > | |
| Tests: ${{needs.build-info.outputs.default-python-version}}: | |
| ${{needs.build-info.outputs.parallel-test-types-list-as-string}} | |
| run: breeze testing tests --run-in-parallel | |
| - name: > | |
| Post Tests success: ${{needs.build-info.outputs.default-python-version}}:Boto" | |
| uses: ./.github/actions/post_tests_success | |
| if: success() | |
| - name: > | |
| Post Tests failure: ${{needs.build-info.outputs.default-python-version}}:Boto" | |
| uses: ./.github/actions/post_tests_failure | |
| if: failure() | |
| tests-postgres-in-progress-features-disabled: | |
| timeout-minutes: 130 | |
| name: > | |
| InProgressDisabledPostgres${{needs.build-info.outputs.default-postgres-version}}, | |
| Py${{needs.build-info.outputs.default-python-version}}: | |
| ${{needs.build-info.outputs.parallel-test-types-list-as-string}} | |
| runs-on: "${{needs.build-info.outputs.runs-on}}" | |
| needs: [build-info, wait-for-ci-images] | |
| env: | |
| RUNS_ON: "${{needs.build-info.outputs.runs-on}}" | |
| PARALLEL_TEST_TYPES: "${{needs.build-info.outputs.parallel-test-types-list-as-string}}" | |
| PR_LABELS: "${{needs.build-info.outputs.pull-request-labels}}" | |
| FULL_TESTS_NEEDED: "${{needs.build-info.outputs.full-tests-needed}}" | |
| DEBUG_RESOURCES: "${{needs.build-info.outputs.debug-resources}}" | |
| BACKEND: "postgres" | |
| PYTHON_MAJOR_MINOR_VERSION: "${{needs.build-info.outputs.default-python-version}}" | |
| PYTHON_VERSION: "${needs.build-info.outputs.default-python-version}}" | |
| POSTGRES_VERSION: "${{needs.build-info.outputs.default-postgres-version}}" | |
| BACKEND_VERSION: "${{needs.build-info.outputs.default-postgres-version}}" | |
| AIRFLOW_ENABLE_AIP_44: "false" | |
| JOB_ID: > | |
| postgres-in-progress-disabled-${{needs.build-info.outputs.default-python-version}}- | |
| ${{needs.build-info.outputs.default-postgres-version}} | |
| COVERAGE: "${{needs.build-info.outputs.run-coverage}}" | |
| if: needs.build-info.outputs.run-tests == 'true' && needs.build-info.outputs.run-amazon-tests == 'true' | |
| steps: | |
| - name: Cleanup repo | |
| shell: bash | |
| run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" | |
| - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" | |
| uses: actions/checkout@v3 | |
| with: | |
| persist-credentials: false | |
| - name: > | |
| Prepare breeze & CI image: ${{needs.build-info.outputs.default-python-version}}:${{env.IMAGE_TAG}} | |
| uses: ./.github/actions/prepare_breeze_and_image | |
| - name: > | |
| Tests: ${{needs.build-info.outputs.default-python-version}}: | |
| ${{needs.build-info.outputs.parallel-test-types-list-as-string}} | |
| run: breeze testing tests --run-in-parallel | |
| - name: > | |
| Post Tests success: ${{needs.build-info.outputs.default-python-version}}:FeaturesDisabled" | |
| uses: ./.github/actions/post_tests_success | |
| if: success() | |
| - name: > | |
| Post Tests failure: ${{needs.build-info.outputs.default-python-version}}:FeaturesDisabled" | |
| uses: ./.github/actions/post_tests_failure | |
| if: failure() | |
| tests-mysql: | |
| timeout-minutes: 130 | |
| name: > | |
| MySQL${{matrix.mysql-version}}, Py${{matrix.python-version}}: | |
| ${{needs.build-info.outputs.parallel-test-types-list-as-string}} | |
| runs-on: "${{needs.build-info.outputs.runs-on}}" | |
| needs: [build-info, wait-for-ci-images] | |
| strategy: | |
| matrix: | |
| python-version: "${{fromJson(needs.build-info.outputs.python-versions)}}" | |
| mysql-version: "${{fromJson(needs.build-info.outputs.mysql-versions)}}" | |
| exclude: "${{fromJson(needs.build-info.outputs.mysql-exclude)}}" | |
| fail-fast: false | |
| env: | |
| RUNS_ON: "${{needs.build-info.outputs.runs-on}}" | |
| PR_LABELS: "${{needs.build-info.outputs.pull-request-labels}}" | |
| FULL_TESTS_NEEDED: "${{needs.build-info.outputs.full-tests-needed}}" | |
| PARALLEL_TEST_TYPES: "${{needs.build-info.outputs.parallel-test-types-list-as-string}}" | |
| DEBUG_RESOURCES: "${{needs.build-info.outputs.debug-resources}}" | |
| BACKEND: "mysql" | |
| PYTHON_MAJOR_MINOR_VERSION: "${{matrix.python-version}}" | |
| MYSQL_VERSION: "${{matrix.mysql-version}}" | |
| BACKEND_VERSION: "${{matrix.mysql-version}}" | |
| JOB_ID: "mysql-${{matrix.mysql-version}}-${{matrix.python-version}}" | |
| if: needs.build-info.outputs.run-tests == 'true' | |
| steps: | |
| - name: Cleanup repo | |
| shell: bash | |
| run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" | |
| - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" | |
| uses: actions/checkout@v3 | |
| with: | |
| persist-credentials: false | |
| - name: "Prepare breeze & CI image: ${{matrix.python-version}}:${{env.IMAGE_TAG}}" | |
| uses: ./.github/actions/prepare_breeze_and_image | |
| - name: > | |
| Migration Tests: | |
| ${{matrix.python-version}}:${{needs.build-info.outputs.parallel-test-types-list-as-string}} | |
| uses: ./.github/actions/migration_tests | |
| - name: > | |
| Tests: ${{matrix.python-version}}:${{needs.build-info.outputs.parallel-test-types-list-as-string}} | |
| run: breeze testing tests --run-in-parallel | |
| - name: > | |
| Post Tests success: MySQL" | |
| uses: ./.github/actions/post_tests_success | |
| if: success() | |
| - name: > | |
| Post Tests failure: MySQL" | |
| uses: ./.github/actions/post_tests_failure | |
| if: failure() | |
| tests-mssql: | |
| timeout-minutes: 130 | |
| name: > | |
| MSSQL${{matrix.mssql-version}}, Py${{matrix.python-version}}: | |
| ${{needs.build-info.outputs.parallel-test-types-list-as-string}} | |
| runs-on: "${{needs.build-info.outputs.runs-on}}" | |
| needs: [build-info, wait-for-ci-images] | |
| strategy: | |
| matrix: | |
| python-version: "${{fromJson(needs.build-info.outputs.python-versions)}}" | |
| mssql-version: "${{fromJson(needs.build-info.outputs.mssql-versions)}}" | |
| exclude: "${{fromJson(needs.build-info.outputs.mssql-exclude)}}" | |
| fail-fast: false | |
| env: | |
| RUNS_ON: "${{needs.build-info.outputs.runs-on}}" | |
| PARALLEL_TEST_TYPES: "${{needs.build-info.outputs.parallel-test-types-list-as-string}}" | |
| PR_LABELS: "${{needs.build-info.outputs.pull-request-labels}}" | |
| FULL_TESTS_NEEDED: "${{needs.build-info.outputs.full-tests-needed}}" | |
| DEBUG_RESOURCES: "${{needs.build-info.outputs.debug-resources}}" | |
| BACKEND: "mssql" | |
| PYTHON_MAJOR_MINOR_VERSION: "${{matrix.python-version}}" | |
| MSSQL_VERSION: "${{matrix.mssql-version}}" | |
| BACKEND_VERSION: "${{matrix.mssql-version}}" | |
| JOB_ID: "mssql-${{matrix.mssql-version}}-${{matrix.python-version}}" | |
| COVERAGE: "${{needs.build-info.outputs.run-coverage}}" | |
| # The below (and corresponding selective checks code) can be removed once | |
| # https://github.com/apache/airflow/issues/31575 is fixed. | |
| # This is a temporary workaround for flaky tests that occur in MSSQL tests for public runners | |
| PARALLELISM: "${{needs.build-info.outputs.mssql-parallelism}}" | |
| if: > | |
| needs.build-info.outputs.run-tests == 'true' && | |
| (needs.build-info.outputs.runs-on == 'self-hosted' || | |
| needs.build-info.outputs.full-tests-needed == 'true') | |
| steps: | |
| - name: Cleanup repo | |
| shell: bash | |
| run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" | |
| - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" | |
| uses: actions/checkout@v3 | |
| with: | |
| persist-credentials: false | |
| - name: "Prepare breeze & CI image: ${{matrix.python-version}}:${{env.IMAGE_TAG}}" | |
| uses: ./.github/actions/prepare_breeze_and_image | |
| - name: > | |
| Migration Tests: | |
| ${{matrix.python-version}}:${{needs.build-info.outputs.parallel-test-types-list-as-string}} | |
| uses: ./.github/actions/migration_tests | |
| - name: > | |
| Tests: ${{matrix.python-version}}:${{needs.build-info.outputs.parallel-test-types-list-as-string}} | |
| run: breeze testing tests --run-in-parallel | |
| - name: > | |
| Post Tests success: MsSQL" | |
| uses: ./.github/actions/post_tests_success | |
| if: success() | |
| - name: > | |
| Post Tests failure: MsSQL" | |
| uses: ./.github/actions/post_tests_failure | |
| if: failure() | |
| tests-sqlite: | |
| timeout-minutes: 130 | |
| name: > | |
| Sqlite Py${{matrix.python-version}}: ${{needs.build-info.outputs.parallel-test-types-list-as-string}} | |
| runs-on: "${{needs.build-info.outputs.runs-on}}" | |
| needs: [build-info, wait-for-ci-images] | |
| strategy: | |
| matrix: | |
| python-version: ${{ fromJson(needs.build-info.outputs.python-versions) }} | |
| exclude: ${{ fromJson(needs.build-info.outputs.sqlite-exclude) }} | |
| fail-fast: false | |
| if: needs.build-info.outputs.run-tests == 'true' | |
| env: | |
| RUNS_ON: "${{needs.build-info.outputs.runs-on}}" | |
| PARALLEL_TEST_TYPES: "${{needs.build-info.outputs.parallel-test-types-list-as-string}}" | |
| PR_LABELS: "${{needs.build-info.outputs.pull-request-labels}}" | |
| PYTHON_MAJOR_MINOR_VERSION: "${{matrix.python-version}}" | |
| FULL_TESTS_NEEDED: "${{needs.build-info.outputs.full-tests-needed}}" | |
| DEBUG_RESOURCES: "${{needs.build-info.outputs.debug-resources}}" | |
| BACKEND: "sqlite" | |
| BACKEND_VERSION: "" | |
| JOB_ID: "sqlite-${{matrix.python-version}}" | |
| COVERAGE: "${{needs.build-info.outputs.run-coverage}}" | |
| steps: | |
| - name: Cleanup repo | |
| shell: bash | |
| run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" | |
| - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" | |
| uses: actions/checkout@v3 | |
| with: | |
| persist-credentials: false | |
| - name: "Prepare breeze & CI image: ${{matrix.python-version}}:${{env.IMAGE_TAG}}" | |
| uses: ./.github/actions/prepare_breeze_and_image | |
| - name: > | |
| Migration Tests: | |
| ${{matrix.python-version}}:${{needs.build-info.outputs.parallel-test-types-list-as-string}} | |
| uses: ./.github/actions/migration_tests | |
| - name: > | |
| Tests: ${{matrix.python-version}}:${{needs.build-info.outputs.parallel-test-types-list-as-string}} | |
| run: breeze testing tests --run-in-parallel | |
| - name: > | |
| Post Tests success: Sqlite" | |
| uses: ./.github/actions/post_tests_success | |
| if: success() | |
| - name: > | |
| Post Tests failure: Sqlite" | |
| uses: ./.github/actions/post_tests_failure | |
| if: failure() | |
| tests-integration-postgres: | |
| timeout-minutes: 130 | |
| name: Integration Tests Postgres | |
| runs-on: "${{needs.build-info.outputs.runs-on}}" | |
| needs: [build-info, wait-for-ci-images] | |
| env: | |
| RUNS_ON: "${{needs.build-info.outputs.runs-on}}" | |
| PARALLEL_TEST_TYPES: "${{needs.build-info.outputs.parallel-test-types-list-as-string}}" | |
| PR_LABELS: "${{needs.build-info.outputs.pull-request-labels}}" | |
| FULL_TESTS_NEEDED: "${{needs.build-info.outputs.full-tests-needed}}" | |
| DEBUG_RESOURCES: "${{needs.build-info.outputs.debug-resources}}" | |
| BACKEND: "postgres" | |
| PYTHON_MAJOR_MINOR_VERSION: "${{needs.build-info.outputs.default-python-version}}" | |
| POSTGRES_VERSION: "${{needs.build-info.outputs.default-postgres-version}}" | |
| BACKEND_VERSION: "${{needs.build-info.outputs.default-postgres-version}}" | |
| JOB_ID: "integration" | |
| COVERAGE: "${{needs.build-info.outputs.run-coverage}}" | |
| SKIP_PROVIDER_TESTS: "${{needs.build-info.outputs.skip-provider-tests}}" | |
| if: needs.build-info.outputs.run-tests == 'true' | |
| steps: | |
| - name: Cleanup repo | |
| shell: bash | |
| run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" | |
| - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" | |
| uses: actions/checkout@v3 | |
| with: | |
| persist-credentials: false | |
| - name: "Prepare breeze & CI image: ${{env.PYTHON_MAJOR_MINOR_VERSION}}:${{env.IMAGE_TAG}}" | |
| uses: ./.github/actions/prepare_breeze_and_image | |
| - name: "Integration Tests Postgres: cassandra" | |
| run: | | |
| breeze testing integration-tests --integration cassandra | |
| breeze down | |
| if: needs.build-info.outputs.runs-on != 'self-hosted' | |
| - name: "Integration Tests Postgres: mongo" | |
| run: | | |
| breeze testing integration-tests --integration mongo | |
| breeze down | |
| if: needs.build-info.outputs.runs-on != 'self-hosted' | |
| - name: "Integration Tests Postgres: pinot" | |
| run: | | |
| breeze testing integration-tests --integration pinot | |
| breeze down | |
| if: needs.build-info.outputs.runs-on != 'self-hosted' | |
| - name: "Integration Tests Postgres: celery" | |
| run: | | |
| breeze testing integration-tests --integration celery | |
| breeze down | |
| if: needs.build-info.outputs.runs-on != 'self-hosted' | |
| - name: "Integration Tests Postgres: trino, kerberos" | |
| run: | | |
| breeze testing integration-tests --integration trino --integration kerberos | |
| breeze down | |
| if: needs.build-info.outputs.runs-on != 'self-hosted' | |
| - name: "Integration Tests Postgres: Kafka" | |
| run: | | |
| breeze testing integration-tests --integration kafka | |
| breeze down | |
| if: needs.build-info.outputs.runs-on != 'self-hosted' | |
| - name: "Integration Tests Postgres: all-testable" | |
| run: breeze testing integration-tests --integration all-testable | |
| if: needs.build-info.outputs.runs-on == 'self-hosted' | |
| - name: > | |
| Post Tests success: Integration Postgres" | |
| uses: ./.github/actions/post_tests_success | |
| if: success() | |
| - name: > | |
| Post Tests failure: Integration Postgres" | |
| uses: ./.github/actions/post_tests_failure | |
| if: failure() | |
| tests-integration-mysql: | |
| timeout-minutes: 130 | |
| name: Integration Tests MySQL | |
| runs-on: "${{needs.build-info.outputs.runs-on}}" | |
| needs: [build-info, wait-for-ci-images] | |
| env: | |
| RUNS_ON: "${{needs.build-info.outputs.runs-on}}" | |
| PARALLEL_TEST_TYPES: "${{needs.build-info.outputs.parallel-test-types-list-as-string}}" | |
| PR_LABELS: "${{needs.build-info.outputs.pull-request-labels}}" | |
| FULL_TESTS_NEEDED: "${{needs.build-info.outputs.full-tests-needed}}" | |
| DEBUG_RESOURCES: "${{needs.build-info.outputs.debug-resources}}" | |
| BACKEND: "mysql" | |
| PYTHON_MAJOR_MINOR_VERSION: "${{needs.build-info.outputs.default-python-version}}" | |
| MYSQL_VERSION: "${{needs.build-info.outputs.default-mysql-version}}" | |
| BACKEND_VERSION: "${{needs.build-info.outputs.default-mysql-version}}" | |
| JOB_ID: "integration" | |
| COVERAGE: "${{needs.build-info.outputs.run-coverage}}" | |
| SKIP_PROVIDER_TESTS: "${{needs.build-info.outputs.skip-provider-tests}}" | |
| if: needs.build-info.outputs.run-tests == 'true' | |
| steps: | |
| - name: Cleanup repo | |
| shell: bash | |
| run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" | |
| if: needs.build-info.outputs.runs-on == 'self-hosted' | |
| - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" | |
| uses: actions/checkout@v3 | |
| with: | |
| persist-credentials: false | |
| if: needs.build-info.outputs.runs-on == 'self-hosted' | |
| - name: "Prepare breeze & CI image: ${{env.PYTHON_MAJOR_MINOR_VERSION}}:${{env.IMAGE_TAG}}" | |
| uses: ./.github/actions/prepare_breeze_and_image | |
| if: needs.build-info.outputs.runs-on == 'self-hosted' | |
| - name: "Integration Tests MySQL: all-testable" | |
| run: breeze testing integration-tests --integration all-testable | |
| if: needs.build-info.outputs.runs-on == 'self-hosted' | |
| - name: > | |
| Post Tests success: Integration MySQL" | |
| uses: ./.github/actions/post_tests_success | |
| if: success() && needs.build-info.outputs.runs-on == 'self-hosted' | |
| - name: > | |
| Post Tests failure: Integration MySQL" | |
| uses: ./.github/actions/post_tests_failure | |
| if: failure() && needs.build-info.outputs.runs-on == 'self-hosted' | |
| tests-quarantined: | |
| timeout-minutes: 60 | |
| name: "Quarantined tests" | |
| runs-on: "${{needs.build-info.outputs.runs-on}}" | |
| continue-on-error: true | |
| needs: [build-info, wait-for-ci-images] | |
| env: | |
| RUNS_ON: "${{needs.build-info.outputs.runs-on}}" | |
| TEST_TYPE: "Quarantined" | |
| PR_LABELS: "${{needs.build-info.outputs.pull-request-labels}}" | |
| PYTHON_MAJOR_MINOR_VERSION: "${{needs.build-info.outputs.default-python-version}}" | |
| DEBUG_RESOURCES: "${{needs.build-info.outputs.debug-resources}}" | |
| JOB_ID: "quarantined-${{needs.build-info.outputs.default-python-version}}" | |
| COVERAGE: "${{needs.build-info.outputs.run-coverage}}" | |
| if: needs.build-info.outputs.run-tests == 'true' && needs.build-info.outputs.runs-on == 'self-hosted' | |
| steps: | |
| - name: Cleanup repo | |
| shell: bash | |
| run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" | |
| - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" | |
| uses: actions/checkout@v3 | |
| with: | |
| persist-credentials: false | |
| - name: > | |
| Prepare breeze & CI image: ${{needs.build-info.outputs.default-python-version}}:${{env.IMAGE_TAG}} | |
| uses: ./.github/actions/prepare_breeze_and_image | |
| - name: > | |
| Tests: postgres:${{needs.build-info.outputs.default-python-version}}:Quarantined | |
| run: breeze testing tests || true | |
| env: | |
| BACKEND: "postgres" | |
| BACKEND_VERSION: "11" | |
| POSTGRES_VERSION: "11" | |
| - name: > | |
| Tests: mysql:${{needs.build-info.outputs.default-python-version}}:Quarantined | |
| run: breeze testing tests || true | |
| env: | |
| BACKEND: "mysql" | |
| BACKEND_VERSION: ${{needs.build-info.outputs.default-mysql-version}} | |
| MYSQL_VERSION: ${{needs.build-info.outputs.default-mysql-version}} | |
| - name: > | |
| Tests: mssql:${{needs.build-info.outputs.default-python-version}}:Quarantined | |
| run: breeze testing tests || true | |
| env: | |
| BACKEND: "mssql" | |
| BACKEND_VERSION: ${{needs.build-info.outputs.default-mssql-version}} | |
| MSSQL_VERSION: ${{needs.build-info.outputs.default-mssql-version}} | |
| - name: > | |
| Tests: sqlite:${{needs.build-info.outputs.default-python-version}}:Quarantined | |
| run: breeze testing tests || true | |
| env: | |
| BACKEND: "sqlite" | |
| BACKEND_VERSION: "" | |
| MSSQL_VERSION: "" | |
| - name: > | |
| Post Tests success: Quarantined" | |
| uses: ./.github/actions/post_tests_success | |
| if: success() | |
| - name: > | |
| Post Tests failure: Quarantined" | |
| uses: ./.github/actions/post_tests_failure | |
| if: failure() | |
| summarize-warnings: | |
| timeout-minutes: 15 | |
| name: "Summarize warnings" | |
| runs-on: "${{needs.build-info.outputs.runs-on}}" | |
| needs: | |
| - build-info | |
| - tests-postgres | |
| - tests-sqlite | |
| - tests-mysql | |
| - tests-quarantined | |
| - tests-integration-postgres | |
| - tests-integration-mysql | |
| env: | |
| RUNS_ON: "${{needs.build-info.outputs.runs-on}}" | |
| steps: | |
| - name: Cleanup repo | |
| run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" | |
| - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" | |
| uses: actions/checkout@v3 | |
| with: | |
| persist-credentials: false | |
| submodules: recursive | |
| - name: "Download all artifacts from the current build" | |
| uses: actions/download-artifact@v3 | |
| with: | |
| path: ./artifacts | |
| - name: "Summarize all warnings" | |
| run: | | |
| ls -R ./artifacts/ | |
| cat ./artifacts/test-warnings*/* | sort | uniq | |
| echo | |
| echo Total number of unique warnings $(cat ./artifacts/test-warnings*/* | sort | uniq | wc -l) | |
| wait-for-prod-images: | |
| timeout-minutes: 80 | |
| name: "Wait for & verify PROD images" | |
| runs-on: "${{needs.build-info.outputs.runs-on}}" | |
| needs: [build-info, wait-for-ci-images, build-prod-images] | |
| if: needs.build-info.outputs.image-build == 'true' | |
| env: | |
| RUNS_ON: "${{needs.build-info.outputs.runs-on}}" | |
| BACKEND: sqlite | |
| PYTHON_MAJOR_MINOR_VERSION: "${{needs.build-info.outputs.default-python-version}}" | |
| # Force more parallelism for pull even on public images | |
| PARALLELISM: 6 | |
| steps: | |
| - name: Cleanup repo | |
| run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" | |
| - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" | |
| uses: actions/checkout@v3 | |
| with: | |
| persist-credentials: false | |
| - name: "Install Breeze" | |
| uses: ./.github/actions/breeze | |
| - name: Wait for PROD images ${{ env.PYTHON_VERSIONS }}:${{ env.IMAGE_TAG }} | |
| # We wait for the images to be available either from "build-images.yml' run as pull_request_target | |
| # or from build-prod-images above. | |
| # We are utilising single job to wait for all images because this job merely waits | |
| # For the images to be available. | |
| run: breeze prod-image pull --wait-for-image --run-in-parallel | |
| env: | |
| PYTHON_VERSIONS: ${{ needs.build-info.outputs.python-versions-list-as-string }} | |
| DEBUG_RESOURCES: ${{needs.build-info.outputs.debug-resources}} | |
| - name: Verify PROD images ${{ env.PYTHON_VERSIONS }}:${{ env.IMAGE_TAG }} | |
| # We pull images again (which is a NOOP) but this time we also verify the images | |
| # Having it as a separate step allows us to see if the problem was with waiting or verification | |
| run: breeze prod-image pull --verify --run-in-parallel | |
| env: | |
| PYTHON_VERSIONS: ${{ needs.build-info.outputs.python-versions-list-as-string }} | |
| DEBUG_RESOURCES: ${{needs.build-info.outputs.debug-resources}} | |
| - name: "Fix ownership" | |
| run: breeze ci fix-ownership | |
| if: always() | |
| test-docker-compose-quick-start: | |
| timeout-minutes: 60 | |
| name: "Test docker-compose quick start" | |
| runs-on: "${{needs.build-info.outputs.runs-on}}" | |
| needs: [build-info, wait-for-prod-images] | |
| if: needs.build-info.outputs.image-build == 'true' | |
| env: | |
| PYTHON_MAJOR_MINOR_VERSION: "${{needs.build-info.outputs.default-python-version}}" | |
| steps: | |
| - name: Cleanup repo | |
| run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" | |
| - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" | |
| uses: actions/checkout@v3 | |
| with: | |
| fetch-depth: 2 | |
| persist-credentials: false | |
| - name: > | |
| Prepare breeze & PROD image: | |
| ${{needs.build-info.outputs.default-python-version}}:${{env.IMAGE_TAG}} | |
| uses: ./.github/actions/prepare_breeze_and_image | |
| with: | |
| pull-image-type: 'PROD' | |
| - name: "Test docker-compose quick start" | |
| run: breeze testing docker-compose-tests | |
| - name: "Fix ownership" | |
| run: breeze ci fix-ownership | |
| if: always() | |
| tests-kubernetes: | |
| timeout-minutes: 240 | |
| name: "Helm: ${{matrix.executor}} - ${{needs.build-info.outputs.kubernetes-versions-list-as-string}}" | |
| runs-on: "${{needs.build-info.outputs.runs-on}}" | |
| needs: [build-info, wait-for-prod-images] | |
| strategy: | |
| matrix: | |
| executor: [KubernetesExecutor, CeleryExecutor, LocalExecutor] | |
| fail-fast: false | |
| env: | |
| RUNS_ON: "${{needs.build-info.outputs.runs-on}}" | |
| DEBUG_RESOURCES: ${{needs.build-info.outputs.debug-resources}} | |
| if: > | |
| ( needs.build-info.outputs.run-kubernetes-tests == 'true' || | |
| needs.build-info.outputs.needs-helm-tests == 'true' ) && | |
| needs.build-info.outputs.default-branch == 'main' | |
| steps: | |
| - name: Cleanup repo | |
| run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" | |
| - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" | |
| uses: actions/checkout@v3 | |
| with: | |
| persist-credentials: false | |
| - name: "Install Breeze" | |
| uses: ./.github/actions/breeze | |
| id: breeze | |
| - name: Pull PROD images ${{ env.PYTHON_VERSIONS }}:${{ env.IMAGE_TAG }} | |
| run: breeze prod-image pull --run-in-parallel --tag-as-latest | |
| env: | |
| PYTHON_VERSIONS: ${{ needs.build-info.outputs.python-versions-list-as-string }} | |
| - name: "Cache bin folder with tools for kubernetes testing" | |
| uses: actions/cache@v3 | |
| with: | |
| path: ".build/.k8s-env" | |
| key: "\ | |
| k8s-env-${{steps.breeze.outputs.host-python-version}}-\ | |
| ${{ hashFiles('scripts/ci/kubernetes/k8s_requirements.txt','setup.cfg',\ | |
| 'setup.py','pyproject.toml','generated/provider_dependencies.json') }}" | |
| - name: Run complete K8S tests ${{needs.build-info.outputs.kubernetes-combos-list-as-string}} | |
| run: breeze k8s run-complete-tests --run-in-parallel --upgrade | |
| env: | |
| PYTHON_VERSIONS: ${{ needs.build-info.outputs.python-versions-list-as-string }} | |
| KUBERNETES_VERSIONS: ${{needs.build-info.outputs.kubernetes-versions-list-as-string}} | |
| EXECUTOR: ${{matrix.executor}} | |
| VERBOSE: false | |
| - name: Upload KinD logs on failure ${{needs.build-info.outputs.kubernetes-combos-list-as-string}} | |
| uses: actions/upload-artifact@v3 | |
| if: failure() || cancelled() | |
| with: | |
| name: kind-logs-${{matrix.executor}} | |
| path: /tmp/kind_logs_* | |
| retention-days: 7 | |
| - name: > | |
| Upload test resource logs on failure ${{needs.build-info.outputs.kubernetes-combos-list-as-string}} | |
| uses: actions/upload-artifact@v3 | |
| if: failure() || cancelled() | |
| with: | |
| name: k8s-test-resources-${{matrix.executor}} | |
| path: /tmp/k8s_test_resources_* | |
| retention-days: 7 | |
| - name: "Delete clusters just in case they are left" | |
| run: breeze k8s delete-cluster --all | |
| if: always() | |
| - name: "Fix ownership" | |
| run: breeze ci fix-ownership | |
| if: always() | |
| constraints: | |
| permissions: | |
| contents: write | |
| packages: write | |
| timeout-minutes: 80 | |
| name: "Constraints" | |
| runs-on: "${{needs.build-info.outputs.runs-on}}" | |
| needs: | |
| - build-info | |
| - docs | |
| - wait-for-ci-images | |
| - wait-for-prod-images | |
| - static-checks | |
| - tests-sqlite | |
| - tests-mysql | |
| - tests-postgres | |
| - tests-integration-postgres | |
| - tests-integration-mysql | |
| - push-early-buildx-cache-to-github-registry | |
| env: | |
| RUNS_ON: "${{needs.build-info.outputs.runs-on}}" | |
| DEBUG_RESOURCES: ${{needs.build-info.outputs.debug-resources}} | |
| if: needs.build-info.outputs.upgrade-to-newer-dependencies != 'false' | |
| steps: | |
| - name: Cleanup repo | |
| run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" | |
| - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" | |
| uses: actions/checkout@v3 | |
| with: | |
| persist-credentials: false | |
| submodules: recursive | |
| - name: "Install Breeze" | |
| uses: ./.github/actions/breeze | |
| - name: Pull CI images ${{ env.PYTHON_VERSIONS }}:${{ env.IMAGE_TAG }} | |
| run: breeze ci-image pull --run-in-parallel --tag-as-latest | |
| env: | |
| PYTHON_VERSIONS: ${{ needs.build-info.outputs.python-versions-list-as-string }} | |
| - name: "Generate constraints" | |
| run: | | |
| breeze release-management generate-constraints --run-in-parallel \ | |
| --airflow-constraints-mode constraints-source-providers | |
| breeze release-management generate-constraints \ | |
| --run-in-parallel --airflow-constraints-mode constraints-no-providers | |
| breeze release-management generate-constraints \ | |
| --run-in-parallel --airflow-constraints-mode constraints | |
| env: | |
| PYTHON_VERSIONS: ${{ needs.build-info.outputs.python-versions-list-as-string }} | |
| - name: "Set constraints branch name" | |
| id: constraints-branch | |
| run: ./scripts/ci/constraints/ci_branch_constraints.sh >> ${GITHUB_OUTPUT} | |
| if: needs.build-info.outputs.canary-run == 'true' | |
| - name: Checkout ${{ steps.constraints-branch.outputs.branch }} | |
| uses: actions/checkout@v3 | |
| if: needs.build-info.outputs.canary-run == 'true' | |
| with: | |
| path: "repo" | |
| ref: ${{ steps.constraints-branch.outputs.branch }} | |
| persist-credentials: false | |
| - name: "Commit changed constraint files for ${{needs.build-info.outputs.python-versions}}" | |
| run: ./scripts/ci/constraints/ci_commit_constraints.sh | |
| if: needs.build-info.outputs.canary-run == 'true' | |
| - name: "Push changes" | |
| uses: ./.github/actions/github-push-action | |
| if: needs.build-info.outputs.canary-run == 'true' | |
| with: | |
| github_token: ${{ secrets.GITHUB_TOKEN }} | |
| branch: ${{ steps.constraints-branch.outputs.branch }} | |
| directory: "repo" | |
| - name: "Fix ownership" | |
| run: breeze ci fix-ownership | |
| if: always() | |
| # Push BuildX cache to GitHub Registry in Apache repository, if all tests are successful and build | |
| # is executed as result of direct push to "main" or one of the "vX-Y-test" branches | |
| # It rebuilds all images using just-pushed constraints using buildx and pushes them to registry | |
| # It will automatically check if a new python image was released and will pull the latest one if needed | |
| push-buildx-cache-to-github-registry: | |
| timeout-minutes: 50 | |
| name: "Push Image Cache" | |
| runs-on: "${{needs.build-info.outputs.runs-on}}" | |
| needs: | |
| - build-info | |
| - constraints | |
| - docs | |
| if: needs.build-info.outputs.canary-run == 'true' | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| platform: ["linux/amd64", "linux/arm64"] | |
| env: | |
| RUNS_ON: "${{needs.build-info.outputs.runs-on}}" | |
| VERSION_SUFFIX_FOR_PYPI: "dev0" | |
| steps: | |
| - name: Cleanup repo | |
| run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" | |
| - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" | |
| uses: actions/checkout@v3 | |
| with: | |
| persist-credentials: false | |
| - name: > | |
| Prepare breeze & CI image: ${{needs.build-info.outputs.default-python-version}}:${{env.IMAGE_TAG}} | |
| uses: ./.github/actions/prepare_breeze_and_image | |
| env: | |
| # Always use default Python version of CI image for preparing packages | |
| PYTHON_MAJOR_MINOR_VERSION: "${{needs.build-info.outputs.default-python-version}}" | |
| - name: "Cleanup dist and context file" | |
| run: rm -fv ./dist/* ./docker-context-files/* | |
| - name: "Prepare airflow package for PROD build" | |
| run: breeze release-management prepare-airflow-package --package-format wheel | |
| env: | |
| VERSION_SUFFIX_FOR_PYPI: "dev0" | |
| - name: "Prepare providers packages for PROD build" | |
| run: > | |
| breeze release-management prepare-provider-packages | |
| --package-list-file ./scripts/ci/installed_providers.txt | |
| --package-format wheel | |
| env: | |
| VERSION_SUFFIX_FOR_PYPI: "dev0" | |
| if: needs.build-info.outputs.default-branch == 'main' | |
| - name: "Start ARM instance" | |
| run: ./scripts/ci/images/ci_start_arm_instance_and_connect_to_docker.sh | |
| if: matrix.platform == 'linux/arm64' | |
| - name: "Push CI cache ${{ matrix.platform }}" | |
| run: > | |
| breeze ci-image build | |
| --builder airflow_cache | |
| --prepare-buildx-cache | |
| --run-in-parallel | |
| --force-build | |
| --platform ${{ matrix.platform }} | |
| env: | |
| DEBUG_RESOURCES: ${{needs.build-info.outputs.debug-resources}} | |
| COMMIT_SHA: ${{ github.sha }} | |
| - name: "Push CI latest image ${{ matrix.platform }}" | |
| run: > | |
| breeze ci-image build | |
| --tag-as-latest --push --run-in-parallel --platform ${{ matrix.platform }} | |
| env: | |
| DEBUG_RESOURCES: ${{needs.build-info.outputs.debug-resources}} | |
| COMMIT_SHA: ${{ github.sha }} | |
| if: matrix.platform == 'linux/amd64' | |
| - name: "Move dist packages to docker-context files" | |
| run: mv -v ./dist/*.whl ./docker-context-files | |
| - name: "Push PROD cache ${{ matrix.python-version }} ${{ matrix.platform }}" | |
| run: > | |
| breeze prod-image build | |
| --builder airflow_cache | |
| --install-packages-from-context | |
| --run-in-parallel | |
| --prepare-buildx-cache | |
| --platform ${{ matrix.platform }} | |
| env: | |
| COMMIT_SHA: ${{ github.sha }} | |
| - name: "Push PROD latest image ${{ matrix.platform }}" | |
| run: > | |
| breeze prod-image build --tag-as-latest --install-packages-from-context | |
| --push --run-in-parallel --platform ${{ matrix.platform }} | |
| env: | |
| DEBUG_RESOURCES: ${{needs.build-info.outputs.debug-resources}} | |
| COMMIT_SHA: ${{ github.sha }} | |
| if: matrix.platform == 'linux/amd64' | |
| - name: "Stop ARM instance" | |
| run: ./scripts/ci/images/ci_stop_arm_instance.sh | |
| if: always() && matrix.platform == 'linux/arm64' | |
| - name: "Fix ownership" | |
| run: breeze ci fix-ownership | |
| if: always() | |
| build-ci-arm-images: | |
| timeout-minutes: 50 | |
| name: > | |
| Build CI ARM images | |
| ${{needs.build-info.outputs.all-python-versions-list-as-string}} | |
| runs-on: "${{needs.build-info.outputs.runs-on}}" | |
| needs: | |
| - build-info | |
| - wait-for-ci-images | |
| - wait-for-prod-images | |
| - static-checks | |
| - tests-sqlite | |
| - tests-mysql | |
| - tests-postgres | |
| - tests-integration-postgres | |
| - tests-integration-mysql | |
| env: | |
| DEFAULT_BRANCH: ${{ needs.build-info.outputs.default-branch }} | |
| DEFAULT_CONSTRAINTS_BRANCH: ${{ needs.build-info.outputs.default-constraints-branch }} | |
| RUNS_ON: "${{needs.build-info.outputs.runs-on}}" | |
| # Force more parallelism for build even on small instances | |
| PARALLELISM: 6 | |
| if: > | |
| needs.build-info.outputs.upgrade-to-newer-dependencies != 'false' && | |
| needs.build-info.outputs.in-workflow-build == 'true' && | |
| needs.build-info.outputs.canary-run != 'true' | |
| steps: | |
| - name: Cleanup repo | |
| run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" | |
| - uses: actions/checkout@v3 | |
| with: | |
| ref: ${{ needs.build-info.outputs.targetCommitSha }} | |
| persist-credentials: false | |
| submodules: recursive | |
| - name: "Install Breeze" | |
| uses: ./.github/actions/breeze | |
| - name: "Start ARM instance" | |
| run: ./scripts/ci/images/ci_start_arm_instance_and_connect_to_docker.sh | |
| - name: > | |
| Build CI ARM images ${{ env.IMAGE_TAG }} | |
| ${{needs.build-info.outputs.all-python-versions-list-as-string}}:${{env.IMAGE_TAG}} | |
| run: > | |
| breeze ci-image build --run-in-parallel --builder airflow_cache --platform "linux/arm64" | |
| env: | |
| UPGRADE_TO_NEWER_DEPENDENCIES: ${{ needs.build-info.outputs.upgrade-to-newer-dependencies }} | |
| DOCKER_CACHE: ${{ needs.build-info.outputs.cache-directive }} | |
| PYTHON_VERSIONS: ${{needs.build-info.outputs.all-python-versions-list-as-string}} | |
| DEBUG_RESOURCES: ${{needs.build-info.outputs.debug-resources}} | |
| COMMIT_SHA: ${{ github.sha }} | |
| - name: "Stop ARM instance" | |
| run: ./scripts/ci/images/ci_stop_arm_instance.sh | |
| if: always() | |
| - name: "Fix ownership" | |
| run: breeze ci fix-ownership | |
| if: always() |