Test providers RC releases #63
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| --- | |
| name: Test providers RC releases | |
| on: # yamllint disable-line rule:truthy | |
| schedule: | |
| - cron: "0 0 * * *" | |
| workflow_dispatch: | |
| inputs: | |
| rc_testing_branch: | |
| # If a branch is given, the workflow will use it for deployment and testing. | |
| # If no branch is provided, the workflow will create a new rc testing branch | |
| # for deployment and testing. | |
| description: | | |
| rc_testing_branch: existing testing branch | |
| (Either rc_testing_branch or issue_url is required, and you cannot give both.) | |
| required: false | |
| default: '' | |
| issue_url: | |
| description: | | |
| issue_url: the GitHub issue URL that tracks the status of Providers release | |
| (Either rc_testing_branch or issue_url is required, and you cannot give both.) | |
| required: false | |
| base_git_rev: | |
| description: 'The base git revision to test Providers RCs' | |
| required: false | |
| type: string | |
| default: 'main' | |
| jobs: | |
| check-rc-testing-announcement: | |
| runs-on: 'ubuntu-20.04' | |
| if: github.event_name == 'schedule' | |
| env: | |
| GH_TOKEN: ${{ github.token }} | |
| steps: | |
| - name: Checkout apache-airflow | |
| uses: actions/checkout@v3 | |
| with: | |
| repository: 'apache/airflow' | |
| - name: Parse the latest 100 GitHub issues from apache-airflow to check providers testing announcement | |
| id: parse-airflow-gh-issues | |
| run: | | |
| # The default limit is 30. Set it to 100 for retrieving more issues | |
| rc_issue_url=`gh issue list \ | |
| --json createdAt,title,url \ | |
| --limit 100 \ | |
| --jq 'map( | |
| select( | |
| .title | | |
| contains ("Status of testing Providers that were prepared on ") | |
| ) | |
| ) | .[0].url'` | |
| echo "rc_issue_url=$rc_issue_url" >> $GITHUB_OUTPUT | |
| - name: Checkout current repo | |
| uses: actions/checkout@v3 | |
| if: steps.parse-airflow-gh-issues.outputs.rc_issue_url != '' | |
| - name: Parse the latest GitHub pull requests for checking existing RC provider testing pull request | |
| id: parse-current-repo | |
| if: steps.parse-airflow-gh-issues.outputs.rc_issue_url != '' | |
| run: | | |
| # The default limit is 30. Set it to 100 for retrieving more pull requests | |
| rc_issue_url="${{ steps.parse-airflow-gh-issues.outputs.rc_issue_url }}" | |
| testing_pr_url=`gh pr list \ | |
| --json createdAt,title,url \ | |
| --limit 100 \ | |
| --state open \ | |
| --state closed \ | |
| --jq 'map( | |
| select( | |
| .title | | |
| contains ( | |
| "[DO NOT MERGE] Test RC provider packages for $rc_issue_url" | |
| ) | |
| ) | |
| ) | .[0].url'` | |
| echo "testing_pr_url=$testing_pr_url" >> $GITHUB_OUTPUT | |
| - name: Export rc_issue_url | |
| id: export-rc-issue-url | |
| run: | | |
| rc_issue_url="${{ steps.parse-airflow-gh-issues.outputs.rc_issue_url }}" | |
| testing_pr_url="${{ steps.parse-current-repo.outputs.testing_pr_url }}" | |
| if [ "$rc_issue_url" == "" ] ; then | |
| echo "No RC providers testing announcement found on apache-airflow" | |
| elif [ "$testing_pr_url" != "" ] ; then | |
| echo "Branch for testing RC providers has been created" | |
| rc_issue_url="" | |
| fi | |
| echo "rc_issue_url=$rc_issue_url" >> $GITHUB_OUTPUT | |
| outputs: | |
| rc_issue_url: ${{ steps.export-rc-issue-url.outputs.rc_issue_url }} | |
| validate-manual-input: | |
| runs-on: 'ubuntu-20.04' | |
| if: github.event_name == 'workflow_dispatch' | |
| steps: | |
| - name: Validate user input | |
| if: | | |
| (inputs.rc_testing_branch == '' && inputs.issue_url == '') || | |
| (inputs.rc_testing_branch != '' && inputs.issue_url != '') | |
| run: | | |
| echo "Either rc_testing_branch or issue_url is required, and you cannot give both." | |
| exit 1 | |
| create-branch-for-testing-rc-release: | |
| needs: [validate-manual-input, check-rc-testing-announcement] | |
| runs-on: 'ubuntu-20.04' | |
| if: | | |
| always() && | |
| ( | |
| (github.event_name == 'workflow_dispatch' && inputs.issue_url != '') || | |
| (github.event_name == 'schedule' && needs.check-rc-testing-announcement.outputs.rc_issue_url != '') | |
| ) | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v3 | |
| with: | |
| ref: ${{ inputs.base_git_rev }} | |
| - name: Install dev dependency | |
| run: | | |
| python3 -m pip install -r dev/integration_test_scripts/requirements.txt | |
| - name: Setup Github Actions git user | |
| run: | | |
| git config --global user.email "[email protected]" | |
| git config --global user.name "GitHub Actions" | |
| - name: Export GitHub RC provider testing url | |
| id: export-rc-issue-url | |
| run: | | |
| if [ "${{ inputs.issue_url }}" != "" ] ; then | |
| rc_issue_url="${{ inputs.issue_url }}" | |
| else | |
| rc_issue_url="${{ needs.check-rc-testing-announcement.outputs.rc_issue_url }}" | |
| fi | |
| echo "rc_issue_url=$rc_issue_url" | |
| echo "rc_issue_url=$rc_issue_url" >> $GITHUB_OUTPUT | |
| - name: Create RC branch | |
| id: create_rc_branch | |
| run: | | |
| current_timestamp=`date +%Y-%m-%dT%H-%M-%S%Z` | |
| echo "Current timestamp is" $current_timestamp | |
| branch_name="rc-test-$current_timestamp" | |
| git checkout -b $branch_name | |
| echo "rc_testing_branch=$branch_name" | |
| echo "rc_testing_branch=$branch_name" >> $GITHUB_OUTPUT | |
| - name: Update project dependencies to use RC providers | |
| run: | | |
| rc_issue_url="${{ steps.export-rc-issue-url.outputs.rc_issue_url }}" | |
| echo "Updating setup.cfg with RC provider packages on $rc_issue_url" | |
| python3 dev/integration_test_scripts/replace_dependencies.py --issue-url $rc_issue_url | |
| - name: Commit changes and create a pull request | |
| env: | |
| GH_TOKEN: ${{ github.token }} | |
| run: | | |
| rc_issue_url="${{ steps.export-rc-issue-url.outputs.rc_issue_url }}" | |
| git add setup.cfg | |
| git commit -m "Updating setup.cfg with RC provider packages on $rc_issue_url" | |
| git push origin ${{ steps.create_rc_branch.outputs.rc_testing_branch }} | |
| gh pr create --title "[DO NOT MERGE] Test RC provider packages for $rc_issue_url" \ | |
| --fill | |
| echo "git_rev=$(git rev-parse HEAD)" >> $GITHUB_OUTPUT | |
| outputs: | |
| rc_testing_branch: ${{ steps.create_rc_branch.outputs.rc_testing_branch }} | |
| export-rc-testing-branch-name: | |
| needs: [validate-manual-input, create-branch-for-testing-rc-release] | |
| if: | | |
| always() && | |
| needs.create-branch-for-testing-rc-release.result == 'success' || | |
| (needs.validate-manual-input.result == 'success' && inputs.rc_testing_branch ) | |
| runs-on: 'ubuntu-20.04' | |
| steps: | |
| - name: export rc_testing_branch | |
| id: export-rc-testing-branch-name-step | |
| run: | | |
| if [ "${{ inputs.rc_testing_branch }}" == "" ]; then | |
| rc_testing_branch=${{ needs.create-branch-for-testing-rc-release.outputs.rc_testing_branch }} | |
| else | |
| rc_testing_branch=${{ inputs.rc_testing_branch }} | |
| fi | |
| echo "rc_testing_branch=$rc_testing_branch" >> $GITHUB_OUTPUT | |
| outputs: | |
| rc_testing_branch: ${{ steps.export-rc-testing-branch-name-step.outputs.rc_testing_branch }} | |
| deploy-rc-testing-branch-to-astro-cloud: | |
| needs: export-rc-testing-branch-name | |
| if: | | |
| always() && | |
| needs.export-rc-testing-branch-name.result == 'success' | |
| uses: ./.github/workflows/deploy-to-astro-cloud-reuse-wf.yaml | |
| with: | |
| git_rev: ${{ needs.export-rc-testing-branch-name.outputs.rc_testing_branch }} | |
| environment_to_deploy: 'providers-integration-tests' | |
| secrets: | |
| docker_registry: ${{ secrets.DOCKER_REGISTRY }} | |
| organization_id: ${{ secrets.ORGANIZATION_ID }} | |
| deployment_id: ${{ secrets.PROVIDER_INTEGRATION_TESTS_DEPLOYMENT_ID }} | |
| astronomer_key_id: ${{ secrets.PROVIDER_INTEGRATION_TESTS_ASTRONOMER_KEY_ID }} | |
| astronomer_key_secret: ${{ secrets.PROVIDER_INTEGRATION_TESTS_ASTRONOMER_KEY_SECRET }} | |
| SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} | |
| wait-for-deployment-to-be-ready-and-trigger-master-dag: | |
| needs: [deploy-rc-testing-branch-to-astro-cloud, export-rc-testing-branch-name] | |
| if: | | |
| always() && | |
| needs.deploy-rc-testing-branch-to-astro-cloud.result == 'success' | |
| runs-on: 'ubuntu-20.04' | |
| steps: | |
| - name: Sleep for 30 minutes | |
| run: | | |
| echo "Current timestamp is" `date` | |
| echo "Sleeping for 1800 seconds (30 minutes)" | |
| echo "allowing the deployed image to be updated across all Airflow components." | |
| sleep 1800 | |
| - name: checkout | |
| uses: actions/checkout@v3 | |
| with: | |
| ref: ${{ needs.export-rc-testing-branch-name.outputs.rc_testing_branch }} | |
| - name: Trigger master dag | |
| run: | | |
| python3 dev/integration_test_scripts/trigger_dag.py \ | |
| ${{ secrets.PROVIDER_INTEGRATION_TESTS_DEPLOYMENT_ID }} \ | |
| ${{ secrets.PROVIDER_INTEGRATION_TESTS_ASTRONOMER_KEY_ID }} \ | |
| ${{ secrets.PROVIDER_INTEGRATION_TESTS_ASTRONOMER_KEY_SECRET }} |