From d9b329dadb1a9561a8be73f6d28d5896e40df0f5 Mon Sep 17 00:00:00 2001 From: Wei Lee Date: Sat, 1 Jul 2023 08:27:27 +0800 Subject: [PATCH] fix triggering failure due to astro_cloud subdomain change (#1222) * ci(github-actions): fix triggering failure due to astro_cloud subdomain change * ci(github-actions): extract common wait and trigger logic as a reusable workflow --- .../workflows/deploy-integration-tests.yaml | 62 ++++++------------- .github/workflows/test-rc-release.yaml | 29 +++------ .github/workflows/trigger-dag-reuse-wf.yaml | 61 ++++++++++++++++++ Makefile | 2 +- dev/integration_test_scripts/trigger_dag.py | 17 +++-- 5 files changed, 102 insertions(+), 69 deletions(-) create mode 100644 .github/workflows/trigger-dag-reuse-wf.yaml diff --git a/.github/workflows/deploy-integration-tests.yaml b/.github/workflows/deploy-integration-tests.yaml index 477bb5467..2d64626ac 100644 --- a/.github/workflows/deploy-integration-tests.yaml +++ b/.github/workflows/deploy-integration-tests.yaml @@ -48,28 +48,15 @@ jobs: github.event_name == 'schedule' || (github.event_name == 'workflow_dispatch' && inputs.dags_to_trigger_after_deployment != '') needs: deploy-to-providers-integration-tests - runs-on: 'ubuntu-20.04' - steps: - - name: Sleep for 30 minutes - run: | - sleep_time=1800 - echo "Current timestamp is" `date` - echo "Sleeping for $sleep_time seconds" - echo "allowing the deployed image to be updated across all Airflow components.." - sleep 1800 - - - name: checkout - uses: actions/checkout@v3 - with: - ref: ${{ inputs.git_rev }} - - - name: Trigger DAG(s) - run: | - python3 dev/integration_test_scripts/trigger_dag.py \ - ${{ secrets.PROVIDER_INTEGRATION_TESTS_DEPLOYMENT_ID }} \ - ${{ secrets.PROVIDER_INTEGRATION_TESTS_ASTRONOMER_KEY_ID }} \ - ${{ secrets.PROVIDER_INTEGRATION_TESTS_ASTRONOMER_KEY_SECRET }} \ - --dag-ids "${{ inputs.dags_to_trigger_after_deployment }}" + uses: ./.github/workflows/trigger-dag-reuse-wf.yaml + with: + git_rev: ${{ inputs.git_rev }} + dags_to_trigger_after_deployment: ${{ inputs.dags_to_trigger_after_deployment }} + secrets: + astro_subdomain: ${{ secrets.ASTRO_SUBDOMAIN }} + deployment_id: ${{ secrets.PROVIDER_INTEGRATION_TESTS_DEPLOYMENT_ID }} + astronomer_key_id: ${{ secrets.PROVIDER_INTEGRATION_TESTS_ASTRONOMER_KEY_ID }} + astronomer_key_secret: ${{ secrets.PROVIDER_INTEGRATION_TESTS_ASTRONOMER_KEY_SECRET }} deploy-to-providers-integration-tests-on-KE: if: | @@ -92,25 +79,12 @@ jobs: github.event_name == 'schedule' || (github.event_name == 'workflow_dispatch' && inputs.dags_to_trigger_after_deployment != '') needs: deploy-to-providers-integration-tests-on-KE - runs-on: 'ubuntu-20.04' - steps: - - name: Sleep and wait for the deployment to be ready - run: | - sleep_time=1800 - echo "Current timestamp is" `date` - echo "Sleeping for $sleep_time seconds" - echo "allowing the deployed image to be updated across all Airflow components.." - sleep $sleep_time - - - name: checkout - uses: actions/checkout@v3 - with: - ref: ${{ inputs.git_rev }} - - - name: Trigger DAG(s) - run: | - python3 dev/integration_test_scripts/trigger_dag.py \ - ${{ secrets.PROVIDER_INTEGRATION_TESTS_ON_KE_DEPLOYMENT_ID }} \ - ${{ secrets.PROVIDER_INTEGRATION_TESTS_ON_KE_ASTRONOMER_KEY_ID }} \ - ${{ secrets.PROVIDER_INTEGRATION_TESTS_ON_KE_ASTRONOMER_KEY_SECRET }} \ - --dag-ids "${{ inputs.dags_to_trigger_after_deployment }}" + uses: ./.github/workflows/trigger-dag-reuse-wf.yaml + with: + git_rev: ${{ inputs.git_rev }} + dags_to_trigger_after_deployment: ${{ inputs.dags_to_trigger_after_deployment }} + secrets: + astro_subdomain: ${{ secrets.ASTRO_SUBDOMAIN }} + deployment_id: ${{ secrets.PROVIDER_INTEGRATION_TESTS_DEPLOYMENT_ID }} + astronomer_key_id: ${{ secrets.PROVIDER_INTEGRATION_TESTS_ASTRONOMER_KEY_ID }} + astronomer_key_secret: ${{ secrets.PROVIDER_INTEGRATION_TESTS_ASTRONOMER_KEY_SECRET }} diff --git a/.github/workflows/test-rc-release.yaml b/.github/workflows/test-rc-release.yaml index d37a9a59e..d844a9ed8 100644 --- a/.github/workflows/test-rc-release.yaml +++ b/.github/workflows/test-rc-release.yaml @@ -219,23 +219,12 @@ jobs: if: | always() && needs.deploy-rc-testing-branch-to-astro-cloud.result == 'success' - runs-on: 'ubuntu-20.04' - steps: - - name: Sleep for 30 minutes - run: | - echo "Current timestamp is" `date` - echo "Sleeping for 1800 seconds (30 minutes)" - echo "allowing the deployed image to be updated across all Airflow components." - sleep 1800 - - - name: checkout - uses: actions/checkout@v3 - with: - ref: ${{ needs.export-rc-testing-branch-name.outputs.rc_testing_branch }} - - - name: Trigger master dag - run: | - python3 dev/integration_test_scripts/trigger_dag.py \ - ${{ secrets.PROVIDER_INTEGRATION_TESTS_DEPLOYMENT_ID }} \ - ${{ secrets.PROVIDER_INTEGRATION_TESTS_ASTRONOMER_KEY_ID }} \ - ${{ secrets.PROVIDER_INTEGRATION_TESTS_ASTRONOMER_KEY_SECRET }} + uses: ./.github/workflows/trigger-dag-reuse-wf.yaml + with: + git_rev: ${{ needs.export-rc-testing-branch-name.outputs.rc_testing_branch }} + dags_to_trigger_after_deployment: "example_master_dag" + secrets: + astro_subdomain: ${{ secrets.ASTRO_SUBDOMAIN }} + deployment_id: ${{ secrets.PROVIDER_INTEGRATION_TESTS_DEPLOYMENT_ID }} + astronomer_key_id: ${{ secrets.PROVIDER_INTEGRATION_TESTS_ASTRONOMER_KEY_ID }} + astronomer_key_secret: ${{ secrets.PROVIDER_INTEGRATION_TESTS_ASTRONOMER_KEY_SECRET }} diff --git a/.github/workflows/trigger-dag-reuse-wf.yaml b/.github/workflows/trigger-dag-reuse-wf.yaml new file mode 100644 index 000000000..6730297b0 --- /dev/null +++ b/.github/workflows/trigger-dag-reuse-wf.yaml @@ -0,0 +1,61 @@ +--- +name: Wait for deployment and trigger dags + +on: # yamllint disable-line rule:truthy + workflow_call: + inputs: + git_rev: + description: 'The git revision to deploy' + required: false + type: string + default: '' + wait_time: + description: 'seconds to wait (default 1800 seconds = 30 minutes)' + required: false + type: number + default: 1800 + dags_to_trigger_after_deployment: + description: | + Comma separated list of dag_ids to trigger after deployment + (e.g. "example_async_adf_run_pipeline, example_async_batch") + required: false + type: string + default: '' + secrets: + astro_subdomain: + description: 'astro cloud subdomain' + required: true + deployment_id: + description: 'astro cloud deployment_id' + required: true + astronomer_key_id: + description: 'astro cloud astronomer_key_id' + required: true + astronomer_key_secret: + description: 'astro cloud astronomer_key_secret' + required: true + +jobs: + wait-for-deployment-to-be-ready-and-trigger-dag: + runs-on: 'ubuntu-20.04' + steps: + - name: Sleep and wait for astro cloud deployment + run: | + echo "Current timestamp is" `date` + echo "Sleeping for ${{ inputs.wait_time }}" + echo "allowing the deployed image to be updated across all Airflow components." + sleep ${{ inputs.wait_time }} + + - name: Checkout + uses: actions/checkout@v3 + with: + ref: ${{ inputs.git_rev }} + + - name: Trigger DAG(s) + run: | + python3 dev/integration_test_scripts/trigger_dag.py \ + ${{ secrets.astro_subdomain }} \ + ${{ secrets.deployment_id }} \ + ${{ secrets.astronomer_key_id }} \ + ${{ secrets.astronomer_key_secret }} \ + --dag-ids "${{ inputs.dags_to_trigger_after_deployment }}" diff --git a/Makefile b/Makefile index 73699afd7..b317437ad 100644 --- a/Makefile +++ b/Makefile @@ -87,7 +87,7 @@ test-rc-deps: ## Test providers RC by building an image with given dependencies python3 dev/integration_test_scripts/replace_dependencies.py --issue-url '$(ISSUE_URL)' echo "Building and deploying image to Astro Cloud" cd ".circleci/integration-tests/" && \ - bash script.sh 'astro-cloud' '$(DOCKER_REGISTRY)' '$(ORGANIZATION_ID)' '$(DEPLOYMENT_ID)' '$(ASTRONOMER_KEY_ID)' '$(ASTRONOMER_KEY_SECRET)' + bash script.sh 'astro-cloud' '$(ASTRO_SUBDOMAIN)' '$(DOCKER_REGISTRY)' '$(ORGANIZATION_ID)' '$(DEPLOYMENT_ID)' '$(ASTRONOMER_KEY_ID)' '$(ASTRONOMER_KEY_SECRET)' $(eval current_timestamp := $(shell date)) echo "Current timestamp is" $(current_timestamp) echo "Sleeping for 1800 seconds (30 minutes) allowing the deployed image to be updated across all Airflow components.." diff --git a/dev/integration_test_scripts/trigger_dag.py b/dev/integration_test_scripts/trigger_dag.py index 36ba7e44f..543eb38e5 100644 --- a/dev/integration_test_scripts/trigger_dag.py +++ b/dev/integration_test_scripts/trigger_dag.py @@ -27,17 +27,20 @@ def get_access_token(api_key_id: str, api_key_secret: str) -> str: return response_json["access_token"] -def trigger_dag_runs(dag_ids: list[str], deployment_id: str, bearer_token: str) -> None: +def trigger_dag_runs( + *, dag_ids: list[str], astro_subdomain: str, deployment_id: str, bearer_token: str +) -> None: """ Triggers the DAG using Airflow REST API. + :param dag_ids: list of dag_id to trigger + :param astro_subdomain: subdomain of the Astro Cloud (e.g., https://.astronomer.run/) :param deployment_id: ID of the Astro Cloud deployment. Using this, we generate the short deployment ID needed for the construction of Airflow endpoint to hit :param bearer_token: bearer token to be used for authentication with the Airflow REST API - :param dag_ids: list of dag_id to trigger """ short_deployment_id = f"d{deployment_id[-7:]}" - integration_tests_deployment_url = f"https://e2etesting.astronomer.run/{short_deployment_id}" + integration_tests_deployment_url = f"https://{astro_subdomain}.astronomer.run/{short_deployment_id}" headers = { "Content-Type": "application/json", "Cache-Control": "no-cache", @@ -61,6 +64,7 @@ def trigger_dag_runs(dag_ids: list[str], deployment_id: str, bearer_token: str) if __name__ == "__main__": parser = argparse.ArgumentParser() + parser.add_argument("astro_subdomain", help="subdomain of the Astro Cloud", type=str) parser.add_argument("deployment_id", help="ID of the deployment in Astro Cloud", type=str) parser.add_argument("astronomer_key_id", help="Key ID of the Astro Cloud deployment", type=str) parser.add_argument("astronomer_key_secret", help="Key secret of the Astro Cloud deployment", type=str) @@ -81,4 +85,9 @@ def trigger_dag_runs(dag_ids: list[str], deployment_id: str, bearer_token: str) input_dag_ids = args.dag_ids dag_ids = [dag_id.strip() for dag_id in input_dag_ids.split(",")] - trigger_dag_runs(dag_ids, args.deployment_id, token) + trigger_dag_runs( + dag_ids=dag_ids, + astro_subdomain=args.astro_subdomain, + deployment_id=args.deployment_id, + bearer_token=token, + )