fix triggering failure due to astro_cloud subdomain change (#1222)

* ci(github-actions): fix triggering failure due to astro_cloud subdomain change
* ci(github-actions): extract common wait and trigger logic as a reusable workflow

.github/workflows/deploy-integration-tests.yaml

@@ -48,28 +48,15 @@ jobs:
       github.event_name == 'schedule' ||
       (github.event_name == 'workflow_dispatch' && inputs.dags_to_trigger_after_deployment != '')
     needs: deploy-to-providers-integration-tests
-    runs-on: 'ubuntu-20.04'
-    steps:
-      - name: Sleep for 30 minutes
-        run: |
-          sleep_time=1800
-          echo "Current timestamp is" `date`
-          echo "Sleeping for $sleep_time seconds"
-          echo "allowing the deployed image to be updated across all Airflow components.."
-          sleep 1800
-
-      - name: checkout
-        uses: actions/checkout@v3
-        with:
-          ref: ${{ inputs.git_rev }}
-
-      - name: Trigger DAG(s)
-        run: |
-          python3 dev/integration_test_scripts/trigger_dag.py \
-            ${{ secrets.PROVIDER_INTEGRATION_TESTS_DEPLOYMENT_ID }} \
-            ${{ secrets.PROVIDER_INTEGRATION_TESTS_ASTRONOMER_KEY_ID }} \
-            ${{ secrets.PROVIDER_INTEGRATION_TESTS_ASTRONOMER_KEY_SECRET }} \
-            --dag-ids "${{ inputs.dags_to_trigger_after_deployment }}"
+    uses: ./.github/workflows/trigger-dag-reuse-wf.yaml
+    with:
+      git_rev: ${{ inputs.git_rev }}
+      dags_to_trigger_after_deployment: ${{ inputs.dags_to_trigger_after_deployment }}
+    secrets:
+      astro_subdomain: ${{ secrets.ASTRO_SUBDOMAIN }}
+      deployment_id: ${{ secrets.PROVIDER_INTEGRATION_TESTS_DEPLOYMENT_ID }}
+      astronomer_key_id: ${{ secrets.PROVIDER_INTEGRATION_TESTS_ASTRONOMER_KEY_ID }}
+      astronomer_key_secret: ${{ secrets.PROVIDER_INTEGRATION_TESTS_ASTRONOMER_KEY_SECRET }}
 
   deploy-to-providers-integration-tests-on-KE:
     if: |
@@ -92,25 +79,12 @@ jobs:
       github.event_name == 'schedule' ||
       (github.event_name == 'workflow_dispatch' && inputs.dags_to_trigger_after_deployment != '')
     needs: deploy-to-providers-integration-tests-on-KE
-    runs-on: 'ubuntu-20.04'
-    steps:
-      - name: Sleep and wait for the deployment to be ready
-        run: |
-          sleep_time=1800
-          echo "Current timestamp is" `date`
-          echo "Sleeping for $sleep_time seconds"
-          echo "allowing the deployed image to be updated across all Airflow components.."
-          sleep $sleep_time
-
-      - name: checkout
-        uses: actions/checkout@v3
-        with:
-          ref: ${{ inputs.git_rev }}
-
-      - name: Trigger DAG(s)
-        run: |
-          python3 dev/integration_test_scripts/trigger_dag.py \
-            ${{ secrets.PROVIDER_INTEGRATION_TESTS_ON_KE_DEPLOYMENT_ID }} \
-            ${{ secrets.PROVIDER_INTEGRATION_TESTS_ON_KE_ASTRONOMER_KEY_ID }} \
-            ${{ secrets.PROVIDER_INTEGRATION_TESTS_ON_KE_ASTRONOMER_KEY_SECRET }} \
-            --dag-ids "${{ inputs.dags_to_trigger_after_deployment }}"
+    uses: ./.github/workflows/trigger-dag-reuse-wf.yaml
+    with:
+      git_rev: ${{ inputs.git_rev }}
+      dags_to_trigger_after_deployment: ${{ inputs.dags_to_trigger_after_deployment }}
+    secrets:
+      astro_subdomain: ${{ secrets.ASTRO_SUBDOMAIN }}
+      deployment_id: ${{ secrets.PROVIDER_INTEGRATION_TESTS_DEPLOYMENT_ID }}
+      astronomer_key_id: ${{ secrets.PROVIDER_INTEGRATION_TESTS_ASTRONOMER_KEY_ID }}
+      astronomer_key_secret: ${{ secrets.PROVIDER_INTEGRATION_TESTS_ASTRONOMER_KEY_SECRET }}

.github/workflows/test-rc-release.yaml

@@ -219,23 +219,12 @@ jobs:
     if: |
       always() &&
       needs.deploy-rc-testing-branch-to-astro-cloud.result == 'success'
-    runs-on: 'ubuntu-20.04'
-    steps:
-      - name: Sleep for 30 minutes
-        run: |
-          echo "Current timestamp is" `date`
-          echo "Sleeping for 1800 seconds (30 minutes)"
-          echo "allowing the deployed image to be updated across all Airflow components."
-          sleep 1800
-
-      - name: checkout
-        uses: actions/checkout@v3
-        with:
-          ref: ${{ needs.export-rc-testing-branch-name.outputs.rc_testing_branch }}
-
-      - name: Trigger master dag
-        run: |
-          python3 dev/integration_test_scripts/trigger_dag.py \
-            ${{ secrets.PROVIDER_INTEGRATION_TESTS_DEPLOYMENT_ID }} \
-            ${{ secrets.PROVIDER_INTEGRATION_TESTS_ASTRONOMER_KEY_ID }} \
-            ${{ secrets.PROVIDER_INTEGRATION_TESTS_ASTRONOMER_KEY_SECRET }}
+    uses: ./.github/workflows/trigger-dag-reuse-wf.yaml
+    with:
+      git_rev: ${{ needs.export-rc-testing-branch-name.outputs.rc_testing_branch }}
+      dags_to_trigger_after_deployment: "example_master_dag"
+    secrets:
+      astro_subdomain: ${{ secrets.ASTRO_SUBDOMAIN }}
+      deployment_id: ${{ secrets.PROVIDER_INTEGRATION_TESTS_DEPLOYMENT_ID }}
+      astronomer_key_id: ${{ secrets.PROVIDER_INTEGRATION_TESTS_ASTRONOMER_KEY_ID }}
+      astronomer_key_secret: ${{ secrets.PROVIDER_INTEGRATION_TESTS_ASTRONOMER_KEY_SECRET }}

.github/workflows/trigger-dag-reuse-wf.yaml

@@ -0,0 +1,61 @@
+---
+name: Wait for deployment and trigger dags
+
+on:  # yamllint disable-line rule:truthy
+  workflow_call:
+    inputs:
+      git_rev:
+        description: 'The git revision to deploy'
+        required: false
+        type: string
+        default: ''
+      wait_time:
+        description: 'seconds to wait (default 1800 seconds = 30 minutes)'
+        required: false
+        type: number
+        default: 1800
+      dags_to_trigger_after_deployment:
+        description: |
+          Comma separated list of dag_ids to trigger after deployment
+          (e.g. "example_async_adf_run_pipeline, example_async_batch")
+        required: false
+        type: string
+        default: ''
+    secrets:
+      astro_subdomain:
+        description: 'astro cloud subdomain'
+        required: true
+      deployment_id:
+        description: 'astro cloud deployment_id'
+        required: true
+      astronomer_key_id:
+        description: 'astro cloud astronomer_key_id'
+        required: true
+      astronomer_key_secret:
+        description: 'astro cloud astronomer_key_secret'
+        required: true
+
+jobs:
+  wait-for-deployment-to-be-ready-and-trigger-dag:
+    runs-on: 'ubuntu-20.04'
+    steps:
+      - name: Sleep and wait for astro cloud deployment
+        run: |
+          echo "Current timestamp is" `date`
+          echo "Sleeping for ${{ inputs.wait_time }}"
+          echo "allowing the deployed image to be updated across all Airflow components."
+          sleep ${{ inputs.wait_time }}
+
+      - name: Checkout
+        uses: actions/checkout@v3
+        with:
+          ref: ${{ inputs.git_rev }}
+
+      - name: Trigger DAG(s)
+        run: |
+          python3 dev/integration_test_scripts/trigger_dag.py \
+            ${{ secrets.astro_subdomain }} \
+            ${{ secrets.deployment_id }} \
+            ${{ secrets.astronomer_key_id }} \
+            ${{ secrets.astronomer_key_secret }} \
+            --dag-ids "${{ inputs.dags_to_trigger_after_deployment }}"

Makefile

@@ -87,7 +87,7 @@ test-rc-deps: ## Test providers RC by building an image with given dependencies
 	python3 dev/integration_test_scripts/replace_dependencies.py --issue-url '$(ISSUE_URL)'
 	echo "Building and deploying image to Astro Cloud"
 	cd ".circleci/integration-tests/" && \
-	 bash script.sh 'astro-cloud' '$(DOCKER_REGISTRY)' '$(ORGANIZATION_ID)' '$(DEPLOYMENT_ID)' '$(ASTRONOMER_KEY_ID)' '$(ASTRONOMER_KEY_SECRET)'
+	 bash script.sh 'astro-cloud' '$(ASTRO_SUBDOMAIN)' '$(DOCKER_REGISTRY)' '$(ORGANIZATION_ID)' '$(DEPLOYMENT_ID)' '$(ASTRONOMER_KEY_ID)' '$(ASTRONOMER_KEY_SECRET)'
 	$(eval current_timestamp := $(shell date))
 	echo "Current timestamp is" $(current_timestamp)
 	echo "Sleeping for 1800 seconds (30 minutes) allowing the deployed image to be updated across all Airflow components.."

dev/integration_test_scripts/trigger_dag.py

@@ -27,17 +27,20 @@ def get_access_token(api_key_id: str, api_key_secret: str) -> str:
     return response_json["access_token"]
 
 
-def trigger_dag_runs(dag_ids: list[str], deployment_id: str, bearer_token: str) -> None:
+def trigger_dag_runs(
+    *, dag_ids: list[str], astro_subdomain: str, deployment_id: str, bearer_token: str
+) -> None:
     """
     Triggers the DAG using Airflow REST API.
 
+    :param dag_ids: list of dag_id to trigger
+    :param astro_subdomain: subdomain of the Astro Cloud  (e.g., https://<subdomain>.astronomer.run/)
     :param deployment_id: ID of the Astro Cloud deployment. Using this, we generate the short deployment ID needed
         for the construction of Airflow endpoint to hit
     :param bearer_token: bearer token to be used for authentication with the Airflow REST API
-    :param dag_ids: list of dag_id to trigger
     """
     short_deployment_id = f"d{deployment_id[-7:]}"
-    integration_tests_deployment_url = f"https://e2etesting.astronomer.run/{short_deployment_id}"
+    integration_tests_deployment_url = f"https://{astro_subdomain}.astronomer.run/{short_deployment_id}"
     headers = {
         "Content-Type": "application/json",
         "Cache-Control": "no-cache",
@@ -61,6 +64,7 @@ def trigger_dag_runs(dag_ids: list[str], deployment_id: str, bearer_token: str)
 
 if __name__ == "__main__":
     parser = argparse.ArgumentParser()
+    parser.add_argument("astro_subdomain", help="subdomain of the Astro Cloud", type=str)
     parser.add_argument("deployment_id", help="ID of the deployment in Astro Cloud", type=str)
     parser.add_argument("astronomer_key_id", help="Key ID of the Astro Cloud deployment", type=str)
     parser.add_argument("astronomer_key_secret", help="Key secret of the Astro Cloud deployment", type=str)
@@ -81,4 +85,9 @@ def trigger_dag_runs(dag_ids: list[str], deployment_id: str, bearer_token: str)
     input_dag_ids = args.dag_ids
     dag_ids = [dag_id.strip() for dag_id in input_dag_ids.split(",")]
 
-    trigger_dag_runs(dag_ids, args.deployment_id, token)
+    trigger_dag_runs(
+        dag_ids=dag_ids,
+        astro_subdomain=args.astro_subdomain,
+        deployment_id=args.deployment_id,
+        bearer_token=token,
+    )