Merge pull request #254 from godatadriven/dbt-probe-workflows

Adding probe workflow for dbt Core and Cloud
godatadriven · Sep 24, 2024 · 6ce98d5 · 6ce98d5
2 parents 3819123 + 912d9d0
commit 6ce98d5
Show file tree

Hide file tree

Showing 2 changed files with 99 additions and 0 deletions.
diff --git a/.github/workflows/dbt_artifact_probes.yml b/.github/workflows/dbt_artifact_probes.yml
@@ -0,0 +1,72 @@
+---
+    name: dbt Artifacts probe
+
+    on:
+      schedule:
+        - cron: '0 6 * * *'
+      workflow_dispatch:
+
+    env:
+      DBT_PROFILES_DIR: dbt_project
+      DBT_PROJECT_DIR: dbt_project
+      POETRY_VERSION: "1.8.3"
+      POETRY_VIRTUALENVS_IN_PROJECT: true
+
+    jobs:
+        dbt-cloud:
+            runs-on: ubuntu-latest
+            steps:
+                - name: Checkout
+                  uses: actions/checkout@v4
+
+                - name: Determine python version
+                  id: python-version
+                  run: |
+                      export PYTHON_VERSION=$(cat .python-version)
+                      echo "PYTHON_VERSION: $PYTHON_VERSION"
+                      echo "PYTHON_VERSION=$PYTHON_VERSION" >> $GITHUB_OUTPUT
+
+                - name: Setup Python
+                  uses: ./.github/actions/setup_python_env
+                  with:
+                    poetry-version: ${{ env.POETRY_VERSION }}
+                    python-version: ${{ steps.python-version.outputs.PYTHON_VERSION }}
+
+                - name: Trigger dbt Cloud job and download artifacts
+                  run: ./scripts/get_dbt_cloud_artifacts.sh
+                  env:
+                    DBT_ACCOUNT_ID: ${{ secrets.DBT_ACCOUNT_ID }}
+                    DBT_CLOUD_JOB_ID: ${{ secrets.DBT_CLOUD_JOB_ID }}
+                    DBT_TOKEN_VALUE: ${{ secrets.DBT_TOKEN_VALUE }}
+
+                - name: Run `dbt-bouncer`
+                  run: poetry run dbt-bouncer --config-file ./dbt-bouncer-example.yml
+
+        dbt-core:
+            runs-on: ubuntu-latest
+            steps:
+                - name: Checkout
+                  uses: actions/checkout@v4
+
+                - name: Determine python version
+                  id: python-version
+                  run: |
+                      export PYTHON_VERSION=$(cat .python-version)
+                      echo "PYTHON_VERSION: $PYTHON_VERSION"
+                      echo "PYTHON_VERSION=$PYTHON_VERSION" >> $GITHUB_OUTPUT
+
+                - name: Setup Python
+                  uses: ./.github/actions/setup_python_env
+                  with:
+                    poetry-version: ${{ env.POETRY_VERSION }}
+                    python-version: ${{ steps.python-version.outputs.PYTHON_VERSION }}
+
+                - name: Install latest dbt-core and generate dbt artifacts
+                  run: poetry run pip install dbt-core -U --pre
+
+                - name: Regenerate dbt artifacts and run `dbt-bouncer`
+                  run: |
+                    poetry run dbt deps
+                    poetry run dbt build
+                    poetry run dbt docs generate
+                    poetry run dbt-bouncer --config-file ./dbt-bouncer-example.yml
diff --git a/scripts/get_dbt_cloud_artifacts.sh b/scripts/get_dbt_cloud_artifacts.sh
@@ -0,0 +1,27 @@
+set -euo
+
+RESPONSE=$(curl -H "Authorization: Token $DBT_TOKEN_VALUE" -H "Content-Type:application/json" -d '{"cause":"Triggered by GitHub Actions"}' https://cloud.getdbt.com/api/v2/accounts/$DBT_ACCOUNT_ID/jobs/$DBT_CLOUD_JOB_ID/run/);
+STATUSCODE=$(echo "$RESPONSE" | jq '.status.code');
+
+echo "Starting dbt cloud job:";
+if [ $STATUSCODE != 200 ];
+    then echo "$RESPONSE" && bash -c "exit 1";
+fi;
+
+RUN_ID=$(echo $RESPONSE | jq '.data.id');
+echo "Run_id: $RUN_ID";
+while true;
+do
+    sleep 5;
+    WAITING=$(curl -s -G -H "Authorization:Token $DBT_TOKEN_VALUE" -H "Content-Type:application/json" "https://cloud.getdbt.com/api/v2/accounts/$DBT_ACCOUNT_ID/runs/$RUN_ID/");
+    echo "Job status: $(echo $WAITING | jq '.data.status_humanized')";
+    if ( $(echo $WAITING | jq '.data.is_complete') );
+    then
+        break;
+    fi;
+done;
+
+mkdir -p ./tmp
+curl -H "Authorization: Token $DBT_TOKEN_VALUE" https://cloud.getdbt.com/api/v2/accounts/$DBT_ACCOUNT_ID/runs/$RUN_ID/artifacts/catalog.json > './dbt_project/target/catalog.json'
+curl -H "Authorization: Token $DBT_TOKEN_VALUE" https://cloud.getdbt.com/api/v2/accounts/$DBT_ACCOUNT_ID/runs/$RUN_ID/artifacts/manifest.json > './dbt_project/target/manifest.json'
+curl -H "Authorization: Token $DBT_TOKEN_VALUE" https://cloud.getdbt.com/api/v2/accounts/$DBT_ACCOUNT_ID/runs/$RUN_ID/artifacts/run_results.json > './dbt_project/target/run_results.json'