Skip to content

Commit

Permalink
Add CI/CD
Browse files Browse the repository at this point in the history
  • Loading branch information
BAntonellini committed Jul 1, 2024
1 parent bac2f45 commit 1bc92f4
Show file tree
Hide file tree
Showing 5 changed files with 84 additions and 0 deletions.
53 changes: 53 additions & 0 deletions .github/workflows/main.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,53 @@
name: Test template

on:
workflow_dispatch:
pull_request:

jobs:
build:
runs-on: ubuntu-latest

steps:
- name: Checkout code
uses: actions/checkout@v3

- name: Set up Python
uses: actions/setup-python@v4
with:
python-version: 3.10

- name: Install dependencies
run: |
python -m pip install --upgrade pip
pip install -r ci_requirements.txt
- name: Run Copier
run: |
copier -d setup_dbt_project=True -d setup_dbt_profile=True -d setup_ci_cd=True -d setup_precommit=True
-d setup_airflow_dag=True -d dbt_project_dir="." -d dbt_project_name="balboa" -d is_new_project=True
-d airflow_profile_path="automate/dbt" -d yml_dags_path="orchestrate/dag_yml_definitions"
-d dags_path="orchestrate/dags" -a ci_answers.yml copy [email protected]:datacoves/setup_template.git ./generated/
# run dbt debug and dbt compile
- name: Run dbt commands
run: |
cd ./generated
dbt debug
dbt compile
# run `dbt-coves generate airflow dags`
- name: Generate dags
run: |
dbt-coves generate airflow-dags --yml-path generated/orchestrate/dag_yml_definitions/sample_dag.yml
--dags-path generated/orchestrate/dags
# fill Airflow DagBag with the result of generated dags
- name: Load dags
run: python test_load_dagbag.py

# run pre-commit
- name: Run pre-commit
run: |
cd ./generated
pre-commit run --all-files
7 changes: 7 additions & 0 deletions ci_answers.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
ci_provider: github
dags_path: orchestrate/dags
dbt_adapter: snowflake
use_dbt_checkpoint: true
use_sqlfluff: true
use_yamllint: true
yml_dags_path: orchestrate/dag_yml_definitions
6 changes: 6 additions & 0 deletions ci_requirements.txt
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
# dbt-coves==1.8.0
git+https://github.com/datacoves/dbt-coves.git@DCV-2515-dbt-coves-setup-datacoves
dbt-core==1.7.17
dbt-snowflake==1.7.5
pre-commit==3.7.1
apache-airflow==2.8.4
6 changes: 6 additions & 0 deletions copier.yml
Original file line number Diff line number Diff line change
@@ -1,3 +1,9 @@
_exclude:
- ci_answers.yml
- .github/
- ci_requirements.txt
- test_load_dagbag.py

ci_provider:
type: str
choices:
Expand Down
12 changes: 12 additions & 0 deletions test_load_dagbag.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@
from airflow.models import DagBag

dagbag = DagBag(dag_folder="generated/orchestrate/dags")

if dagbag.import_errors:
print("The following DAGs present import errors:")
for error in dagbag.import_errors:
print(error)
print("Review the stack above for details")


assert len(dagbag.import_errors) == 0

0 comments on commit 1bc92f4

Please sign in to comment.