diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml new file mode 100644 index 0000000..84ed2d9 --- /dev/null +++ b/.github/workflows/main.yml @@ -0,0 +1,53 @@ +name: Test template + +on: + workflow_dispatch: + pull_request: + +jobs: + build: + runs-on: ubuntu-latest + + steps: + - name: Checkout code + uses: actions/checkout@v3 + + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: 3.10 + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install -r ci_requirements.txt + + - name: Run Copier + run: | + copier -d setup_dbt_project=True -d setup_dbt_profile=True -d setup_ci_cd=True -d setup_precommit=True + -d setup_airflow_dag=True -d dbt_project_dir="." -d dbt_project_name="balboa" -d is_new_project=True + -d airflow_profile_path="automate/dbt" -d yml_dags_path="orchestrate/dag_yml_definitions" + -d dags_path="orchestrate/dags" -a ci_answers.yml copy git@github.com:datacoves/setup_template.git ./generated/ + + # run dbt debug and dbt compile + - name: Run dbt commands + run: | + cd ./generated + dbt debug + dbt compile + + # run `dbt-coves generate airflow dags` + - name: Generate dags + run: | + dbt-coves generate airflow-dags --yml-path generated/orchestrate/dag_yml_definitions/sample_dag.yml + --dags-path generated/orchestrate/dags + + # fill Airflow DagBag with the result of generated dags + - name: Load dags + run: python test_load_dagbag.py + + # run pre-commit + - name: Run pre-commit + run: | + cd ./generated + pre-commit run --all-files diff --git a/ci_answers.yml b/ci_answers.yml new file mode 100644 index 0000000..240dbb9 --- /dev/null +++ b/ci_answers.yml @@ -0,0 +1,7 @@ +ci_provider: github +dags_path: orchestrate/dags +dbt_adapter: snowflake +use_dbt_checkpoint: true +use_sqlfluff: true +use_yamllint: true +yml_dags_path: orchestrate/dag_yml_definitions diff --git a/ci_requirements.txt b/ci_requirements.txt new file mode 100644 index 0000000..fae31ce --- /dev/null +++ b/ci_requirements.txt @@ -0,0 +1,6 @@ +# dbt-coves==1.8.0 +git+https://github.com/datacoves/dbt-coves.git@DCV-2515-dbt-coves-setup-datacoves +dbt-core==1.7.17 +dbt-snowflake==1.7.5 +pre-commit==3.7.1 +apache-airflow==2.8.4 \ No newline at end of file diff --git a/copier.yml b/copier.yml index dd41284..805bd85 100644 --- a/copier.yml +++ b/copier.yml @@ -1,3 +1,9 @@ +_exclude: + - ci_answers.yml + - .github/ + - ci_requirements.txt + - test_load_dagbag.py + ci_provider: type: str choices: diff --git a/test_load_dagbag.py b/test_load_dagbag.py new file mode 100644 index 0000000..244f7e6 --- /dev/null +++ b/test_load_dagbag.py @@ -0,0 +1,12 @@ +from airflow.models import DagBag + +dagbag = DagBag(dag_folder="generated/orchestrate/dags") + +if dagbag.import_errors: + print("The following DAGs present import errors:") + for error in dagbag.import_errors: + print(error) + print("Review the stack above for details") + + +assert len(dagbag.import_errors) == 0