From 8986064019a1d4e3d5fae77680d4cc4eeb84f2f4 Mon Sep 17 00:00:00 2001 From: Zachary Suarez <40529691+suarzac@users.noreply.github.com> Date: Thu, 6 Jun 2024 13:47:28 -0700 Subject: [PATCH] Update proj-to-bundle method to include trigger. (#130) * Update proj-to-bundle method to include trigger. * Add Tests and run MAKE FMT, COV * updating dependencies (#129) (cherry picked from commit 372db6e0fbc70910bb33266db320387102fcfca9) --------- Co-authored-by: suarzac Co-authored-by: Ashok Singamaneni --- brickflow/cli/projects.py | 4 +++- brickflow/codegen/databricks_bundle.py | 1 + tests/codegen/expected_bundles/dev_bundle_monorepo.yml | 6 +++++- tests/codegen/expected_bundles/dev_bundle_polyrepo.yml | 6 +++++- .../expected_bundles/dev_bundle_polyrepo_with_auto_libs.yml | 5 +++++ tests/codegen/expected_bundles/local_bundle.yml | 6 +++++- .../codegen/expected_bundles/local_bundle_prefix_suffix.yml | 6 +++++- tests/codegen/sample_workflow.py | 4 ++++ 8 files changed, 33 insertions(+), 5 deletions(-) diff --git a/brickflow/cli/projects.py b/brickflow/cli/projects.py index 52fa2c1a..0783d1f5 100644 --- a/brickflow/cli/projects.py +++ b/brickflow/cli/projects.py @@ -40,7 +40,9 @@ class BrickflowProject(BaseModel): name: str path_from_repo_root_to_project_root: str = "." # used for mono repo - path_project_root_to_workflows_dir: str # used for repos with multiple batches of workflows + path_project_root_to_workflows_dir: ( + str # used for repos with multiple batches of workflows + ) deployment_mode: str = BrickflowDeployMode.BUNDLE.value brickflow_version: str = DEFAULT_BRICKFLOW_VERSION_MODE enable_plugins: bool = False diff --git a/brickflow/codegen/databricks_bundle.py b/brickflow/codegen/databricks_bundle.py index 7220e896..5608ce06 100644 --- a/brickflow/codegen/databricks_bundle.py +++ b/brickflow/codegen/databricks_bundle.py @@ -696,6 +696,7 @@ def proj_to_bundle(self) -> DatabricksAssetBundles: email_notifications=workflow.email_notifications, notification_settings=workflow.notification_settings, webhook_notifications=workflow.webhook_notifications, + trigger=workflow.trigger, ) jobs[workflow_name] = job diff --git a/tests/codegen/expected_bundles/dev_bundle_monorepo.yml b/tests/codegen/expected_bundles/dev_bundle_monorepo.yml index a16d9895..c73bb076 100644 --- a/tests/codegen/expected_bundles/dev_bundle_monorepo.yml +++ b/tests/codegen/expected_bundles/dev_bundle_monorepo.yml @@ -9,11 +9,15 @@ targets: notification_settings: null webhook_notifications: null timeout_seconds: null - health: + health: rules: - metric: "RUN_DURATION_SECONDS" op: "GREATER_THAN" value: 7200.0 + trigger: + pause_status: UNPAUSED + file_arrival: + url: git_source: git_commit: a git_provider: github diff --git a/tests/codegen/expected_bundles/dev_bundle_polyrepo.yml b/tests/codegen/expected_bundles/dev_bundle_polyrepo.yml index debf7279..c8ca5d48 100644 --- a/tests/codegen/expected_bundles/dev_bundle_polyrepo.yml +++ b/tests/codegen/expected_bundles/dev_bundle_polyrepo.yml @@ -9,11 +9,15 @@ targets: notification_settings: null webhook_notifications: null timeout_seconds: null - health: + health: rules: - metric: "RUN_DURATION_SECONDS" op: "GREATER_THAN" value: 7200.0 + trigger: + pause_status: UNPAUSED + file_arrival: + url: git_source: git_commit: a git_provider: github diff --git a/tests/codegen/expected_bundles/dev_bundle_polyrepo_with_auto_libs.yml b/tests/codegen/expected_bundles/dev_bundle_polyrepo_with_auto_libs.yml index 9f828510..12be15ee 100644 --- a/tests/codegen/expected_bundles/dev_bundle_polyrepo_with_auto_libs.yml +++ b/tests/codegen/expected_bundles/dev_bundle_polyrepo_with_auto_libs.yml @@ -7,6 +7,7 @@ targets: some_wf: email_notifications: null health: null + trigger: null git_source: git_commit: a git_provider: github @@ -76,6 +77,10 @@ targets: - metric: "RUN_DURATION_SECONDS" op: "GREATER_THAN" value: 7200.0 + trigger: + pause_status: UNPAUSED + file_arrival: + url: git_source: git_commit: a git_provider: github diff --git a/tests/codegen/expected_bundles/local_bundle.yml b/tests/codegen/expected_bundles/local_bundle.yml index bfa72b88..617a36de 100644 --- a/tests/codegen/expected_bundles/local_bundle.yml +++ b/tests/codegen/expected_bundles/local_bundle.yml @@ -9,11 +9,15 @@ targets: notification_settings: null webhook_notifications: null timeout_seconds: null - health: + health: rules: - metric: "RUN_DURATION_SECONDS" op: "GREATER_THAN" value: 7200.0 + trigger: + pause_status: UNPAUSED + file_arrival: + url: git_source: null job_clusters: [] max_concurrent_runs: 1.0 diff --git a/tests/codegen/expected_bundles/local_bundle_prefix_suffix.yml b/tests/codegen/expected_bundles/local_bundle_prefix_suffix.yml index e3bc14d7..647c74b2 100644 --- a/tests/codegen/expected_bundles/local_bundle_prefix_suffix.yml +++ b/tests/codegen/expected_bundles/local_bundle_prefix_suffix.yml @@ -9,11 +9,15 @@ targets: notification_settings: null webhook_notifications: null timeout_seconds: null - health: + health: rules: - metric: "RUN_DURATION_SECONDS" op: "GREATER_THAN" value: 7200.0 + trigger: + pause_status: UNPAUSED + file_arrival: + url: git_source: null job_clusters: [] max_concurrent_runs: 1.0 diff --git a/tests/codegen/sample_workflow.py b/tests/codegen/sample_workflow.py index 18801863..2776873b 100644 --- a/tests/codegen/sample_workflow.py +++ b/tests/codegen/sample_workflow.py @@ -32,6 +32,10 @@ {"metric": "RUN_DURATION_SECONDS", "op": "GREATER_THAN", "value": 7200.0} ] }, # type: ignore + trigger={ + "file_arrival": {"url": ""}, + "pause_status": "UNPAUSED", + }, # type: ignore )