Update proj-to-bundle method to include trigger. (#130)

* Update proj-to-bundle method to include trigger. * Add Tests and run MAKE FMT, COV * updating dependencies (#129) (cherry picked from commit 372db6e) --------- Co-authored-by: suarzac <[email protected]> Co-authored-by: Ashok Singamaneni <[email protected]>
Nike-Inc · Jun 6, 2024 · 8986064 · 8986064
1 parent 372db6e
commit 8986064
Show file tree

Hide file tree

Showing 8 changed files with 33 additions and 5 deletions.
diff --git a/brickflow/cli/projects.py b/brickflow/cli/projects.py
@@ -40,7 +40,9 @@
 class BrickflowProject(BaseModel):
     name: str
     path_from_repo_root_to_project_root: str = "."  # used for mono repo
-    path_project_root_to_workflows_dir: str  # used for repos with multiple batches of workflows
+    path_project_root_to_workflows_dir: (
+        str  # used for repos with multiple batches of workflows
+    )
     deployment_mode: str = BrickflowDeployMode.BUNDLE.value
     brickflow_version: str = DEFAULT_BRICKFLOW_VERSION_MODE
     enable_plugins: bool = False

diff --git a/brickflow/codegen/databricks_bundle.py b/brickflow/codegen/databricks_bundle.py
@@ -696,6 +696,7 @@ def proj_to_bundle(self) -> DatabricksAssetBundles:
                 email_notifications=workflow.email_notifications,
                 notification_settings=workflow.notification_settings,
                 webhook_notifications=workflow.webhook_notifications,
+                trigger=workflow.trigger,
             )
             jobs[workflow_name] = job
 

diff --git a/tests/codegen/expected_bundles/dev_bundle_monorepo.yml b/tests/codegen/expected_bundles/dev_bundle_monorepo.yml
@@ -9,11 +9,15 @@ targets:
           notification_settings: null
           webhook_notifications: null
           timeout_seconds: null
-          health: 
+          health:
             rules:
               - metric: "RUN_DURATION_SECONDS"
                 op: "GREATER_THAN"
                 value: 7200.0
+          trigger:
+            pause_status: UNPAUSED
+            file_arrival:
+              url: <my_url>
           git_source:
             git_commit: a
             git_provider: github

diff --git a/tests/codegen/expected_bundles/dev_bundle_polyrepo.yml b/tests/codegen/expected_bundles/dev_bundle_polyrepo.yml
@@ -9,11 +9,15 @@ targets:
           notification_settings: null
           webhook_notifications: null
           timeout_seconds: null
-          health: 
+          health:
             rules:
               - metric: "RUN_DURATION_SECONDS"
                 op: "GREATER_THAN"
                 value: 7200.0
+          trigger:
+            pause_status: UNPAUSED
+            file_arrival:
+              url: <my_url>
           git_source:
             git_commit: a
             git_provider: github

diff --git a/tests/codegen/expected_bundles/dev_bundle_polyrepo_with_auto_libs.yml b/tests/codegen/expected_bundles/dev_bundle_polyrepo_with_auto_libs.yml
@@ -7,6 +7,7 @@ targets:
         some_wf:
           email_notifications: null
           health: null
+          trigger: null
           git_source:
             git_commit: a
             git_provider: github
@@ -76,6 +77,10 @@ targets:
               - metric: "RUN_DURATION_SECONDS"
                 op: "GREATER_THAN"
                 value: 7200.0
+          trigger:
+            pause_status: UNPAUSED
+            file_arrival:
+              url: <my_url>
           git_source:
             git_commit: a
             git_provider: github

diff --git a/tests/codegen/expected_bundles/local_bundle.yml b/tests/codegen/expected_bundles/local_bundle.yml
@@ -9,11 +9,15 @@ targets:
           notification_settings: null
           webhook_notifications: null
           timeout_seconds: null
-          health: 
+          health:
             rules:
               - metric: "RUN_DURATION_SECONDS"
                 op: "GREATER_THAN"
                 value: 7200.0
+          trigger:
+            pause_status: UNPAUSED
+            file_arrival:
+              url: <my_url>
           git_source: null
           job_clusters: []
           max_concurrent_runs: 1.0

diff --git a/tests/codegen/expected_bundles/local_bundle_prefix_suffix.yml b/tests/codegen/expected_bundles/local_bundle_prefix_suffix.yml
@@ -9,11 +9,15 @@ targets:
           notification_settings: null
           webhook_notifications: null
           timeout_seconds: null
-          health: 
+          health:
             rules:
               - metric: "RUN_DURATION_SECONDS"
                 op: "GREATER_THAN"
                 value: 7200.0
+          trigger:
+            pause_status: UNPAUSED
+            file_arrival:
+              url: <my_url>
           git_source: null
           job_clusters: []
           max_concurrent_runs: 1.0

diff --git a/tests/codegen/sample_workflow.py b/tests/codegen/sample_workflow.py
@@ -32,6 +32,10 @@
             {"metric": "RUN_DURATION_SECONDS", "op": "GREATER_THAN", "value": 7200.0}
         ]
     },  # type: ignore
+    trigger={
+        "file_arrival": {"url": "<my_url>"},
+        "pause_status": "UNPAUSED",
+    },  # type: ignore
 )