Skip to content

Commit

Permalink
add databricks workflow dag to master dag
Browse files Browse the repository at this point in the history
  • Loading branch information
vatsrahul1001 committed Aug 16, 2024
1 parent 194d4f8 commit 4cbc39e
Show file tree
Hide file tree
Showing 2 changed files with 9 additions and 4 deletions.
5 changes: 4 additions & 1 deletion .circleci/integration-tests/master_dag.py
Original file line number Diff line number Diff line change
Expand Up @@ -310,7 +310,10 @@ def prepare_dag_dependency(task_info, execution_time):
chain(*kubernetes_trigger_tasks)

# Databricks DAG
databricks_task_info = [{"databricks_dag": "example_async_databricks"}, {"databricks_workflow_dag": "example_databricks_workflow"}]
databricks_task_info = [
{"databricks_dag": "example_async_databricks"},
{"databricks_workflow_dag": "example_databricks_workflow"},
]
databricks_trigger_tasks, ids = prepare_dag_dependency(databricks_task_info, "{{ ds }}")
dag_run_ids.extend(ids)
chain(*databricks_trigger_tasks)
Expand Down
8 changes: 5 additions & 3 deletions .github/scripts/refactor_dag.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,19 +2,21 @@

import argparse


def remove_lines_after(file_path, target_line):
with open(file_path, 'r') as input_file:
with open(file_path) as input_file:
lines = input_file.readlines()

with open(file_path, 'w') as output_file:
with open(file_path, "w") as output_file:
for line in lines:
if target_line in line:
break
output_file.write(line)


if __name__ == "__main__":
parser = argparse.ArgumentParser()
parser.add_argument("file_path", help="file path", type=str)
args = parser.parse_args()
target_line = "from tests.system.utils import get_test_run"
remove_lines_after(args.file_path, target_line)
remove_lines_after(args.file_path, target_line)

0 comments on commit 4cbc39e

Please sign in to comment.