From 4cbc39ec3614e70ca1004d19c20c54021e00dc51 Mon Sep 17 00:00:00 2001 From: vatsrahul1001 Date: Fri, 16 Aug 2024 11:39:51 +0530 Subject: [PATCH] add databricks workflow dag to master dag --- .circleci/integration-tests/master_dag.py | 5 ++++- .github/scripts/refactor_dag.py | 8 +++++--- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/.circleci/integration-tests/master_dag.py b/.circleci/integration-tests/master_dag.py index 6563c04fe..a3d8c4577 100644 --- a/.circleci/integration-tests/master_dag.py +++ b/.circleci/integration-tests/master_dag.py @@ -310,7 +310,10 @@ def prepare_dag_dependency(task_info, execution_time): chain(*kubernetes_trigger_tasks) # Databricks DAG - databricks_task_info = [{"databricks_dag": "example_async_databricks"}, {"databricks_workflow_dag": "example_databricks_workflow"}] + databricks_task_info = [ + {"databricks_dag": "example_async_databricks"}, + {"databricks_workflow_dag": "example_databricks_workflow"}, + ] databricks_trigger_tasks, ids = prepare_dag_dependency(databricks_task_info, "{{ ds }}") dag_run_ids.extend(ids) chain(*databricks_trigger_tasks) diff --git a/.github/scripts/refactor_dag.py b/.github/scripts/refactor_dag.py index 4223c0842..6df56723b 100644 --- a/.github/scripts/refactor_dag.py +++ b/.github/scripts/refactor_dag.py @@ -2,19 +2,21 @@ import argparse + def remove_lines_after(file_path, target_line): - with open(file_path, 'r') as input_file: + with open(file_path) as input_file: lines = input_file.readlines() - with open(file_path, 'w') as output_file: + with open(file_path, "w") as output_file: for line in lines: if target_line in line: break output_file.write(line) + if __name__ == "__main__": parser = argparse.ArgumentParser() parser.add_argument("file_path", help="file path", type=str) args = parser.parse_args() target_line = "from tests.system.utils import get_test_run" - remove_lines_after(args.file_path, target_line) \ No newline at end of file + remove_lines_after(args.file_path, target_line)