-
Notifications
You must be signed in to change notification settings - Fork 1
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
* Create databricks doc page * Fix reference to host and cluster id * Update ADF docs * schedule_interval -> schedule * Remove k8 reference * Alpha order sidebar * Update datahub docs * Fix Alpha order of Notifications * Change Data Set to dataset --------- Co-authored-by: Mayra Pena <[email protected]> Co-authored-by: Noel Gomez <[email protected]>
- Loading branch information
3 people
authored
Sep 11, 2024
1 parent
621a0e7
commit fb316ed
Showing
25 changed files
with
356 additions
and
139 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -24,10 +24,8 @@ TRANSFORM_CONFIG = { | |
| spec=k8s.V1PodSpec( | ||
| containers=[ | ||
| k8s.V1Container( | ||
| name="transform", | ||
| # Replace with your image repo and tag | ||
| name="base", | ||
| image="<IMAGE REPO>:<IMAGE TAG>", | ||
| bash_command="echo SUCCESS!", | ||
| ) | ||
| ] | ||
| ) | ||
|
|
@@ -46,43 +44,41 @@ TRANSFORM_CONFIG = { | |
| schedule_interval="0 0 1 */12 *", | ||
| tags=["version_2"], | ||
| catchup=False, | ||
| yaml_sample_dag={ | ||
| "schedule_interval": "0 0 1 */12 *", | ||
| "tags": ["version_4"], | ||
| "catchup": False, | ||
| "default_args": { | ||
| "start_date": datetime.datetime(2023, 1, 1, 0, 0), | ||
| "owner": "airflow", | ||
| "email": "[email protected]", | ||
| "email_on_failure": True, | ||
| }, | ||
| }, | ||
| ) | ||
| def custommize_worker_dag(): | ||
| def yaml_teams_dag(): | ||
| transform = DatacovesBashOperator( | ||
| task_id="transform", executor_config=TRANSFORM_CONFIG | ||
| task_id="transform", | ||
| bash_command="echo SUCCESS!", | ||
| executor_config=TRANSFORM_CONFIG, | ||
| ) | ||
|
|
||
|
|
||
| dag = customize_worker_dag() | ||
| dag = yaml_teams_dag() | ||
| ``` | ||
|
|
||
| ### YAML version | ||
| In the yml dag you can configure the image. | ||
|
|
||
| ```yaml | ||
| ... | ||
|
|
||
| # DAG Tasks | ||
| nodes: | ||
| ... | ||
| transform: | ||
| description: "Sample DAG with custom image" | ||
| schedule_interval: "0 0 1 */12 *" | ||
| tags: | ||
| - version_2 | ||
| default_args: | ||
| start_date: 2023-01-01 | ||
| owner: Noel Gomez | ||
| email: [email protected] | ||
| email_on_failure: true | ||
| catchup: false | ||
|
|
||
| # DAG Tasks | ||
| nodes: | ||
| transform: | ||
| operator: operators.datacoves.bash.DatacovesBashOperator | ||
| type: task | ||
| config: | ||
| # Replace with your custom docker image <IMAGE REPO>:<IMAGE TAG> | ||
| image: <IMAGE REPO>:<IMAGE TAG> | ||
|
|
||
| bash_command: "echo SUCCESS!" | ||
| ... | ||
|
|
||
| bash_command: "echo SUCCESS!" | ||
| ``` | ||
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -12,11 +12,11 @@ In the following example, we're requesting a minimum of 8Gb of memory and 1000m | |
|
|
||
| ```python | ||
| import datetime | ||
|
|
||
| from airflow.decorators import dag | ||
| from kubernetes.client import models as k8s | ||
| from operators.datacoves.bash import DatacovesBashOperator | ||
|
|
||
| # Configuration for Kubernetes Pod Override with Resource Requests | ||
| TRANSFORM_CONFIG = { | ||
| "pod_override": k8s.V1Pod( | ||
| spec=k8s.V1PodSpec( | ||
|
|
@@ -32,7 +32,6 @@ TRANSFORM_CONFIG = { | |
| ), | ||
| } | ||
|
|
||
|
|
||
| @dag( | ||
| default_args={ | ||
| "start_date": datetime.datetime(2023, 1, 1, 0, 0), | ||
|
|
@@ -41,43 +40,41 @@ TRANSFORM_CONFIG = { | |
| "email_on_failure": True, | ||
| }, | ||
| description="Sample DAG with custom resources", | ||
| schedule_interval="0 0 1 */12 *", | ||
| schedule="0 0 1 */12 *", | ||
| tags=["version_2"], | ||
| catchup=False, | ||
| yaml_sample_dag={ | ||
| "schedule_interval": "0 0 1 */12 *", | ||
| "tags": ["version_4"], | ||
| "catchup": False, | ||
| "default_args": { | ||
| "start_date": datetime.datetime(2023, 1, 1, 0, 0), | ||
| "owner": "airflow", | ||
| "email": "[email protected]", | ||
| "email_on_failure": True, | ||
| }, | ||
| }, | ||
| ) | ||
| def request_resources_dag(): | ||
| transform = DatacovesBashOperator( | ||
| task_id="transform", executor_config=TRANSFORM_CONFIG | ||
| transform_task = DatacovesBashOperator( | ||
| task_id="transform", | ||
| executor_config=TRANSFORM_CONFIG | ||
| ) | ||
|
|
||
|
|
||
| dag = request_resources_dag() | ||
| ``` | ||
|
|
||
| ### YAML version | ||
| In the yml DAG you can configure the memory and cpu resources. | ||
|
|
||
| ```yaml | ||
| description: "Sample DAG with custom resources" | ||
| schedule_interval: "0 0 1 */12 *" | ||
| tags: | ||
| - version_2 | ||
| default_args: | ||
| start_date: 2023-01-01 | ||
| owner: Noel Gomez | ||
| email: [email protected] | ||
| email_on_failure: true | ||
| catchup: false | ||
|
|
||
| # DAG Tasks | ||
| nodes: | ||
| ... | ||
| transform: | ||
| operator: operators.datacoves.bash.DatacovesBashOperator | ||
| type: task | ||
| config: | ||
| resources: | ||
| memory: 8Gi | ||
| cpu: 1000m | ||
| ... | ||
| ``` | ||
Oops, something went wrong.