From 09b4c06483a1cc4a295d3f65dafe3b37d522a527 Mon Sep 17 00:00:00 2001 From: Leon Luttenberger Date: Tue, 20 Aug 2024 20:15:02 -0500 Subject: [PATCH] chore: Add linter for YAML files (#227) --- .github/ISSUE_TEMPLATE/config.yaml | 2 +- .github/workflows/check-integration-tests.yml | 2 +- .../module-checks-dev-requirements-change.yml | 4 +- .github/workflows/module-checks.yml | 2 +- .github/workflows/mwaa-requirements.check.yml | 2 +- .github/workflows/one-click-template-lint.yml | 2 +- .github/workflows/yaml-lint-check.yml | 27 ++++++++ .yamllint.yaml | 9 +++ examples/manifests/deployment.yaml | 2 +- examples/manifests/event-bus-modules.yaml | 4 +- examples/manifests/fmops-modules.yaml | 2 +- examples/manifests/networking-modules.yaml | 2 +- examples/manifests/personas-modules.yaml | 2 +- .../sagemaker-endpoints-modules.yaml | 2 +- examples/manifests/sagemaker-hugging-face.yml | 2 +- .../sagemaker-model-monitoring-modules.yaml | 2 +- ...sagemaker-model-package-group-modules.yaml | 6 +- ...odel-package-promote-pipeline-modules.yaml | 2 +- .../manifests/sagemaker-notebook-modules.yaml | 6 +- .../bedrock-finetuning-modules.yaml | 2 +- .../bedrock-finetuning-sfn/deployment.yaml | 2 +- .../networking-modules.yaml | 2 +- manifests/fmops-qna-rag/deployment.yaml | 2 +- .../fmops-qna-rag/networking-modules.yaml | 2 +- manifests/fmops-qna-rag/qna-rag-modules.yaml | 4 +- manifests/fmops-qna-rag/storage-modules.yaml | 2 +- .../deployment.yaml | 2 +- .../images-modules.yaml | 2 +- .../mlflow-modules.yaml | 2 +- .../networking-modules.yaml | 2 +- .../sagemaker-studio-modules.yaml | 6 +- .../mlops-sagemaker-multiacc/deployment.yaml | 2 +- .../kernels-modules.yaml | 2 +- .../networking-modules.yaml | 2 +- .../sagemaker-model-cicd-modules.yaml | 2 +- .../sagemaker-studio-modules.yaml | 6 +- .../sagemaker-templates-modules.yaml | 2 +- .../storage-modules.yaml | 2 +- manifests/mlops-sagemaker/deployment.yaml | 2 +- .../mlops-sagemaker/kernels-modules.yaml | 2 +- .../mlops-sagemaker/networking-modules.yaml | 2 +- .../sagemaker-studio-modules.yaml | 6 +- .../sagemaker-templates-modules.yaml | 2 +- .../mlops-sagemaker/storage-modules.yaml | 2 +- manifests/mwaa-ml-training/deployment.yaml | 2 +- .../mwaa-ml-training/mwaa-dag-modules.yaml | 2 +- manifests/mwaa-ml-training/mwaa-modules.yaml | 2 +- .../mwaa-ml-training/networking-modules.yaml | 2 +- manifests/ray-on-eks/base-modules.yaml | 2 +- manifests/ray-on-eks/core-modules.yaml | 8 +-- manifests/ray-on-eks/deployment.yaml | 2 +- manifests/ray-on-eks/images-modules.yaml | 2 +- manifests/ray-on-eks/ray-cluster-modules.yaml | 6 +- .../ray-on-eks/ray-operator-modules.yaml | 2 +- modules/eks/ray-cluster/deployspec.yaml | 14 ++-- modules/eks/ray-image/deployspec.yaml | 2 +- modules/eks/ray-image/modulestack.yaml | 2 +- modules/eks/ray-operator/deployspec.yaml | 48 +++++++------- modules/eks/ray-operator/modulestack.yaml | 4 +- modules/examples/airflow-dags/deployspec.yaml | 64 +++++++++---------- modules/examples/event-bus/deployspec.yaml | 24 +++---- modules/examples/personas/deployspec.yaml | 18 +++--- modules/fmops/qna-rag/deployspec.yaml | 2 +- modules/mlflow/mlflow-fargate/deployspec.yaml | 20 +++--- modules/mlflow/mlflow-image/deployspec.yaml | 20 +++--- .../sagemaker-endpoint/deployspec.yaml | 24 +++---- .../deployspec.yaml | 24 +++---- .../deployspec.yaml | 24 +++---- .../deployspec.yaml | 24 +++---- .../sagemaker-notebook/deployspec.yaml | 24 +++---- .../sagemaker-studio/deployspec.yaml | 2 +- .../deployspec.yaml | 20 +++--- .../build_app/.pre-commit-config.yaml | 36 +++++------ .../build_app/.pre-commit-config.yaml | 36 +++++------ one-click-launch.yaml | 2 +- seedfarmer.yaml | 2 +- 76 files changed, 324 insertions(+), 288 deletions(-) create mode 100644 .github/workflows/yaml-lint-check.yml create mode 100644 .yamllint.yaml diff --git a/.github/ISSUE_TEMPLATE/config.yaml b/.github/ISSUE_TEMPLATE/config.yaml index ec4bb386..3ba13e0c 100644 --- a/.github/ISSUE_TEMPLATE/config.yaml +++ b/.github/ISSUE_TEMPLATE/config.yaml @@ -1 +1 @@ -blank_issues_enabled: false \ No newline at end of file +blank_issues_enabled: false diff --git a/.github/workflows/check-integration-tests.yml b/.github/workflows/check-integration-tests.yml index 7df4b5cc..fecca57a 100644 --- a/.github/workflows/check-integration-tests.yml +++ b/.github/workflows/check-integration-tests.yml @@ -57,4 +57,4 @@ jobs: MODULE_PATH: ${{ matrix.modules.module_path }} with: filename: .github/ISSUE_TEMPLATE/missing_integ_tests_issue.md - update_existing: false \ No newline at end of file + update_existing: false diff --git a/.github/workflows/module-checks-dev-requirements-change.yml b/.github/workflows/module-checks-dev-requirements-change.yml index 38425c48..713b53f9 100644 --- a/.github/workflows/module-checks-dev-requirements-change.yml +++ b/.github/workflows/module-checks-dev-requirements-change.yml @@ -2,7 +2,7 @@ name: Module Checks Dev Requirements Change on: push: - branches: + branches: - "main" paths: - "requirements-dev.txt" @@ -95,4 +95,4 @@ jobs: run: cd $MODULE_PATH/ && pytest - name: NPM Test if: ${{ steps.determine-language.outputs.language == 'typescript' }} - run: cd $MODULE_PATH/ && npm test \ No newline at end of file + run: cd $MODULE_PATH/ && npm test diff --git a/.github/workflows/module-checks.yml b/.github/workflows/module-checks.yml index 77107122..444a4a1b 100644 --- a/.github/workflows/module-checks.yml +++ b/.github/workflows/module-checks.yml @@ -2,7 +2,7 @@ name: Module Checks on: push: - branches: + branches: - "main" paths: - "modules/**" diff --git a/.github/workflows/mwaa-requirements.check.yml b/.github/workflows/mwaa-requirements.check.yml index 6472a816..4ad9a75d 100644 --- a/.github/workflows/mwaa-requirements.check.yml +++ b/.github/workflows/mwaa-requirements.check.yml @@ -2,7 +2,7 @@ name: MWAA Requirements Check on: push: - branches: + branches: - "main" paths: - "data/mwaa/requirements/**" diff --git a/.github/workflows/one-click-template-lint.yml b/.github/workflows/one-click-template-lint.yml index deed270b..dc2cf250 100644 --- a/.github/workflows/one-click-template-lint.yml +++ b/.github/workflows/one-click-template-lint.yml @@ -2,7 +2,7 @@ name: Check One-Click Launch Template on: push: - branches: + branches: - "main" paths: - "one-click-launch.yaml" diff --git a/.github/workflows/yaml-lint-check.yml b/.github/workflows/yaml-lint-check.yml new file mode 100644 index 00000000..2e041557 --- /dev/null +++ b/.github/workflows/yaml-lint-check.yml @@ -0,0 +1,27 @@ +name: Check YAML Files + +on: + workflow_dispatch: + + push: + branches: + - "main" + + pull_request: + branches: + - "main" + - "release/*" + - "stable" + +jobs: + lint: + name: YAML Lint + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Install yamllint + run: pip install yamllint + + - name: Lint YAML files + run: yamllint --format github . diff --git a/.yamllint.yaml b/.yamllint.yaml new file mode 100644 index 00000000..94582f6b --- /dev/null +++ b/.yamllint.yaml @@ -0,0 +1,9 @@ +--- +extends: default + +ignore-from-file: .gitignore + +rules: + line-length: disable + document-start: disable + truthy: disable diff --git a/examples/manifests/deployment.yaml b/examples/manifests/deployment.yaml index 149a8cf9..09f751ab 100644 --- a/examples/manifests/deployment.yaml +++ b/examples/manifests/deployment.yaml @@ -33,4 +33,4 @@ targetAccountMappings: envVariable: TOOLING_ACCOUNT regionMappings: - region: us-east-1 - default: true \ No newline at end of file + default: true diff --git a/examples/manifests/event-bus-modules.yaml b/examples/manifests/event-bus-modules.yaml index ecba7fb2..d4b849d5 100644 --- a/examples/manifests/event-bus-modules.yaml +++ b/examples/manifests/event-bus-modules.yaml @@ -5,6 +5,6 @@ parameters: - name: event_bus_name value: mlops-bus - name: source_accounts - value: '["123123123123"]' # Accounts that must have permissions to put events (source accounts) + value: '["123123123123"]' # Accounts that must have permissions to put events (source accounts) - name: tags - value: '{"test": "test"}' \ No newline at end of file + value: '{"test": "test"}' diff --git a/examples/manifests/fmops-modules.yaml b/examples/manifests/fmops-modules.yaml index 39ecfb2f..b65efdcb 100644 --- a/examples/manifests/fmops-modules.yaml +++ b/examples/manifests/fmops-modules.yaml @@ -17,4 +17,4 @@ parameters: moduleMetadata: group: networking name: networking - key: PrivateSubnetIds \ No newline at end of file + key: PrivateSubnetIds diff --git a/examples/manifests/networking-modules.yaml b/examples/manifests/networking-modules.yaml index ab4090de..9acea2b7 100644 --- a/examples/manifests/networking-modules.yaml +++ b/examples/manifests/networking-modules.yaml @@ -3,4 +3,4 @@ path: git::https://github.com/awslabs/idf-modules.git//modules/network/basic-cdk targetAccount: primary parameters: - name: internet-accessible - value: True \ No newline at end of file + value: True diff --git a/examples/manifests/personas-modules.yaml b/examples/manifests/personas-modules.yaml index 05be819e..fc789086 100644 --- a/examples/manifests/personas-modules.yaml +++ b/examples/manifests/personas-modules.yaml @@ -2,4 +2,4 @@ name: personas path: git::https://github.com/awslabs/aiops-modules.git//modules/examples/personas?ref=release/1.4.0&depth=1 parameters: - name: bucket-name - value: my-bucket \ No newline at end of file + value: my-bucket diff --git a/examples/manifests/sagemaker-endpoints-modules.yaml b/examples/manifests/sagemaker-endpoints-modules.yaml index d5b2a63d..85f8c42e 100644 --- a/examples/manifests/sagemaker-endpoints-modules.yaml +++ b/examples/manifests/sagemaker-endpoints-modules.yaml @@ -38,4 +38,4 @@ parameters: - name: managed_instance_scaling value: True - name: scaling_max_instance_count - value: 10 \ No newline at end of file + value: 10 diff --git a/examples/manifests/sagemaker-hugging-face.yml b/examples/manifests/sagemaker-hugging-face.yml index c806f9d3..f3a0215c 100644 --- a/examples/manifests/sagemaker-hugging-face.yml +++ b/examples/manifests/sagemaker-hugging-face.yml @@ -21,4 +21,4 @@ parameters: moduleMetadata: group: networking name: networking - key: PrivateSubnetIds \ No newline at end of file + key: PrivateSubnetIds diff --git a/examples/manifests/sagemaker-model-monitoring-modules.yaml b/examples/manifests/sagemaker-model-monitoring-modules.yaml index 8d4c926c..cb789468 100644 --- a/examples/manifests/sagemaker-model-monitoring-modules.yaml +++ b/examples/manifests/sagemaker-model-monitoring-modules.yaml @@ -38,4 +38,4 @@ parameters: moduleMetadata: group: networking name: networking - key: PrivateSubnetIds \ No newline at end of file + key: PrivateSubnetIds diff --git a/examples/manifests/sagemaker-model-package-group-modules.yaml b/examples/manifests/sagemaker-model-package-group-modules.yaml index 742273a0..124a2947 100644 --- a/examples/manifests/sagemaker-model-package-group-modules.yaml +++ b/examples/manifests/sagemaker-model-package-group-modules.yaml @@ -11,7 +11,7 @@ parameters: name: event-bus key: EventBusArn - name: target_account_ids - value: '["444333222555"]' # Accounts that must have read-only permissions on the model pkg group + value: '["444333222555"]' # Accounts that must have read-only permissions on the model pkg group - name: sagemaker_project_id value: 123123 - name: sagemaker_project_name @@ -26,8 +26,8 @@ parameters: - name: model_package_group_description value: Test model package group module - Target - name: target_account_ids - value: '["111222333444"]' # Accounts that must have read-only permissions on the model pkg group + value: '["111222333444"]' # Accounts that must have read-only permissions on the model pkg group - name: sagemaker_project_id value: 123123 - name: sagemaker_project_name - value: test \ No newline at end of file + value: test diff --git a/examples/manifests/sagemaker-model-package-promote-pipeline-modules.yaml b/examples/manifests/sagemaker-model-package-promote-pipeline-modules.yaml index 727f85fe..2cd4eec8 100644 --- a/examples/manifests/sagemaker-model-package-promote-pipeline-modules.yaml +++ b/examples/manifests/sagemaker-model-package-promote-pipeline-modules.yaml @@ -21,4 +21,4 @@ parameters: moduleMetadata: group: registry name: target-model-package-group - key: SagemakerModelPackageGroupName \ No newline at end of file + key: SagemakerModelPackageGroupName diff --git a/examples/manifests/sagemaker-notebook-modules.yaml b/examples/manifests/sagemaker-notebook-modules.yaml index d91df920..27be9e48 100644 --- a/examples/manifests/sagemaker-notebook-modules.yaml +++ b/examples/manifests/sagemaker-notebook-modules.yaml @@ -5,15 +5,15 @@ parameters: value: dummy - name: instance_type value: ml.t2.xlarge - - name: subnet_ids # Optional parameter, you can remove it safely + - name: subnet_ids # Optional parameter, you can remove it safely valueFrom: moduleMetadata: group: networking name: networking key: PrivateSubnetIds - - name: vpc_id # Optional parameter, you can remove it safely + - name: vpc_id # Optional parameter, you can remove it safely valueFrom: moduleMetadata: group: networking name: networking - key: VpcId \ No newline at end of file + key: VpcId diff --git a/manifests/bedrock-finetuning-sfn/bedrock-finetuning-modules.yaml b/manifests/bedrock-finetuning-sfn/bedrock-finetuning-modules.yaml index dc1a31e9..fc2f157c 100644 --- a/manifests/bedrock-finetuning-sfn/bedrock-finetuning-modules.yaml +++ b/manifests/bedrock-finetuning-sfn/bedrock-finetuning-modules.yaml @@ -17,4 +17,4 @@ parameters: name: networking key: PrivateSubnetIds - name: removal-policy - value: DESTROY \ No newline at end of file + value: DESTROY diff --git a/manifests/bedrock-finetuning-sfn/deployment.yaml b/manifests/bedrock-finetuning-sfn/deployment.yaml index dc8ff40d..18c99296 100644 --- a/manifests/bedrock-finetuning-sfn/deployment.yaml +++ b/manifests/bedrock-finetuning-sfn/deployment.yaml @@ -14,4 +14,4 @@ targetAccountMappings: default: true regionMappings: - region: us-east-1 - default: true \ No newline at end of file + default: true diff --git a/manifests/bedrock-finetuning-sfn/networking-modules.yaml b/manifests/bedrock-finetuning-sfn/networking-modules.yaml index ab4090de..9acea2b7 100644 --- a/manifests/bedrock-finetuning-sfn/networking-modules.yaml +++ b/manifests/bedrock-finetuning-sfn/networking-modules.yaml @@ -3,4 +3,4 @@ path: git::https://github.com/awslabs/idf-modules.git//modules/network/basic-cdk targetAccount: primary parameters: - name: internet-accessible - value: True \ No newline at end of file + value: True diff --git a/manifests/fmops-qna-rag/deployment.yaml b/manifests/fmops-qna-rag/deployment.yaml index 16c917bc..a5699f59 100644 --- a/manifests/fmops-qna-rag/deployment.yaml +++ b/manifests/fmops-qna-rag/deployment.yaml @@ -16,4 +16,4 @@ targetAccountMappings: default: true regionMappings: - region: us-east-1 - default: true \ No newline at end of file + default: true diff --git a/manifests/fmops-qna-rag/networking-modules.yaml b/manifests/fmops-qna-rag/networking-modules.yaml index 610ff772..ea0841b2 100644 --- a/manifests/fmops-qna-rag/networking-modules.yaml +++ b/manifests/fmops-qna-rag/networking-modules.yaml @@ -2,4 +2,4 @@ name: networking path: git::https://github.com/awslabs/idf-modules.git//modules/network/basic-cdk?ref=release/1.11.0&depth=1 parameters: - name: internet-accessible - value: True \ No newline at end of file + value: True diff --git a/manifests/fmops-qna-rag/qna-rag-modules.yaml b/manifests/fmops-qna-rag/qna-rag-modules.yaml index 6fd82bd9..5578a20b 100644 --- a/manifests/fmops-qna-rag/qna-rag-modules.yaml +++ b/manifests/fmops-qna-rag/qna-rag-modules.yaml @@ -2,7 +2,7 @@ name: qna-rag path: git::https://github.com/awslabs/aiops-modules.git//modules/fmops/qna-rag?ref=release/1.4.0&depth=1 parameters: - name: cognito-pool-id - #Replace below value with valid congnito pool id + # Replace below value with valid congnito pool id value: us-east-1_XXXXX - name: os-domain-endpoint valueFrom: @@ -21,4 +21,4 @@ parameters: moduleMetadata: group: networking name: networking - key: VpcId \ No newline at end of file + key: VpcId diff --git a/manifests/fmops-qna-rag/storage-modules.yaml b/manifests/fmops-qna-rag/storage-modules.yaml index ec3908fc..b189f91e 100644 --- a/manifests/fmops-qna-rag/storage-modules.yaml +++ b/manifests/fmops-qna-rag/storage-modules.yaml @@ -16,4 +16,4 @@ parameters: moduleMetadata: group: networking name: networking - key: PrivateSubnetIds \ No newline at end of file + key: PrivateSubnetIds diff --git a/manifests/mlflow-experiments-tracking/deployment.yaml b/manifests/mlflow-experiments-tracking/deployment.yaml index 39d88c92..25aef103 100644 --- a/manifests/mlflow-experiments-tracking/deployment.yaml +++ b/manifests/mlflow-experiments-tracking/deployment.yaml @@ -20,4 +20,4 @@ targetAccountMappings: default: true regionMappings: - region: us-east-1 - default: true \ No newline at end of file + default: true diff --git a/manifests/mlflow-experiments-tracking/images-modules.yaml b/manifests/mlflow-experiments-tracking/images-modules.yaml index cce96437..63454dc8 100644 --- a/manifests/mlflow-experiments-tracking/images-modules.yaml +++ b/manifests/mlflow-experiments-tracking/images-modules.yaml @@ -7,4 +7,4 @@ parameters: moduleMetadata: group: storage name: ecr-mlflow - key: EcrRepositoryName \ No newline at end of file + key: EcrRepositoryName diff --git a/manifests/mlflow-experiments-tracking/mlflow-modules.yaml b/manifests/mlflow-experiments-tracking/mlflow-modules.yaml index 3434cdbd..e09e79fa 100644 --- a/manifests/mlflow-experiments-tracking/mlflow-modules.yaml +++ b/manifests/mlflow-experiments-tracking/mlflow-modules.yaml @@ -51,4 +51,4 @@ parameters: name: mlflow-mysql key: SecurityGroupId - name: efs-removal-policy - value: DESTROY \ No newline at end of file + value: DESTROY diff --git a/manifests/mlflow-experiments-tracking/networking-modules.yaml b/manifests/mlflow-experiments-tracking/networking-modules.yaml index ab4090de..9acea2b7 100644 --- a/manifests/mlflow-experiments-tracking/networking-modules.yaml +++ b/manifests/mlflow-experiments-tracking/networking-modules.yaml @@ -3,4 +3,4 @@ path: git::https://github.com/awslabs/idf-modules.git//modules/network/basic-cdk targetAccount: primary parameters: - name: internet-accessible - value: True \ No newline at end of file + value: True diff --git a/manifests/mlflow-experiments-tracking/sagemaker-studio-modules.yaml b/manifests/mlflow-experiments-tracking/sagemaker-studio-modules.yaml index 85fea57e..2de6c876 100644 --- a/manifests/mlflow-experiments-tracking/sagemaker-studio-modules.yaml +++ b/manifests/mlflow-experiments-tracking/sagemaker-studio-modules.yaml @@ -16,10 +16,10 @@ parameters: key: PrivateSubnetIds - name: data_science_users value: - - ds-user-1 + - ds-user-1 - name: lead_data_science_users value: - - lead-ds-user-1 + - lead-ds-user-1 - name: server_lifecycle_name value: studio-auto-shutdown - name: studio_bucket_name @@ -27,4 +27,4 @@ parameters: - name: retain_efs value: 'False' - name: enable_custom_sagemaker_projects - value: 'True' \ No newline at end of file + value: 'True' diff --git a/manifests/mlops-sagemaker-multiacc/deployment.yaml b/manifests/mlops-sagemaker-multiacc/deployment.yaml index 5c06bd77..1a3e1840 100644 --- a/manifests/mlops-sagemaker-multiacc/deployment.yaml +++ b/manifests/mlops-sagemaker-multiacc/deployment.yaml @@ -43,4 +43,4 @@ targetAccountMappings: envVariable: PROD_ACCOUNT regionMappings: - region: us-east-1 - default: true \ No newline at end of file + default: true diff --git a/manifests/mlops-sagemaker-multiacc/kernels-modules.yaml b/manifests/mlops-sagemaker-multiacc/kernels-modules.yaml index 2fc3c496..1f68ae27 100644 --- a/manifests/mlops-sagemaker-multiacc/kernels-modules.yaml +++ b/manifests/mlops-sagemaker-multiacc/kernels-modules.yaml @@ -37,4 +37,4 @@ parameters: - name: kernel-user-gid value: '0' - name: kernel-user-home-mount-path - value: /root \ No newline at end of file + value: /root diff --git a/manifests/mlops-sagemaker-multiacc/networking-modules.yaml b/manifests/mlops-sagemaker-multiacc/networking-modules.yaml index 188c90ab..a08b05ba 100644 --- a/manifests/mlops-sagemaker-multiacc/networking-modules.yaml +++ b/manifests/mlops-sagemaker-multiacc/networking-modules.yaml @@ -17,4 +17,4 @@ path: git::https://github.com/awslabs/idf-modules.git//modules/network/basic-cdk targetAccount: prod parameters: - name: internet-accessible - value: True \ No newline at end of file + value: True diff --git a/manifests/mlops-sagemaker-multiacc/sagemaker-model-cicd-modules.yaml b/manifests/mlops-sagemaker-multiacc/sagemaker-model-cicd-modules.yaml index b01ae7bc..5fabfe9d 100644 --- a/manifests/mlops-sagemaker-multiacc/sagemaker-model-cicd-modules.yaml +++ b/manifests/mlops-sagemaker-multiacc/sagemaker-model-cicd-modules.yaml @@ -1,4 +1,4 @@ -name: model # replace with name of the ML model you prefer +name: model # replace with name of the ML model you prefer path: git::https://github.com/awslabs/aiops-modules.git//modules/sagemaker/sagemaker-model-cicd?ref=release/1.4.0&depth=1 targetAccount: tooling parameters: diff --git a/manifests/mlops-sagemaker-multiacc/sagemaker-studio-modules.yaml b/manifests/mlops-sagemaker-multiacc/sagemaker-studio-modules.yaml index f76be6b6..3090a5be 100644 --- a/manifests/mlops-sagemaker-multiacc/sagemaker-studio-modules.yaml +++ b/manifests/mlops-sagemaker-multiacc/sagemaker-studio-modules.yaml @@ -16,10 +16,10 @@ parameters: key: PrivateSubnetIds - name: data_science_users value: - - ds-user-1 + - ds-user-1 - name: lead_data_science_users value: - - lead-ds-user-1 + - lead-ds-user-1 - name: server_lifecycle_name value: studio-auto-shutdown - name: studio_bucket_name @@ -27,4 +27,4 @@ parameters: - name: retain_efs value: 'False' - name: enable_custom_sagemaker_projects - value: 'True' \ No newline at end of file + value: 'True' diff --git a/manifests/mlops-sagemaker-multiacc/sagemaker-templates-modules.yaml b/manifests/mlops-sagemaker-multiacc/sagemaker-templates-modules.yaml index ab203626..9d2409ef 100644 --- a/manifests/mlops-sagemaker-multiacc/sagemaker-templates-modules.yaml +++ b/manifests/mlops-sagemaker-multiacc/sagemaker-templates-modules.yaml @@ -61,4 +61,4 @@ parameters: moduleMetadata: group: networking name: networking-prod - key: PrivateSubnetIds \ No newline at end of file + key: PrivateSubnetIds diff --git a/manifests/mlops-sagemaker-multiacc/storage-modules.yaml b/manifests/mlops-sagemaker-multiacc/storage-modules.yaml index 82c63255..8c4c1463 100644 --- a/manifests/mlops-sagemaker-multiacc/storage-modules.yaml +++ b/manifests/mlops-sagemaker-multiacc/storage-modules.yaml @@ -18,4 +18,4 @@ parameters: - name: encryption-type value: KMS - name: retention-type - value: DESTROY \ No newline at end of file + value: DESTROY diff --git a/manifests/mlops-sagemaker/deployment.yaml b/manifests/mlops-sagemaker/deployment.yaml index 5b7d97da..b2553b24 100644 --- a/manifests/mlops-sagemaker/deployment.yaml +++ b/manifests/mlops-sagemaker/deployment.yaml @@ -20,4 +20,4 @@ targetAccountMappings: default: true regionMappings: - region: us-east-1 - default: true \ No newline at end of file + default: true diff --git a/manifests/mlops-sagemaker/kernels-modules.yaml b/manifests/mlops-sagemaker/kernels-modules.yaml index 1a257fa9..a35b5b24 100644 --- a/manifests/mlops-sagemaker/kernels-modules.yaml +++ b/manifests/mlops-sagemaker/kernels-modules.yaml @@ -37,4 +37,4 @@ parameters: - name: kernel-user-gid value: '0' - name: kernel-user-home-mount-path - value: /root \ No newline at end of file + value: /root diff --git a/manifests/mlops-sagemaker/networking-modules.yaml b/manifests/mlops-sagemaker/networking-modules.yaml index ab4090de..9acea2b7 100644 --- a/manifests/mlops-sagemaker/networking-modules.yaml +++ b/manifests/mlops-sagemaker/networking-modules.yaml @@ -3,4 +3,4 @@ path: git::https://github.com/awslabs/idf-modules.git//modules/network/basic-cdk targetAccount: primary parameters: - name: internet-accessible - value: True \ No newline at end of file + value: True diff --git a/manifests/mlops-sagemaker/sagemaker-studio-modules.yaml b/manifests/mlops-sagemaker/sagemaker-studio-modules.yaml index 85fea57e..2de6c876 100644 --- a/manifests/mlops-sagemaker/sagemaker-studio-modules.yaml +++ b/manifests/mlops-sagemaker/sagemaker-studio-modules.yaml @@ -16,10 +16,10 @@ parameters: key: PrivateSubnetIds - name: data_science_users value: - - ds-user-1 + - ds-user-1 - name: lead_data_science_users value: - - lead-ds-user-1 + - lead-ds-user-1 - name: server_lifecycle_name value: studio-auto-shutdown - name: studio_bucket_name @@ -27,4 +27,4 @@ parameters: - name: retain_efs value: 'False' - name: enable_custom_sagemaker_projects - value: 'True' \ No newline at end of file + value: 'True' diff --git a/manifests/mlops-sagemaker/sagemaker-templates-modules.yaml b/manifests/mlops-sagemaker/sagemaker-templates-modules.yaml index fdc91e96..5801f19b 100644 --- a/manifests/mlops-sagemaker/sagemaker-templates-modules.yaml +++ b/manifests/mlops-sagemaker/sagemaker-templates-modules.yaml @@ -73,4 +73,4 @@ parameters: moduleMetadata: group: sagemaker-studio name: studio - key: StudioDomainArn \ No newline at end of file + key: StudioDomainArn diff --git a/manifests/mlops-sagemaker/storage-modules.yaml b/manifests/mlops-sagemaker/storage-modules.yaml index 6e853383..dd367740 100644 --- a/manifests/mlops-sagemaker/storage-modules.yaml +++ b/manifests/mlops-sagemaker/storage-modules.yaml @@ -18,4 +18,4 @@ parameters: - name: encryption-type value: SSE - name: retention-type - value: DESTROY \ No newline at end of file + value: DESTROY diff --git a/manifests/mwaa-ml-training/deployment.yaml b/manifests/mwaa-ml-training/deployment.yaml index 919edd3d..1b4852ef 100644 --- a/manifests/mwaa-ml-training/deployment.yaml +++ b/manifests/mwaa-ml-training/deployment.yaml @@ -16,4 +16,4 @@ targetAccountMappings: default: true regionMappings: - region: us-east-1 - default: true \ No newline at end of file + default: true diff --git a/manifests/mwaa-ml-training/mwaa-dag-modules.yaml b/manifests/mwaa-ml-training/mwaa-dag-modules.yaml index cef026e6..d910edff 100644 --- a/manifests/mwaa-ml-training/mwaa-dag-modules.yaml +++ b/manifests/mwaa-ml-training/mwaa-dag-modules.yaml @@ -25,4 +25,4 @@ parameters: # moduleMetadata: # group: optionals # name: datalake-buckets - # key: FullAccessPolicyArn \ No newline at end of file + # key: FullAccessPolicyArn diff --git a/manifests/mwaa-ml-training/mwaa-modules.yaml b/manifests/mwaa-ml-training/mwaa-modules.yaml index c2231691..d5a6057d 100644 --- a/manifests/mwaa-ml-training/mwaa-modules.yaml +++ b/manifests/mwaa-ml-training/mwaa-modules.yaml @@ -19,4 +19,4 @@ parameters: - name: airflow-version value: "2.9.2" - name: custom-requirements-path - value: data/mwaa/requirements/requirements.txt \ No newline at end of file + value: data/mwaa/requirements/requirements.txt diff --git a/manifests/mwaa-ml-training/networking-modules.yaml b/manifests/mwaa-ml-training/networking-modules.yaml index ab4090de..9acea2b7 100644 --- a/manifests/mwaa-ml-training/networking-modules.yaml +++ b/manifests/mwaa-ml-training/networking-modules.yaml @@ -3,4 +3,4 @@ path: git::https://github.com/awslabs/idf-modules.git//modules/network/basic-cdk targetAccount: primary parameters: - name: internet-accessible - value: True \ No newline at end of file + value: True diff --git a/manifests/ray-on-eks/base-modules.yaml b/manifests/ray-on-eks/base-modules.yaml index 35d9b71a..f9f2e8f5 100644 --- a/manifests/ray-on-eks/base-modules.yaml +++ b/manifests/ray-on-eks/base-modules.yaml @@ -23,4 +23,4 @@ parameters: - name: Encryption value: KMS_MANAGED - name: RemovalPolicy - value: DESTROY \ No newline at end of file + value: DESTROY diff --git a/manifests/ray-on-eks/core-modules.yaml b/manifests/ray-on-eks/core-modules.yaml index 14b94b20..09a9410b 100644 --- a/manifests/ray-on-eks/core-modules.yaml +++ b/manifests/ray-on-eks/core-modules.yaml @@ -65,7 +65,7 @@ parameters: - name: EksAddons value: # Autoscaling - deploy_cluster_autoscaler: True + deploy_cluster_autoscaler: True deploy_metrics_server: True # Observability deploy_cloudwatch_observability_addon: True @@ -103,10 +103,10 @@ parameters: - name: DraImportPath valueFrom: parameterValue: draImportPath - - name: FsxVersion - value : "2.15" + - name: FsxVersion + value: "2.15" - name: Namespace valueFrom: parameterValue: rayNamespaceName - name: ImportPolicy - value: "NEW_CHANGED_DELETED" \ No newline at end of file + value: "NEW_CHANGED_DELETED" diff --git a/manifests/ray-on-eks/deployment.yaml b/manifests/ray-on-eks/deployment.yaml index 1b33c1c9..6a7d3b61 100644 --- a/manifests/ray-on-eks/deployment.yaml +++ b/manifests/ray-on-eks/deployment.yaml @@ -27,4 +27,4 @@ targetAccountMappings: draExportPath: /ray/export/ regionMappings: - region: us-east-1 - default: true \ No newline at end of file + default: true diff --git a/manifests/ray-on-eks/images-modules.yaml b/manifests/ray-on-eks/images-modules.yaml index a626936d..53930232 100644 --- a/manifests/ray-on-eks/images-modules.yaml +++ b/manifests/ray-on-eks/images-modules.yaml @@ -7,4 +7,4 @@ parameters: moduleMetadata: group: base name: ray-ecr - key: EcrRepositoryName \ No newline at end of file + key: EcrRepositoryName diff --git a/manifests/ray-on-eks/ray-cluster-modules.yaml b/manifests/ray-on-eks/ray-cluster-modules.yaml index 3a44b577..9706e586 100644 --- a/manifests/ray-on-eks/ray-cluster-modules.yaml +++ b/manifests/ray-on-eks/ray-cluster-modules.yaml @@ -63,13 +63,13 @@ parameters: name: ray key: ImageUri - name: WorkerTolerations - value: # make sure to match w/ the taints on the GPU Nodegroup + value: # make sure to match w/ the taints on the GPU Nodegroup - key: "nvidia.com/gpu" value: "true" # operator: "Equal" effect: "NoSchedule" - name: WorkerLabels - value: # make sure to match w/ the labels on the GPU Nodegroup + value: # make sure to match w/ the labels on the GPU Nodegroup usage: gpu - name: PvcName valueFrom: @@ -79,4 +79,4 @@ parameters: key: PersistentVolumeClaimName - name: DraExportPath valueFrom: - parameterValue: draExportPath \ No newline at end of file + parameterValue: draExportPath diff --git a/manifests/ray-on-eks/ray-operator-modules.yaml b/manifests/ray-on-eks/ray-operator-modules.yaml index 0ed5ca79..f28e7662 100644 --- a/manifests/ray-on-eks/ray-operator-modules.yaml +++ b/manifests/ray-on-eks/ray-operator-modules.yaml @@ -57,4 +57,4 @@ parameters: moduleMetadata: group: base name: buckets - key: ArtifactsBucketName \ No newline at end of file + key: ArtifactsBucketName diff --git a/modules/eks/ray-cluster/deployspec.yaml b/modules/eks/ray-cluster/deployspec.yaml index 6b60ee21..cd899a3f 100644 --- a/modules/eks/ray-cluster/deployspec.yaml +++ b/modules/eks/ray-cluster/deployspec.yaml @@ -3,18 +3,18 @@ deploy: phases: install: commands: - - npm install -g aws-cdk@2.145.0 - - pip install -r requirements.txt + - npm install -g aws-cdk@2.145.0 + - pip install -r requirements.txt build: commands: - - cdk deploy --all --require-approval never --progress events --app "python app.py" --outputs-file ./cdk-exports.json - - seedfarmer metadata convert -f cdk-exports.json || true + - cdk deploy --all --require-approval never --progress events --app "python app.py" --outputs-file ./cdk-exports.json + - seedfarmer metadata convert -f cdk-exports.json || true destroy: phases: install: commands: - - npm install -g aws-cdk@2.145.0 - - pip install -r requirements.txt + - npm install -g aws-cdk@2.145.0 + - pip install -r requirements.txt build: commands: - - cdk destroy --force --all --app "python app.py" + - cdk destroy --force --all --app "python app.py" diff --git a/modules/eks/ray-image/deployspec.yaml b/modules/eks/ray-image/deployspec.yaml index 3fe6098e..8059b40a 100644 --- a/modules/eks/ray-image/deployspec.yaml +++ b/modules/eks/ray-image/deployspec.yaml @@ -18,4 +18,4 @@ destroy: build: commands: - aws ecr delete-repository --repository-name ${SEEDFARMER_PARAMETER_ECR_REPO_NAME} --force -# build_type: BUILD_GENERAL1_LARGE \ No newline at end of file +# build_type: BUILD_GENERAL1_LARGE diff --git a/modules/eks/ray-image/modulestack.yaml b/modules/eks/ray-image/modulestack.yaml index d19cf57f..be36fdfc 100644 --- a/modules/eks/ray-image/modulestack.yaml +++ b/modules/eks/ray-image/modulestack.yaml @@ -39,4 +39,4 @@ Resources: - !Sub "arn:${AWS::Partition}:ecr:${AWS::Region}:${AWS::AccountId}:repository/${ECRRepoName}" Version: 2012-10-17 PolicyName: "modulespecific-policy" - Roles: [!Ref RoleName] \ No newline at end of file + Roles: [!Ref RoleName] diff --git a/modules/eks/ray-operator/deployspec.yaml b/modules/eks/ray-operator/deployspec.yaml index c55146e4..08efd345 100644 --- a/modules/eks/ray-operator/deployspec.yaml +++ b/modules/eks/ray-operator/deployspec.yaml @@ -3,35 +3,35 @@ deploy: phases: install: commands: - - npm install -g aws-cdk@2.145.0 - - pip install -r requirements.txt + - npm install -g aws-cdk@2.145.0 + - pip install -r requirements.txt build: commands: - - cdk deploy --all --require-approval never --progress events --app "python app.py" --outputs-file ./cdk-exports.json - - seedfarmer metadata convert -f cdk-exports.json || true - - export EKS_SERVICE_ACCOUNT_ROLE=$(cat SEEDFARMER_MODULE_METADATA | jq -r ".EksServiceAccountRoleArn") - # Assume the RAY_CLUSTER_ADMIN_ROLE and add our new service account role as a user in the aws_auth ConfigMap - - eval $(aws sts assume-role --role-arn ${SEEDFARMER_PARAMETER_EKS_CLUSTER_ADMIN_ROLE_ARN} --role-session-name test | jq -r '.Credentials | "export AWS_ACCESS_KEY_ID=\(.AccessKeyId)\nexport AWS_SECRET_ACCESS_KEY=\(.SecretAccessKey)\nexport AWS_SESSION_TOKEN=\(.SessionToken)\n"') - - | - eksctl get iamidentitymapping --cluster ${SEEDFARMER_PARAMETER_EKS_CLUSTER_NAME} --arn ${EKS_SERVICE_ACCOUNT_ROLE} \ - && echo "IAM Identity Mapping already found" \ - || eksctl create iamidentitymapping --cluster ${SEEDFARMER_PARAMETER_EKS_CLUSTER_NAME} --arn ${EKS_SERVICE_ACCOUNT_ROLE} --username ${SEEDFARMER_MODULE_NAME} - # Un-Assume the EKS_CLUSTER_ADMIN_ROLE or the rest of the deploy will fail - - unset AWS_ACCESS_KEY_ID && unset AWS_SECRET_ACCESS_KEY && unset AWS_SESSION_TOKEN + - cdk deploy --all --require-approval never --progress events --app "python app.py" --outputs-file ./cdk-exports.json + - seedfarmer metadata convert -f cdk-exports.json || true + - export EKS_SERVICE_ACCOUNT_ROLE=$(cat SEEDFARMER_MODULE_METADATA | jq -r ".EksServiceAccountRoleArn") + # Assume the RAY_CLUSTER_ADMIN_ROLE and add our new service account role as a user in the aws_auth ConfigMap + - eval $(aws sts assume-role --role-arn ${SEEDFARMER_PARAMETER_EKS_CLUSTER_ADMIN_ROLE_ARN} --role-session-name test | jq -r '.Credentials | "export AWS_ACCESS_KEY_ID=\(.AccessKeyId)\nexport AWS_SECRET_ACCESS_KEY=\(.SecretAccessKey)\nexport AWS_SESSION_TOKEN=\(.SessionToken)\n"') + - | + eksctl get iamidentitymapping --cluster ${SEEDFARMER_PARAMETER_EKS_CLUSTER_NAME} --arn ${EKS_SERVICE_ACCOUNT_ROLE} \ + && echo "IAM Identity Mapping already found" \ + || eksctl create iamidentitymapping --cluster ${SEEDFARMER_PARAMETER_EKS_CLUSTER_NAME} --arn ${EKS_SERVICE_ACCOUNT_ROLE} --username ${SEEDFARMER_MODULE_NAME} + # Un-Assume the EKS_CLUSTER_ADMIN_ROLE or the rest of the deploy will fail + - unset AWS_ACCESS_KEY_ID && unset AWS_SECRET_ACCESS_KEY && unset AWS_SESSION_TOKEN destroy: phases: install: commands: - - npm install -g aws-cdk@2.145.0 - - pip install -r requirements.txt + - npm install -g aws-cdk@2.145.0 + - pip install -r requirements.txt build: commands: - # Assume the EKS_CLUSTER_ADMIN_ROLE and delete our service account role from the aws_auth ConfigMap - - eval $(aws sts assume-role --role-arn ${SEEDFARMER_PARAMETER_EKS_CLUSTER_ADMIN_ROLE_ARN} --role-session-name test | jq -r '.Credentials | "export AWS_ACCESS_KEY_ID=\(.AccessKeyId)\nexport AWS_SECRET_ACCESS_KEY=\(.SecretAccessKey)\nexport AWS_SESSION_TOKEN=\(.SessionToken)\n"') - - | - eksctl get iamidentitymapping --cluster ${SEEDFARMER_PARAMETER_EKS_CLUSTER_NAME} --arn ${EKS_SERVICE_ACCOUNT_ROLE} \ - && eksctl delete iamidentitymapping --cluster ${SEEDFARMER_PARAMETER_EKS_CLUSTER_NAME} --arn ${EKS_SERVICE_ACCOUNT_ROLE} \ - || echo "IAM Identity Mapping not found" - # Un-Assume the EKS_CLUSTER_ADMIN_ROLE or the rest of the deploy will fail - - unset AWS_ACCESS_KEY_ID && unset AWS_SECRET_ACCESS_KEY && unset AWS_SESSION_TOKEN - - cdk destroy --force --all --app "python app.py" + # Assume the EKS_CLUSTER_ADMIN_ROLE and delete our service account role from the aws_auth ConfigMap + - eval $(aws sts assume-role --role-arn ${SEEDFARMER_PARAMETER_EKS_CLUSTER_ADMIN_ROLE_ARN} --role-session-name test | jq -r '.Credentials | "export AWS_ACCESS_KEY_ID=\(.AccessKeyId)\nexport AWS_SECRET_ACCESS_KEY=\(.SecretAccessKey)\nexport AWS_SESSION_TOKEN=\(.SessionToken)\n"') + - | + eksctl get iamidentitymapping --cluster ${SEEDFARMER_PARAMETER_EKS_CLUSTER_NAME} --arn ${EKS_SERVICE_ACCOUNT_ROLE} \ + && eksctl delete iamidentitymapping --cluster ${SEEDFARMER_PARAMETER_EKS_CLUSTER_NAME} --arn ${EKS_SERVICE_ACCOUNT_ROLE} \ + || echo "IAM Identity Mapping not found" + # Un-Assume the EKS_CLUSTER_ADMIN_ROLE or the rest of the deploy will fail + - unset AWS_ACCESS_KEY_ID && unset AWS_SECRET_ACCESS_KEY && unset AWS_SESSION_TOKEN + - cdk destroy --force --all --app "python app.py" diff --git a/modules/eks/ray-operator/modulestack.yaml b/modules/eks/ray-operator/modulestack.yaml index d6a4ce8e..aa272b57 100644 --- a/modules/eks/ray-operator/modulestack.yaml +++ b/modules/eks/ray-operator/modulestack.yaml @@ -58,8 +58,8 @@ Resources: - "logs:CreateLogGroup" - "logs:Describe*" - "logs:DescribeLogGroups" - Resource: - - !Sub "arn:${AWS::Partition}:logs::${AWS::AccountId}:log-group/*" + Resource: + - !Sub "arn:${AWS::Partition}:logs::${AWS::AccountId}:log-group/*" - Effect: Allow Action: - "iam:Get*" diff --git a/modules/examples/airflow-dags/deployspec.yaml b/modules/examples/airflow-dags/deployspec.yaml index 9fea97a3..8f06528c 100644 --- a/modules/examples/airflow-dags/deployspec.yaml +++ b/modules/examples/airflow-dags/deployspec.yaml @@ -3,43 +3,43 @@ deploy: phases: install: commands: - - npm install -g aws-cdk@2.137.0 - - pip install -r requirements.txt + - npm install -g aws-cdk@2.137.0 + - pip install -r requirements.txt build: commands: - - > - echo "SEEDFARMER_MODULE_METADATA: ${SEEDFARMER_MODULE_METADATA}" - - > - echo "SEEDFARMER_PARAMETER_DAG_BUCKET_NAME: ${SEEDFARMER_PARAMETER_DAG_BUCKET_NAME}" - - > - echo "SEEDFARMER_PARAMETER_DAG_PATH: ${SEEDFARMER_PARAMETER_DAG_PATH}" - - > - echo "SEEDFARMER_PARAMETER_MWAA_EXEC_ROLE_ARN: ${SEEDFARMER_PARAMETER_MWAA_EXEC_ROLE_ARN}" - - cdk deploy --require-approval never --progress events --app "python app.py" --outputs-file ./cdk-exports.json - - seedfarmer metadata convert -f cdk-exports.json || true - - export MLOPS_BUCKET=$(cat SEEDFARMER_MODULE_METADATA | jq -r ".MlOpsBucket") - - export DAG_IAM_ROLE=$(cat SEEDFARMER_MODULE_METADATA | jq -r ".DagRoleArn") - - export SAGEMAKER_IAM_ROLE=$(cat SEEDFARMER_MODULE_METADATA | jq -r ".SageMakerExecutionRole") - - > - echo "MLOPS_BUCKET: ${MLOPS_BUCKET}" - - > - echo "DAG_IAM_ROLE: ${DAG_IAM_ROLE}" - - > - echo "SAGEMAKER_IAM_ROLE: ${SAGEMAKER_IAM_ROLE}" - - sed -i "s/MLOPS_S3_BUCKET/${MLOPS_BUCKET}/g" dags/config.py - - sed -i "s/AWS_REGION_NAME/${AWS_DEFAULT_REGION}/g" dags/config.py - - sed -i "s~DAG_IAM_ROLE~${DAG_IAM_ROLE}~g" dags/config.py - - sed -i "s~SAGEMAKER_IAM_ROLE~${SAGEMAKER_IAM_ROLE}~g" dags/config.py - - aws s3 cp --recursive dags/ s3://$SEEDFARMER_PARAMETER_DAG_BUCKET_NAME/$SEEDFARMER_PARAMETER_DAG_PATH/mlops/ + - > + echo "SEEDFARMER_MODULE_METADATA: ${SEEDFARMER_MODULE_METADATA}" + - > + echo "SEEDFARMER_PARAMETER_DAG_BUCKET_NAME: ${SEEDFARMER_PARAMETER_DAG_BUCKET_NAME}" + - > + echo "SEEDFARMER_PARAMETER_DAG_PATH: ${SEEDFARMER_PARAMETER_DAG_PATH}" + - > + echo "SEEDFARMER_PARAMETER_MWAA_EXEC_ROLE_ARN: ${SEEDFARMER_PARAMETER_MWAA_EXEC_ROLE_ARN}" + - cdk deploy --require-approval never --progress events --app "python app.py" --outputs-file ./cdk-exports.json + - seedfarmer metadata convert -f cdk-exports.json || true + - export MLOPS_BUCKET=$(cat SEEDFARMER_MODULE_METADATA | jq -r ".MlOpsBucket") + - export DAG_IAM_ROLE=$(cat SEEDFARMER_MODULE_METADATA | jq -r ".DagRoleArn") + - export SAGEMAKER_IAM_ROLE=$(cat SEEDFARMER_MODULE_METADATA | jq -r ".SageMakerExecutionRole") + - > + echo "MLOPS_BUCKET: ${MLOPS_BUCKET}" + - > + echo "DAG_IAM_ROLE: ${DAG_IAM_ROLE}" + - > + echo "SAGEMAKER_IAM_ROLE: ${SAGEMAKER_IAM_ROLE}" + - sed -i "s/MLOPS_S3_BUCKET/${MLOPS_BUCKET}/g" dags/config.py + - sed -i "s/AWS_REGION_NAME/${AWS_DEFAULT_REGION}/g" dags/config.py + - sed -i "s~DAG_IAM_ROLE~${DAG_IAM_ROLE}~g" dags/config.py + - sed -i "s~SAGEMAKER_IAM_ROLE~${SAGEMAKER_IAM_ROLE}~g" dags/config.py + - aws s3 cp --recursive dags/ s3://$SEEDFARMER_PARAMETER_DAG_BUCKET_NAME/$SEEDFARMER_PARAMETER_DAG_PATH/mlops/ destroy: phases: install: commands: - - npm install -g aws-cdk@2.137.0 - - pip install -r requirements.txt + - npm install -g aws-cdk@2.137.0 + - pip install -r requirements.txt build: commands: - - export MLOPS_BUCKET=$(echo ${SEEDFARMER_MODULE_METADATA} | jq -r ".MlOpsBucket") - - aws s3 rm --recursive s3://$SEEDFARMER_PARAMETER_DAG_BUCKET_NAME/$SEEDFARMER_PARAMETER_DAG_PATH/mlops/ - - aws s3 rm --recursive s3://$MLOPS_BUCKET/ - - cdk destroy --force --app "python app.py" + - export MLOPS_BUCKET=$(echo ${SEEDFARMER_MODULE_METADATA} | jq -r ".MlOpsBucket") + - aws s3 rm --recursive s3://$SEEDFARMER_PARAMETER_DAG_BUCKET_NAME/$SEEDFARMER_PARAMETER_DAG_PATH/mlops/ + - aws s3 rm --recursive s3://$MLOPS_BUCKET/ + - cdk destroy --force --app "python app.py" diff --git a/modules/examples/event-bus/deployspec.yaml b/modules/examples/event-bus/deployspec.yaml index c0f572cc..b2e83736 100644 --- a/modules/examples/event-bus/deployspec.yaml +++ b/modules/examples/event-bus/deployspec.yaml @@ -3,24 +3,24 @@ deploy: phases: install: commands: - - env - # Install whatever additional build libraries - - npm install -g aws-cdk@2.126.0 - - pip install -r requirements.txt + - env + # Install whatever additional build libraries + - npm install -g aws-cdk@2.126.0 + - pip install -r requirements.txt build: commands: # execute the CDK - - cdk deploy --require-approval never --progress events --app "python app.py" --outputs-file ./cdk-exports.json - # Export metadata - - seedfarmer metadata convert -f cdk-exports.json || true + - cdk deploy --require-approval never --progress events --app "python app.py" --outputs-file ./cdk-exports.json + # Export metadata + - seedfarmer metadata convert -f cdk-exports.json || true destroy: phases: install: commands: - # Install whatever additional build libraries - - npm install -g aws-cdk@2.126.0 - - pip install -r requirements.txt + # Install whatever additional build libraries + - npm install -g aws-cdk@2.126.0 + - pip install -r requirements.txt build: commands: - # execute the CDK - - cdk destroy --force --app "python app.py" \ No newline at end of file + # execute the CDK + - cdk destroy --force --app "python app.py" diff --git a/modules/examples/personas/deployspec.yaml b/modules/examples/personas/deployspec.yaml index 3ca3063a..6df85ebc 100644 --- a/modules/examples/personas/deployspec.yaml +++ b/modules/examples/personas/deployspec.yaml @@ -3,20 +3,20 @@ deploy: phases: install: commands: - - npm install -g aws-cdk@2.137.0 - - pip install -r requirements.txt + - npm install -g aws-cdk@2.137.0 + - pip install -r requirements.txt build: commands: - - cdk deploy --require-approval never --progress events --app "python app.py" --outputs-file ./cdk-exports_personas.json - # Export metadata - - seedfarmer metadata convert -f cdk-exports_personas.json || true + - cdk deploy --require-approval never --progress events --app "python app.py" --outputs-file ./cdk-exports_personas.json + # Export metadata + - seedfarmer metadata convert -f cdk-exports_personas.json || true destroy: phases: install: commands: - - npm install -g aws-cdk@2.137.0 - - pip install -r requirements.txt + - npm install -g aws-cdk@2.137.0 + - pip install -r requirements.txt build: commands: - # execute the CDK - - cdk destroy --force --app "python app.py" \ No newline at end of file + # execute the CDK + - cdk destroy --force --app "python app.py" diff --git a/modules/fmops/qna-rag/deployspec.yaml b/modules/fmops/qna-rag/deployspec.yaml index 5b17ea82..cfd48ccf 100644 --- a/modules/fmops/qna-rag/deployspec.yaml +++ b/modules/fmops/qna-rag/deployspec.yaml @@ -18,4 +18,4 @@ destroy: - pip install -r requirements.txt build: commands: - - cdk destroy --force --app "python app.py" \ No newline at end of file + - cdk destroy --force --app "python app.py" diff --git a/modules/mlflow/mlflow-fargate/deployspec.yaml b/modules/mlflow/mlflow-fargate/deployspec.yaml index 436f58a5..81c4b751 100644 --- a/modules/mlflow/mlflow-fargate/deployspec.yaml +++ b/modules/mlflow/mlflow-fargate/deployspec.yaml @@ -5,26 +5,26 @@ deploy: phases: install: commands: - - npm install -g aws-cdk@2.129.0 - - pip install -r requirements.txt + - npm install -g aws-cdk@2.129.0 + - pip install -r requirements.txt build: commands: - - cdk deploy --require-approval never --progress events --app "python app.py" --outputs-file ./cdk-exports.json - # Export metadata - - seedfarmer metadata convert -f cdk-exports.json || true + - cdk deploy --require-approval never --progress events --app "python app.py" --outputs-file ./cdk-exports.json + # Export metadata + - seedfarmer metadata convert -f cdk-exports.json || true post_build: commands: - - echo "Build successful" + - echo "Build successful" destroy: phases: install: commands: - - npm install -g aws-cdk@2.129.0 - - pip install -r requirements.txt + - npm install -g aws-cdk@2.129.0 + - pip install -r requirements.txt build: commands: - - cdk destroy --force --app "python app.py" + - cdk destroy --force --app "python app.py" post_build: commands: - - echo "Destroy successful" + - echo "Destroy successful" diff --git a/modules/mlflow/mlflow-image/deployspec.yaml b/modules/mlflow/mlflow-image/deployspec.yaml index 1b7cdebc..8cdbded2 100644 --- a/modules/mlflow/mlflow-image/deployspec.yaml +++ b/modules/mlflow/mlflow-image/deployspec.yaml @@ -5,26 +5,26 @@ deploy: phases: install: commands: - - npm install -g aws-cdk@2.130.0 - - pip install -r requirements.txt + - npm install -g aws-cdk@2.130.0 + - pip install -r requirements.txt build: commands: - - cdk deploy --require-approval never --progress events --app "python app.py" --outputs-file ./cdk-exports.json - # Export metadata - - seedfarmer metadata convert -f cdk-exports.json || true + - cdk deploy --require-approval never --progress events --app "python app.py" --outputs-file ./cdk-exports.json + # Export metadata + - seedfarmer metadata convert -f cdk-exports.json || true post_build: commands: - - echo "Build successful" + - echo "Build successful" destroy: phases: install: commands: - - npm install -g aws-cdk@2.130.0 - - pip install -r requirements.txt + - npm install -g aws-cdk@2.130.0 + - pip install -r requirements.txt build: commands: - - cdk destroy --force --app "python app.py" + - cdk destroy --force --app "python app.py" post_build: commands: - - echo "Destroy successful" + - echo "Destroy successful" diff --git a/modules/sagemaker/sagemaker-endpoint/deployspec.yaml b/modules/sagemaker/sagemaker-endpoint/deployspec.yaml index f98da702..c5a1aebc 100644 --- a/modules/sagemaker/sagemaker-endpoint/deployspec.yaml +++ b/modules/sagemaker/sagemaker-endpoint/deployspec.yaml @@ -3,23 +3,23 @@ deploy: phases: install: commands: - - env - # Install whatever additional build libraries - - npm install -g aws-cdk@2.126.0 - - pip install -r requirements.txt + - env + # Install whatever additional build libraries + - npm install -g aws-cdk@2.126.0 + - pip install -r requirements.txt build: commands: - - cdk deploy --require-approval never --progress events --app "python app.py" --outputs-file ./cdk-exports.json - # Export metadata - - seedfarmer metadata convert -f cdk-exports.json || true + - cdk deploy --require-approval never --progress events --app "python app.py" --outputs-file ./cdk-exports.json + # Export metadata + - seedfarmer metadata convert -f cdk-exports.json || true destroy: phases: install: commands: - # Install whatever additional build libraries - - npm install -g aws-cdk@2.126.0 - - pip install -r requirements.txt + # Install whatever additional build libraries + - npm install -g aws-cdk@2.126.0 + - pip install -r requirements.txt build: commands: - # execute the CDK - - cdk destroy --force --app "python app.py" \ No newline at end of file + # execute the CDK + - cdk destroy --force --app "python app.py" diff --git a/modules/sagemaker/sagemaker-model-monitoring/deployspec.yaml b/modules/sagemaker/sagemaker-model-monitoring/deployspec.yaml index add0f851..c5a1aebc 100644 --- a/modules/sagemaker/sagemaker-model-monitoring/deployspec.yaml +++ b/modules/sagemaker/sagemaker-model-monitoring/deployspec.yaml @@ -3,23 +3,23 @@ deploy: phases: install: commands: - - env - # Install whatever additional build libraries - - npm install -g aws-cdk@2.126.0 - - pip install -r requirements.txt + - env + # Install whatever additional build libraries + - npm install -g aws-cdk@2.126.0 + - pip install -r requirements.txt build: commands: - - cdk deploy --require-approval never --progress events --app "python app.py" --outputs-file ./cdk-exports.json - # Export metadata - - seedfarmer metadata convert -f cdk-exports.json || true + - cdk deploy --require-approval never --progress events --app "python app.py" --outputs-file ./cdk-exports.json + # Export metadata + - seedfarmer metadata convert -f cdk-exports.json || true destroy: phases: install: commands: - # Install whatever additional build libraries - - npm install -g aws-cdk@2.126.0 - - pip install -r requirements.txt + # Install whatever additional build libraries + - npm install -g aws-cdk@2.126.0 + - pip install -r requirements.txt build: commands: - # execute the CDK - - cdk destroy --force --app "python app.py" + # execute the CDK + - cdk destroy --force --app "python app.py" diff --git a/modules/sagemaker/sagemaker-model-package-group/deployspec.yaml b/modules/sagemaker/sagemaker-model-package-group/deployspec.yaml index c0f572cc..b2e83736 100644 --- a/modules/sagemaker/sagemaker-model-package-group/deployspec.yaml +++ b/modules/sagemaker/sagemaker-model-package-group/deployspec.yaml @@ -3,24 +3,24 @@ deploy: phases: install: commands: - - env - # Install whatever additional build libraries - - npm install -g aws-cdk@2.126.0 - - pip install -r requirements.txt + - env + # Install whatever additional build libraries + - npm install -g aws-cdk@2.126.0 + - pip install -r requirements.txt build: commands: # execute the CDK - - cdk deploy --require-approval never --progress events --app "python app.py" --outputs-file ./cdk-exports.json - # Export metadata - - seedfarmer metadata convert -f cdk-exports.json || true + - cdk deploy --require-approval never --progress events --app "python app.py" --outputs-file ./cdk-exports.json + # Export metadata + - seedfarmer metadata convert -f cdk-exports.json || true destroy: phases: install: commands: - # Install whatever additional build libraries - - npm install -g aws-cdk@2.126.0 - - pip install -r requirements.txt + # Install whatever additional build libraries + - npm install -g aws-cdk@2.126.0 + - pip install -r requirements.txt build: commands: - # execute the CDK - - cdk destroy --force --app "python app.py" \ No newline at end of file + # execute the CDK + - cdk destroy --force --app "python app.py" diff --git a/modules/sagemaker/sagemaker-model-package-promote-pipeline/deployspec.yaml b/modules/sagemaker/sagemaker-model-package-promote-pipeline/deployspec.yaml index c0f572cc..b2e83736 100644 --- a/modules/sagemaker/sagemaker-model-package-promote-pipeline/deployspec.yaml +++ b/modules/sagemaker/sagemaker-model-package-promote-pipeline/deployspec.yaml @@ -3,24 +3,24 @@ deploy: phases: install: commands: - - env - # Install whatever additional build libraries - - npm install -g aws-cdk@2.126.0 - - pip install -r requirements.txt + - env + # Install whatever additional build libraries + - npm install -g aws-cdk@2.126.0 + - pip install -r requirements.txt build: commands: # execute the CDK - - cdk deploy --require-approval never --progress events --app "python app.py" --outputs-file ./cdk-exports.json - # Export metadata - - seedfarmer metadata convert -f cdk-exports.json || true + - cdk deploy --require-approval never --progress events --app "python app.py" --outputs-file ./cdk-exports.json + # Export metadata + - seedfarmer metadata convert -f cdk-exports.json || true destroy: phases: install: commands: - # Install whatever additional build libraries - - npm install -g aws-cdk@2.126.0 - - pip install -r requirements.txt + # Install whatever additional build libraries + - npm install -g aws-cdk@2.126.0 + - pip install -r requirements.txt build: commands: - # execute the CDK - - cdk destroy --force --app "python app.py" \ No newline at end of file + # execute the CDK + - cdk destroy --force --app "python app.py" diff --git a/modules/sagemaker/sagemaker-notebook/deployspec.yaml b/modules/sagemaker/sagemaker-notebook/deployspec.yaml index c0f572cc..b2e83736 100644 --- a/modules/sagemaker/sagemaker-notebook/deployspec.yaml +++ b/modules/sagemaker/sagemaker-notebook/deployspec.yaml @@ -3,24 +3,24 @@ deploy: phases: install: commands: - - env - # Install whatever additional build libraries - - npm install -g aws-cdk@2.126.0 - - pip install -r requirements.txt + - env + # Install whatever additional build libraries + - npm install -g aws-cdk@2.126.0 + - pip install -r requirements.txt build: commands: # execute the CDK - - cdk deploy --require-approval never --progress events --app "python app.py" --outputs-file ./cdk-exports.json - # Export metadata - - seedfarmer metadata convert -f cdk-exports.json || true + - cdk deploy --require-approval never --progress events --app "python app.py" --outputs-file ./cdk-exports.json + # Export metadata + - seedfarmer metadata convert -f cdk-exports.json || true destroy: phases: install: commands: - # Install whatever additional build libraries - - npm install -g aws-cdk@2.126.0 - - pip install -r requirements.txt + # Install whatever additional build libraries + - npm install -g aws-cdk@2.126.0 + - pip install -r requirements.txt build: commands: - # execute the CDK - - cdk destroy --force --app "python app.py" \ No newline at end of file + # execute the CDK + - cdk destroy --force --app "python app.py" diff --git a/modules/sagemaker/sagemaker-studio/deployspec.yaml b/modules/sagemaker/sagemaker-studio/deployspec.yaml index 028e8124..b5031d8a 100644 --- a/modules/sagemaker/sagemaker-studio/deployspec.yaml +++ b/modules/sagemaker/sagemaker-studio/deployspec.yaml @@ -40,7 +40,7 @@ destroy: - echo $EFS_ID - echo $DOMAIN_ID - > - if [[ $RETAIN_EFS == "FALSE" ]]; then + if [[ $RETAIN_EFS == "FALSE" ]]; then echo "DELETING EFS" python scripts/delete_efs.py ${EFS_ID} ${DOMAIN_ID} || true fi; diff --git a/modules/sagemaker/sagemaker-templates-service-catalog/deployspec.yaml b/modules/sagemaker/sagemaker-templates-service-catalog/deployspec.yaml index f95cdd70..581faad9 100644 --- a/modules/sagemaker/sagemaker-templates-service-catalog/deployspec.yaml +++ b/modules/sagemaker/sagemaker-templates-service-catalog/deployspec.yaml @@ -3,21 +3,21 @@ deploy: phases: install: commands: - - env - # Install whatever additional build libraries - - npm install -g aws-cdk@2.130.0 - - pip install -r requirements.txt + - env + # Install whatever additional build libraries + - npm install -g aws-cdk@2.130.0 + - pip install -r requirements.txt build: commands: - - cdk deploy --require-approval never --progress events --app "python app.py" --outputs-file ./cdk-exports.json + - cdk deploy --require-approval never --progress events --app "python app.py" --outputs-file ./cdk-exports.json destroy: phases: install: commands: - # Install whatever additional build libraries - - npm install -g aws-cdk@2.130.0 - - pip install -r requirements.txt + # Install whatever additional build libraries + - npm install -g aws-cdk@2.130.0 + - pip install -r requirements.txt build: commands: - # execute the CDK - - cdk destroy --force --app "python app.py" \ No newline at end of file + # execute the CDK + - cdk destroy --force --app "python app.py" diff --git a/modules/sagemaker/sagemaker-templates-service-catalog/templates/finetune_llm_evaluation/seed_code/build_app/.pre-commit-config.yaml b/modules/sagemaker/sagemaker-templates-service-catalog/templates/finetune_llm_evaluation/seed_code/build_app/.pre-commit-config.yaml index 7a9c7e1c..a2ad1b4e 100644 --- a/modules/sagemaker/sagemaker-templates-service-catalog/templates/finetune_llm_evaluation/seed_code/build_app/.pre-commit-config.yaml +++ b/modules/sagemaker/sagemaker-templates-service-catalog/templates/finetune_llm_evaluation/seed_code/build_app/.pre-commit-config.yaml @@ -16,36 +16,36 @@ # SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. repos: -- repo: https://github.com/pre-commit/pre-commit-hooks + - repo: https://github.com/pre-commit/pre-commit-hooks rev: v4.3.0 hooks: - - id: check-added-large-files - - id: check-json - - id: check-merge-conflict - # - id: check-yaml - - id: end-of-file-fixer - - id: requirements-txt-fixer - - id: trailing-whitespace -- repo: https://github.com/psf/black + - id: check-added-large-files + - id: check-json + - id: check-merge-conflict + # - id: check-yaml + - id: end-of-file-fixer + - id: requirements-txt-fixer + - id: trailing-whitespace + - repo: https://github.com/psf/black rev: 22.6.0 hooks: - - id: black + - id: black args: ["--line-length=120"] -- repo: https://gitlab.com/PyCQA/flake8 + - repo: https://gitlab.com/PyCQA/flake8 rev: 3.9.2 hooks: - - id: flake8 + - id: flake8 args: ["--ignore=E231,E501,F841,W503,F403,E266,W605,F541,F401,E302", "--exclude=app.py", "--max-line-length=120"] -- repo: https://github.com/Lucas-C/pre-commit-hooks + - repo: https://github.com/Lucas-C/pre-commit-hooks rev: v1.2.0 hooks: - - id: forbid-crlf - - id: remove-crlf - - id: insert-license + - id: forbid-crlf + - id: remove-crlf + - id: insert-license files: \.(py|yaml)$ -- repo: local + - repo: local hooks: - - id: clear-jupyter-notebooks + - id: clear-jupyter-notebooks name: clear-jupyter-notebooks entry: bash -c 'find . -type f -name "*.ipynb" -exec jupyter nbconvert --ClearOutputPreprocessor.enabled=True --inplace "{}" \; && git add . && exit 0' language: system diff --git a/modules/sagemaker/sagemaker-templates-service-catalog/templates/xgboost_abalone/seed_code/build_app/.pre-commit-config.yaml b/modules/sagemaker/sagemaker-templates-service-catalog/templates/xgboost_abalone/seed_code/build_app/.pre-commit-config.yaml index 7a9c7e1c..e12eaf27 100644 --- a/modules/sagemaker/sagemaker-templates-service-catalog/templates/xgboost_abalone/seed_code/build_app/.pre-commit-config.yaml +++ b/modules/sagemaker/sagemaker-templates-service-catalog/templates/xgboost_abalone/seed_code/build_app/.pre-commit-config.yaml @@ -16,36 +16,36 @@ # SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. repos: -- repo: https://github.com/pre-commit/pre-commit-hooks + - repo: https://github.com/pre-commit/pre-commit-hooks rev: v4.3.0 hooks: - - id: check-added-large-files - - id: check-json - - id: check-merge-conflict - # - id: check-yaml - - id: end-of-file-fixer - - id: requirements-txt-fixer - - id: trailing-whitespace -- repo: https://github.com/psf/black + - id: check-added-large-files + - id: check-json + - id: check-merge-conflict + # - id: check-yaml + - id: end-of-file-fixer + - id: requirements-txt-fixer + - id: trailing-whitespace + - repo: https://github.com/psf/black rev: 22.6.0 hooks: - - id: black + - id: black args: ["--line-length=120"] -- repo: https://gitlab.com/PyCQA/flake8 + - repo: https://gitlab.com/PyCQA/flake8 rev: 3.9.2 hooks: - - id: flake8 + - id: flake8 args: ["--ignore=E231,E501,F841,W503,F403,E266,W605,F541,F401,E302", "--exclude=app.py", "--max-line-length=120"] -- repo: https://github.com/Lucas-C/pre-commit-hooks + - repo: https://github.com/Lucas-C/pre-commit-hooks rev: v1.2.0 hooks: - - id: forbid-crlf - - id: remove-crlf - - id: insert-license + - id: forbid-crlf + - id: remove-crlf + - id: insert-license files: \.(py|yaml)$ -- repo: local + - repo: local hooks: - - id: clear-jupyter-notebooks + - id: clear-jupyter-notebooks name: clear-jupyter-notebooks entry: bash -c 'find . -type f -name "*.ipynb" -exec jupyter nbconvert --ClearOutputPreprocessor.enabled=True --inplace "{}" \; && git add . && exit 0' language: system diff --git a/one-click-launch.yaml b/one-click-launch.yaml index e1c601f7..6818e06c 100644 --- a/one-click-launch.yaml +++ b/one-click-launch.yaml @@ -249,7 +249,7 @@ Resources: "UniqueId": "$UUID", "Status": "$STATUS", "Reason": "$STATUS", - "Data": "Deployment of solution has finished or stopped. See status." + "Data": "Deployment of solution has finished or stopped. See status." } EOF cat /tmp/payload.json diff --git a/seedfarmer.yaml b/seedfarmer.yaml index a3c7789e..53f8da51 100644 --- a/seedfarmer.yaml +++ b/seedfarmer.yaml @@ -1,4 +1,4 @@ project: aiops description: This is for local testing - intended for contributions manifestValidationFailOnUnknownFields: true -#projectPolicyPath: +# projectPolicyPath: