diff --git a/CHANGELOG.md b/CHANGELOG.md index aeaf78c..0f07144 100755 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,12 +5,19 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [2.1.2] - 2023-04-17 + +### Updated + +- The S3 bucket policy on the logging S3 bucket to grant access to the logging service principal (logging.s3.amazonaws.com) for access log delivery. +- Python libraries. + ## [2.1.1] - 2023-01-11 ### Updated -- Python runtime 3.10. +- Python runtime 3.9. - Python libraries. ## [2.1.0] - 2022-11-30 diff --git a/deployment/build-s3-dist.sh b/deployment/build-s3-dist.sh index 6e24e3e..540c055 100755 --- a/deployment/build-s3-dist.sh +++ b/deployment/build-s3-dist.sh @@ -69,8 +69,8 @@ echo "cd $source_dir" cd $source_dir # setup lambda layers (building sagemaker layer using lambda build environment for python 3.8) -echo 'docker run -v "$source_dir"/lib/blueprints/byom/lambdas/sagemaker_layer:/var/task lambci/lambda:build-python3.8 /bin/bash -c "cat requirements.txt; pip3 install -r requirements.txt -t ./python; exit"' -docker run -v "$source_dir"/lib/blueprints/byom/lambdas/sagemaker_layer:/var/task lambci/lambda:build-python3.8 /bin/bash -c "cat requirements.txt; pip3 install -r requirements.txt -t ./python; exit" +echo 'docker run --entrypoint /bin/bash -v "$source_dir"/lib/blueprints/byom/lambdas/sagemaker_layer:/var/task public.ecr.aws/lambda/python:3.9 -c "cat requirements.txt; pip3 install -r requirements.txt -t ./python; exit"' +docker run --entrypoint /bin/bash -v "$source_dir"/lib/blueprints/byom/lambdas/sagemaker_layer:/var/task public.ecr.aws/lambda/python:3.9 -c "cat requirements.txt; pip3 install -r requirements.txt -t ./python; exit" # Remove tests and cache stuff (to reduce size) find "$source_dir"/lib/blueprints/byom/lambdas/sagemaker_layer/python -type d -name "tests" -exec rm -rfv {} + @@ -115,30 +115,30 @@ echo "npm install -g aws-cdk@$cdk_version" npm install -g aws-cdk@$cdk_version #Run 'cdk synth for BYOM blueprints -echo "cdk synth DataQualityModelMonitorStack > lib/blueprints/byom/byom_data_quality_monitor.yaml --path-metadata false --version-reporting false" -cdk synth DataQualityModelMonitorStack > lib/blueprints/byom/byom_data_quality_monitor.yaml --path-metadata false --version-reporting false -echo "cdk synth ModelQualityModelMonitorStack > lib/blueprints/byom/byom_model_quality_monitor.yaml --path-metadata false --version-reporting false" -cdk synth ModelQualityModelMonitorStack > lib/blueprints/byom/byom_model_quality_monitor.yaml --path-metadata false --version-reporting false -echo "cdk synth ModelBiasModelMonitorStack > lib/blueprints/byom/byom_model_bias_monitor.yaml --path-metadata false --version-reporting false" -cdk synth ModelBiasModelMonitorStack > lib/blueprints/byom/byom_model_bias_monitor.yaml --path-metadata false --version-reporting false -echo "cdk synth ModelExplainabilityModelMonitorStack > lib/blueprints/byom/byom_model_explainability_monitor.yaml --path-metadata false --version-reporting false" -cdk synth ModelExplainabilityModelMonitorStack > lib/blueprints/byom/byom_model_explainability_monitor.yaml --path-metadata false --version-reporting false -echo "cdk synth SingleAccountCodePipelineStack > lib/blueprints/byom/single_account_codepipeline.yaml --path-metadata false --version-reporting false" -cdk synth SingleAccountCodePipelineStack > lib/blueprints/byom/single_account_codepipeline.yaml --path-metadata false --version-reporting false -echo "cdk synth MultiAccountCodePipelineStack > lib/blueprints/byom/multi_account_codepipeline.yaml --path-metadata false --version-reporting false" -cdk synth MultiAccountCodePipelineStack > lib/blueprints/byom/multi_account_codepipeline.yaml --path-metadata false --version-reporting false -echo "cdk synth BYOMRealtimePipelineStack > lib/blueprints/byom/byom_realtime_inference_pipeline.yaml --path-metadata false --version-reporting false" -cdk synth BYOMRealtimePipelineStack > lib/blueprints/byom/byom_realtime_inference_pipeline.yaml --path-metadata false --version-reporting false -echo "cdk synth BYOMCustomAlgorithmImageBuilderStack > lib/blueprints/byom/byom_custom_algorithm_image_builder.yaml --path-metadata false --version-reporting false" -cdk synth BYOMCustomAlgorithmImageBuilderStack > lib/blueprints/byom/byom_custom_algorithm_image_builder.yaml --path-metadata false --version-reporting false -echo "cdk synth BYOMBatchStack > lib/blueprints/byom/byom_batch_pipeline.yaml --path-metadata false --version-reporting false" -cdk synth BYOMBatchStack > lib/blueprints/byom/byom_batch_pipeline.yaml --path-metadata false --version-reporting false -echo "cdk synth AutopilotJobStack > lib/blueprints/byom/autopilot_training_pipeline.yaml --path-metadata false --version-reporting false" -cdk synth AutopilotJobStack > lib/blueprints/byom/autopilot_training_pipeline.yaml --path-metadata false --version-reporting false -echo "cdk synth TrainingJobStack > lib/blueprints/byom/model_training_pipeline.yaml --path-metadata false --version-reporting false" -cdk synth TrainingJobStack > lib/blueprints/byom/model_training_pipeline.yaml --path-metadata false --version-reporting false -echo "cdk synth HyperparamaterTunningJobStack > lib/blueprints/byom/model_hyperparameter_tunning_pipeline.yaml --path-metadata false --version-reporting false" -cdk synth HyperparamaterTunningJobStack > lib/blueprints/byom/model_hyperparameter_tunning_pipeline.yaml --path-metadata false --version-reporting false +echo "cdk synth DataQualityModelMonitorStack > lib/blueprints/byom/byom_data_quality_monitor.yaml --path-metadata false --version-reporting false --generate-bootstrap-version-rule false" +cdk synth DataQualityModelMonitorStack > lib/blueprints/byom/byom_data_quality_monitor.yaml --path-metadata false --version-reporting false --generate-bootstrap-version-rule false +echo "cdk synth ModelQualityModelMonitorStack > lib/blueprints/byom/byom_model_quality_monitor.yaml --path-metadata false --version-reporting false --generate-bootstrap-version-rule false" +cdk synth ModelQualityModelMonitorStack > lib/blueprints/byom/byom_model_quality_monitor.yaml --path-metadata false --version-reporting false --generate-bootstrap-version-rule false +echo "cdk synth ModelBiasModelMonitorStack > lib/blueprints/byom/byom_model_bias_monitor.yaml --path-metadata false --version-reporting false --generate-bootstrap-version-rule false" +cdk synth ModelBiasModelMonitorStack > lib/blueprints/byom/byom_model_bias_monitor.yaml --path-metadata false --version-reporting false --generate-bootstrap-version-rule false +echo "cdk synth ModelExplainabilityModelMonitorStack > lib/blueprints/byom/byom_model_explainability_monitor.yaml --path-metadata false --version-reporting false --generate-bootstrap-version-rule false" +cdk synth ModelExplainabilityModelMonitorStack > lib/blueprints/byom/byom_model_explainability_monitor.yaml --path-metadata false --version-reporting false --generate-bootstrap-version-rule false +echo "cdk synth SingleAccountCodePipelineStack > lib/blueprints/byom/single_account_codepipeline.yaml --path-metadata false --version-reporting false --generate-bootstrap-version-rule false" +cdk synth SingleAccountCodePipelineStack > lib/blueprints/byom/single_account_codepipeline.yaml --path-metadata false --version-reporting false --generate-bootstrap-version-rule false +echo "cdk synth MultiAccountCodePipelineStack > lib/blueprints/byom/multi_account_codepipeline.yaml --path-metadata false --version-reporting false --generate-bootstrap-version-rule false" +cdk synth MultiAccountCodePipelineStack > lib/blueprints/byom/multi_account_codepipeline.yaml --path-metadata false --version-reporting false --generate-bootstrap-version-rule false +echo "cdk synth BYOMRealtimePipelineStack > lib/blueprints/byom/byom_realtime_inference_pipeline.yaml --path-metadata false --version-reporting false --generate-bootstrap-version-rule false" +cdk synth BYOMRealtimePipelineStack > lib/blueprints/byom/byom_realtime_inference_pipeline.yaml --path-metadata false --version-reporting false --generate-bootstrap-version-rule false +echo "cdk synth BYOMCustomAlgorithmImageBuilderStack > lib/blueprints/byom/byom_custom_algorithm_image_builder.yaml --path-metadata false --version-reporting false --generate-bootstrap-version-rule false" +cdk synth BYOMCustomAlgorithmImageBuilderStack > lib/blueprints/byom/byom_custom_algorithm_image_builder.yaml --path-metadata false --version-reporting false --generate-bootstrap-version-rule false +echo "cdk synth BYOMBatchStack > lib/blueprints/byom/byom_batch_pipeline.yaml --path-metadata false --version-reporting false --generate-bootstrap-version-rule false" +cdk synth BYOMBatchStack > lib/blueprints/byom/byom_batch_pipeline.yaml --path-metadata false --version-reporting false --generate-bootstrap-version-rule false +echo "cdk synth AutopilotJobStack > lib/blueprints/byom/autopilot_training_pipeline.yaml --path-metadata false --version-reporting false --generate-bootstrap-version-rule false" +cdk synth AutopilotJobStack > lib/blueprints/byom/autopilot_training_pipeline.yaml --path-metadata false --version-reporting false --generate-bootstrap-version-rule false +echo "cdk synth TrainingJobStack > lib/blueprints/byom/model_training_pipeline.yaml --path-metadata false --version-reporting false --generate-bootstrap-version-rule false" +cdk synth TrainingJobStack > lib/blueprints/byom/model_training_pipeline.yaml --path-metadata false --version-reporting false --generate-bootstrap-version-rule false +echo "cdk synth HyperparamaterTunningJobStack > lib/blueprints/byom/model_hyperparameter_tunning_pipeline.yaml --path-metadata false --version-reporting false --generate-bootstrap-version-rule false" +cdk synth HyperparamaterTunningJobStack > lib/blueprints/byom/model_hyperparameter_tunning_pipeline.yaml --path-metadata false --version-reporting false --generate-bootstrap-version-rule false # Replace %%VERSION%% in other templates replace="s/%%VERSION%%/$3/g" @@ -168,10 +168,10 @@ echo "sed -i -e $replace lib/blueprints/byom/model_hyperparameter_tunning_pipeli sed -i -e $replace lib/blueprints/byom/model_hyperparameter_tunning_pipeline.yaml # Run 'cdk synth' for main templates to generate raw solution outputs -echo "cdk synth mlops-workload-orchestrator-single-account --path-metadata false --version-reporting false --output=$staging_dist_dir" -cdk synth mlops-workload-orchestrator-single-account --path-metadata false --version-reporting false --output=$staging_dist_dir -echo "cdk synth mlops-workload-orchestrator-multi-account --path-metadata false --version-reporting false --output=$staging_dist_dir" -cdk synth mlops-workload-orchestrator-multi-account --path-metadata false --version-reporting false --output=$staging_dist_dir +echo "cdk synth mlops-workload-orchestrator-single-account --path-metadata false --version-reporting false --generate-bootstrap-version-rule false --output=$staging_dist_dir" +cdk synth mlops-workload-orchestrator-single-account --path-metadata false --version-reporting false --generate-bootstrap-version-rule false --output=$staging_dist_dir +echo "cdk synth mlops-workload-orchestrator-multi-account --path-metadata false --version-reporting false --generate-bootstrap-version-rule false --output=$staging_dist_dir" +cdk synth mlops-workload-orchestrator-multi-account --path-metadata false --version-reporting false --generate-bootstrap-version-rule false --output=$staging_dist_dir # Remove unnecessary output files echo "cd $staging_dist_dir" diff --git a/source/lambdas/solution_helper/lambda_function.py b/source/lambdas/solution_helper/lambda_function.py index 1e79ae2..1b97c89 100644 --- a/source/lambdas/solution_helper/lambda_function.py +++ b/source/lambdas/solution_helper/lambda_function.py @@ -19,6 +19,9 @@ logger = logging.getLogger(__name__) helper = CfnResource(json_logging=True, log_level="INFO") +# requests.post timeout in seconds +REQUST_TIMEOUT = 60 + def _sanitize_data(resource_properties): # Define allowed keys. You need to update this list with new metrics @@ -42,7 +45,11 @@ def _sanitize_data(resource_properties): resource_properties.pop("UUID", None) # send only allowed metrics - sanitized_data = {key: resource_properties[key] for key in allowed_keys if key in resource_properties} + sanitized_data = { + key: resource_properties[key] + for key in allowed_keys + if key in resource_properties + } return sanitized_data @@ -63,9 +70,16 @@ def _send_anonymous_metrics(request_type, resource_properties): } logger.info(f"Sending payload: {payload}") - response = requests.post("https://metrics.awssolutionsbuilder.com/generic", json=payload, headers=headers) + response = requests.post( + "https://metrics.awssolutionsbuilder.com/generic", + json=payload, + headers=headers, + timeout=REQUST_TIMEOUT, + ) # log the response - logger.info(f"Response from the metrics endpoint: {response.status_code} {response.reason}") + logger.info( + f"Response from the metrics endpoint: {response.status_code} {response.reason}" + ) # raise error if response is an 404, 503, 500, 403 etc. response.raise_for_status() return response diff --git a/source/lib/blueprints/byom/lambdas/create_model_training_job/tests/fixtures/training_fixtures.py b/source/lib/blueprints/byom/lambdas/create_model_training_job/tests/fixtures/training_fixtures.py index 332173a..17dcc53 100644 --- a/source/lib/blueprints/byom/lambdas/create_model_training_job/tests/fixtures/training_fixtures.py +++ b/source/lib/blueprints/byom/lambdas/create_model_training_job/tests/fixtures/training_fixtures.py @@ -35,7 +35,12 @@ def mocked_common_env_vars(): "CONTENT_TYPE": "csv", "USE_SPOT_INSTANCES": "True", "HYPERPARAMETERS": json.dumps( - dict(eval_metric="auc", objective="binary:logistic", num_round=400, rate_drop=0.3) + dict( + eval_metric="auc", + objective="binary:logistic", + num_round=400, + rate_drop=0.3, + ) ), "TAGS": json.dumps([{"pipeline": "training"}]), } @@ -87,7 +92,24 @@ def mocked_hyperparameters(mocked_training_job_env_vars): @pytest.fixture() -def mocked_estimator_config(mocked_training_job_env_vars): +def mocked_sagemaker_session(): + region = "us-east-1" + boto_mock = Mock(name="boto_session", region_name=region) + sms = Mock( + name="sagemaker_session", + boto_session=boto_mock, + boto_region_name=region, + config=None, + local_mode=False, + s3_resource=None, + s3_client=None, + ) + sms.sagemaker_config = {} + return sms + + +@pytest.fixture() +def mocked_estimator_config(mocked_training_job_env_vars, mocked_sagemaker_session): return dict( image_uri=os.environ["IMAGE_URI"], role=os.environ["ROLE_ARN"], @@ -95,17 +117,19 @@ def mocked_estimator_config(mocked_training_job_env_vars): instance_type=os.environ["INSTANCE_TYPE"], volume_size=int(os.environ["INSTANCE_VOLUME_SIZE"]), output_path=f"s3://{os.environ['ASSETS_BUCKET']}/{os.environ['JOB_OUTPUT_LOCATION']}", - sagemaker_session=Mock(), + sagemaker_session=mocked_sagemaker_session, ) @pytest.fixture() def mocked_data_channels(mocked_training_job_env_vars): train_input = TrainingInput( - f"s3://{os.environ['ASSETS_BUCKET']}/{os.environ['TRAINING_DATA_KEY']}", content_type=os.environ["CONTENT_TYPE"] + f"s3://{os.environ['ASSETS_BUCKET']}/{os.environ['TRAINING_DATA_KEY']}", + content_type=os.environ["CONTENT_TYPE"], ) validation_input = TrainingInput( - f"s3://{os.environ['ASSETS_BUCKET']}/{os.environ['TRAINING_DATA_KEY']}", content_type=os.environ["CONTENT_TYPE"] + f"s3://{os.environ['ASSETS_BUCKET']}/{os.environ['TRAINING_DATA_KEY']}", + content_type=os.environ["CONTENT_TYPE"], ) data_channels = {"train": train_input, "validation": validation_input} diff --git a/source/lib/blueprints/byom/lambdas/create_model_training_job/tests/test_create_model_training.py b/source/lib/blueprints/byom/lambdas/create_model_training_job/tests/test_create_model_training.py index 27546e9..34d0a25 100644 --- a/source/lib/blueprints/byom/lambdas/create_model_training_job/tests/test_create_model_training.py +++ b/source/lib/blueprints/byom/lambdas/create_model_training_job/tests/test_create_model_training.py @@ -24,10 +24,16 @@ mocked_job_name, mocked_raw_search_grid, mocked_hyperparameter_ranges, + mocked_sagemaker_session, ) -def test_create_estimator(mocked_estimator_config, mocked_hyperparameters, mocked_data_channels, mocked_job_name): +def test_create_estimator( + mocked_estimator_config, + mocked_hyperparameters, + mocked_data_channels, + mocked_job_name, +): job = SolutionModelTraining( job_name=mocked_job_name, estimator_config=mocked_estimator_config, @@ -69,14 +75,17 @@ def test_create_hyperparameter_tuner( assert tuner.max_jobs == 10 assert tuner.strategy == "Bayesian" assert tuner.objective_type == "Maximize" - TestCase().assertDictEqual(tuner._hyperparameter_ranges, mocked_hyperparameter_ranges) + TestCase().assertDictEqual( + tuner._hyperparameter_ranges, mocked_hyperparameter_ranges + ) def test_format_search_grid(mocked_raw_search_grid): formeated_grid = SolutionModelTraining.format_search_grid(mocked_raw_search_grid) # assert a Continuous parameter TestCase().assertListEqual( - mocked_raw_search_grid["eta"][1], [formeated_grid["eta"].min_value, formeated_grid["eta"].max_value] + mocked_raw_search_grid["eta"][1], + [formeated_grid["eta"].min_value, formeated_grid["eta"].max_value], ) # assert an Integer parameter TestCase().assertListEqual( @@ -84,7 +93,9 @@ def test_format_search_grid(mocked_raw_search_grid): [formeated_grid["max_depth"].min_value, formeated_grid["max_depth"].max_value], ) # assert a Categorical parameter - TestCase().assertListEqual(mocked_raw_search_grid["optimizer"][1], formeated_grid["optimizer"].values) + TestCase().assertListEqual( + mocked_raw_search_grid["optimizer"][1], formeated_grid["optimizer"].values + ) @patch("model_training_helper.SolutionModelTraining._create_hyperparameter_tuner") @@ -127,7 +138,9 @@ def test_create_training_job( @patch("model_training_helper.SolutionModelTraining._create_estimator") @patch("main.Session") @patch("main.get_client") -def test_handler_training_job(mocked_client, mocked_session, mocked_create_estimator, mocked_training_job_env_vars): +def test_handler_training_job( + mocked_client, mocked_session, mocked_create_estimator, mocked_training_job_env_vars +): mocked_client.boto_region_name = "us-east-1" from main import handler diff --git a/source/lib/blueprints/byom/lambdas/create_update_cf_stackset/tests/fixtures/stackset_fixtures.py b/source/lib/blueprints/byom/lambdas/create_update_cf_stackset/tests/fixtures/stackset_fixtures.py index 2381021..4ae4c9e 100644 --- a/source/lib/blueprints/byom/lambdas/create_update_cf_stackset/tests/fixtures/stackset_fixtures.py +++ b/source/lib/blueprints/byom/lambdas/create_update_cf_stackset/tests/fixtures/stackset_fixtures.py @@ -48,6 +48,9 @@ def mocked_cp_success_message(): def mocked_cp_failure_message(): return "StackSet Job Failed" +@pytest.fixture() +def mocked_describe_response(): + return {'StackInstance':{"StackInstanceStatus": {"DetailedStatus": "SUCCEEDED"}}} @pytest.fixture() def mocked_cp_continuation_message(): diff --git a/source/lib/blueprints/byom/lambdas/create_update_cf_stackset/tests/test_create_update_cf_stackset.py b/source/lib/blueprints/byom/lambdas/create_update_cf_stackset/tests/test_create_update_cf_stackset.py index 5f3aa01..6c2e52e 100644 --- a/source/lib/blueprints/byom/lambdas/create_update_cf_stackset/tests/test_create_update_cf_stackset.py +++ b/source/lib/blueprints/byom/lambdas/create_update_cf_stackset/tests/test_create_update_cf_stackset.py @@ -14,6 +14,7 @@ import json import tempfile import pytest +from unittest.mock import patch, Mock from botocore.stub import Stubber import botocore.session from tests.fixtures.stackset_fixtures import ( @@ -31,6 +32,7 @@ mocked_decoded_parameters, mocked_codepipeline_event, mocked_invalid_user_parms, + mocked_describe_response ) from moto import mock_cloudformation, mock_s3 from unittest.mock import patch @@ -75,21 +77,13 @@ def test_create_stackset_and_instances( cf_client, ) stacksets = cf_client.list_stack_sets() - # print(stacksets) # assert one StackSet has been created assert len(stacksets["Summaries"]) == 1 # assert the created name has the passed name assert stacksets["Summaries"][0]["StackSetName"] == stackset_name # assert the status of the stackset is ACTIVE assert stacksets["Summaries"][0]["Status"] == "ACTIVE" - - # describe stackset instance - instance = cf_client.describe_stack_instance( - StackSetName=stackset_name, - StackInstanceAccount=mocked_account_ids[0], - StackInstanceRegion=mocked_regions[0], - ) - assert instance["ResponseMetadata"]["HTTPStatusCode"] == 200 + assert stacksets["ResponseMetadata"]["HTTPStatusCode"] == 200 # assert the function will throw an exception with pytest.raises(Exception): @@ -138,29 +132,21 @@ def test_get_stackset_instance_status_client_error( get_stackset_instance_status(stackset_name, mocked_account_ids[0], mocked_regions[0], cf_client) -@mock_cloudformation +@patch("boto3.client") def test_get_stackset_instance_status( + patched_client, stackset_name, - mocked_template, - mocked_template_parameters, - mocked_org_ids, mocked_account_ids, mocked_regions, + mocked_describe_response ): cf_client = boto3.client("cloudformation", region_name=mocked_regions[0]) - # create a mocked stackset and instance - create_stackset_and_instances( - stackset_name, - mocked_template, - json.loads(mocked_template_parameters), - mocked_org_ids, - mocked_regions, - cf_client, - ) - # should throw an KeyError exception - with pytest.raises(KeyError): - get_stackset_instance_status(stackset_name, mocked_account_ids[0], mocked_regions[0], cf_client) + patched_client().describe_stack_instance.return_value = mocked_describe_response + response = get_stackset_instance_status(stackset_name, mocked_account_ids[0], mocked_regions[0], cf_client) + + assert response == "SUCCEEDED" + @mock_cloudformation diff --git a/source/lib/blueprints/byom/lambdas/sagemaker_layer/requirements.txt b/source/lib/blueprints/byom/lambdas/sagemaker_layer/requirements.txt index 079679d..c6a4b55 100644 --- a/source/lib/blueprints/byom/lambdas/sagemaker_layer/requirements.txt +++ b/source/lib/blueprints/byom/lambdas/sagemaker_layer/requirements.txt @@ -1,4 +1,4 @@ -botocore==1.29.46 -boto3==1.26.46 -awscli==1.27.46 -sagemaker==2.128.0 +botocore==1.29.75 +boto3==1.26.75 +awscli==1.27.75 +sagemaker==2.146.0 diff --git a/source/lib/blueprints/byom/pipeline_definitions/deploy_actions.py b/source/lib/blueprints/byom/pipeline_definitions/deploy_actions.py index 033012e..9a07690 100644 --- a/source/lib/blueprints/byom/pipeline_definitions/deploy_actions.py +++ b/source/lib/blueprints/byom/pipeline_definitions/deploy_actions.py @@ -64,7 +64,7 @@ def sagemaker_layer(scope, blueprint_bucket): scope, "sagemakerlayer", code=lambda_.Code.from_bucket(blueprint_bucket, "blueprints/byom/lambdas/sagemaker_layer.zip"), - compatible_runtimes=[lambda_.Runtime.PYTHON_3_8], + compatible_runtimes=[lambda_.Runtime.PYTHON_3_9], ) @@ -137,7 +137,7 @@ def batch_transform( batch_transform_lambda = lambda_.Function( scope, id, - runtime=lambda_.Runtime.PYTHON_3_8, + runtime=lambda_.Runtime.PYTHON_3_9, handler=lambda_handler, layers=[sm_layer], role=lambda_role, @@ -334,7 +334,7 @@ def create_baseline_job_lambda( create_baseline_job_lambda = lambda_.Function( scope, "create_data_baseline_job", - runtime=lambda_.Runtime.PYTHON_3_8, + runtime=lambda_.Runtime.PYTHON_3_9, handler=lambda_handler, role=lambda_role, code=lambda_.Code.from_bucket(blueprint_bucket, "blueprints/byom/lambdas/create_baseline_job.zip"), @@ -414,7 +414,7 @@ def create_stackset_action( create_update_cf_stackset_lambda = lambda_.Function( scope, f"{action_name}_stackset_lambda", - runtime=lambda_.Runtime.PYTHON_3_8, + runtime=lambda_.Runtime.PYTHON_3_9, handler="main.lambda_handler", role=lambda_role, code=lambda_.Code.from_bucket(blueprint_bucket, "blueprints/byom/lambdas/create_update_cf_stackset.zip"), @@ -507,7 +507,7 @@ def create_invoke_lambda_custom_resource( id, code=lambda_.Code.from_bucket(blueprint_bucket, "blueprints/byom/lambdas/invoke_lambda_custom_resource.zip"), handler="index.handler", - runtime=lambda_.Runtime.PYTHON_3_8, + runtime=lambda_.Runtime.PYTHON_3_9, timeout=core.Duration.minutes(5), ) @@ -557,7 +557,7 @@ def create_copy_assets_lambda(scope, blueprint_repository_bucket_name): "CustomResourceLambda", code=lambda_.Code.from_asset("lambdas/custom_resource"), handler="index.on_event", - runtime=lambda_.Runtime.PYTHON_3_8, + runtime=lambda_.Runtime.PYTHON_3_9, memory_size=256, environment={ "SOURCE_BUCKET": source_bucket, @@ -592,7 +592,7 @@ def create_solution_helper(scope): "SolutionHelper", code=lambda_.Code.from_asset("lambdas/solution_helper"), handler="lambda_function.handler", - runtime=lambda_.Runtime.PYTHON_3_8, + runtime=lambda_.Runtime.PYTHON_3_9, timeout=core.Duration.minutes(5), ) @@ -742,7 +742,7 @@ def autopilot_training_job( autopilot_lambda = lambda_.Function( scope, id, - runtime=lambda_.Runtime.PYTHON_3_8, + runtime=lambda_.Runtime.PYTHON_3_9, handler=lambda_handler, layers=[sm_layer], role=lambda_role, @@ -863,7 +863,7 @@ def model_training_job( training_lambda = lambda_.Function( scope, id, - runtime=lambda_.Runtime.PYTHON_3_8, + runtime=lambda_.Runtime.PYTHON_3_9, handler=lambda_handler, layers=[sm_layer], role=lambda_role, diff --git a/source/lib/blueprints/byom/realtime_inference_pipeline.py b/source/lib/blueprints/byom/realtime_inference_pipeline.py index fd0b5df..6424c9b 100644 --- a/source/lib/blueprints/byom/realtime_inference_pipeline.py +++ b/source/lib/blueprints/byom/realtime_inference_pipeline.py @@ -68,9 +68,10 @@ def __init__(self, scope: core.Construct, id: str, **kwargs) -> None: self, "BYOMInference", lambda_function_props={ - "runtime": lambda_.Runtime.PYTHON_3_8, + "runtime": lambda_.Runtime.PYTHON_3_9, "handler": "main.handler", "code": lambda_.Code.from_bucket(blueprint_bucket, "blueprints/byom/lambdas/inference.zip"), + "timeout": core.Duration.minutes(5) }, api_gateway_props={ "defaultMethodOptions": { diff --git a/source/lib/mlops_orchestrator_stack.py b/source/lib/mlops_orchestrator_stack.py index 3c7d997..5f2d23f 100644 --- a/source/lib/mlops_orchestrator_stack.py +++ b/source/lib/mlops_orchestrator_stack.py @@ -29,7 +29,6 @@ from lib.conditional_resource import ConditionalResources from lib.blueprints.byom.pipeline_definitions.helpers import ( suppress_s3_access_policy, - apply_secure_bucket_policy, suppress_lambda_policies, suppress_sns, ) @@ -51,12 +50,18 @@ from lib.blueprints.byom.pipeline_definitions.configure_multi_account import ( configure_multi_account_parameters_permissions, ) -from lib.blueprints.byom.pipeline_definitions.sagemaker_model_registry import create_sagemaker_model_registry -from lib.blueprints.byom.pipeline_definitions.cdk_context_value import get_cdk_context_value +from lib.blueprints.byom.pipeline_definitions.sagemaker_model_registry import ( + create_sagemaker_model_registry, +) +from lib.blueprints.byom.pipeline_definitions.cdk_context_value import ( + get_cdk_context_value, +) class MLOpsStack(core.Stack): - def __init__(self, scope: core.Construct, id: str, *, multi_account=False, **kwargs) -> None: + def __init__( + self, scope: core.Construct, id: str, *, multi_account=False, **kwargs + ) -> None: super().__init__(scope, id, **kwargs) # Get stack parameters: @@ -74,16 +79,24 @@ def __init__(self, scope: core.Construct, id: str, *, multi_account=False, **kwa allow_detailed_error_message = pf.create_detailed_error_message_parameter(self) # Conditions - git_address_provided = cf.create_git_address_provided_condition(self, git_address) + git_address_provided = cf.create_git_address_provided_condition( + self, git_address + ) # client provided an existing S3 bucket name, to be used for assets - existing_bucket_provided = cf.create_existing_bucket_provided_condition(self, existing_bucket) + existing_bucket_provided = cf.create_existing_bucket_provided_condition( + self, existing_bucket + ) # client provided an existing Amazon ECR name - existing_ecr_provided = cf.create_existing_ecr_provided_condition(self, existing_ecr_repo) + existing_ecr_provided = cf.create_existing_ecr_provided_condition( + self, existing_ecr_repo + ) # client wants the solution to create model registry - model_registry_condition = cf.create_model_registry_condition(self, create_model_registry) + model_registry_condition = cf.create_model_registry_condition( + self, create_model_registry + ) # S3 bucket needs to be created for assets create_new_bucket = cf.create_new_bucket_condition(self, existing_bucket) @@ -101,13 +114,18 @@ def __init__(self, scope: core.Construct, id: str, *, multi_account=False, **kwa encryption=s3.BucketEncryption.S3_MANAGED, block_public_access=s3.BlockPublicAccess.BLOCK_ALL, versioned=True, + enforce_ssl=True, ) - # Apply secure transfer bucket policy - apply_secure_bucket_policy(access_logs_bucket) + # remove ACL to add bucket policy instead + access_logs_bucket.node.default_child.add_deletion_override( + "Properties.AccessControl" + ) # This is a logging bucket. - access_logs_bucket.node.default_child.cfn_options.metadata = suppress_s3_access_policy() + access_logs_bucket.node.default_child.cfn_options.metadata = ( + suppress_s3_access_policy() + ) # Import user provide S3 bucket, if any. s3.Bucket.from_bucket_arn is used instead of # s3.Bucket.from_bucket_name to allow cross account bucket. @@ -118,7 +136,9 @@ def __init__(self, scope: core.Construct, id: str, *, multi_account=False, **kwa ) # Create the resource if existing_bucket_provided condition is True - core.Aspects.of(client_existing_bucket).add(ConditionalResources(existing_bucket_provided)) + core.Aspects.of(client_existing_bucket).add( + ConditionalResources(existing_bucket_provided) + ) # Import user provided Amazon ECR repository @@ -126,7 +146,9 @@ def __init__(self, scope: core.Construct, id: str, *, multi_account=False, **kwa self, "ClientExistingECRReo", existing_ecr_repo.value_as_string ) # Create the resource if existing_ecr_provided condition is True - core.Aspects.of(client_erc_repo).add(ConditionalResources(existing_ecr_provided)) + core.Aspects.of(client_erc_repo).add( + ConditionalResources(existing_ecr_provided) + ) # Creating assets bucket so that users can upload ML Models to it. assets_bucket = s3.Bucket( @@ -137,11 +159,9 @@ def __init__(self, scope: core.Construct, id: str, *, multi_account=False, **kwa server_access_logs_bucket=access_logs_bucket, server_access_logs_prefix="assets_bucket_access_logs", block_public_access=s3.BlockPublicAccess.BLOCK_ALL, + enforce_ssl=True, ) - # Apply secure transport bucket policy - apply_secure_bucket_policy(assets_bucket) - # Create the resource if create_new_bucket condition is True core.Aspects.of(assets_bucket).add(ConditionalResources(create_new_bucket)) @@ -152,6 +172,39 @@ def __init__(self, scope: core.Construct, id: str, *, multi_account=False, **kwa assets_bucket.bucket_name, ).to_string() + blueprints_bucket_name = "blueprint-repository-" + str(uuid.uuid4()) + blueprint_repository_bucket = s3.Bucket( + self, + blueprints_bucket_name, + encryption=s3.BucketEncryption.S3_MANAGED, + server_access_logs_bucket=access_logs_bucket, + server_access_logs_prefix=blueprints_bucket_name, + block_public_access=s3.BlockPublicAccess.BLOCK_ALL, + versioned=True, + enforce_ssl=True, + ) + + # add S3 Server Access Logs Policy + access_logs_bucket.add_to_resource_policy( + iam.PolicyStatement( + effect=iam.Effect.ALLOW, + actions=["s3:PutObject"], + principals=[iam.ServicePrincipal("logging.s3.amazonaws.com")], + resources=[ + f"{access_logs_bucket.bucket_arn}/*", + ], + conditions={ + "ArnLike": { + "aws:SourceArn": [ + f"arn:{core.Aws.PARTITION}:s3:::{assets_s3_bucket_name}", + blueprint_repository_bucket.bucket_arn, + ] + }, + "StringEquals": {"aws:SourceAccount": core.Aws.ACCOUNT_ID}, + }, + ) + ) + # Creating Amazon ECR repository ecr_repo = ecr.Repository(self, "ECRRepo", image_scan_on_push=True) @@ -172,27 +225,18 @@ def __init__(self, scope: core.Construct, id: str, *, multi_account=False, **kwa ecr_repo.repository_arn, ).to_string() - blueprints_bucket_name = "blueprint-repository-" + str(uuid.uuid4()) - blueprint_repository_bucket = s3.Bucket( - self, - blueprints_bucket_name, - encryption=s3.BucketEncryption.S3_MANAGED, - server_access_logs_bucket=access_logs_bucket, - server_access_logs_prefix=blueprints_bucket_name, - block_public_access=s3.BlockPublicAccess.BLOCK_ALL, - versioned=True, - ) - # Apply secure transport bucket policy - apply_secure_bucket_policy(blueprint_repository_bucket) - # create sns topic and subscription mlops_notifications_topic = sns.Topic( self, "MLOpsNotificationsTopic", ) - mlops_notifications_topic.node.default_child.cfn_options.metadata = suppress_sns() + mlops_notifications_topic.node.default_child.cfn_options.metadata = ( + suppress_sns() + ) mlops_notifications_topic.add_subscription( - subscriptions.EmailSubscription(email_address=notification_email.value_as_string) + subscriptions.EmailSubscription( + email_address=notification_email.value_as_string + ) ) # grant EventBridge permissions to publish messages @@ -216,9 +260,13 @@ def __init__(self, scope: core.Construct, id: str, *, multi_account=False, **kwa # creating SageMaker Model registry # use the first 8 characters as a unique_id to be appended to the model_package_group_name - unique_id = core.Fn.select(0, core.Fn.split("-", create_id_function.get_att_string("UUID"))) + unique_id = core.Fn.select( + 0, core.Fn.split("-", create_id_function.get_att_string("UUID")) + ) model_package_group_name = f"mlops-model-registry-{unique_id}" - model_registry = create_sagemaker_model_registry(self, "SageMakerModelRegistry", model_package_group_name) + model_registry = create_sagemaker_model_registry( + self, "SageMakerModelRegistry", model_package_group_name + ) # only create based on the condition model_registry.cfn_options.condition = model_registry_condition @@ -227,7 +275,9 @@ def __init__(self, scope: core.Construct, id: str, *, multi_account=False, **kwa model_registry.node.add_dependency(create_id_function) # Custom resource to copy source bucket content to blueprints bucket - custom_resource_lambda_fn = create_copy_assets_lambda(self, blueprint_repository_bucket.bucket_name) + custom_resource_lambda_fn = create_copy_assets_lambda( + self, blueprint_repository_bucket.bucket_name + ) # grant permission to upload file to the blueprints bucket blueprint_repository_bucket.grant_write(custom_resource_lambda_fn) @@ -246,12 +296,18 @@ def __init__(self, scope: core.Construct, id: str, *, multi_account=False, **kwa # Cloudformation policy setup orchestrator_policy = create_orchestrator_policy( - self, pipeline_stack_name, ecr_repo_name, blueprint_repository_bucket, assets_s3_bucket_name + self, + pipeline_stack_name, + ecr_repo_name, + blueprint_repository_bucket, + assets_s3_bucket_name, ) orchestrator_policy.attach_to_role(cloudformation_role) # Lambda function IAM setup - lambda_passrole_policy = iam.PolicyStatement(actions=["iam:passrole"], resources=[cloudformation_role.role_arn]) + lambda_passrole_policy = iam.PolicyStatement( + actions=["iam:passrole"], resources=[cloudformation_role.role_arn] + ) # create sagemaker layer sm_layer = sagemaker_layer(self, blueprint_repository_bucket) # make sure the sagemaker code is uploaded first to the blueprints bucket @@ -261,7 +317,7 @@ def __init__(self, scope: core.Construct, id: str, *, multi_account=False, **kwa self, "PipelineOrchestration", lambda_function_props={ - "runtime": lambda_.Runtime.PYTHON_3_8, + "runtime": lambda_.Runtime.PYTHON_3_9, "handler": "index.handler", "code": lambda_.Code.from_asset("lambdas/pipeline_orchestration"), "layers": [sm_layer], @@ -278,15 +334,25 @@ def __init__(self, scope: core.Construct, id: str, *, multi_account=False, **kwa ) # add lambda suppressions - provisioner_apigw_lambda.lambda_function.node.default_child.cfn_options.metadata = suppress_lambda_policies() + provisioner_apigw_lambda.lambda_function.node.default_child.cfn_options.metadata = ( + suppress_lambda_policies() + ) - provision_resource = provisioner_apigw_lambda.api_gateway.root.add_resource("provisionpipeline") + provision_resource = provisioner_apigw_lambda.api_gateway.root.add_resource( + "provisionpipeline" + ) provision_resource.add_method("POST") - status_resource = provisioner_apigw_lambda.api_gateway.root.add_resource("pipelinestatus") + status_resource = provisioner_apigw_lambda.api_gateway.root.add_resource( + "pipelinestatus" + ) status_resource.add_method("POST") blueprint_repository_bucket.grant_read(provisioner_apigw_lambda.lambda_function) - provisioner_apigw_lambda.lambda_function.add_to_role_policy(lambda_passrole_policy) - orchestrator_policy.attach_to_role(provisioner_apigw_lambda.lambda_function.role) + provisioner_apigw_lambda.lambda_function.add_to_role_policy( + lambda_passrole_policy + ) + orchestrator_policy.attach_to_role( + provisioner_apigw_lambda.lambda_function.role + ) # Environment variables setup provisioner_apigw_lambda.lambda_function.add_environment( @@ -299,31 +365,47 @@ def __init__(self, scope: core.Construct, id: str, *, multi_account=False, **kwa provisioner_apigw_lambda.lambda_function.add_environment( key="ACCESS_BUCKET", value=str(access_logs_bucket.bucket_name) ) - provisioner_apigw_lambda.lambda_function.add_environment(key="ASSETS_BUCKET", value=str(assets_s3_bucket_name)) + provisioner_apigw_lambda.lambda_function.add_environment( + key="ASSETS_BUCKET", value=str(assets_s3_bucket_name) + ) provisioner_apigw_lambda.lambda_function.add_environment( key="CFN_ROLE_ARN", value=str(cloudformation_role.role_arn) ) - provisioner_apigw_lambda.lambda_function.add_environment(key="PIPELINE_STACK_NAME", value=pipeline_stack_name) + provisioner_apigw_lambda.lambda_function.add_environment( + key="PIPELINE_STACK_NAME", value=pipeline_stack_name + ) provisioner_apigw_lambda.lambda_function.add_environment( key="NOTIFICATION_EMAIL", value=notification_email.value_as_string ) - provisioner_apigw_lambda.lambda_function.add_environment(key="REGION", value=core.Aws.REGION) - provisioner_apigw_lambda.lambda_function.add_environment(key="IS_MULTI_ACCOUNT", value=str(multi_account)) + provisioner_apigw_lambda.lambda_function.add_environment( + key="REGION", value=core.Aws.REGION + ) + provisioner_apigw_lambda.lambda_function.add_environment( + key="IS_MULTI_ACCOUNT", value=str(multi_account) + ) provisioner_apigw_lambda.lambda_function.add_environment( key="USE_MODEL_REGISTRY", value=use_model_registry.value_as_string ) provisioner_apigw_lambda.lambda_function.add_environment( - key="ALLOW_DETAILED_ERROR_MESSAGE", value=allow_detailed_error_message.value_as_string + key="ALLOW_DETAILED_ERROR_MESSAGE", + value=allow_detailed_error_message.value_as_string, ) provisioner_apigw_lambda.lambda_function.add_environment( - key="MLOPS_NOTIFICATIONS_SNS_TOPIC", value=mlops_notifications_topic.topic_arn + key="MLOPS_NOTIFICATIONS_SNS_TOPIC", + value=mlops_notifications_topic.topic_arn, + ) + provisioner_apigw_lambda.lambda_function.add_environment( + key="ECR_REPO_NAME", value=ecr_repo_name ) - provisioner_apigw_lambda.lambda_function.add_environment(key="ECR_REPO_NAME", value=ecr_repo_name) - provisioner_apigw_lambda.lambda_function.add_environment(key="ECR_REPO_ARN", value=ecr_repo_arn) + provisioner_apigw_lambda.lambda_function.add_environment( + key="ECR_REPO_ARN", value=ecr_repo_arn + ) - provisioner_apigw_lambda.lambda_function.add_environment(key="LOG_LEVEL", value="DEBUG") + provisioner_apigw_lambda.lambda_function.add_environment( + key="LOG_LEVEL", value="DEBUG" + ) cfn_policy_for_lambda = orchestrator_policy.node.default_child cfn_policy_for_lambda.cfn_options.metadata = { "cfn_nag": { @@ -346,7 +428,9 @@ def __init__(self, scope: core.Construct, id: str, *, multi_account=False, **kwa repo_name_split = core.Fn.split("/", git_address.value_as_string) repo_name = core.Fn.select(5, repo_name_split) # getting codecommit repo cdk object using 'from_repository_name' - repo = codecommit.Repository.from_repository_name(self, "AWSMLOpsFrameworkRepository", repo_name) + repo = codecommit.Repository.from_repository_name( + self, "AWSMLOpsFrameworkRepository", repo_name + ) codebuild_project = codebuild.PipelineProject( self, "Take config file", @@ -398,10 +482,14 @@ def __init__(self, scope: core.Construct, id: str, *, multi_account=False, **kwa cross_account_keys=False, ) codecommit_pipeline.add_to_role_policy( - create_invoke_lambda_policy([provisioner_apigw_lambda.lambda_function.function_arn]) + create_invoke_lambda_policy( + [provisioner_apigw_lambda.lambda_function.function_arn] + ) ) codebuild_project.add_to_role_policy( - create_invoke_lambda_policy([provisioner_apigw_lambda.lambda_function.function_arn]) + create_invoke_lambda_policy( + [provisioner_apigw_lambda.lambda_function.function_arn] + ) ) pipeline_child_nodes = codecommit_pipeline.node.find_all() pipeline_child_nodes[1].node.default_child.cfn_options.metadata = { @@ -420,17 +508,25 @@ def __init__(self, scope: core.Construct, id: str, *, multi_account=False, **kwa } # custom resource for operational metrics### - metrics_mapping = core.CfnMapping(self, "AnonymousData", mapping={"SendAnonymousData": {"Data": "Yes"}}) + metrics_mapping = core.CfnMapping( + self, "AnonymousData", mapping={"SendAnonymousData": {"Data": "Yes"}} + ) metrics_condition = core.CfnCondition( self, "AnonymousDatatoAWS", - expression=core.Fn.condition_equals(metrics_mapping.find_in_map("SendAnonymousData", "Data"), "Yes"), + expression=core.Fn.condition_equals( + metrics_mapping.find_in_map("SendAnonymousData", "Data"), "Yes" + ), ) # If user chooses Git as pipeline provision type, create codepipeline with Git repo as source core.Aspects.of(repo).add(ConditionalResources(git_address_provided)) - core.Aspects.of(codecommit_pipeline).add(ConditionalResources(git_address_provided)) - core.Aspects.of(codebuild_project).add(ConditionalResources(git_address_provided)) + core.Aspects.of(codecommit_pipeline).add( + ConditionalResources(git_address_provided) + ) + core.Aspects.of(codebuild_project).add( + ConditionalResources(git_address_provided) + ) # Create Template Interface paramaters_list = [ @@ -444,11 +540,21 @@ def __init__(self, scope: core.Construct, id: str, *, multi_account=False, **kwa ] paramaters_labels = { - f"{notification_email.logical_id}": {"default": "Notification Email (Required)"}, - f"{git_address.logical_id}": {"default": "CodeCommit Repo URL Address (Optional)"}, - f"{existing_bucket.logical_id}": {"default": "Name of an Existing S3 Bucket (Optional)"}, - f"{existing_ecr_repo.logical_id}": {"default": "Name of an Existing Amazon ECR repository (Optional)"}, - f"{use_model_registry.logical_id}": {"default": "Do you want to use SageMaker Model Registry?"}, + f"{notification_email.logical_id}": { + "default": "Notification Email (Required)" + }, + f"{git_address.logical_id}": { + "default": "CodeCommit Repo URL Address (Optional)" + }, + f"{existing_bucket.logical_id}": { + "default": "Name of an Existing S3 Bucket (Optional)" + }, + f"{existing_ecr_repo.logical_id}": { + "default": "Name of an Existing Amazon ECR repository (Optional)" + }, + f"{use_model_registry.logical_id}": { + "default": "Do you want to use SageMaker Model Registry?" + }, f"{create_model_registry.logical_id}": { "default": "Do you want the solution to create a SageMaker's model package group?" }, @@ -460,7 +566,11 @@ def __init__(self, scope: core.Construct, id: str, *, multi_account=False, **kwa # configure mutli-account parameters and permissions is_delegated_admin = None if multi_account: - paramaters_list, paramaters_labels, is_delegated_admin = configure_multi_account_parameters_permissions( + ( + paramaters_list, + paramaters_labels, + is_delegated_admin, + ) = configure_multi_account_parameters_permissions( self, assets_bucket, blueprint_repository_bucket, @@ -489,7 +599,9 @@ def __init__(self, scope: core.Construct, id: str, *, multi_account=False, **kwa ).to_string(), "Region": core.Aws.REGION, "IsMultiAccount": str(multi_account), - "IsDelegatedAccount": is_delegated_admin if multi_account else core.Aws.NO_VALUE, + "IsDelegatedAccount": is_delegated_admin + if multi_account + else core.Aws.NO_VALUE, "UseModelRegistry": use_model_registry.value_as_string, "SolutionId": get_cdk_context_value(self, "SolutionId"), "Version": get_cdk_context_value(self, "Version"), diff --git a/source/requirements-test.txt b/source/requirements-test.txt index 6b64a08..2b76da3 100644 --- a/source/requirements-test.txt +++ b/source/requirements-test.txt @@ -1,6 +1,6 @@ -sagemaker==2.128.0 +sagemaker==2.146.0 boto3==1.26.46 crhelper==2.0.6 pytest==7.2.0 pytest-cov==4.0.0 -moto[all]==4.0.7 \ No newline at end of file +moto[all]==4.1.3 \ No newline at end of file