From 0087aedf4d58c931ec3b46d081501da43f4b89b5 Mon Sep 17 00:00:00 2001 From: Steven Moy Date: Fri, 10 Mar 2023 18:41:00 -0800 Subject: [PATCH 1/2] EN-1848 Extend the templatized variable preference to group, policy, role, user and permission sets --- .../aws/iam/group/template_generation.py | 24 +++++++++-- .../aws/iam/policy/template_generation.py | 23 +++++++++-- .../aws/iam/role/template_generation.py | 22 +++------- .../aws/iam/user/template_generation.py | 41 +++++++++++++++---- .../permission_set/template_generation.py | 32 ++++++++++++--- iambic/plugins/v0_1_0/aws/utils.py | 17 +++++++- 6 files changed, 121 insertions(+), 38 deletions(-) diff --git a/iambic/plugins/v0_1_0/aws/iam/group/template_generation.py b/iambic/plugins/v0_1_0/aws/iam/group/template_generation.py index a066351dd..959feffda 100644 --- a/iambic/plugins/v0_1_0/aws/iam/group/template_generation.py +++ b/iambic/plugins/v0_1_0/aws/iam/group/template_generation.py @@ -6,7 +6,6 @@ from typing import TYPE_CHECKING import aiofiles - from iambic.core import noq_json as json from iambic.core.logger import log from iambic.core.models import ExecutionMessage @@ -32,7 +31,11 @@ list_groups, ) from iambic.plugins.v0_1_0.aws.models import AWSAccount -from iambic.plugins.v0_1_0.aws.utils import get_aws_account_map, normalize_boto3_resp +from iambic.plugins.v0_1_0.aws.utils import ( + calculate_import_preference, + get_aws_account_map, + normalize_boto3_resp, +) if TYPE_CHECKING: from iambic.plugins.v0_1_0.aws.iambic_plugin import AWSConfig @@ -216,6 +219,11 @@ async def create_templated_group( # noqa: C901 config.min_accounts_required_for_wildcard_included_accounts ) + # calculate preference based on existing template + prefer_templatized = calculate_import_preference( + existing_template_map.get(group_name) + ) + # Generate the params used for attribute creation group_template_params = {"identifier": group_name} group_template_properties = {"group_name": group_name} @@ -272,12 +280,20 @@ async def create_templated_group( # noqa: C901 if managed_policy_resources: group_template_properties["managed_policies"] = await group_dict_attribute( - aws_account_map, num_of_accounts, managed_policy_resources, False + aws_account_map, + num_of_accounts, + managed_policy_resources, + False, + prefer_templatized=prefer_templatized, ) if inline_policy_document_resources: group_template_properties["inline_policies"] = await group_dict_attribute( - aws_account_map, num_of_accounts, inline_policy_document_resources, False + aws_account_map, + num_of_accounts, + inline_policy_document_resources, + False, + prefer_templatized=prefer_templatized, ) file_path = get_templated_group_file_path( diff --git a/iambic/plugins/v0_1_0/aws/iam/policy/template_generation.py b/iambic/plugins/v0_1_0/aws/iam/policy/template_generation.py index 57c3879ac..3978a08fa 100644 --- a/iambic/plugins/v0_1_0/aws/iam/policy/template_generation.py +++ b/iambic/plugins/v0_1_0/aws/iam/policy/template_generation.py @@ -28,7 +28,11 @@ list_managed_policies, ) from iambic.plugins.v0_1_0.aws.models import AWSAccount -from iambic.plugins.v0_1_0.aws.utils import get_aws_account_map, normalize_boto3_resp +from iambic.plugins.v0_1_0.aws.utils import ( + calculate_import_preference, + get_aws_account_map, + normalize_boto3_resp, +) if TYPE_CHECKING: from iambic.plugins.v0_1_0.aws.iambic_plugin import AWSConfig @@ -193,6 +197,11 @@ async def create_templated_managed_policy( # noqa: C901 managed_policy_ref["account_id"] ] = normalize_boto3_resp(content_dict) + # calculate preference based on existing template + prefer_templatized = calculate_import_preference( + existing_template_map.get(managed_policy_name) + ) + # Generate the params used for attribute creation template_properties = {"policy_name": managed_policy_name} @@ -248,7 +257,11 @@ async def create_templated_managed_policy( # noqa: C901 template_properties["path"] = path template_properties["policy_document"] = await group_dict_attribute( - aws_account_map, num_of_accounts, policy_document_resources, True + aws_account_map, + num_of_accounts, + policy_document_resources, + True, + prefer_templatized=prefer_templatized, ) if description_resources: @@ -258,7 +271,11 @@ async def create_templated_managed_policy( # noqa: C901 if tag_resources: tags = await group_dict_attribute( - aws_account_map, num_of_accounts, tag_resources, True + aws_account_map, + num_of_accounts, + tag_resources, + True, + prefer_templatized=prefer_templatized, ) if isinstance(tags, dict): tags = [tags] diff --git a/iambic/plugins/v0_1_0/aws/iam/role/template_generation.py b/iambic/plugins/v0_1_0/aws/iam/role/template_generation.py index dfefd4533..960fd575a 100644 --- a/iambic/plugins/v0_1_0/aws/iam/role/template_generation.py +++ b/iambic/plugins/v0_1_0/aws/iam/role/template_generation.py @@ -33,7 +33,11 @@ list_roles, ) from iambic.plugins.v0_1_0.aws.models import AWSAccount -from iambic.plugins.v0_1_0.aws.utils import get_aws_account_map, normalize_boto3_resp +from iambic.plugins.v0_1_0.aws.utils import ( + calculate_import_preference, + get_aws_account_map, + normalize_boto3_resp, +) if TYPE_CHECKING: from iambic.plugins.v0_1_0.aws.iambic_plugin import AWSConfig @@ -226,22 +230,6 @@ async def _account_id_to_role_map(role_refs): return account_id_to_role_map -def calculate_import_preference(existing_template): - prefer_templatized = False - try: - if existing_template: - # this is expensive just to compute if - # the existing template is already bias toward templatized. - existing_template_in_str = existing_template.json() - prefer_templatized = "{{" in existing_template_in_str - except Exception as exc_info: - # We are willing to tolerate exception because - # we are calculating preference from possibly bad templates on disk - log_params = {"exc_info": str(exc_info)} - log.error("cannot calculate preference from existing template", **log_params) - return prefer_templatized - - async def create_templated_role( # noqa: C901 aws_account_map: dict[str, AWSAccount], role_name: str, diff --git a/iambic/plugins/v0_1_0/aws/iam/user/template_generation.py b/iambic/plugins/v0_1_0/aws/iam/user/template_generation.py index 4d7af67c9..0bba65bd4 100644 --- a/iambic/plugins/v0_1_0/aws/iam/user/template_generation.py +++ b/iambic/plugins/v0_1_0/aws/iam/user/template_generation.py @@ -6,7 +6,6 @@ from typing import TYPE_CHECKING import aiofiles - from iambic.core import noq_json as json from iambic.core.logger import log from iambic.core.models import ExecutionMessage @@ -34,7 +33,11 @@ list_users, ) from iambic.plugins.v0_1_0.aws.models import AWSAccount -from iambic.plugins.v0_1_0.aws.utils import get_aws_account_map, normalize_boto3_resp +from iambic.plugins.v0_1_0.aws.utils import ( + calculate_import_preference, + get_aws_account_map, + normalize_boto3_resp, +) if TYPE_CHECKING: from iambic.plugins.v0_1_0.aws.iambic_plugin import AWSConfig @@ -233,6 +236,11 @@ async def create_templated_user( # noqa: C901 config.min_accounts_required_for_wildcard_included_accounts ) + # calculate preference based on existing template + prefer_templatized = calculate_import_preference( + existing_template_map.get(user_name) + ) + # Generate the params used for attribute creation user_template_params = {"identifier": user_name} user_template_properties = {"user_name": user_name} @@ -322,7 +330,10 @@ async def create_templated_user( # noqa: C901 if permissions_boundary_resources: user_template_properties["permissions_boundary"] = await group_dict_attribute( - aws_account_map, num_of_accounts, permissions_boundary_resources + aws_account_map, + num_of_accounts, + permissions_boundary_resources, + prefer_templatized=prefer_templatized, ) if description_resources: @@ -332,21 +343,37 @@ async def create_templated_user( # noqa: C901 if managed_policy_resources: user_template_properties["managed_policies"] = await group_dict_attribute( - aws_account_map, num_of_accounts, managed_policy_resources, False + aws_account_map, + num_of_accounts, + managed_policy_resources, + False, + prefer_templatized=prefer_templatized, ) if inline_policy_document_resources: user_template_properties["inline_policies"] = await group_dict_attribute( - aws_account_map, num_of_accounts, inline_policy_document_resources, False + aws_account_map, + num_of_accounts, + inline_policy_document_resources, + False, + prefer_templatized=prefer_templatized, ) if group_resources: user_template_properties["groups"] = await group_dict_attribute( - aws_account_map, num_of_accounts, group_resources, False + aws_account_map, + num_of_accounts, + group_resources, + False, + prefer_templatized=prefer_templatized, ) if tag_resources: tags = await group_dict_attribute( - aws_account_map, num_of_accounts, tag_resources, True + aws_account_map, + num_of_accounts, + tag_resources, + True, + prefer_templatized=prefer_templatized, ) if isinstance(tags, dict): tags = [tags] diff --git a/iambic/plugins/v0_1_0/aws/identity_center/permission_set/template_generation.py b/iambic/plugins/v0_1_0/aws/identity_center/permission_set/template_generation.py index 631ab3c94..073e0d3f6 100644 --- a/iambic/plugins/v0_1_0/aws/identity_center/permission_set/template_generation.py +++ b/iambic/plugins/v0_1_0/aws/identity_center/permission_set/template_generation.py @@ -7,7 +7,6 @@ from typing import TYPE_CHECKING, Union import aiofiles - from iambic.core import noq_json as json from iambic.core.logger import log from iambic.core.models import ExecutionMessage @@ -32,7 +31,11 @@ get_permission_set_users_and_groups, ) from iambic.plugins.v0_1_0.aws.models import AWSAccount -from iambic.plugins.v0_1_0.aws.utils import get_aws_account_map, normalize_boto3_resp +from iambic.plugins.v0_1_0.aws.utils import ( + calculate_import_preference, + get_aws_account_map, + normalize_boto3_resp, +) # TODO: Update all grouping functions to support org grouping once multiple orgs with IdentityCenter is functional # TODO: Update partial import to support permission set only being deleted on a single org @@ -154,6 +157,11 @@ async def create_templated_permission_set( # noqa: C901 permission_set_ref["account_id"] ] = normalize_boto3_resp(content_dict) + # calculate preference based on existing template + prefer_templatized = calculate_import_preference( + existing_template_map.get(permission_set_name) + ) + # Generate the params used for attribute creation template_params = {"identifier": permission_set_name, "access_rules": []} template_properties = {"name": permission_set_name} @@ -294,7 +302,10 @@ async def create_templated_permission_set( # noqa: C901 if permissions_boundary_resources: template_properties["permissions_boundary"] = await group_dict_attribute( - aws_account_map, num_of_accounts, permissions_boundary_resources + aws_account_map, + num_of_accounts, + permissions_boundary_resources, + prefer_templatized=prefer_templatized, ) if description_resources: @@ -304,7 +315,11 @@ async def create_templated_permission_set( # noqa: C901 if managed_policy_resources: template_properties["managed_policies"] = await group_dict_attribute( - aws_account_map, num_of_accounts, managed_policy_resources, False + aws_account_map, + num_of_accounts, + managed_policy_resources, + False, + prefer_templatized=prefer_templatized, ) if customer_managed_policy_ref_resources: @@ -315,16 +330,21 @@ async def create_templated_permission_set( # noqa: C901 num_of_accounts, customer_managed_policy_ref_resources, False, + prefer_templatized=prefer_templatized, ) if tag_resources: template_properties["tags"] = await group_dict_attribute( - aws_account_map, num_of_accounts, tag_resources, False + aws_account_map, + num_of_accounts, + tag_resources, + False, + prefer_templatized=prefer_templatized, ) if inline_policy_resources: template_properties["inline_policy"] = json.loads( - await group_int_or_str_attribute( + await group_int_or_str_attribute( # hm... other (role, user) inline policies use group_dict_attribute, not sure the reason aws_account_map, num_of_accounts, inline_policy_resources, diff --git a/iambic/plugins/v0_1_0/aws/utils.py b/iambic/plugins/v0_1_0/aws/utils.py index 1b7b68299..4b91fefc1 100644 --- a/iambic/plugins/v0_1_0/aws/utils.py +++ b/iambic/plugins/v0_1_0/aws/utils.py @@ -8,7 +8,6 @@ import boto3 from botocore.exceptions import ClientError, NoCredentialsError - from iambic.core.iambic_enum import IambicManaged from iambic.core.logger import log from iambic.core.utils import aio_wrapper, camel_to_snake @@ -17,6 +16,22 @@ from iambic.plugins.v0_1_0.aws.iambic_plugin import AWSConfig +def calculate_import_preference(existing_template): + prefer_templatized = False + try: + if existing_template: + # this is expensive just to compute if + # the existing template is already bias toward templatized. + existing_template_in_str = existing_template.json() + prefer_templatized = "{{" in existing_template_in_str + except Exception as exc_info: + # We are willing to tolerate exception because + # we are calculating preference from possibly bad templates on disk + log_params = {"exc_info": str(exc_info)} + log.error("cannot calculate preference from existing template", **log_params) + return prefer_templatized + + async def boto_crud_call(boto_fnc, **kwargs) -> Union[list, dict]: """Responsible for calls to boto. Adds async support and error handling :param boto_fnc: From d9f41cebaaf5d6843257f5c88f3f6e729b88f797 Mon Sep 17 00:00:00 2001 From: Steven Moy Date: Fri, 10 Mar 2023 18:57:07 -0800 Subject: [PATCH 2/2] Remove debug code --- iambic/core/template_generation.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/iambic/core/template_generation.py b/iambic/core/template_generation.py index f67de66c2..f0bd88a57 100644 --- a/iambic/core/template_generation.py +++ b/iambic/core/template_generation.py @@ -4,6 +4,7 @@ from typing import Union import xxhash + from iambic.core import noq_json as json from iambic.core.context import ctx from iambic.core.logger import log @@ -477,8 +478,6 @@ def create_or_update_template( # iambic-specific knowledge requires us to load the existing template # because it will not be reflected by AWS API. if existing_template := existing_template_map.get(identifier, None): - if identifier == "{{account_name}}_administrator": - pass merged_template = merge_model( new_template, existing_template, all_provider_children )