From cb3db27bf09ca6b12aef5de1ea2e27d08e0fe80c Mon Sep 17 00:00:00 2001 From: Will-NOQ <102608210+Will-NOQ@users.noreply.github.com> Date: Fri, 14 Jul 2023 18:22:43 -0500 Subject: [PATCH] Fixes to the way roles are resolved as part of iambic detect. --- iambic/core/detect.py | 10 +- iambic/core/models.py | 3 +- .../aws/iam/role/template_generation.py | 93 ++++++++++++------- iambic/plugins/v0_1_0/aws/iam/role/utils.py | 13 ++- 4 files changed, 78 insertions(+), 41 deletions(-) diff --git a/iambic/core/detect.py b/iambic/core/detect.py index 92a268dcf..9b6704c62 100644 --- a/iambic/core/detect.py +++ b/iambic/core/detect.py @@ -1,7 +1,9 @@ +from __future__ import annotations + from collections import defaultdict from typing import Type -from iambic.core.models import ProviderChild, BaseTemplate +from iambic.core.models import BaseTemplate, ProviderChild from iambic.core.utils import evaluate_on_provider @@ -25,13 +27,15 @@ def group_detect_messages(group_by: str, messages: list) -> dict: def generate_template_output( excluded_provider_ids: list[str], provider_child_map: dict[str, ProviderChild], - template: Type[BaseTemplate] + template: Type[BaseTemplate], ) -> dict[str, dict]: provider_children_value_map = dict() for provider_child_id, provider_child in provider_child_map.items(): if provider_child_id in excluded_provider_ids: continue - elif not evaluate_on_provider(template, provider_child, exclude_import_only=False): + elif not evaluate_on_provider( + template, provider_child, exclude_import_only=False + ): continue if provider_child_value := template.apply_resource_dict(provider_child): diff --git a/iambic/core/models.py b/iambic/core/models.py index 38e40a931..5a4c7a383 100644 --- a/iambic/core/models.py +++ b/iambic/core/models.py @@ -39,12 +39,13 @@ LiteralScalarString, apply_to_provider, create_commented_map, + get_rendered_template_str_value, get_writable_directory, simplify_dt, snake_to_camelcap, sort_dict, transform_comments, - yaml, get_rendered_template_str_value, + yaml, ) if TYPE_CHECKING: diff --git a/iambic/plugins/v0_1_0/aws/iam/role/template_generation.py b/iambic/plugins/v0_1_0/aws/iam/role/template_generation.py index e177508a0..3e9e5b6c9 100644 --- a/iambic/plugins/v0_1_0/aws/iam/role/template_generation.py +++ b/iambic/plugins/v0_1_0/aws/iam/role/template_generation.py @@ -1,5 +1,6 @@ from __future__ import annotations +import asyncio import itertools import os from collections import defaultdict @@ -8,13 +9,19 @@ import aiofiles from iambic.core import noq_json as json +from iambic.core.detect import generate_template_output, group_detect_messages from iambic.core.logger import log from iambic.core.models import ExecutionMessage from iambic.core.template_generation import ( create_or_update_template, delete_orphaned_templates, ) -from iambic.core.utils import NoqSemaphore, normalize_dict_keys, resource_file_upsert +from iambic.core.utils import ( + NoqSemaphore, + get_rendered_template_str_value, + normalize_dict_keys, + resource_file_upsert, +) from iambic.plugins.v0_1_0.aws.event_bridge.models import RoleMessageDetails from iambic.plugins.v0_1_0.aws.iam.policy.models import AssumeRolePolicyDocument from iambic.plugins.v0_1_0.aws.iam.role.models import ( @@ -23,7 +30,7 @@ RoleProperties, ) from iambic.plugins.v0_1_0.aws.iam.role.utils import ( - get_role_across_accounts, + get_role, get_role_inline_policies, get_role_managed_policies, list_role_tags, @@ -143,20 +150,39 @@ async def generate_account_role_resource_files( async def generate_role_resource_file_for_all_accounts( exe_message: ExecutionMessage, - aws_accounts: list[AWSAccount], role_name: str, + aws_account_map: dict[str, AWSAccount], + updated_account_ids: list[str], + iambic_template: AwsIamRoleTemplate, ) -> list: + async def get_role_for_account(aws_account: AWSAccount): + iam_client = await aws_account.get_boto3_client("iam") + account_role_name = get_rendered_template_str_value(role_name, aws_account) + role = await get_role(account_role_name, iam_client) + role["PermissionsBoundary"]["PolicyArn"] = role["PermissionsBoundary"].pop( + "PermissionsBoundaryArn" + ) + return {aws_account.account_id: role} + account_resource_dir_map = { aws_account.account_id: get_response_dir(exe_message, aws_account) - for aws_account in aws_accounts + for aws_account in aws_account_map.values() } role_resource_file_upsert_semaphore = NoqSemaphore(resource_file_upsert, 10) messages = [] response = [] - role_across_accounts = await get_role_across_accounts( - aws_accounts, role_name, False + role_across_accounts = generate_template_output( + updated_account_ids, aws_account_map, iambic_template ) + updated_account_roles = await asyncio.gather( + *[ + get_role_for_account(aws_account_map[account_id]) + for account_id in updated_account_ids + ] + ) + for updated_account_role in updated_account_roles: + role_across_accounts.update(updated_account_role) role_across_accounts = {k: v for k, v in role_across_accounts.items() if v} log.debug( @@ -450,7 +476,7 @@ async def create_templated_role( # noqa: C901 AwsIamRoleTemplate, role_template_params, RoleProperties(**role_template_properties), - list(aws_account_map.values()), + [aws_account_map[role_ref["account_id"]] for role_ref in role_refs], ) except Exception as e: log_params = { @@ -491,18 +517,21 @@ async def collect_aws_roles( ) if detect_messages: - aws_accounts = list(aws_account_map.values()) generate_role_resource_file_for_all_accounts_semaphore = NoqSemaphore( generate_role_resource_file_for_all_accounts, 45 ) + grouped_detect_messages = group_detect_messages("role_name", detect_messages) tasks = [ { "exe_message": exe_message, - "aws_accounts": aws_accounts, - "role_name": role.role_name, + "aws_account_map": aws_account_map, + "role_name": role_name, + "updated_account_ids": [ + message.account_id for message in role_messages + ], + "iambic_template": existing_template_map.get(role_name), } - for role in detect_messages - if not role.delete + for role_name, role_messages in grouped_detect_messages.items() ] # Remove deleted or mark templates for update @@ -519,15 +548,6 @@ async def collect_aws_roles( ): # It's the only account for the template so delete it existing_template.delete() - else: - # There are other accounts for the template so re-eval the template - tasks.append( - { - "exe_message": exe_message, - "aws_accounts": aws_accounts, - "role_name": existing_template.properties.role_name, - } - ) account_role_list = ( await generate_role_resource_file_for_all_accounts_semaphore.process(tasks) @@ -568,19 +588,24 @@ async def collect_aws_roles( } ) - log.info( - "Setting inline policies in role templates", - accounts=list(aws_account_map.keys()), - ) - await set_role_resource_inline_policies_semaphore.process(messages) - log.info( - "Setting managed policies in role templates", - accounts=list(aws_account_map.keys()), - ) - await set_role_resource_managed_policies_semaphore.process(messages) - log.info("Setting tags in role templates", accounts=list(aws_account_map.keys())) - await set_role_resource_tags_semaphore.process(messages) - log.info("Finished retrieving role details", accounts=list(aws_account_map.keys())) + if not detect_messages: + log.info( + "Setting inline policies in role templates", + accounts=list(aws_account_map.keys()), + ) + await set_role_resource_inline_policies_semaphore.process(messages) + log.info( + "Setting managed policies in role templates", + accounts=list(aws_account_map.keys()), + ) + await set_role_resource_managed_policies_semaphore.process(messages) + log.info( + "Setting tags in role templates", accounts=list(aws_account_map.keys()) + ) + await set_role_resource_tags_semaphore.process(messages) + log.info( + "Finished retrieving role details", accounts=list(aws_account_map.keys()) + ) account_role_output = json.dumps(account_roles) with open( diff --git a/iambic/plugins/v0_1_0/aws/iam/role/utils.py b/iambic/plugins/v0_1_0/aws/iam/role/utils.py index de886da8a..2b233564f 100644 --- a/iambic/plugins/v0_1_0/aws/iam/role/utils.py +++ b/iambic/plugins/v0_1_0/aws/iam/role/utils.py @@ -10,7 +10,11 @@ from iambic.core.context import ctx from iambic.core.logger import log from iambic.core.models import ProposedChange, ProposedChangeType -from iambic.core.utils import aio_wrapper, plugin_apply_wrapper +from iambic.core.utils import ( + aio_wrapper, + get_rendered_template_str_value, + plugin_apply_wrapper, +) from iambic.plugins.v0_1_0.aws.models import AWSAccount from iambic.plugins.v0_1_0.aws.utils import boto_crud_call, paginated_search @@ -99,7 +103,7 @@ async def get_role_managed_policies(role_name: str, iam_client) -> list[dict[str else: break - return policies + return [{"PolicyArn": policy["PolicyArn"]} for policy in policies] async def get_role(role_name: str, iam_client, include_policies: bool = True) -> dict: @@ -107,6 +111,8 @@ async def get_role(role_name: str, iam_client, include_policies: bool = True) -> current_role = (await boto_crud_call(iam_client.get_role, RoleName=role_name))[ "Role" ] + current_role.get("PermissionsBoundary", {}).pop("PermissionsBoundaryType", None) + if include_policies: current_role["ManagedPolicies"] = await get_role_managed_policies( role_name, iam_client @@ -129,9 +135,10 @@ async def get_role_across_accounts( ) -> dict: async def get_role_for_account(aws_account: AWSAccount): iam_client = await aws_account.get_boto3_client("iam") + account_role_name = get_rendered_template_str_value(role_name, aws_account) return { aws_account.account_id: await get_role( - role_name, iam_client, include_policies + account_role_name, iam_client, include_policies ) }