Merge pull request #229 from noqdev/fix/en-1848-account-id-substituti…

…on-2

EN-1848 Extend the templatized variable preference to group, policy, …

iambic/core/template_generation.py

@@ -4,6 +4,7 @@
 from typing import Union
 
 import xxhash
+
 from iambic.core import noq_json as json
 from iambic.core.context import ctx
 from iambic.core.logger import log
@@ -477,8 +478,6 @@ def create_or_update_template(
     # iambic-specific knowledge requires us to load the existing template
     # because it will not be reflected by AWS API.
     if existing_template := existing_template_map.get(identifier, None):
-        if identifier == "{{account_name}}_administrator":
-            pass
         merged_template = merge_model(
             new_template, existing_template, all_provider_children
         )

iambic/plugins/v0_1_0/aws/iam/group/template_generation.py

@@ -6,7 +6,6 @@
 from typing import TYPE_CHECKING
 
 import aiofiles
-
 from iambic.core import noq_json as json
 from iambic.core.logger import log
 from iambic.core.models import ExecutionMessage
@@ -32,7 +31,11 @@
     list_groups,
 )
 from iambic.plugins.v0_1_0.aws.models import AWSAccount
-from iambic.plugins.v0_1_0.aws.utils import get_aws_account_map, normalize_boto3_resp
+from iambic.plugins.v0_1_0.aws.utils import (
+    calculate_import_preference,
+    get_aws_account_map,
+    normalize_boto3_resp,
+)
 
 if TYPE_CHECKING:
     from iambic.plugins.v0_1_0.aws.iambic_plugin import AWSConfig
@@ -216,6 +219,11 @@ async def create_templated_group(  # noqa: C901
         config.min_accounts_required_for_wildcard_included_accounts
     )
 
+    # calculate preference based on existing template
+    prefer_templatized = calculate_import_preference(
+        existing_template_map.get(group_name)
+    )
+
     # Generate the params used for attribute creation
     group_template_params = {"identifier": group_name}
     group_template_properties = {"group_name": group_name}
@@ -272,12 +280,20 @@ async def create_templated_group(  # noqa: C901
 
     if managed_policy_resources:
         group_template_properties["managed_policies"] = await group_dict_attribute(
-            aws_account_map, num_of_accounts, managed_policy_resources, False
+            aws_account_map,
+            num_of_accounts,
+            managed_policy_resources,
+            False,
+            prefer_templatized=prefer_templatized,
         )
 
     if inline_policy_document_resources:
         group_template_properties["inline_policies"] = await group_dict_attribute(
-            aws_account_map, num_of_accounts, inline_policy_document_resources, False
+            aws_account_map,
+            num_of_accounts,
+            inline_policy_document_resources,
+            False,
+            prefer_templatized=prefer_templatized,
         )
 
     file_path = get_templated_group_file_path(

iambic/plugins/v0_1_0/aws/iam/policy/template_generation.py

@@ -28,7 +28,11 @@
     list_managed_policies,
 )
 from iambic.plugins.v0_1_0.aws.models import AWSAccount
-from iambic.plugins.v0_1_0.aws.utils import get_aws_account_map, normalize_boto3_resp
+from iambic.plugins.v0_1_0.aws.utils import (
+    calculate_import_preference,
+    get_aws_account_map,
+    normalize_boto3_resp,
+)
 
 if TYPE_CHECKING:
     from iambic.plugins.v0_1_0.aws.iambic_plugin import AWSConfig
@@ -193,6 +197,11 @@ async def create_templated_managed_policy(  # noqa: C901
                 managed_policy_ref["account_id"]
             ] = normalize_boto3_resp(content_dict)
 
+    # calculate preference based on existing template
+    prefer_templatized = calculate_import_preference(
+        existing_template_map.get(managed_policy_name)
+    )
+
     # Generate the params used for attribute creation
     template_properties = {"policy_name": managed_policy_name}
 
@@ -248,7 +257,11 @@ async def create_templated_managed_policy(  # noqa: C901
         template_properties["path"] = path
 
     template_properties["policy_document"] = await group_dict_attribute(
-        aws_account_map, num_of_accounts, policy_document_resources, True
+        aws_account_map,
+        num_of_accounts,
+        policy_document_resources,
+        True,
+        prefer_templatized=prefer_templatized,
     )
 
     if description_resources:
@@ -258,7 +271,11 @@ async def create_templated_managed_policy(  # noqa: C901
 
     if tag_resources:
         tags = await group_dict_attribute(
-            aws_account_map, num_of_accounts, tag_resources, True
+            aws_account_map,
+            num_of_accounts,
+            tag_resources,
+            True,
+            prefer_templatized=prefer_templatized,
         )
         if isinstance(tags, dict):
             tags = [tags]

iambic/plugins/v0_1_0/aws/iam/role/template_generation.py

@@ -33,7 +33,11 @@
     list_roles,
 )
 from iambic.plugins.v0_1_0.aws.models import AWSAccount
-from iambic.plugins.v0_1_0.aws.utils import get_aws_account_map, normalize_boto3_resp
+from iambic.plugins.v0_1_0.aws.utils import (
+    calculate_import_preference,
+    get_aws_account_map,
+    normalize_boto3_resp,
+)
 
 if TYPE_CHECKING:
     from iambic.plugins.v0_1_0.aws.iambic_plugin import AWSConfig
@@ -226,22 +230,6 @@ async def _account_id_to_role_map(role_refs):
     return account_id_to_role_map
 
 
-def calculate_import_preference(existing_template):
-    prefer_templatized = False
-    try:
-        if existing_template:
-            # this is expensive just to compute if
-            # the existing template is already bias toward templatized.
-            existing_template_in_str = existing_template.json()
-            prefer_templatized = "{{" in existing_template_in_str
-    except Exception as exc_info:
-        # We are willing to tolerate exception because
-        # we are calculating preference from possibly bad templates on disk
-        log_params = {"exc_info": str(exc_info)}
-        log.error("cannot calculate preference from existing template", **log_params)
-    return prefer_templatized
-
-
 async def create_templated_role(  # noqa: C901
     aws_account_map: dict[str, AWSAccount],
     role_name: str,

iambic/plugins/v0_1_0/aws/iam/user/template_generation.py

@@ -6,7 +6,6 @@
 from typing import TYPE_CHECKING
 
 import aiofiles
-
 from iambic.core import noq_json as json
 from iambic.core.logger import log
 from iambic.core.models import ExecutionMessage
@@ -34,7 +33,11 @@
     list_users,
 )
 from iambic.plugins.v0_1_0.aws.models import AWSAccount
-from iambic.plugins.v0_1_0.aws.utils import get_aws_account_map, normalize_boto3_resp
+from iambic.plugins.v0_1_0.aws.utils import (
+    calculate_import_preference,
+    get_aws_account_map,
+    normalize_boto3_resp,
+)
 
 if TYPE_CHECKING:
     from iambic.plugins.v0_1_0.aws.iambic_plugin import AWSConfig
@@ -233,6 +236,11 @@ async def create_templated_user(  # noqa: C901
         config.min_accounts_required_for_wildcard_included_accounts
     )
 
+    # calculate preference based on existing template
+    prefer_templatized = calculate_import_preference(
+        existing_template_map.get(user_name)
+    )
+
     # Generate the params used for attribute creation
     user_template_params = {"identifier": user_name}
     user_template_properties = {"user_name": user_name}
@@ -322,7 +330,10 @@ async def create_templated_user(  # noqa: C901
 
     if permissions_boundary_resources:
         user_template_properties["permissions_boundary"] = await group_dict_attribute(
-            aws_account_map, num_of_accounts, permissions_boundary_resources
+            aws_account_map,
+            num_of_accounts,
+            permissions_boundary_resources,
+            prefer_templatized=prefer_templatized,
         )
 
     if description_resources:
@@ -332,21 +343,37 @@ async def create_templated_user(  # noqa: C901
 
     if managed_policy_resources:
         user_template_properties["managed_policies"] = await group_dict_attribute(
-            aws_account_map, num_of_accounts, managed_policy_resources, False
+            aws_account_map,
+            num_of_accounts,
+            managed_policy_resources,
+            False,
+            prefer_templatized=prefer_templatized,
         )
 
     if inline_policy_document_resources:
         user_template_properties["inline_policies"] = await group_dict_attribute(
-            aws_account_map, num_of_accounts, inline_policy_document_resources, False
+            aws_account_map,
+            num_of_accounts,
+            inline_policy_document_resources,
+            False,
+            prefer_templatized=prefer_templatized,
         )
 
     if group_resources:
         user_template_properties["groups"] = await group_dict_attribute(
-            aws_account_map, num_of_accounts, group_resources, False
+            aws_account_map,
+            num_of_accounts,
+            group_resources,
+            False,
+            prefer_templatized=prefer_templatized,
         )
     if tag_resources:
         tags = await group_dict_attribute(
-            aws_account_map, num_of_accounts, tag_resources, True
+            aws_account_map,
+            num_of_accounts,
+            tag_resources,
+            True,
+            prefer_templatized=prefer_templatized,
         )
         if isinstance(tags, dict):
             tags = [tags]

iambic/plugins/v0_1_0/aws/identity_center/permission_set/template_generation.py

@@ -7,7 +7,6 @@
 from typing import TYPE_CHECKING, Union
 
 import aiofiles
-
 from iambic.core import noq_json as json
 from iambic.core.logger import log
 from iambic.core.models import ExecutionMessage
@@ -32,7 +31,11 @@
     get_permission_set_users_and_groups,
 )
 from iambic.plugins.v0_1_0.aws.models import AWSAccount
-from iambic.plugins.v0_1_0.aws.utils import get_aws_account_map, normalize_boto3_resp
+from iambic.plugins.v0_1_0.aws.utils import (
+    calculate_import_preference,
+    get_aws_account_map,
+    normalize_boto3_resp,
+)
 
 # TODO: Update all grouping functions to support org grouping once multiple orgs with IdentityCenter is functional
 # TODO: Update partial import to support permission set only being deleted on a single org
@@ -154,6 +157,11 @@ async def create_templated_permission_set(  # noqa: C901
                 permission_set_ref["account_id"]
             ] = normalize_boto3_resp(content_dict)
 
+    # calculate preference based on existing template
+    prefer_templatized = calculate_import_preference(
+        existing_template_map.get(permission_set_name)
+    )
+
     # Generate the params used for attribute creation
     template_params = {"identifier": permission_set_name, "access_rules": []}
     template_properties = {"name": permission_set_name}
@@ -294,7 +302,10 @@ async def create_templated_permission_set(  # noqa: C901
 
     if permissions_boundary_resources:
         template_properties["permissions_boundary"] = await group_dict_attribute(
-            aws_account_map, num_of_accounts, permissions_boundary_resources
+            aws_account_map,
+            num_of_accounts,
+            permissions_boundary_resources,
+            prefer_templatized=prefer_templatized,
         )
 
     if description_resources:
@@ -304,7 +315,11 @@ async def create_templated_permission_set(  # noqa: C901
 
     if managed_policy_resources:
         template_properties["managed_policies"] = await group_dict_attribute(
-            aws_account_map, num_of_accounts, managed_policy_resources, False
+            aws_account_map,
+            num_of_accounts,
+            managed_policy_resources,
+            False,
+            prefer_templatized=prefer_templatized,
         )
 
     if customer_managed_policy_ref_resources:
@@ -315,16 +330,21 @@ async def create_templated_permission_set(  # noqa: C901
             num_of_accounts,
             customer_managed_policy_ref_resources,
             False,
+            prefer_templatized=prefer_templatized,
         )
 
     if tag_resources:
         template_properties["tags"] = await group_dict_attribute(
-            aws_account_map, num_of_accounts, tag_resources, False
+            aws_account_map,
+            num_of_accounts,
+            tag_resources,
+            False,
+            prefer_templatized=prefer_templatized,
         )
 
     if inline_policy_resources:
         template_properties["inline_policy"] = json.loads(
-            await group_int_or_str_attribute(
+            await group_int_or_str_attribute(  # hm... other (role, user) inline policies use group_dict_attribute, not sure the reason
                 aws_account_map,
                 num_of_accounts,
                 inline_policy_resources,

iambic/plugins/v0_1_0/aws/utils.py

@@ -8,7 +8,6 @@
 
 import boto3
 from botocore.exceptions import ClientError, NoCredentialsError
-
 from iambic.core.iambic_enum import IambicManaged
 from iambic.core.logger import log
 from iambic.core.utils import aio_wrapper, camel_to_snake
@@ -17,6 +16,22 @@
     from iambic.plugins.v0_1_0.aws.iambic_plugin import AWSConfig
 
 
+def calculate_import_preference(existing_template):
+    prefer_templatized = False
+    try:
+        if existing_template:
+            # this is expensive just to compute if
+            # the existing template is already bias toward templatized.
+            existing_template_in_str = existing_template.json()
+            prefer_templatized = "{{" in existing_template_in_str
+    except Exception as exc_info:
+        # We are willing to tolerate exception because
+        # we are calculating preference from possibly bad templates on disk
+        log_params = {"exc_info": str(exc_info)}
+        log.error("cannot calculate preference from existing template", **log_params)
+    return prefer_templatized
+
+
 async def boto_crud_call(boto_fnc, **kwargs) -> Union[list, dict]:
     """Responsible for calls to boto. Adds async support and error handling
     :param boto_fnc: