From dd1861d576f3c8f593e56c8aecfaff5418466fb8 Mon Sep 17 00:00:00 2001 From: Will-NOQ <102608210+Will-NOQ@users.noreply.github.com> Date: Mon, 27 Feb 2023 11:50:40 -0600 Subject: [PATCH 01/32] git plan/apply is now the only way to plan and apply via the command line which are now just plan/apply --- .../aws/role/test_update_template.py | 1 - .../aws/user/test_update_template.py | 1 - .../okta/group/test_okta_group.py | 14 +-- functional_tests/okta/user/test_okta_user.py | 20 ++- functional_tests/test_google.py | 11 +- iambic/config/wizard.py | 1 - iambic/core/utils.py | 3 +- iambic/lambda/app.py | 24 +--- iambic/main.py | 119 ++---------------- .../plugins/v0_1_0/aws/iam/policy/models.py | 5 +- iambic/plugins/v0_1_0/aws/iam/policy/utils.py | 1 + iambic/plugins/v0_1_0/aws/iam/role/models.py | 3 +- .../aws/iam/user/template_generation.py | 1 + .../permission_set/template_generation.py | 1 - .../identity_center/permission_set/utils.py | 1 - iambic/plugins/v0_1_0/aws/models.py | 7 +- .../plugins/v0_1_0/example/iambic_plugin.py | 3 +- iambic/plugins/v0_1_0/okta/app/utils.py | 4 +- iambic/plugins/v0_1_0/okta/models.py | 3 +- iambic/plugins/v0_1_0/okta/utils.py | 3 +- .../test_template_generation.py | 1 + test/config/test_utils.py | 1 - test/google/group/test_models.py | 1 + test/plugins/v0_1_0/github/test_github_app.py | 1 + .../v0_1_0/github/test_github_plugin.py | 3 +- test/test_main.py | 4 +- 26 files changed, 54 insertions(+), 183 deletions(-) diff --git a/functional_tests/aws/role/test_update_template.py b/functional_tests/aws/role/test_update_template.py index c4673c320..751c322ef 100644 --- a/functional_tests/aws/role/test_update_template.py +++ b/functional_tests/aws/role/test_update_template.py @@ -4,7 +4,6 @@ from unittest import IsolatedAsyncioTestCase import dateparser - from functional_tests.aws.role.utils import generate_role_template_from_base from functional_tests.conftest import IAMBIC_TEST_DETAILS from iambic.core.context import ctx diff --git a/functional_tests/aws/user/test_update_template.py b/functional_tests/aws/user/test_update_template.py index cd1a45355..53069cecc 100644 --- a/functional_tests/aws/user/test_update_template.py +++ b/functional_tests/aws/user/test_update_template.py @@ -4,7 +4,6 @@ from unittest import IsolatedAsyncioTestCase import dateparser - from functional_tests.aws.user.utils import generate_user_template_from_base from functional_tests.conftest import IAMBIC_TEST_DETAILS from iambic.core.context import ctx diff --git a/functional_tests/okta/group/test_okta_group.py b/functional_tests/okta/group/test_okta_group.py index a24fafbbe..8297eee21 100644 --- a/functional_tests/okta/group/test_okta_group.py +++ b/functional_tests/okta/group/test_okta_group.py @@ -6,7 +6,7 @@ from functional_tests.conftest import IAMBIC_TEST_DETAILS from iambic.core.iambic_enum import IambicManaged from iambic.core.parser import load_templates -from iambic.main import run_apply +from iambic.main import run_force_apply def test_okta_group(): @@ -32,8 +32,7 @@ def test_okta_group(): temp_file.write(iambic_functional_test_group_yaml) # Create group - run_apply( - True, + run_force_apply( temp_config_filename, [test_group_fp], temp_templates_directory, @@ -53,8 +52,7 @@ def test_okta_group(): ) + datetime.timedelta(days=1) # Write new template, apply, and confirm access removed group_template.write() - run_apply( - True, + run_force_apply( temp_config_filename, [test_group_fp], temp_templates_directory, @@ -76,8 +74,7 @@ def test_okta_group(): 0 ].username = "this_user_should_not_exist@example.com" group_template.write() - run_apply( - True, + run_force_apply( temp_config_filename, [test_group_fp], temp_templates_directory, @@ -94,8 +91,7 @@ def test_okta_group(): # Set expiry for the entire group group_template.expires_at = "yesterday" group_template.write() - run_apply( - True, + run_force_apply( temp_config_filename, [test_group_fp], temp_templates_directory, diff --git a/functional_tests/okta/user/test_okta_user.py b/functional_tests/okta/user/test_okta_user.py index 6cb556585..6539d8acd 100644 --- a/functional_tests/okta/user/test_okta_user.py +++ b/functional_tests/okta/user/test_okta_user.py @@ -7,7 +7,7 @@ from functional_tests.conftest import IAMBIC_TEST_DETAILS from iambic.core.iambic_enum import IambicManaged from iambic.core.parser import load_templates -from iambic.main import run_apply +from iambic.main import run_force_apply def test_okta_user(): @@ -34,8 +34,7 @@ def test_okta_user(): temp_file.write(iambic_functional_test_user_yaml) # Create user - run_apply( - True, + run_force_apply( temp_config_filename, [test_user_fp], temp_templates_directory, @@ -50,8 +49,7 @@ def test_okta_user(): user_template.write() # Sleep to give profile time to propagate time.sleep(30) - run_apply( - True, + run_force_apply( temp_config_filename, [test_user_fp], temp_templates_directory, @@ -71,8 +69,7 @@ def test_okta_user(): orig_first_name = user_template.properties.profile["firstName"] user_template.properties.profile["firstName"] = "shouldNotWork" user_template.write() - run_apply( - True, + run_force_apply( temp_config_filename, [test_user_fp], temp_templates_directory, @@ -86,8 +83,7 @@ def test_okta_user(): user_template.properties.profile["firstName"] = orig_first_name user_template.write() - run_apply( - True, + run_force_apply( temp_config_filename, [test_user_fp], temp_templates_directory, @@ -99,8 +95,7 @@ def test_okta_user(): datetime.timezone.utc ) - datetime.timedelta(days=1) user_template.write() - run_apply( - True, + run_force_apply( temp_config_filename, [test_user_fp], temp_templates_directory, @@ -110,8 +105,7 @@ def test_okta_user(): # Needed to really delete the user and file user_template.force_delete = True user_template.write() - run_apply( - True, + run_force_apply( temp_config_filename, [test_user_fp], temp_templates_directory, diff --git a/functional_tests/test_google.py b/functional_tests/test_google.py index 70bb4b261..eff8e7b94 100644 --- a/functional_tests/test_google.py +++ b/functional_tests/test_google.py @@ -5,7 +5,7 @@ from functional_tests.conftest import IAMBIC_TEST_DETAILS from iambic.core.parser import load_templates -from iambic.main import run_apply +from iambic.main import run_force_apply def test_google(): @@ -30,8 +30,7 @@ def test_google(): temp_file.write(iambic_functional_test_group_yaml) # Create group - run_apply( - True, + run_force_apply( IAMBIC_TEST_DETAILS.config_path, [test_group_fp], IAMBIC_TEST_DETAILS.template_dir_path, @@ -51,8 +50,7 @@ def test_google(): ) + datetime.timedelta(days=1) # Write new template, apply, and confirm access removed group_template.write() - run_apply( - True, + run_force_apply( IAMBIC_TEST_DETAILS.config_path, [test_group_fp], IAMBIC_TEST_DETAILS.template_dir_path, @@ -62,8 +60,7 @@ def test_google(): # Set expiry for the entire group group_template.expires_at = datetime.datetime.now() - datetime.timedelta(days=1) group_template.write() - run_apply( - True, + run_force_apply( IAMBIC_TEST_DETAILS.config_path, [test_group_fp], IAMBIC_TEST_DETAILS.template_dir_path, diff --git a/iambic/config/wizard.py b/iambic/config/wizard.py index 8ee4dc4f9..43356b1a1 100644 --- a/iambic/config/wizard.py +++ b/iambic/config/wizard.py @@ -11,7 +11,6 @@ import botocore import questionary from botocore.exceptions import ClientError, NoCredentialsError - from iambic.config.dynamic_config import ( CURRENT_IAMBIC_VERSION, Config, diff --git a/iambic/core/utils.py b/iambic/core/utils.py index a595f3bb5..885b5ff4f 100644 --- a/iambic/core/utils.py +++ b/iambic/core/utils.py @@ -16,13 +16,12 @@ import aiofiles from asgiref.sync import sync_to_async -from ruamel.yaml import YAML - from iambic.core import noq_json as json from iambic.core.context import ExecutionContext from iambic.core.exceptions import RateLimitException from iambic.core.iambic_enum import IambicManaged from iambic.core.logger import log +from ruamel.yaml import YAML if TYPE_CHECKING: from iambic.core.models import ProposedChange diff --git a/iambic/lambda/app.py b/iambic/lambda/app.py index e52563e6f..3b62d42af 100644 --- a/iambic/lambda/app.py +++ b/iambic/lambda/app.py @@ -10,15 +10,7 @@ from iambic.config.dynamic_config import load_config from iambic.config.utils import resolve_config_template_path from iambic.core.models import BaseModel -from iambic.main import ( - run_apply, - run_clone_repos, - run_detect, - run_git_apply, - run_git_plan, - run_import, - run_plan, -) +from iambic.main import run_apply, run_clone_repos, run_detect, run_import, run_plan REPO_BASE_PATH = os.path.expanduser("~/.iambic/repos/") PLAN_OUTPUT_PATH = os.environ.get("PLAN_OUTPUT_PATH", None) @@ -83,21 +75,15 @@ def run_handler(event=None, context=None): context = {"command": "import"} lambda_context = LambdaContext(**context) - config_path = asyncio.run(resolve_config_template_path(REPO_BASE_PATH)) - asyncio.run(load_config(config_path)) - match lambda_context.command: case LambdaCommand.run_import.value: + config_path = asyncio.run(resolve_config_template_path(REPO_BASE_PATH)) + asyncio.run(load_config(config_path)) return run_import(REPO_BASE_PATH, config_path) - case LambdaCommand.run_plan.value: - return run_plan([], REPO_BASE_PATH) - case LambdaCommand.run_apply.value: - return run_apply(True, config_path, [], REPO_BASE_PATH) case LambdaCommand.run_detect.value: return run_detect(REPO_BASE_PATH) case LambdaCommand.run_git_apply.value: - return run_git_apply( - config_path, + return run_apply( False, FROM_SHA, TO_SHA, @@ -105,7 +91,7 @@ def run_handler(event=None, context=None): output_path=PLAN_OUTPUT_PATH, ) case LambdaCommand.run_git_plan.value: - return run_git_plan(config_path, PLAN_OUTPUT_PATH, repo_dir=REPO_BASE_PATH) + return run_plan(PLAN_OUTPUT_PATH, repo_dir=REPO_BASE_PATH) case LambdaCommand.run_clone_git_repos.value: return run_clone_repos(REPO_BASE_PATH) case _: diff --git a/iambic/main.py b/iambic/main.py index 98cc94c55..96ad4abf5 100644 --- a/iambic/main.py +++ b/iambic/main.py @@ -47,41 +47,6 @@ def cli(): ... -@cli.command() -@click.option( - "--template", - "-t", - "templates", - required=False, - multiple=True, - type=click.Path(exists=True), - help="The template file path(s) to apply. Example: ./resources/aws/roles/engineering.yaml", -) -@click.option( - "--repo-dir", - "-d", - "repo_dir", - required=False, - type=click.Path(exists=True), - default=os.getenv("IAMBIC_REPO_DIR"), - help="The repo directory containing the templates. Example: ~/iambic-templates", -) -def plan(templates: list[str], repo_dir: str): - run_plan(templates, repo_dir=repo_dir) - - -def run_plan(templates: list[str], repo_dir: str = str(pathlib.Path.cwd())): - if not templates: - templates = asyncio.run(gather_templates(repo_dir)) - - config_path = asyncio.run(resolve_config_template_path(repo_dir)) - config = asyncio.run(load_config(config_path)) - - asyncio.run(flag_expired_resources(templates)) - ctx.eval_only = True - output_proposed_changes(asyncio.run(config.run_apply(load_templates(templates)))) - - @cli.command() @click.option( "--template", @@ -155,76 +120,24 @@ def run_clone_repos(repo_dir: str = str(pathlib.Path.cwd())): asyncio.run(clone_git_repos(config, repo_dir)) -@cli.command() -@click.option( - "--force", - "-f", - is_flag=True, - show_default=True, - help="Apply changes without asking for permission?", -) -@click.option( - "--config", - "-c", - "config_path", - type=click.Path(exists=True), - help="The config.yaml file path to apply. Example: ./prod/config.yaml", -) -@click.option( - "--template", - "-t", - "templates", - required=False, - multiple=True, - type=click.Path(exists=True), - help="The template file path(s) to apply. Example: ./aws/roles/engineering.yaml", -) -@click.option( - "--repo-dir", - "-d", - "repo_dir", - required=False, - type=click.Path(exists=True), - default=os.getenv("IAMBIC_REPO_DIR"), - help="The repo directory containing the templates. Example: ~/iambic-templates", -) -def apply(force: bool, config_path: str, templates: list[str], repo_dir: str): - run_apply(force, config_path, templates, repo_dir=repo_dir) - - -def run_apply( - force: bool, +def run_force_apply( config_path: str, templates: list[str], repo_dir: str = str(pathlib.Path.cwd()), ): + ctx.eval_only = False + if not templates: templates = asyncio.run(gather_templates(repo_dir)) if not config_path: config_path = asyncio.run(resolve_config_template_path(repo_dir)) config = asyncio.run(load_config(config_path)) - ctx.eval_only = not force templates = load_templates(templates) asyncio.run(flag_expired_resources([template.file_path for template in templates])) - template_changes = asyncio.run(config.run_apply(templates)) - output_proposed_changes(template_changes) - - if ctx.eval_only and template_changes and click.confirm("Proceed?"): - ctx.eval_only = False - asyncio.run(config.run_apply(templates)) - # This was here before, but I don't think it's needed. Leaving it here for now to see if anything breaks. - # asyncio.run(config.run_detect_changes(repo_dir)) + asyncio.run(config.run_apply(templates)) -@cli.command(name="git-apply") -@click.option( - "--config", - "-c", - "config_path", - type=click.Path(exists=True), - help="The config.yaml file path to apply. Example: ./prod/config.yaml", -) @click.option( "--repo-dir", "-d", @@ -261,16 +174,14 @@ def run_apply( type=click.Path(exists=True), help="The location to output the plan Example: ./proposed_changes.yaml", ) -def git_apply( - config_path: str, +def apply( repo_dir: str, allow_dirty: bool, from_sha: str, to_sha: str, plan_output: str, ): - run_git_apply( - config_path, + run_apply( allow_dirty, from_sha, to_sha, @@ -279,8 +190,7 @@ def git_apply( ) -def run_git_apply( - config_path: str, +def run_apply( allow_dirty: bool, from_sha: str, to_sha: str, @@ -313,14 +223,6 @@ def run_git_apply( raise SystemExit(1) -@cli.command() -@click.option( - "--config", - "-c", - "config_path", - type=click.Path(exists=True), - help="The config.yaml file path to apply. Example: ./prod/config.yaml", -) @click.option( "--plan-output", "-o", @@ -337,12 +239,11 @@ def run_git_apply( default=os.getenv("IAMBIC_REPO_DIR"), help="The repo directory containing the templates. Example: ~/iambic-templates", ) -def git_plan(config_path: str, plan_output: str, repo_dir: str): - run_git_plan(config_path, plan_output, repo_dir=repo_dir) +def plan(plan_output: str, repo_dir: str): + run_plan(plan_output, repo_dir=repo_dir) -def run_git_plan( - config_path: str, +def run_plan( output_path: str, repo_dir: str = str(pathlib.Path.cwd()), ): diff --git a/iambic/plugins/v0_1_0/aws/iam/policy/models.py b/iambic/plugins/v0_1_0/aws/iam/policy/models.py index db240bee8..fdd13979c 100644 --- a/iambic/plugins/v0_1_0/aws/iam/policy/models.py +++ b/iambic/plugins/v0_1_0/aws/iam/policy/models.py @@ -6,9 +6,6 @@ from typing import List, Optional, Union import botocore -from jinja2 import BaseLoader, Environment -from pydantic import Field, constr, validator - from iambic.core.context import ExecutionContext from iambic.core.iambic_enum import IambicManaged from iambic.core.logger import log @@ -35,6 +32,8 @@ Tag, ) from iambic.plugins.v0_1_0.aws.utils import boto_crud_call +from jinja2 import BaseLoader, Environment +from pydantic import Field, constr, validator AWS_MANAGED_POLICY_TEMPLATE_TYPE = "NOQ::IAM::ManagedPolicy" diff --git a/iambic/plugins/v0_1_0/aws/iam/policy/utils.py b/iambic/plugins/v0_1_0/aws/iam/policy/utils.py index 6c499af85..206ebbac7 100644 --- a/iambic/plugins/v0_1_0/aws/iam/policy/utils.py +++ b/iambic/plugins/v0_1_0/aws/iam/policy/utils.py @@ -4,6 +4,7 @@ from botocore.exceptions import ClientError from deepdiff import DeepDiff + from iambic.core import noq_json as json from iambic.core.context import ExecutionContext from iambic.core.logger import log diff --git a/iambic/plugins/v0_1_0/aws/iam/role/models.py b/iambic/plugins/v0_1_0/aws/iam/role/models.py index a6f06edbf..40693ce3d 100644 --- a/iambic/plugins/v0_1_0/aws/iam/role/models.py +++ b/iambic/plugins/v0_1_0/aws/iam/role/models.py @@ -5,8 +5,6 @@ from typing import Callable, Optional, Union import botocore -from pydantic import Field, constr, validator - from iambic.core.context import ExecutionContext from iambic.core.iambic_enum import IambicManaged from iambic.core.logger import log @@ -40,6 +38,7 @@ Tag, ) from iambic.plugins.v0_1_0.aws.utils import boto_crud_call, remove_expired_resources +from pydantic import Field, constr, validator AWS_IAM_ROLE_TEMPLATE_TYPE = "NOQ::AWS::IAM::Role" diff --git a/iambic/plugins/v0_1_0/aws/iam/user/template_generation.py b/iambic/plugins/v0_1_0/aws/iam/user/template_generation.py index 6384155b3..5eb02fc36 100644 --- a/iambic/plugins/v0_1_0/aws/iam/user/template_generation.py +++ b/iambic/plugins/v0_1_0/aws/iam/user/template_generation.py @@ -7,6 +7,7 @@ from typing import TYPE_CHECKING import aiofiles + from iambic.core import noq_json as json from iambic.core.logger import log from iambic.core.template_generation import ( diff --git a/iambic/plugins/v0_1_0/aws/identity_center/permission_set/template_generation.py b/iambic/plugins/v0_1_0/aws/identity_center/permission_set/template_generation.py index 6eeeb72a2..a0066c3fc 100644 --- a/iambic/plugins/v0_1_0/aws/identity_center/permission_set/template_generation.py +++ b/iambic/plugins/v0_1_0/aws/identity_center/permission_set/template_generation.py @@ -8,7 +8,6 @@ from typing import TYPE_CHECKING, Union import aiofiles - from iambic.core import noq_json as json from iambic.core.logger import log from iambic.core.template_generation import ( diff --git a/iambic/plugins/v0_1_0/aws/identity_center/permission_set/utils.py b/iambic/plugins/v0_1_0/aws/identity_center/permission_set/utils.py index e22653861..a7a9a504b 100644 --- a/iambic/plugins/v0_1_0/aws/identity_center/permission_set/utils.py +++ b/iambic/plugins/v0_1_0/aws/identity_center/permission_set/utils.py @@ -4,7 +4,6 @@ from botocore.exceptions import ClientError from deepdiff import DeepDiff - from iambic.core import noq_json as json from iambic.core.context import ExecutionContext from iambic.core.logger import log diff --git a/iambic/plugins/v0_1_0/aws/models.py b/iambic/plugins/v0_1_0/aws/models.py index b64c41f9b..01da83634 100644 --- a/iambic/plugins/v0_1_0/aws/models.py +++ b/iambic/plugins/v0_1_0/aws/models.py @@ -7,6 +7,10 @@ import boto3 import botocore +from pydantic import BaseModel as PydanticBaseModel +from pydantic import Field, constr, validator +from ruamel.yaml import YAML, yaml_object + from iambic.core.context import ExecutionContext from iambic.core.iambic_enum import IambicManaged from iambic.core.logger import log @@ -34,9 +38,6 @@ legacy_paginated_search, set_org_account_variables, ) -from pydantic import BaseModel as PydanticBaseModel -from pydantic import Field, constr, validator -from ruamel.yaml import YAML, yaml_object yaml = YAML() diff --git a/iambic/plugins/v0_1_0/example/iambic_plugin.py b/iambic/plugins/v0_1_0/example/iambic_plugin.py index 5c3c2e58b..4e20b8e5a 100644 --- a/iambic/plugins/v0_1_0/example/iambic_plugin.py +++ b/iambic/plugins/v0_1_0/example/iambic_plugin.py @@ -1,11 +1,10 @@ from __future__ import annotations -from pydantic import BaseModel - from iambic.core.iambic_plugin import ProviderPlugin from iambic.plugins.v0_1_0 import PLUGIN_VERSION from iambic.plugins.v0_1_0.example.handlers import import_example_resources, load from iambic.plugins.v0_1_0.example.local_file.models import ExampleLocalFileTemplate +from pydantic import BaseModel class ExampleConfig(BaseModel): diff --git a/iambic/plugins/v0_1_0/okta/app/utils.py b/iambic/plugins/v0_1_0/okta/app/utils.py index f1293396f..d6b4b1142 100644 --- a/iambic/plugins/v0_1_0/okta/app/utils.py +++ b/iambic/plugins/v0_1_0/okta/app/utils.py @@ -4,6 +4,8 @@ import functools from typing import TYPE_CHECKING, List +import okta.models as models + from iambic.core.context import ExecutionContext from iambic.core.logger import log from iambic.core.models import ProposedChange, ProposedChangeType @@ -12,8 +14,6 @@ from iambic.plugins.v0_1_0.okta.models import App, Assignment, Group from iambic.plugins.v0_1_0.okta.utils import handle_okta_fn -import okta.models as models - if TYPE_CHECKING: from iambic.plugins.v0_1_0.okta.iambic_plugin import OktaOrganization diff --git a/iambic/plugins/v0_1_0/okta/models.py b/iambic/plugins/v0_1_0/okta/models.py index 05cdf2b71..d9572a55e 100644 --- a/iambic/plugins/v0_1_0/okta/models.py +++ b/iambic/plugins/v0_1_0/okta/models.py @@ -3,9 +3,10 @@ from enum import Enum from typing import Any, List, Optional, Union -from iambic.core.models import BaseModel, ExpiryModel from pydantic import Field +from iambic.core.models import BaseModel, ExpiryModel + # Reference: https://www.guidodiepen.nl/2019/02/implementing-a-simple-plugin-framework-in-python/ diff --git a/iambic/plugins/v0_1_0/okta/utils.py b/iambic/plugins/v0_1_0/okta/utils.py index b3bb4cbea..9a720b3df 100644 --- a/iambic/plugins/v0_1_0/okta/utils.py +++ b/iambic/plugins/v0_1_0/okta/utils.py @@ -4,9 +4,10 @@ import json import okta.models as models +from okta.errors.okta_api_error import OktaAPIError + from iambic.core.exceptions import RateLimitException from iambic.plugins.v0_1_0.okta.exceptions import UserProfileNotUpdatableYet -from okta.errors.okta_api_error import OktaAPIError async def generate_user_profile(user: models.User): diff --git a/test/aws/identity_center/test_template_generation.py b/test/aws/identity_center/test_template_generation.py index 54050ddf4..1c88995dc 100644 --- a/test/aws/identity_center/test_template_generation.py +++ b/test/aws/identity_center/test_template_generation.py @@ -1,6 +1,7 @@ from __future__ import annotations import pytest + from iambic.plugins.v0_1_0.aws.identity_center.permission_set.template_generation import ( _sorted_and_clean_access_rules, ) diff --git a/test/config/test_utils.py b/test/config/test_utils.py index 649448384..db6dace74 100644 --- a/test/config/test_utils.py +++ b/test/config/test_utils.py @@ -5,7 +5,6 @@ from tempfile import TemporaryDirectory import pytest - from iambic.config.dynamic_config import Config from iambic.core.iambic_enum import IambicManaged from iambic.plugins.v0_1_0.aws.iambic_plugin import AWSConfig diff --git a/test/google/group/test_models.py b/test/google/group/test_models.py index 518fd5968..11a5ce0fb 100644 --- a/test/google/group/test_models.py +++ b/test/google/group/test_models.py @@ -3,6 +3,7 @@ from unittest.mock import MagicMock import pytest + from iambic.core.template_generation import merge_model from iambic.plugins.v0_1_0.google.group.models import ( GroupTemplateProperties, diff --git a/test/plugins/v0_1_0/github/test_github_app.py b/test/plugins/v0_1_0/github/test_github_app.py index de6dcd9ee..1af7df936 100644 --- a/test/plugins/v0_1_0/github/test_github_app.py +++ b/test/plugins/v0_1_0/github/test_github_app.py @@ -4,6 +4,7 @@ from unittest.mock import patch import pytest + from iambic.plugins.v0_1_0.github.github_app import ( _get_installation_token, calculate_signature, diff --git a/test/plugins/v0_1_0/github/test_github_plugin.py b/test/plugins/v0_1_0/github/test_github_plugin.py index f4b75d0d2..8b3211d32 100644 --- a/test/plugins/v0_1_0/github/test_github_plugin.py +++ b/test/plugins/v0_1_0/github/test_github_plugin.py @@ -6,9 +6,8 @@ import shutil import tempfile -import pytest - import iambic.plugins.v0_1_0.github.github +import pytest from iambic.config.dynamic_config import load_config TEST_CONFIG_DIR = "config/" diff --git a/test/test_main.py b/test/test_main.py index 06ce02d93..3e68d34b7 100644 --- a/test/test_main.py +++ b/test/test_main.py @@ -6,7 +6,7 @@ import iambic.plugins.v0_1_0.example import pytest -from iambic.main import run_apply +from iambic.main import run_force_apply TEST_TEMPLATE_YAML = """template_type: NOQ::Example::LocalFile name: test_template @@ -65,7 +65,7 @@ def test_run_apply(example_test_filesystem): with open(f"{repo_dir}/{TEST_TEMPLATE_PATH}", "r") as f: before_template_content = "\n".join(f.readlines()) assert "tomorrow" in before_template_content - run_apply(False, config_path, None, repo_dir) + run_force_apply(config_path, [], repo_dir) with open(f"{repo_dir}/{TEST_TEMPLATE_PATH}", "r") as f: after_template_content = "\n".join(f.readlines()) assert "tomorrow" not in after_template_content From 28f50ca4d3b91e14503d15d4c299674476ea8bc3 Mon Sep 17 00:00:00 2001 From: Will-NOQ <102608210+Will-NOQ@users.noreply.github.com> Date: Mon, 27 Feb 2023 11:53:31 -0600 Subject: [PATCH 02/32] Smooth out transition on command update in lambda. --- iambic/lambda/app.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/iambic/lambda/app.py b/iambic/lambda/app.py index 3b62d42af..50e63b1d6 100644 --- a/iambic/lambda/app.py +++ b/iambic/lambda/app.py @@ -82,6 +82,16 @@ def run_handler(event=None, context=None): return run_import(REPO_BASE_PATH, config_path) case LambdaCommand.run_detect.value: return run_detect(REPO_BASE_PATH) + case LambdaCommand.run_apply.value: + return run_apply( + False, + FROM_SHA, + TO_SHA, + repo_dir=REPO_BASE_PATH, + output_path=PLAN_OUTPUT_PATH, + ) + case LambdaCommand.run_plan.value: + return run_plan(PLAN_OUTPUT_PATH, repo_dir=REPO_BASE_PATH) case LambdaCommand.run_git_apply.value: return run_apply( False, From 4ff0fc3f5bfa290cd682da559246d009dc5b8c38 Mon Sep 17 00:00:00 2001 From: Will-NOQ <102608210+Will-NOQ@users.noreply.github.com> Date: Mon, 27 Feb 2023 13:29:30 -0600 Subject: [PATCH 03/32] Optimized our functional tests to only call load_config once. --- functional_tests/conftest.py | 14 +++++------ .../okta/group/test_okta_group.py | 9 ++++--- functional_tests/okta/user/test_okta_user.py | 24 +++++++++---------- functional_tests/test_google.py | 6 ++--- iambic/lambda/app.py | 6 ++--- iambic/main.py | 17 +++---------- iambic/plugins/v0_1_0/github/github.py | 15 +++++++----- test/test_main.py | 6 ++++- 8 files changed, 45 insertions(+), 52 deletions(-) diff --git a/functional_tests/conftest.py b/functional_tests/conftest.py index 4072dd2c5..7de6c8dac 100644 --- a/functional_tests/conftest.py +++ b/functional_tests/conftest.py @@ -10,7 +10,6 @@ from iambic.config.dynamic_config import Config, load_config from iambic.core.context import ctx from iambic.core.logger import log -from iambic.main import run_import from iambic.plugins.v0_1_0.aws.models import AWSAccount if not os.environ.get("GITHUB_ACTIONS", None): @@ -104,18 +103,17 @@ def generate_templates_fixture(request): with open(IAMBIC_TEST_DETAILS.config_path, "w") as temp_file: temp_file.write(all_config) - if not FUNCTIONAL_TEST_TEMPLATE_DIR: - run_import( - IAMBIC_TEST_DETAILS.template_dir_path, - IAMBIC_TEST_DETAILS.config_path, - ) - log.info("Finished generating templates for testing") - log.info("Setting up config for testing") IAMBIC_TEST_DETAILS.config = asyncio.run( load_config(IAMBIC_TEST_DETAILS.config_path) ) + if not FUNCTIONAL_TEST_TEMPLATE_DIR: + asyncio.run( + IAMBIC_TEST_DETAILS.config.run_import(IAMBIC_TEST_DETAILS.template_dir_path) + ) + log.info("Finished generating templates for testing") + for aws_account in IAMBIC_TEST_DETAILS.config.aws.accounts: if aws_account.identity_center_details: IAMBIC_TEST_DETAILS.identity_center_account = aws_account diff --git a/functional_tests/okta/group/test_okta_group.py b/functional_tests/okta/group/test_okta_group.py index 8297eee21..aafa9a6eb 100644 --- a/functional_tests/okta/group/test_okta_group.py +++ b/functional_tests/okta/group/test_okta_group.py @@ -10,7 +10,6 @@ def test_okta_group(): - temp_config_filename = IAMBIC_TEST_DETAILS.config_path temp_templates_directory = IAMBIC_TEST_DETAILS.template_dir_path iambic_functional_test_group_yaml = """template_type: NOQ::Okta::Group @@ -33,7 +32,7 @@ def test_okta_group(): # Create group run_force_apply( - temp_config_filename, + IAMBIC_TEST_DETAILS.config, [test_group_fp], temp_templates_directory, ) @@ -53,7 +52,7 @@ def test_okta_group(): # Write new template, apply, and confirm access removed group_template.write() run_force_apply( - temp_config_filename, + IAMBIC_TEST_DETAILS.config, [test_group_fp], temp_templates_directory, ) @@ -75,7 +74,7 @@ def test_okta_group(): ].username = "this_user_should_not_exist@example.com" group_template.write() run_force_apply( - temp_config_filename, + IAMBIC_TEST_DETAILS.config, [test_group_fp], temp_templates_directory, ) @@ -92,7 +91,7 @@ def test_okta_group(): group_template.expires_at = "yesterday" group_template.write() run_force_apply( - temp_config_filename, + IAMBIC_TEST_DETAILS.config, [test_group_fp], temp_templates_directory, ) diff --git a/functional_tests/okta/user/test_okta_user.py b/functional_tests/okta/user/test_okta_user.py index 6539d8acd..67a97d3c1 100644 --- a/functional_tests/okta/user/test_okta_user.py +++ b/functional_tests/okta/user/test_okta_user.py @@ -3,6 +3,7 @@ import datetime import os import time +import random from functional_tests.conftest import IAMBIC_TEST_DETAILS from iambic.core.iambic_enum import IambicManaged @@ -11,18 +12,17 @@ def test_okta_user(): - temp_config_filename = IAMBIC_TEST_DETAILS.config_path temp_templates_directory = IAMBIC_TEST_DETAILS.template_dir_path - - iambic_functional_test_user_yaml = """template_type: NOQ::Okta::User + username = f"iambic_functional_test_user_{random.randint(0, 1000000)}" + iambic_functional_test_user_yaml = f"""template_type: NOQ::Okta::User properties: - username: iambic_functional_test_user + username: {username} idp_name: development profile: firstName: iambic lastName: functional_test_user - email: iambic_functional_test_user@example.com - login: iambic_functional_test_user@example.com + email: {username}@example.com + login: {username}@example.com status: active """ test_user_fp = os.path.join( @@ -35,7 +35,7 @@ def test_okta_user(): # Create user run_force_apply( - temp_config_filename, + IAMBIC_TEST_DETAILS.config, [test_user_fp], temp_templates_directory, ) @@ -50,7 +50,7 @@ def test_okta_user(): # Sleep to give profile time to propagate time.sleep(30) run_force_apply( - temp_config_filename, + IAMBIC_TEST_DETAILS.config, [test_user_fp], temp_templates_directory, ) @@ -70,7 +70,7 @@ def test_okta_user(): user_template.properties.profile["firstName"] = "shouldNotWork" user_template.write() run_force_apply( - temp_config_filename, + IAMBIC_TEST_DETAILS.config, [test_user_fp], temp_templates_directory, ) @@ -84,7 +84,7 @@ def test_okta_user(): user_template.write() run_force_apply( - temp_config_filename, + IAMBIC_TEST_DETAILS.config, [test_user_fp], temp_templates_directory, ) @@ -96,7 +96,7 @@ def test_okta_user(): ) - datetime.timedelta(days=1) user_template.write() run_force_apply( - temp_config_filename, + IAMBIC_TEST_DETAILS.config, [test_user_fp], temp_templates_directory, ) @@ -106,7 +106,7 @@ def test_okta_user(): user_template.force_delete = True user_template.write() run_force_apply( - temp_config_filename, + IAMBIC_TEST_DETAILS.config, [test_user_fp], temp_templates_directory, ) diff --git a/functional_tests/test_google.py b/functional_tests/test_google.py index eff8e7b94..ee18f3307 100644 --- a/functional_tests/test_google.py +++ b/functional_tests/test_google.py @@ -31,7 +31,7 @@ def test_google(): # Create group run_force_apply( - IAMBIC_TEST_DETAILS.config_path, + IAMBIC_TEST_DETAILS.config, [test_group_fp], IAMBIC_TEST_DETAILS.template_dir_path, ) @@ -51,7 +51,7 @@ def test_google(): # Write new template, apply, and confirm access removed group_template.write() run_force_apply( - IAMBIC_TEST_DETAILS.config_path, + IAMBIC_TEST_DETAILS.config, [test_group_fp], IAMBIC_TEST_DETAILS.template_dir_path, ) @@ -61,7 +61,7 @@ def test_google(): group_template.expires_at = datetime.datetime.now() - datetime.timedelta(days=1) group_template.write() run_force_apply( - IAMBIC_TEST_DETAILS.config_path, + IAMBIC_TEST_DETAILS.config, [test_group_fp], IAMBIC_TEST_DETAILS.template_dir_path, ) diff --git a/iambic/lambda/app.py b/iambic/lambda/app.py index 50e63b1d6..300598fc1 100644 --- a/iambic/lambda/app.py +++ b/iambic/lambda/app.py @@ -10,7 +10,7 @@ from iambic.config.dynamic_config import load_config from iambic.config.utils import resolve_config_template_path from iambic.core.models import BaseModel -from iambic.main import run_apply, run_clone_repos, run_detect, run_import, run_plan +from iambic.main import run_apply, run_clone_repos, run_detect, run_plan REPO_BASE_PATH = os.path.expanduser("~/.iambic/repos/") PLAN_OUTPUT_PATH = os.environ.get("PLAN_OUTPUT_PATH", None) @@ -78,8 +78,8 @@ def run_handler(event=None, context=None): match lambda_context.command: case LambdaCommand.run_import.value: config_path = asyncio.run(resolve_config_template_path(REPO_BASE_PATH)) - asyncio.run(load_config(config_path)) - return run_import(REPO_BASE_PATH, config_path) + config = asyncio.run(load_config(config_path)) + return asyncio.run(config.run_import(REPO_BASE_PATH)) case LambdaCommand.run_detect.value: return run_detect(REPO_BASE_PATH) case LambdaCommand.run_apply.value: diff --git a/iambic/main.py b/iambic/main.py index 96ad4abf5..578f21046 100644 --- a/iambic/main.py +++ b/iambic/main.py @@ -4,11 +4,10 @@ import os import pathlib import warnings -from typing import Optional import click -from iambic.config.dynamic_config import init_plugins, load_config +from iambic.config.dynamic_config import Config, init_plugins, load_config from iambic.config.utils import resolve_config_template_path from iambic.config.wizard import ConfigurationWizard from iambic.core.context import ctx @@ -121,7 +120,7 @@ def run_clone_repos(repo_dir: str = str(pathlib.Path.cwd())): def run_force_apply( - config_path: str, + config: Config, templates: list[str], repo_dir: str = str(pathlib.Path.cwd()), ): @@ -129,9 +128,6 @@ def run_force_apply( if not templates: templates = asyncio.run(gather_templates(repo_dir)) - if not config_path: - config_path = asyncio.run(resolve_config_template_path(repo_dir)) - config = asyncio.run(load_config(config_path)) templates = load_templates(templates) asyncio.run(flag_expired_resources([template.file_path for template in templates])) @@ -281,14 +277,7 @@ def config_discovery(repo_dir: str): help="The repo directory containing the templates. Example: ~/iambic-templates", ) def import_(repo_dir: str): - run_import(repo_dir=repo_dir) - - -def run_import( - repo_dir: str = str(pathlib.Path.cwd()), config_path: Optional[str] = None -): - if not config_path: - config_path = asyncio.run(resolve_config_template_path(repo_dir)) + config_path = asyncio.run(resolve_config_template_path(repo_dir)) config = asyncio.run(load_config(config_path)) asyncio.run(config.run_import(repo_dir)) diff --git a/iambic/plugins/v0_1_0/github/github.py b/iambic/plugins/v0_1_0/github/github.py index cf8924af9..c14c94f9b 100644 --- a/iambic/plugins/v0_1_0/github/github.py +++ b/iambic/plugins/v0_1_0/github/github.py @@ -1,5 +1,6 @@ from __future__ import annotations +import asyncio import json import os import re @@ -14,10 +15,12 @@ import github from github.PullRequest import PullRequest +from iambic.config.dynamic_config import load_config +from iambic.config.utils import resolve_config_template_path from iambic.core.git import Repo, clone_git_repo from iambic.core.logger import log from iambic.core.utils import yaml -from iambic.main import run_detect, run_expire, run_import +from iambic.main import run_detect, run_expire iambic_app = __import__("iambic.lambda.app", globals(), locals(), [], 0) lambda_run_handler = getattr(iambic_app, "lambda").app.run_handler @@ -489,11 +492,11 @@ def handle_import(github_client: github.Github, context: dict[str, Any]) -> None def _handle_import(repo_url: str, default_branch: str) -> None: try: - repo = prepare_local_repo_for_new_commits( - repo_url, get_lambda_repo_path(), "import" - ) - - run_import(get_lambda_repo_path()) + repo_dir = get_lambda_repo_path() + repo = prepare_local_repo_for_new_commits(repo_url, repo_dir, "import") + config_path = asyncio.run(resolve_config_template_path(repo_dir)) + config = asyncio.run(load_config(config_path)) + asyncio.run(config.run_import(repo_dir)) repo.git.add(".") diff_list = repo.head.commit.diff() if len(diff_list) > 0: diff --git a/test/test_main.py b/test/test_main.py index 3e68d34b7..451ce9752 100644 --- a/test/test_main.py +++ b/test/test_main.py @@ -1,11 +1,13 @@ from __future__ import annotations +import asyncio import os import shutil import tempfile import iambic.plugins.v0_1_0.example import pytest +from iambic.config.dynamic_config import load_config from iambic.main import run_force_apply TEST_TEMPLATE_YAML = """template_type: NOQ::Example::LocalFile @@ -65,7 +67,9 @@ def test_run_apply(example_test_filesystem): with open(f"{repo_dir}/{TEST_TEMPLATE_PATH}", "r") as f: before_template_content = "\n".join(f.readlines()) assert "tomorrow" in before_template_content - run_force_apply(config_path, [], repo_dir) + + config = asyncio.run(load_config(config_path)) + run_force_apply(config, [], repo_dir) with open(f"{repo_dir}/{TEST_TEMPLATE_PATH}", "r") as f: after_template_content = "\n".join(f.readlines()) assert "tomorrow" not in after_template_content From 7d391f2f4fa0f7e6c7d2f733393b2d3c08aea2ff Mon Sep 17 00:00:00 2001 From: Will-NOQ <102608210+Will-NOQ@users.noreply.github.com> Date: Mon, 27 Feb 2023 13:35:14 -0600 Subject: [PATCH 04/32] Added missing decorators. --- iambic/main.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/iambic/main.py b/iambic/main.py index 578f21046..a1d7068d7 100644 --- a/iambic/main.py +++ b/iambic/main.py @@ -134,6 +134,7 @@ def run_force_apply( asyncio.run(config.run_apply(templates)) +@cli.command() @click.option( "--repo-dir", "-d", @@ -219,6 +220,7 @@ def run_apply( raise SystemExit(1) +@cli.command() @click.option( "--plan-output", "-o", From 235247ca49b5118e0d28f4f7589a4624e15fdd36 Mon Sep 17 00:00:00 2001 From: Curtis Castrapel Date: Mon, 27 Feb 2023 12:13:03 -0800 Subject: [PATCH 05/32] Fix unit test --- functional_tests/okta/user/test_okta_user.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/functional_tests/okta/user/test_okta_user.py b/functional_tests/okta/user/test_okta_user.py index 67a97d3c1..e81309d2b 100644 --- a/functional_tests/okta/user/test_okta_user.py +++ b/functional_tests/okta/user/test_okta_user.py @@ -2,8 +2,8 @@ import datetime import os -import time import random +import time from functional_tests.conftest import IAMBIC_TEST_DETAILS from iambic.core.iambic_enum import IambicManaged @@ -42,7 +42,7 @@ def test_okta_user(): # Test Reading Template user_template = load_templates([test_user_fp])[0] - assert user_template.properties.username == "iambic_functional_test_user" + assert user_template.properties.username == username # Test Updating Template user_template.properties.profile["firstName"] = "TestNameChange" From 4fb6546aa30e526cbfbcabc9a37e4c2313863fa3 Mon Sep 17 00:00:00 2001 From: Matt Daue Date: Mon, 27 Feb 2023 13:17:34 -0800 Subject: [PATCH 06/32] Replace hard-coded refs of main with default_branch Signed-off-by: Matt Daue --- iambic/plugins/v0_1_0/github/github.py | 26 ++++++++++++++++------ iambic/plugins/v0_1_0/github/github_app.py | 2 +- 2 files changed, 20 insertions(+), 8 deletions(-) diff --git a/iambic/plugins/v0_1_0/github/github.py b/iambic/plugins/v0_1_0/github/github.py index cf8924af9..bb50527f5 100644 --- a/iambic/plugins/v0_1_0/github/github.py +++ b/iambic/plugins/v0_1_0/github/github.py @@ -14,7 +14,7 @@ import github from github.PullRequest import PullRequest -from iambic.core.git import Repo, clone_git_repo +from iambic.core.git import Repo, clone_git_repo, get_remote_default_branch from iambic.core.logger import log from iambic.core.utils import yaml from iambic.main import run_detect, run_expire, run_import @@ -122,7 +122,8 @@ def prepare_local_repo_for_new_commits( repo_config_writer.set_value("user", "email", COMMIT_MESSAGE_USER_EMAIL) repo_config_writer.release() - cloned_repo.git.checkout("-b", f"attempt/{purpose}", "origin/main") + default_branch = get_remote_default_branch(cloned_repo) + cloned_repo.git.checkout("-b", f"attempt/{purpose}", default_branch) return cloned_repo @@ -146,7 +147,8 @@ def prepare_local_repo( cloned_repo = clone_git_repo(repo_url, repo_path, None) for remote in cloned_repo.remotes: remote.fetch() - cloned_repo.git.checkout("-b", "attempt/git-apply", "origin/main") + default_branch = get_remote_default_branch(cloned_repo) + cloned_repo.git.checkout("-b", "attempt/git-apply", default_branch) # Note, this is for local usage, we don't actually # forward this commit upstream @@ -453,7 +455,10 @@ def handle_detect_changes_from_eventbridge( repository_url = context["event"]["repository"]["clone_url"] repo_url = format_github_url(repository_url, github_token) - _handle_detect_changes_from_eventbridge(repo_url, "main") + repo_name = context["repository"] + templates_repo = github_client.get_repo(repo_name) + default_branch = get_remote_default_branch(templates_repo) + _handle_detect_changes_from_eventbridge(repo_url, default_branch) def _handle_detect_changes_from_eventbridge(repo_url: str, default_branch: str) -> None: @@ -484,7 +489,10 @@ def handle_import(github_client: github.Github, context: dict[str, Any]) -> None # repo_name is already in the format {repo_owner}/{repo_short_name} repository_url = context["event"]["repository"]["clone_url"] repo_url = format_github_url(repository_url, github_token) - _handle_import(repo_url, "main") + repo_name = context["repository"] + templates_repo = github_client.get_repo(repo_name) + default_branch = get_remote_default_branch(templates_repo) + _handle_import(repo_url, default_branch) def _handle_import(repo_url: str, default_branch: str) -> None: @@ -514,7 +522,10 @@ def handle_expire(github_client: github.Github, context: dict[str, Any]) -> None # repo_name is already in the format {repo_owner}/{repo_short_name} repository_url = context["event"]["repository"]["clone_url"] repo_url = format_github_url(repository_url, github_token) - _handle_expire(repo_url, "main") + repo_name = context["repository"] + templates_repo = github_client.get_repo(repo_name) + default_branch = get_remote_default_branch(templates_repo) + _handle_expire(repo_url, default_branch) def _handle_expire(repo_url: str, default_branch: str) -> None: @@ -542,7 +553,8 @@ def _handle_expire(repo_url: str, default_branch: str) -> None: log_params = {"proposed_changes": lines} log.info("handle_expire ran", **log_params) - repo.remotes.origin.push(refspec="HEAD:main").raise_if_error() # FIXME + default_branch = get_remote_default_branch(repo) + repo.remotes.origin.push(refspec=f"HEAD:{default_branch}").raise_if_error() # FIXME else: log.info("handle_expire no changes") except Exception as e: diff --git a/iambic/plugins/v0_1_0/github/github_app.py b/iambic/plugins/v0_1_0/github/github_app.py index d4d115176..9c65cf1cf 100644 --- a/iambic/plugins/v0_1_0/github/github_app.py +++ b/iambic/plugins/v0_1_0/github/github_app.py @@ -17,6 +17,7 @@ import jwt from botocore.exceptions import ClientError +from iambic.core.git import get_remote_default_branch import iambic.core.utils import iambic.plugins.v0_1_0.github.github from iambic.core.logger import log @@ -278,7 +279,6 @@ def handle_workflow_run( repository_url = webhook_payload["repository"]["clone_url"] repo_url = format_github_url(repository_url, github_token) - default_branch = "main" repo_name = webhook_payload["repository"]["full_name"] templates_repo = github_client.get_repo(repo_name) default_branch = templates_repo.default_branch From e9513bafe093010a54be4cc7e1e16139d9be98bd Mon Sep 17 00:00:00 2001 From: Steven Moy Date: Mon, 27 Feb 2023 13:43:25 -0800 Subject: [PATCH 07/32] Support partial failure in groups --- .../aws/group/test_update_template.py | 38 ++++++ iambic/plugins/v0_1_0/aws/iam/group/models.py | 61 ++++++---- iambic/plugins/v0_1_0/aws/iam/group/utils.py | 108 +++++++++--------- 3 files changed, 131 insertions(+), 76 deletions(-) diff --git a/functional_tests/aws/group/test_update_template.py b/functional_tests/aws/group/test_update_template.py index ab3e8ba81..fb31899fd 100644 --- a/functional_tests/aws/group/test_update_template.py +++ b/functional_tests/aws/group/test_update_template.py @@ -87,6 +87,44 @@ async def test_update_managed_policies(self): f"{group['ManagedPolicies']} attached to it for group {self.group_name}", ) + async def test_bad_input(self): + self.template.included_accounts = ["*"] + self.template.excluded_accounts = [] + + await self.template.apply(IAMBIC_TEST_DETAILS.config.aws, ctx) + + account_group_mapping = await get_group_across_accounts( + IAMBIC_TEST_DETAILS.config.aws.accounts, self.group_name, False + ) + group_account_ids = [ + account_id for account_id, group in account_group_mapping.items() if group + ] + + self.template.properties.inline_policies.append( + PolicyDocument( + included_accounts=[group_account_ids[0], group_account_ids[1]], + expires_at="tomorrow", + policy_name="test_policy", + statement=[ + { + "action": ["s3:NotARealAction"], + "effect": "BAD_INPUT", + "resource": ["*"], + "expires_at": "tomorrow", + "included_accounts": [group_account_ids[0]], + }, + { + "action": ["s3:AlsoNotARealAction"], + "effect": "BAD_INPUT", + "resource": ["*"], + "expires_at": "tomorrow", + }, + ], + ) + ) + r = await self.template.apply(IAMBIC_TEST_DETAILS.config.aws, ctx) + self.assertEqual(len(r.exceptions_seen), 2) + async def test_create_update_group_all_accounts(self): self.template.included_accounts = ["*"] self.template.excluded_accounts = [] diff --git a/iambic/plugins/v0_1_0/aws/iam/group/models.py b/iambic/plugins/v0_1_0/aws/iam/group/models.py index 0eca7b41e..75f1b123f 100644 --- a/iambic/plugins/v0_1_0/aws/iam/group/models.py +++ b/iambic/plugins/v0_1_0/aws/iam/group/models.py @@ -1,10 +1,11 @@ from __future__ import annotations import asyncio -from itertools import chain from typing import Callable, Optional, Union import botocore +from pydantic import Field, constr, validator + from iambic.core.context import ExecutionContext from iambic.core.iambic_enum import IambicManaged from iambic.core.logger import log @@ -15,6 +16,7 @@ ProposedChange, ProposedChangeType, ) +from iambic.core.utils import plugin_apply_wrapper from iambic.plugins.v0_1_0.aws.iam.group.utils import ( apply_group_inline_policies, apply_group_managed_policies, @@ -30,7 +32,6 @@ AWSTemplate, ) from iambic.plugins.v0_1_0.aws.utils import boto_crud_call, remove_expired_resources -from pydantic import Field, constr, validator AWS_IAM_GROUP_TEMPLATE_TYPE = "NOQ::AWS::IAM::Group" @@ -121,6 +122,7 @@ async def _apply_to_account( # noqa: C901 resource_id=group_name, new_value=dict(**account_group), proposed_changes=[], + exceptions_seen=[], ) log_params = dict( resource_type=self.resource_type, @@ -169,7 +171,7 @@ async def _apply_to_account( # noqa: C901 if group_exists: tasks.extend([]) - supported_update_keys = ["Description", "MaxSessionDuration"] + supported_update_keys = ["Path", "GroupName"] update_resource_log_params = {**log_params} update_group_params = {} for k in supported_update_keys: @@ -180,34 +182,33 @@ async def _apply_to_account( # noqa: C901 old_value=current_group.get(k), new_value=account_group.get(k), ) - update_group_params[k] = current_group.get(k) + update_group_params[f"New{k}"] = account_group.get(k) if update_group_params: log_str = "Out of date resource found." + proposed_changes = [ + ProposedChange( + change_type=ProposedChangeType.UPDATE, + resource_id=group_name, + resource_type=self.resource_type, + ) + ] if context.execute: log.info( f"{log_str} Updating resource...", **update_resource_log_params, ) + apply_awaitable = boto_crud_call( + client.update_group, + RoleName=group_name, + **update_group_params, + ) tasks.append( - boto_crud_call( - client.update_group, - RoleName=group_name, - **{ - k: account_group.get(k) - for k in supported_update_keys - }, - ) + plugin_apply_wrapper(apply_awaitable, proposed_changes) ) else: log.info(log_str, **update_resource_log_params) - account_change_details.proposed_changes.append( - ProposedChange( - change_type=ProposedChangeType.UPDATE, - resource_id=group_name, - resource_type=self.resource_type, - ) - ) + account_change_details.proposed_changes.extend(proposed_changes) else: account_change_details.proposed_changes.append( ProposedChange( @@ -250,14 +251,28 @@ async def _apply_to_account( # noqa: C901 ] ) try: - changes_made = await asyncio.gather(*tasks) + results: list[list[ProposedChange]] = await asyncio.gather( + *tasks, return_exceptions=True + ) + + # separate out the success versus failure calls + exceptions: list[ProposedChange] = [] + changes_made: list[ProposedChange] = [] + for result in results: + for r in result: + if isinstance(r, ProposedChange): + if len(r.exceptions_seen) == 0: + changes_made.append(r) + else: + exceptions.append(r) + except Exception as e: log.exception("Unable to apply changes to resource", error=e, **log_params) return account_change_details if any(changes_made): - account_change_details.proposed_changes.extend( - list(chain.from_iterable(changes_made)) - ) + account_change_details.proposed_changes.extend(changes_made) + if any(exceptions): + account_change_details.exceptions_seen.extend(exceptions) if context.execute: if self.deleted: diff --git a/iambic/plugins/v0_1_0/aws/iam/group/utils.py b/iambic/plugins/v0_1_0/aws/iam/group/utils.py index 834b243c3..e60e1d51d 100644 --- a/iambic/plugins/v0_1_0/aws/iam/group/utils.py +++ b/iambic/plugins/v0_1_0/aws/iam/group/utils.py @@ -2,13 +2,15 @@ import asyncio import json +from itertools import chain from typing import TYPE_CHECKING, Union from deepdiff import DeepDiff + from iambic.core.context import ExecutionContext from iambic.core.logger import log from iambic.core.models import ProposedChange, ProposedChangeType -from iambic.core.utils import aio_wrapper +from iambic.core.utils import aio_wrapper, plugin_apply_wrapper from iambic.plugins.v0_1_0.aws.utils import boto_crud_call, paginated_search if TYPE_CHECKING: @@ -135,23 +137,22 @@ async def apply_group_managed_policies( if new_managed_policies: log_str = "New managed policies discovered." for policy_arn in new_managed_policies: - response.append( + proposed_changes = [ ProposedChange( change_type=ProposedChangeType.ATTACH, resource_id=policy_arn, attribute="managed_policies", ) - ) - if context.execute: - log_str = f"{log_str} Attaching managed policies..." - tasks = [ - boto_crud_call( + ] + response.extend(proposed_changes) + if context.execute: + log_str = f"{log_str} Attaching managed policies..." + apply_awaitable = boto_crud_call( iam_client.attach_group_policy, GroupName=group_name, PolicyArn=policy_arn, ) - for policy_arn in new_managed_policies - ] + tasks.append(plugin_apply_wrapper(apply_awaitable, proposed_changes)) log.info(log_str, managed_policies=new_managed_policies, **log_params) # Delete existing managed policies not in template @@ -163,31 +164,29 @@ async def apply_group_managed_policies( if existing_managed_policies: log_str = "Stale managed policies discovered." for policy_arn in existing_managed_policies: - response.append( + proposed_changes = [ ProposedChange( change_type=ProposedChangeType.DETACH, resource_id=policy_arn, attribute="managed_policies", ) - ) - if context.execute: - log_str = f"{log_str} Detaching managed policies..." - tasks.extend( - [ - boto_crud_call( - iam_client.detach_group_policy, - GroupName=group_name, - PolicyArn=policy_arn, - ) - for policy_arn in existing_managed_policies - ] - ) + ] + response.extend(proposed_changes) + if context.execute: + log_str = f"{log_str} Detaching managed policies..." + apply_awaitable = boto_crud_call( + iam_client.detach_group_policy, + GroupName=group_name, + PolicyArn=policy_arn, + ) + tasks.append(plugin_apply_wrapper(apply_awaitable, proposed_changes)) log.info(log_str, managed_policies=existing_managed_policies, **log_params) if tasks: - await asyncio.gather(*tasks) - - return response + results: list[list[ProposedChange]] = await asyncio.gather(*tasks) + return list(chain.from_iterable(results)) + else: + return response async def apply_group_inline_policies( @@ -212,22 +211,24 @@ async def apply_group_inline_policies( for policy_name in existing_policy_map.keys(): if not template_policy_map.get(policy_name): log_str = "Stale inline policies discovered." + proposed_changes = [ + ProposedChange( + change_type=ProposedChangeType.DELETE, + resource_id=policy_name, + attribute="inline_policies", + ) + ] + response.extend(proposed_changes) + if context.execute: log_str = f"{log_str} Removing inline policy..." - response.append( - ProposedChange( - change_type=ProposedChangeType.DELETE, - resource_id=policy_name, - attribute="inline_policies", - ) - ) - tasks.append( - boto_crud_call( - iam_client.delete_group_policy, - GroupName=group_name, - PolicyName=policy_name, - ) + + apply_awaitable = boto_crud_call( + iam_client.delete_group_policy, + GroupName=group_name, + PolicyName=policy_name, ) + tasks.append(plugin_apply_wrapper(apply_awaitable, proposed_changes)) log.info(log_str, policy_name=policy_name, **log_params) for policy_name, policy_document in template_policy_map.items(): @@ -252,7 +253,7 @@ async def apply_group_inline_policies( log_params["policy_drift"] = policy_drift boto_action = "Updating" resource_existence = "Stale" - response.append( + proposed_changes = [ ProposedChange( change_type=ProposedChangeType.UPDATE, resource_id=policy_name, @@ -261,37 +262,38 @@ async def apply_group_inline_policies( current_value=existing_policy_doc, new_value=policy_document, ) - ) + ] else: boto_action = "Creating" resource_existence = "New" - response.append( + proposed_changes = [ ProposedChange( change_type=ProposedChangeType.CREATE, resource_id=policy_name, attribute="inline_policies", new_value=policy_document, ) - ) + ] + response.extend(proposed_changes) log_str = f"{resource_existence} inline policies discovered." if context.execute and policy_document: log_str = f"{log_str} {boto_action} inline policy..." - tasks.append( - boto_crud_call( - iam_client.put_group_policy, - GroupName=group_name, - PolicyName=policy_name, - PolicyDocument=json.dumps(policy_document), - ) + apply_awaitable = boto_crud_call( + iam_client.put_group_policy, + GroupName=group_name, + PolicyName=policy_name, + PolicyDocument=json.dumps(policy_document), ) + tasks.append(plugin_apply_wrapper(apply_awaitable, proposed_changes)) log.info(log_str, policy_name=policy_name, **log_params) if tasks: - await asyncio.gather(*tasks) - - return response + results: list[list[ProposedChange]] = await asyncio.gather(*tasks) + return list(chain.from_iterable(results)) + else: + return response async def delete_iam_group(group_name: str, iam_client, log_params: dict): From c76c61b39f449aaeb204a79e9f465588afc0daea Mon Sep 17 00:00:00 2001 From: Matt Daue Date: Mon, 27 Feb 2023 13:54:31 -0800 Subject: [PATCH 08/32] Add quick install script Signed-off-by: Matt Daue --- install.sh | 46 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) create mode 100755 install.sh diff --git a/install.sh b/install.sh new file mode 100755 index 000000000..f251f6388 --- /dev/null +++ b/install.sh @@ -0,0 +1,46 @@ +#!/bin/sh + +# Check if docker is installed +if ! command -v docker &> /dev/null +then + echo "Docker is not installed on this system. Please install Docker before running this script. You can install docker by running the following command: curl -fsSL https://get.docker.com -o get-docker.sh && sh get-docker.sh" + exit +fi + +SHELL_NAME=$(ps -p $$ -o args= | awk '{print $1}') +IAMBIC_GIT_REPO="${IAMBIC_GIT_REPO_PATH:-~/iambic_templates}" + +echo "Installing iambic..." +echo "We are creating an iambic git repository in the directory ${IAMBIC_GIT_REPO_PATH}. If you want to change this directory, please edit the DEFAULT_IAMBIC_GIT_REPO variable in the install.sh script." +mkdir -p ${IAMBIC_GIT_REPO_PATH} +DOCKER_ALIAS="alias iambic='docker run -it -u $(id -u):$(id -g) -v ${HOME}/.aws:/app/.aws -e AWS_CONFIG_FILE=/app/.aws/config -e AWS_SHARED_CREDENTIALS_FILE=/app/.aws/credentials -e 'AWS_PROFILE=${AWS_PROFILE}' -v ${IAMBIC_GIT_REPO_PATH}:/templates:Z public.ecr.aws/s2p9s3r8/iambic:latest" + +if [ "$SHELL_NAME" = "bash" ]; then + echo DOCKER_ALIAS >> ~/.bashrc + source ~/.bashrc +else if [ "$SHELL_NAME" = "sh" ]; then + echo DOCKER_ALIAS >> ~/.profile + source ~/.profile +else if [ "$SHELL_NAME" = "zsh" ]; then + echo DOCKER_ALIAS >> ~/.zshrc + source ~/.zshrc +else if [ "$SHELL_NAME" = "ksh" ]; then + echo DOCKER_ALIAS >> ~/.kshrc + source ~/.kshrc +else if [ "$SHELL_NAME" = "dash" ]; then + echo DOCKER_ALIAS >> ~/.profile + source ~/.profile +else if [ "$SHELL_NAME" = "csh" ]; then + echo DOCKER_ALIAS >> ~/.cshrc + source ~/.cshrc +else if [ "$SHELL_NAME" = "tcsh" ]; then + echo DOCKER_ALIAS >> ~/.tcshrc + source ~/.tcshrc +else + echo DOCKER_ALIAS >> ~/.profile + source ~/.profile +fi + +echo "Caching the latest iambic docker container" +${which docker} pull public.ecr.aws/s2p9s3r8/iambic:latest +echo "IAMbic installed successfully. You can now use the 'iambic --help' command to get started with IAMbic." From dd5ba7ce7cc7223bf1d1a2c0fbbb88535774a409 Mon Sep 17 00:00:00 2001 From: Will-NOQ <102608210+Will-NOQ@users.noreply.github.com> Date: Mon, 27 Feb 2023 16:08:12 -0600 Subject: [PATCH 09/32] Updated plan/apply to support templates. Added param to load_config to only load approved plugins. --- .../okta/group/test_okta_group.py | 26 +--- functional_tests/okta/user/test_okta_user.py | 46 ++----- functional_tests/test_google.py | 20 +-- iambic/config/dynamic_config.py | 16 ++- iambic/lambda/app.py | 10 +- iambic/main.py | 119 +++++++++++++----- test/test_main.py | 6 +- 7 files changed, 134 insertions(+), 109 deletions(-) diff --git a/functional_tests/okta/group/test_okta_group.py b/functional_tests/okta/group/test_okta_group.py index aafa9a6eb..c261f910c 100644 --- a/functional_tests/okta/group/test_okta_group.py +++ b/functional_tests/okta/group/test_okta_group.py @@ -6,7 +6,7 @@ from functional_tests.conftest import IAMBIC_TEST_DETAILS from iambic.core.iambic_enum import IambicManaged from iambic.core.parser import load_templates -from iambic.main import run_force_apply +from iambic.main import run_apply def test_okta_group(): @@ -31,11 +31,7 @@ def test_okta_group(): temp_file.write(iambic_functional_test_group_yaml) # Create group - run_force_apply( - IAMBIC_TEST_DETAILS.config, - [test_group_fp], - temp_templates_directory, - ) + run_apply(IAMBIC_TEST_DETAILS.config, [test_group_fp]) # Test Reading Template group_template = load_templates([test_group_fp])[0] @@ -51,11 +47,7 @@ def test_okta_group(): ) + datetime.timedelta(days=1) # Write new template, apply, and confirm access removed group_template.write() - run_force_apply( - IAMBIC_TEST_DETAILS.config, - [test_group_fp], - temp_templates_directory, - ) + run_apply(IAMBIC_TEST_DETAILS.config, [test_group_fp]) group_template = load_templates([test_group_fp])[0] assert len(group_template.properties.members) == 2 @@ -73,11 +65,7 @@ def test_okta_group(): 0 ].username = "this_user_should_not_exist@example.com" group_template.write() - run_force_apply( - IAMBIC_TEST_DETAILS.config, - [test_group_fp], - temp_templates_directory, - ) + run_apply(IAMBIC_TEST_DETAILS.config, [test_group_fp]) if os.path.isfile(proposed_changes_yaml_path): assert os.path.getsize(proposed_changes_yaml_path) == 0 else: @@ -90,11 +78,7 @@ def test_okta_group(): # Set expiry for the entire group group_template.expires_at = "yesterday" group_template.write() - run_force_apply( - IAMBIC_TEST_DETAILS.config, - [test_group_fp], - temp_templates_directory, - ) + run_apply(IAMBIC_TEST_DETAILS.config, [test_group_fp]) group_template = load_templates([test_group_fp])[0] assert group_template.deleted is True diff --git a/functional_tests/okta/user/test_okta_user.py b/functional_tests/okta/user/test_okta_user.py index 67a97d3c1..7c0b30cf4 100644 --- a/functional_tests/okta/user/test_okta_user.py +++ b/functional_tests/okta/user/test_okta_user.py @@ -2,13 +2,13 @@ import datetime import os -import time import random +import time from functional_tests.conftest import IAMBIC_TEST_DETAILS from iambic.core.iambic_enum import IambicManaged from iambic.core.parser import load_templates -from iambic.main import run_force_apply +from iambic.main import run_apply def test_okta_user(): @@ -20,40 +20,32 @@ def test_okta_user(): idp_name: development profile: firstName: iambic - lastName: functional_test_user + lastName: {username} email: {username}@example.com login: {username}@example.com status: active """ test_user_fp = os.path.join( temp_templates_directory, - "resources/okta/development/users/iambic_functional_test_user.yaml", + f"resources/okta/development/users/{username}.yaml", ) with open(test_user_fp, "w") as temp_file: temp_file.write(iambic_functional_test_user_yaml) # Create user - run_force_apply( - IAMBIC_TEST_DETAILS.config, - [test_user_fp], - temp_templates_directory, - ) + run_apply(IAMBIC_TEST_DETAILS.config, [test_user_fp]) # Test Reading Template user_template = load_templates([test_user_fp])[0] - assert user_template.properties.username == "iambic_functional_test_user" + assert user_template.properties.username == username # Test Updating Template user_template.properties.profile["firstName"] = "TestNameChange" user_template.write() # Sleep to give profile time to propagate time.sleep(30) - run_force_apply( - IAMBIC_TEST_DETAILS.config, - [test_user_fp], - temp_templates_directory, - ) + run_apply(IAMBIC_TEST_DETAILS.config, [test_user_fp]) user_template = load_templates([test_user_fp])[0] assert user_template.properties.profile["firstName"] == "TestNameChange" @@ -69,11 +61,7 @@ def test_okta_user(): orig_first_name = user_template.properties.profile["firstName"] user_template.properties.profile["firstName"] = "shouldNotWork" user_template.write() - run_force_apply( - IAMBIC_TEST_DETAILS.config, - [test_user_fp], - temp_templates_directory, - ) + run_apply(IAMBIC_TEST_DETAILS.config, [test_user_fp]) if os.path.isfile(proposed_changes_yaml_path): assert os.path.getsize(proposed_changes_yaml_path) == 0 else: @@ -83,11 +71,7 @@ def test_okta_user(): user_template.properties.profile["firstName"] = orig_first_name user_template.write() - run_force_apply( - IAMBIC_TEST_DETAILS.config, - [test_user_fp], - temp_templates_directory, - ) + run_apply(IAMBIC_TEST_DETAILS.config, [test_user_fp]) user_template = load_templates([test_user_fp])[0] # Expire user @@ -95,18 +79,10 @@ def test_okta_user(): datetime.timezone.utc ) - datetime.timedelta(days=1) user_template.write() - run_force_apply( - IAMBIC_TEST_DETAILS.config, - [test_user_fp], - temp_templates_directory, - ) + run_apply(IAMBIC_TEST_DETAILS.config, [test_user_fp]) user_template = load_templates([test_user_fp])[0] assert user_template.deleted is True # Needed to really delete the user and file user_template.force_delete = True user_template.write() - run_force_apply( - IAMBIC_TEST_DETAILS.config, - [test_user_fp], - temp_templates_directory, - ) + run_apply(IAMBIC_TEST_DETAILS.config, [test_user_fp]) diff --git a/functional_tests/test_google.py b/functional_tests/test_google.py index ee18f3307..4417c5d53 100644 --- a/functional_tests/test_google.py +++ b/functional_tests/test_google.py @@ -5,7 +5,7 @@ from functional_tests.conftest import IAMBIC_TEST_DETAILS from iambic.core.parser import load_templates -from iambic.main import run_force_apply +from iambic.main import run_apply def test_google(): @@ -30,11 +30,7 @@ def test_google(): temp_file.write(iambic_functional_test_group_yaml) # Create group - run_force_apply( - IAMBIC_TEST_DETAILS.config, - [test_group_fp], - IAMBIC_TEST_DETAILS.template_dir_path, - ) + run_apply(IAMBIC_TEST_DETAILS.config, [test_group_fp]) # Test Reading Template group_template = load_templates([test_group_fp])[0] @@ -50,21 +46,13 @@ def test_google(): ) + datetime.timedelta(days=1) # Write new template, apply, and confirm access removed group_template.write() - run_force_apply( - IAMBIC_TEST_DETAILS.config, - [test_group_fp], - IAMBIC_TEST_DETAILS.template_dir_path, - ) + run_apply(IAMBIC_TEST_DETAILS.config, [test_group_fp]) group_template = load_templates([test_group_fp])[0] assert len(group_template.properties.members) == 2 # Set expiry for the entire group group_template.expires_at = datetime.datetime.now() - datetime.timedelta(days=1) group_template.write() - run_force_apply( - IAMBIC_TEST_DETAILS.config, - [test_group_fp], - IAMBIC_TEST_DETAILS.template_dir_path, - ) + run_apply(IAMBIC_TEST_DETAILS.config, [test_group_fp]) group_template = load_templates([test_group_fp])[0] assert group_template.deleted is True diff --git a/iambic/config/dynamic_config.py b/iambic/config/dynamic_config.py index 099471664..165e05d40 100644 --- a/iambic/config/dynamic_config.py +++ b/iambic/config/dynamic_config.py @@ -371,7 +371,11 @@ def write(self, exclude_none=True, exclude_unset=False, exclude_defaults=True): log.info("Config successfully written", config_location=file_path) -async def load_config(config_path: str, configure_plugins: bool = True) -> Config: +async def load_config( + config_path: str, + configure_plugins: bool = True, + approved_plugins_only: bool = False, +) -> Config: """ Load the configuration from the specified file path. @@ -400,6 +404,16 @@ async def load_config(config_path: str, configure_plugins: bool = True) -> Confi ) # Ensure it's a string in case it's a Path for pydantic config_dict = yaml.load(open(config_path)) base_config = Config(file_path=config_path, **config_dict) + if approved_plugins_only: + default_plugins = [ + plugin.location for plugin in Config.__fields__["plugins"].default + ] + base_config.plugins = [ + plugin + for plugin in base_config.plugins + if plugin.location in default_plugins + ] + all_plugins = load_plugins(base_config.plugins) config_fields = {} for plugin in all_plugins: diff --git a/iambic/lambda/app.py b/iambic/lambda/app.py index 300598fc1..6f027d70c 100644 --- a/iambic/lambda/app.py +++ b/iambic/lambda/app.py @@ -10,7 +10,7 @@ from iambic.config.dynamic_config import load_config from iambic.config.utils import resolve_config_template_path from iambic.core.models import BaseModel -from iambic.main import run_apply, run_clone_repos, run_detect, run_plan +from iambic.main import run_clone_repos, run_detect, run_git_apply, run_git_plan REPO_BASE_PATH = os.path.expanduser("~/.iambic/repos/") PLAN_OUTPUT_PATH = os.environ.get("PLAN_OUTPUT_PATH", None) @@ -83,7 +83,7 @@ def run_handler(event=None, context=None): case LambdaCommand.run_detect.value: return run_detect(REPO_BASE_PATH) case LambdaCommand.run_apply.value: - return run_apply( + return run_git_apply( False, FROM_SHA, TO_SHA, @@ -91,9 +91,9 @@ def run_handler(event=None, context=None): output_path=PLAN_OUTPUT_PATH, ) case LambdaCommand.run_plan.value: - return run_plan(PLAN_OUTPUT_PATH, repo_dir=REPO_BASE_PATH) + return run_git_plan(PLAN_OUTPUT_PATH, repo_dir=REPO_BASE_PATH) case LambdaCommand.run_git_apply.value: - return run_apply( + return run_git_apply( False, FROM_SHA, TO_SHA, @@ -101,7 +101,7 @@ def run_handler(event=None, context=None): output_path=PLAN_OUTPUT_PATH, ) case LambdaCommand.run_git_plan.value: - return run_plan(PLAN_OUTPUT_PATH, repo_dir=REPO_BASE_PATH) + return run_git_plan(PLAN_OUTPUT_PATH, repo_dir=REPO_BASE_PATH) case LambdaCommand.run_clone_git_repos.value: return run_clone_repos(REPO_BASE_PATH) case _: diff --git a/iambic/main.py b/iambic/main.py index a1d7068d7..309d229fc 100644 --- a/iambic/main.py +++ b/iambic/main.py @@ -3,6 +3,7 @@ import asyncio import os import pathlib +import sys import warnings import click @@ -119,21 +120,6 @@ def run_clone_repos(repo_dir: str = str(pathlib.Path.cwd())): asyncio.run(clone_git_repos(config, repo_dir)) -def run_force_apply( - config: Config, - templates: list[str], - repo_dir: str = str(pathlib.Path.cwd()), -): - ctx.eval_only = False - - if not templates: - templates = asyncio.run(gather_templates(repo_dir)) - - templates = load_templates(templates) - asyncio.run(flag_expired_resources([template.file_path for template in templates])) - asyncio.run(config.run_apply(templates)) - - @cli.command() @click.option( "--repo-dir", @@ -145,14 +131,31 @@ def run_force_apply( help="The repo directory containing the templates. Example: ~/iambic-templates", ) @click.option( - "--allow-dirty", + "--force", + "-f", is_flag=True, show_default=True, + help="Apply changes without asking for permission?", +) +@click.option( + "--template", + "-t", + "templates", + required=False, + multiple=True, + type=click.Path(exists=True), + help="The template file path(s) to apply. Example: ./aws/roles/engineering.yaml", +) +@click.option( + "--allow-dirty", + is_flag=True, + hidden=True, help="Allow applying changes from a dirty git repo", ) @click.option( "--from-sha", "from_sha", + hidden=True, required=False, type=str, help="The from_sha to calculate diff", @@ -160,6 +163,7 @@ def run_force_apply( @click.option( "--to-sha", "to_sha", + hidden=True, required=False, type=str, help="The to_sha to calculate diff", @@ -173,31 +177,60 @@ def run_force_apply( ) def apply( repo_dir: str, + force: bool, + templates: list[str], allow_dirty: bool, from_sha: str, to_sha: str, plan_output: str, ): - run_apply( - allow_dirty, - from_sha, - to_sha, - repo_dir=repo_dir, - output_path=plan_output, - ) + try: + if from_sha: + assert to_sha, "to_sha is required when from_sha is provided" + assert ( + not templates + ), "templates cannot be provided when from_sha is provided" + run_git_apply( + allow_dirty, + from_sha, + to_sha, + repo_dir=repo_dir, + output_path=plan_output, + ) + else: + assert templates, "templates is a required argument" + assert not to_sha, "to_sha is not supported with templates" + assert not from_sha, "from_sha is not supported with templates" + ctx.eval_only = not force + config_path = asyncio.run(resolve_config_template_path(repo_dir)) + config = asyncio.run(load_config(config_path)) + run_apply(config, templates) + except AssertionError as err: + log.error("Invalid arguments", error=repr(err)) + + +def run_apply(config: Config, templates: list[str]): + templates = load_templates(templates) + asyncio.run(flag_expired_resources([template.file_path for template in templates])) + template_changes = asyncio.run(config.run_apply(templates)) + output_proposed_changes(template_changes) + if ctx.eval_only and template_changes and click.confirm("Proceed?"): + ctx.eval_only = False + asyncio.run(config.run_apply(templates)) + # This was here before, but I don't think it's needed. Leaving it here for now to see if anything breaks. + # asyncio.run(config.run_detect_changes(repo_dir)) -def run_apply( + +def run_git_apply( allow_dirty: bool, from_sha: str, to_sha: str, repo_dir: str = str(pathlib.Path.cwd()), output_path: str = None, ): - ctx.eval_only = False config_path = asyncio.run(resolve_config_template_path(repo_dir)) - asyncio.run(load_config(config_path)) template_changes = asyncio.run( apply_git_changes( @@ -221,10 +254,20 @@ def run_apply( @cli.command() +@click.option( + "--template", + "-t", + "templates", + required=False, + multiple=True, + type=click.Path(exists=True), + help="The template file path(s) to apply. Example: ./resources/aws/roles/engineering.yaml", +) @click.option( "--plan-output", "-o", "plan_output", + hidden=True, type=click.Path(exists=True), help="The location to output the plan Example: ./proposed_changes.yaml", ) @@ -237,11 +280,17 @@ def run_apply( default=os.getenv("IAMBIC_REPO_DIR"), help="The repo directory containing the templates. Example: ~/iambic-templates", ) -def plan(plan_output: str, repo_dir: str): - run_plan(plan_output, repo_dir=repo_dir) +def plan(templates: list, plan_output: str, repo_dir: str): + if plan_output: + run_git_plan(plan_output, repo_dir=repo_dir) + else: + if not templates: + log.error("Invalid arguments", error="templates is a required argument") + raise sys.exit(1) + run_plan(templates, repo_dir=repo_dir) -def run_plan( +def run_git_plan( output_path: str, repo_dir: str = str(pathlib.Path.cwd()), ): @@ -252,6 +301,18 @@ def run_plan( output_proposed_changes(template_changes, output_path=output_path) +def run_plan(templates: list[str], repo_dir: str = str(pathlib.Path.cwd())): + if not templates: + templates = asyncio.run(gather_templates(repo_dir)) + + config_path = asyncio.run(resolve_config_template_path(repo_dir)) + config = asyncio.run(load_config(config_path)) + + asyncio.run(flag_expired_resources(templates)) + ctx.eval_only = True + output_proposed_changes(asyncio.run(config.run_apply(load_templates(templates)))) + + @cli.command() @click.option( "--repo-dir", diff --git a/test/test_main.py b/test/test_main.py index 451ce9752..bafbca06a 100644 --- a/test/test_main.py +++ b/test/test_main.py @@ -8,7 +8,8 @@ import iambic.plugins.v0_1_0.example import pytest from iambic.config.dynamic_config import load_config -from iambic.main import run_force_apply +from iambic.core.utils import gather_templates +from iambic.main import run_apply TEST_TEMPLATE_YAML = """template_type: NOQ::Example::LocalFile name: test_template @@ -69,7 +70,8 @@ def test_run_apply(example_test_filesystem): assert "tomorrow" in before_template_content config = asyncio.run(load_config(config_path)) - run_force_apply(config, [], repo_dir) + templates = asyncio.run(gather_templates(repo_dir)) + run_apply(config, templates) with open(f"{repo_dir}/{TEST_TEMPLATE_PATH}", "r") as f: after_template_content = "\n".join(f.readlines()) assert "tomorrow" not in after_template_content From 1bf11373ef1e5610c4a7f6bf36eb2f67395e20b7 Mon Sep 17 00:00:00 2001 From: Steven Moy Date: Mon, 27 Feb 2023 14:08:22 -0800 Subject: [PATCH 10/32] Move bad input to its own Test class --- .../aws/group/test_update_template.py | 71 ++++++++++++------- 1 file changed, 47 insertions(+), 24 deletions(-) diff --git a/functional_tests/aws/group/test_update_template.py b/functional_tests/aws/group/test_update_template.py index fb31899fd..7df9bb722 100644 --- a/functional_tests/aws/group/test_update_template.py +++ b/functional_tests/aws/group/test_update_template.py @@ -87,7 +87,7 @@ async def test_update_managed_policies(self): f"{group['ManagedPolicies']} attached to it for group {self.group_name}", ) - async def test_bad_input(self): + async def test_create_update_group_all_accounts(self): self.template.included_accounts = ["*"] self.template.excluded_accounts = [] @@ -100,6 +100,13 @@ async def test_bad_input(self): account_id for account_id, group in account_group_mapping.items() if group ] + for account_id in group_account_ids: + self.assertIn( + account_id, + self.all_account_ids, + f"{account_id} not found for group {self.group_name}", + ) + self.template.properties.inline_policies.append( PolicyDocument( included_accounts=[group_account_ids[0], group_account_ids[1]], @@ -108,14 +115,14 @@ async def test_bad_input(self): statement=[ { "action": ["s3:NotARealAction"], - "effect": "BAD_INPUT", + "effect": "Deny", "resource": ["*"], "expires_at": "tomorrow", "included_accounts": [group_account_ids[0]], }, { "action": ["s3:AlsoNotARealAction"], - "effect": "BAD_INPUT", + "effect": "Deny", "resource": ["*"], "expires_at": "tomorrow", }, @@ -123,9 +130,41 @@ async def test_bad_input(self): ) ) r = await self.template.apply(IAMBIC_TEST_DETAILS.config.aws, ctx) - self.assertEqual(len(r.exceptions_seen), 2) + self.assertEqual(len(r.proposed_changes), 2) - async def test_create_update_group_all_accounts(self): + # Set expiration + self.template.properties.inline_policies[1].statement[ + 0 + ].expires_at = dateparser.parse( + "yesterday", settings={"TIMEZONE": "UTC", "RETURN_AS_TIMEZONE_AWARE": True} + ) + r = await self.template.apply(IAMBIC_TEST_DETAILS.config.aws, ctx) + self.assertEqual(len(r.proposed_changes), 1) + + +class UpdateGroupBadInputTestCase(IsolatedAsyncioTestCase): + @classmethod + def setUpClass(cls): + cls.template = asyncio.run( + generate_group_template_from_base(IAMBIC_TEST_DETAILS.template_dir_path) + ) + cls.group_name = cls.template.properties.group_name + cls.all_account_ids = [ + account.account_id for account in IAMBIC_TEST_DETAILS.config.aws.accounts + ] + # Only include the template in half the accounts + # Make the accounts explicit so it's easier to validate account scoped tests + cls.template.included_accounts = cls.all_account_ids[ + : len(cls.all_account_ids) // 2 + ] + asyncio.run(cls.template.apply(IAMBIC_TEST_DETAILS.config.aws, ctx)) + + @classmethod + def tearDownClass(cls): + cls.template.deleted = True + asyncio.run(cls.template.apply(IAMBIC_TEST_DETAILS.config.aws, ctx)) + + async def test_bad_input(self): self.template.included_accounts = ["*"] self.template.excluded_accounts = [] @@ -138,13 +177,6 @@ async def test_create_update_group_all_accounts(self): account_id for account_id, group in account_group_mapping.items() if group ] - for account_id in group_account_ids: - self.assertIn( - account_id, - self.all_account_ids, - f"{account_id} not found for group {self.group_name}", - ) - self.template.properties.inline_policies.append( PolicyDocument( included_accounts=[group_account_ids[0], group_account_ids[1]], @@ -153,14 +185,14 @@ async def test_create_update_group_all_accounts(self): statement=[ { "action": ["s3:NotARealAction"], - "effect": "Deny", + "effect": "BAD_INPUT", "resource": ["*"], "expires_at": "tomorrow", "included_accounts": [group_account_ids[0]], }, { "action": ["s3:AlsoNotARealAction"], - "effect": "Deny", + "effect": "BAD_INPUT", "resource": ["*"], "expires_at": "tomorrow", }, @@ -168,13 +200,4 @@ async def test_create_update_group_all_accounts(self): ) ) r = await self.template.apply(IAMBIC_TEST_DETAILS.config.aws, ctx) - self.assertEqual(len(r.proposed_changes), 2) - - # Set expiration - self.template.properties.inline_policies[1].statement[ - 0 - ].expires_at = dateparser.parse( - "yesterday", settings={"TIMEZONE": "UTC", "RETURN_AS_TIMEZONE_AWARE": True} - ) - r = await self.template.apply(IAMBIC_TEST_DETAILS.config.aws, ctx) - self.assertEqual(len(r.proposed_changes), 1) + self.assertEqual(len(r.exceptions_seen), 2) From 8bab5fa92b8ff848dcefe95365dbc146ffa54d03 Mon Sep 17 00:00:00 2001 From: Matt Daue Date: Mon, 27 Feb 2023 14:16:43 -0800 Subject: [PATCH 11/32] Fix install.sh Signed-off-by: Matt Daue --- install.sh | 46 +++++++++++++++++++++++++++------------------- 1 file changed, 27 insertions(+), 19 deletions(-) diff --git a/install.sh b/install.sh index f251f6388..b5458ee11 100755 --- a/install.sh +++ b/install.sh @@ -1,5 +1,3 @@ -#!/bin/sh - # Check if docker is installed if ! command -v docker &> /dev/null then @@ -7,40 +5,50 @@ then exit fi +if ! command -v git &> /dev/null +then + echo "Git is not installed on this system. Please install Git before running this script. Refer to your operating system's package manager for installation instructions." +fi + SHELL_NAME=$(ps -p $$ -o args= | awk '{print $1}') -IAMBIC_GIT_REPO="${IAMBIC_GIT_REPO_PATH:-~/iambic_templates}" +echo "Detected shell: ${SHELL_NAME}" +IAMBIC_GIT_REPO_PATH="${IAMBIC_GIT_REPO_PATH:-${HOME}/iambic_templates}" echo "Installing iambic..." echo "We are creating an iambic git repository in the directory ${IAMBIC_GIT_REPO_PATH}. If you want to change this directory, please edit the DEFAULT_IAMBIC_GIT_REPO variable in the install.sh script." mkdir -p ${IAMBIC_GIT_REPO_PATH} -DOCKER_ALIAS="alias iambic='docker run -it -u $(id -u):$(id -g) -v ${HOME}/.aws:/app/.aws -e AWS_CONFIG_FILE=/app/.aws/config -e AWS_SHARED_CREDENTIALS_FILE=/app/.aws/credentials -e 'AWS_PROFILE=${AWS_PROFILE}' -v ${IAMBIC_GIT_REPO_PATH}:/templates:Z public.ecr.aws/s2p9s3r8/iambic:latest" +CWD=$(pwd) +cd ${IAMBIC_GIT_REPO_PATH} +$(which git) init . +cd $CWD +DOCKER_ALIAS="alias iambic='docker run -it -u $(id -u):$(id -g) -v ${HOME}/.aws:/app/.aws -e AWS_CONFIG_FILE=/app/.aws/config -e AWS_SHARED_CREDENTIALS_FILE=/app/.aws/credentials -e AWS_PROFILE=${AWS_PROFILE} -v ${IAMBIC_GIT_REPO_PATH}:/templates:Z public.ecr.aws/s2p9s3r8/iambic:latest'" if [ "$SHELL_NAME" = "bash" ]; then - echo DOCKER_ALIAS >> ~/.bashrc + echo "${DOCKER_ALIAS}" >> ~/.bashrc source ~/.bashrc -else if [ "$SHELL_NAME" = "sh" ]; then - echo DOCKER_ALIAS >> ~/.profile +elif [ "$SHELL_NAME" = "sh" ]; then + echo "${DOCKER_ALIAS}" >> ~/.profile source ~/.profile -else if [ "$SHELL_NAME" = "zsh" ]; then - echo DOCKER_ALIAS >> ~/.zshrc +elif [ "$SHELL_NAME" = "zsh" ]; then + echo "${DOCKER_ALIAS}" >> ~/.zshrc source ~/.zshrc -else if [ "$SHELL_NAME" = "ksh" ]; then - echo DOCKER_ALIAS >> ~/.kshrc +elif [ "$SHELL_NAME" = "ksh" ]; then + echo "${DOCKER_ALIAS}" >> ~/.kshrc source ~/.kshrc -else if [ "$SHELL_NAME" = "dash" ]; then - echo DOCKER_ALIAS >> ~/.profile +elif [ "$SHELL_NAME" = "dash" ]; then + echo "${DOCKER_ALIAS}" >> ~/.profile source ~/.profile -else if [ "$SHELL_NAME" = "csh" ]; then - echo DOCKER_ALIAS >> ~/.cshrc +elif [ "$SHELL_NAME" = "csh" ]; then + echo "${DOCKER_ALIAS}" >> ~/.cshrc source ~/.cshrc -else if [ "$SHELL_NAME" = "tcsh" ]; then - echo DOCKER_ALIAS >> ~/.tcshrc +elif [ "$SHELL_NAME" = "tcsh" ]; then + echo "${DOCKER_ALIAS}" >> ~/.tcshrc source ~/.tcshrc else - echo DOCKER_ALIAS >> ~/.profile + echo "${DOCKER_ALIAS}" >> ~/.profile source ~/.profile fi echo "Caching the latest iambic docker container" -${which docker} pull public.ecr.aws/s2p9s3r8/iambic:latest +$( which docker ) pull public.ecr.aws/s2p9s3r8/iambic:latest echo "IAMbic installed successfully. You can now use the 'iambic --help' command to get started with IAMbic." From 5420760be1188916489d089827633928084f0f9b Mon Sep 17 00:00:00 2001 From: Version Auto Bump Date: Mon, 27 Feb 2023 22:25:07 +0000 Subject: [PATCH 12/32] Bump version --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 70b032bfb..f602cc6ca 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "poetry.core.masonry.api" [tool.poetry] name = "iambic" -version = "0.1.50" +version = "0.1.51" description = "The python package used to generate, parse, and execute noqform yaml templates." authors = ["Noq Software "] readme = "README.md" From 8592bef652b401a98d996799a770446721d2e20e Mon Sep 17 00:00:00 2001 From: Matt Daue Date: Mon, 27 Feb 2023 14:35:56 -0800 Subject: [PATCH 13/32] Ensure the shell selector works to write the correct alias Signed-off-by: Matt Daue --- install.sh | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) mode change 100755 => 100644 install.sh diff --git a/install.sh b/install.sh old mode 100755 new mode 100644 index b5458ee11..61197a84e --- a/install.sh +++ b/install.sh @@ -10,7 +10,7 @@ then echo "Git is not installed on this system. Please install Git before running this script. Refer to your operating system's package manager for installation instructions." fi -SHELL_NAME=$(ps -p $$ -o args= | awk '{print $1}') +SHELL_NAME=$(ps -p $$ | tail -1 | awk '{print $NF}') echo "Detected shell: ${SHELL_NAME}" IAMBIC_GIT_REPO_PATH="${IAMBIC_GIT_REPO_PATH:-${HOME}/iambic_templates}" @@ -21,31 +21,39 @@ CWD=$(pwd) cd ${IAMBIC_GIT_REPO_PATH} $(which git) init . cd $CWD -DOCKER_ALIAS="alias iambic='docker run -it -u $(id -u):$(id -g) -v ${HOME}/.aws:/app/.aws -e AWS_CONFIG_FILE=/app/.aws/config -e AWS_SHARED_CREDENTIALS_FILE=/app/.aws/credentials -e AWS_PROFILE=${AWS_PROFILE} -v ${IAMBIC_GIT_REPO_PATH}:/templates:Z public.ecr.aws/s2p9s3r8/iambic:latest'" +DOCKER_ALIAS="alias iambic='docker run -it -u $(id -u):$(id -g) -v ${HOME}/.aws:/app/.aws -e AWS_CONFIG_FILE=/app/.aws/config -e AWS_SHARED_CREDENTIALS_FILE=/app/.aws/credentials -e AWS_PROFILE=${AWS_PROFILE} -v ${CWD}:/templates:Z public.ecr.aws/s2p9s3r8/iambic:latest'" if [ "$SHELL_NAME" = "bash" ]; then echo "${DOCKER_ALIAS}" >> ~/.bashrc + echo "Wrote alias to ~/.bashrc" source ~/.bashrc elif [ "$SHELL_NAME" = "sh" ]; then echo "${DOCKER_ALIAS}" >> ~/.profile + echo "Wrote alias to ~/.profile" source ~/.profile elif [ "$SHELL_NAME" = "zsh" ]; then echo "${DOCKER_ALIAS}" >> ~/.zshrc + echo "Wrote alias to ~/.zshrc" source ~/.zshrc elif [ "$SHELL_NAME" = "ksh" ]; then echo "${DOCKER_ALIAS}" >> ~/.kshrc + echo "Wrote alias to ~/.kshrc" source ~/.kshrc elif [ "$SHELL_NAME" = "dash" ]; then echo "${DOCKER_ALIAS}" >> ~/.profile + echo "Wrote alias to ~/.profile" source ~/.profile -elif [ "$SHELL_NAME" = "csh" ]; then - echo "${DOCKER_ALIAS}" >> ~/.cshrc - source ~/.cshrc elif [ "$SHELL_NAME" = "tcsh" ]; then echo "${DOCKER_ALIAS}" >> ~/.tcshrc + echo "Wrote alias to ~/.tcshrc" source ~/.tcshrc +elif [ "$SHELL_NAME" = "csh" ]; then + echo "${DOCKER_ALIAS}" >> ~/.cshrc + echo "Wrote alias to ~/.cshrc" + source ~/.cshrc else echo "${DOCKER_ALIAS}" >> ~/.profile + echo "Wrote alias to ~/.profile" source ~/.profile fi From ffff1d0acbd57a781b0093bd05a571507a2487d4 Mon Sep 17 00:00:00 2001 From: Matt Daue Date: Mon, 27 Feb 2023 14:47:04 -0800 Subject: [PATCH 14/32] Use ECR_PATH variable so ecr path only mentioned in one place Signed-off-by: Matt Daue --- install.sh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/install.sh b/install.sh index 61197a84e..349d250a1 100644 --- a/install.sh +++ b/install.sh @@ -13,6 +13,7 @@ fi SHELL_NAME=$(ps -p $$ | tail -1 | awk '{print $NF}') echo "Detected shell: ${SHELL_NAME}" IAMBIC_GIT_REPO_PATH="${IAMBIC_GIT_REPO_PATH:-${HOME}/iambic_templates}" +ECR_PATH="public.ecr.aws/s2p9s3r8/iambic:latest" echo "Installing iambic..." echo "We are creating an iambic git repository in the directory ${IAMBIC_GIT_REPO_PATH}. If you want to change this directory, please edit the DEFAULT_IAMBIC_GIT_REPO variable in the install.sh script." @@ -21,7 +22,7 @@ CWD=$(pwd) cd ${IAMBIC_GIT_REPO_PATH} $(which git) init . cd $CWD -DOCKER_ALIAS="alias iambic='docker run -it -u $(id -u):$(id -g) -v ${HOME}/.aws:/app/.aws -e AWS_CONFIG_FILE=/app/.aws/config -e AWS_SHARED_CREDENTIALS_FILE=/app/.aws/credentials -e AWS_PROFILE=${AWS_PROFILE} -v ${CWD}:/templates:Z public.ecr.aws/s2p9s3r8/iambic:latest'" +DOCKER_ALIAS="alias iambic='docker run -it -u $(id -u):$(id -g) -v ${HOME}/.aws:/app/.aws -e AWS_CONFIG_FILE=/app/.aws/config -e AWS_SHARED_CREDENTIALS_FILE=/app/.aws/credentials -e AWS_PROFILE=${AWS_PROFILE} -v ${CWD}:/templates:Z ${ECR_PATH}'" if [ "$SHELL_NAME" = "bash" ]; then echo "${DOCKER_ALIAS}" >> ~/.bashrc @@ -58,5 +59,5 @@ else fi echo "Caching the latest iambic docker container" -$( which docker ) pull public.ecr.aws/s2p9s3r8/iambic:latest +$( which docker ) pull ${ECR_PATH} echo "IAMbic installed successfully. You can now use the 'iambic --help' command to get started with IAMbic." From f77ce0624f374e1290d0810933c7c2ecb1763fc4 Mon Sep 17 00:00:00 2001 From: Matt Daue Date: Mon, 27 Feb 2023 14:48:27 -0800 Subject: [PATCH 15/32] Change to use dash versus _ Signed-off-by: Matt Daue --- install.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/install.sh b/install.sh index 349d250a1..a03b07c30 100644 --- a/install.sh +++ b/install.sh @@ -12,7 +12,7 @@ fi SHELL_NAME=$(ps -p $$ | tail -1 | awk '{print $NF}') echo "Detected shell: ${SHELL_NAME}" -IAMBIC_GIT_REPO_PATH="${IAMBIC_GIT_REPO_PATH:-${HOME}/iambic_templates}" +IAMBIC_GIT_REPO_PATH="${IAMBIC_GIT_REPO_PATH:-${HOME}/iambic-templates}" ECR_PATH="public.ecr.aws/s2p9s3r8/iambic:latest" echo "Installing iambic..." From 430345dc8faeae62d07ea72239793a8d335a8331 Mon Sep 17 00:00:00 2001 From: Version Auto Bump Date: Mon, 27 Feb 2023 22:56:08 +0000 Subject: [PATCH 16/32] Bump version --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index f602cc6ca..62f9310a9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "poetry.core.masonry.api" [tool.poetry] name = "iambic" -version = "0.1.51" +version = "0.1.52" description = "The python package used to generate, parse, and execute noqform yaml templates." authors = ["Noq Software "] readme = "README.md" From 936a2b5ea9d911fd903b002cf73b55a8a62a4176 Mon Sep 17 00:00:00 2001 From: Matt Daue Date: Mon, 27 Feb 2023 15:04:05 -0800 Subject: [PATCH 17/32] Fix AWS PROFILE and CWS env var ref in alias Signed-off-by: Matt Daue --- install.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/install.sh b/install.sh index a03b07c30..e870eb433 100644 --- a/install.sh +++ b/install.sh @@ -22,7 +22,7 @@ CWD=$(pwd) cd ${IAMBIC_GIT_REPO_PATH} $(which git) init . cd $CWD -DOCKER_ALIAS="alias iambic='docker run -it -u $(id -u):$(id -g) -v ${HOME}/.aws:/app/.aws -e AWS_CONFIG_FILE=/app/.aws/config -e AWS_SHARED_CREDENTIALS_FILE=/app/.aws/credentials -e AWS_PROFILE=${AWS_PROFILE} -v ${CWD}:/templates:Z ${ECR_PATH}'" +DOCKER_ALIAS="alias iambic='docker run -it -u $(id -u):$(id -g) -v ${HOME}/.aws:/app/.aws -e AWS_CONFIG_FILE=/app/.aws/config -e AWS_SHARED_CREDENTIALS_FILE=/app/.aws/credentials -e AWS_PROFILE=\${AWS_PROFILE} -v \${CWD}:/templates:Z ${ECR_PATH}'" if [ "$SHELL_NAME" = "bash" ]; then echo "${DOCKER_ALIAS}" >> ~/.bashrc From dc91bb4ff085ec6b31fb9a07517f14f14666a166 Mon Sep 17 00:00:00 2001 From: Version Auto Bump Date: Mon, 27 Feb 2023 23:59:08 +0000 Subject: [PATCH 18/32] Bump version --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 62f9310a9..96a6e6416 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "poetry.core.masonry.api" [tool.poetry] name = "iambic" -version = "0.1.52" +version = "0.1.53" description = "The python package used to generate, parse, and execute noqform yaml templates." authors = ["Noq Software "] readme = "README.md" From 7c4626cfd9b90c3ba7f1faf8b12c6094e016dc04 Mon Sep 17 00:00:00 2001 From: Curtis Castrapel Date: Mon, 27 Feb 2023 16:11:38 -0800 Subject: [PATCH 19/32] EN-1796: Sanitize account name --- .vscode/launch.json | 3 +-- iambic/core/models.py | 5 ++++ iambic/core/utils.py | 15 +++++++++++ .../plugins/v0_1_0/aws/iam/policy/models.py | 3 +++ test/test_utils.py | 26 +++++++++++++++++++ 5 files changed, 50 insertions(+), 2 deletions(-) diff --git a/.vscode/launch.json b/.vscode/launch.json index 9b0c69e20..0d254915b 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -36,9 +36,8 @@ "console": "integratedTerminal", "args": [ "apply", - "--template", - "${env:IAMBIC_TEMPLATE_PATH}" + "../noq-templates/resources/aws/roles/all_accounts/readonly.yaml" ], "envFile": "${workspaceFolder}/.env", "justMyCode": true diff --git a/iambic/core/models.py b/iambic/core/models.py index 628a30387..fd885f977 100644 --- a/iambic/core/models.py +++ b/iambic/core/models.py @@ -34,6 +34,7 @@ from iambic.core.utils import ( apply_to_provider, create_commented_map, + sanitize_string, snake_to_camelcap, sort_dict, transform_comments, @@ -213,6 +214,10 @@ def apply_resource_dict( variables["owner"] = owner rtemplate = Environment(loader=BaseLoader()).from_string(json.dumps(response)) + valid_characters_re = r"[\w_+=,.@-]" + variables = { + k: sanitize_string(v, valid_characters_re) for k, v in variables.items() + } data = rtemplate.render(**variables) return json.loads(data) diff --git a/iambic/core/utils.py b/iambic/core/utils.py index a595f3bb5..6efb0318c 100644 --- a/iambic/core/utils.py +++ b/iambic/core/utils.py @@ -564,3 +564,18 @@ async def __call__(self, func: typing.Callable, *args, **kwargs): log.warning( f"Rate limit hit for {endpoint}. Retrying in {self.wait_time} seconds." ) + + +def sanitize_string(unsanitized_str, valid_characters_re): + """ + This function sanitizes the session name typically passed in an assume_role call, to verify that it's + """ + + sanitized_str = "" + max_length = 64 # Session names have a length limit of 64 characters + for char in unsanitized_str: + if len(sanitized_str) == max_length: + break + if re.match(valid_characters_re, char): + sanitized_str += char + return sanitized_str diff --git a/iambic/plugins/v0_1_0/aws/iam/policy/models.py b/iambic/plugins/v0_1_0/aws/iam/policy/models.py index b96ce6f78..f2b8ab3a7 100644 --- a/iambic/plugins/v0_1_0/aws/iam/policy/models.py +++ b/iambic/plugins/v0_1_0/aws/iam/policy/models.py @@ -18,6 +18,7 @@ ProposedChange, ProposedChangeType, ) +from iambic.core.utils import sanitize_string from iambic.plugins.v0_1_0.aws.iam.models import Path from iambic.plugins.v0_1_0.aws.iam.policy.utils import ( apply_managed_policy_tags, @@ -233,6 +234,8 @@ def apply_resource_dict( variables["account_name"] = aws_account.account_name rtemplate = Environment(loader=BaseLoader()).from_string(json.dumps(response)) + valid_characters_re = r"[\w_+=,.@-]" + variables = [sanitize_string(v, valid_characters_re) for v in variables] data = rtemplate.render(**variables) return json.loads(data) diff --git a/test/test_utils.py b/test/test_utils.py index 52a2a782b..62c41df0e 100644 --- a/test/test_utils.py +++ b/test/test_utils.py @@ -4,6 +4,8 @@ import pytest +from iambic.core.utils import sanitize_string + async def success(): return "success" @@ -20,3 +22,27 @@ async def test_async_gather_behavior(): assert len(results) == 2 assert results[0] == "success" assert isinstance(results[1], Exception) + + +def test_valid_characters(): + # Test that only valid characters are kept + unsanitized_str = "abc123$%^" + valid_characters_re = r"[a-zA-Z0-9]" + expected_output = "abc123" + assert sanitize_string(unsanitized_str, valid_characters_re) == expected_output + + +def test_max_length(): + # Test that the string is truncated to the max length + unsanitized_str = "a" * 100 + valid_characters_re = r"[a-zA-Z0-9]" + expected_output = "a" * 64 + assert sanitize_string(unsanitized_str, valid_characters_re) == expected_output + + +def test_no_valid_characters(): + # Test that an empty string is returned if there are no valid characters + unsanitized_str = "!@#$%^&*()" + valid_characters_re = r"[a-zA-Z0-9]" + expected_output = "" + assert sanitize_string(unsanitized_str, valid_characters_re) == expected_output From 7e472be3c7ec42a93a0e1b254e88811e96069659 Mon Sep 17 00:00:00 2001 From: Curtis Castrapel Date: Mon, 27 Feb 2023 16:12:56 -0800 Subject: [PATCH 20/32] revert launch.json change --- .vscode/launch.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.vscode/launch.json b/.vscode/launch.json index 0d254915b..9b0c69e20 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -36,8 +36,9 @@ "console": "integratedTerminal", "args": [ "apply", + "--template", - "../noq-templates/resources/aws/roles/all_accounts/readonly.yaml" + "${env:IAMBIC_TEMPLATE_PATH}" ], "envFile": "${workspaceFolder}/.env", "justMyCode": true From 76b88b3951172110de573403b062c53eef3d167a Mon Sep 17 00:00:00 2001 From: Curtis Castrapel Date: Mon, 27 Feb 2023 16:17:27 -0800 Subject: [PATCH 21/32] Change comment --- iambic/core/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/iambic/core/utils.py b/iambic/core/utils.py index 6efb0318c..bae912ce2 100644 --- a/iambic/core/utils.py +++ b/iambic/core/utils.py @@ -568,7 +568,7 @@ async def __call__(self, func: typing.Callable, *args, **kwargs): def sanitize_string(unsanitized_str, valid_characters_re): """ - This function sanitizes the session name typically passed in an assume_role call, to verify that it's + This function sanitizes the session name typically passed as a parameter name, to ensure it is valid. """ sanitized_str = "" From c48090e8cf40e634ed80c64c74dc4a1c9bcddbe8 Mon Sep 17 00:00:00 2001 From: Version Auto Bump Date: Tue, 28 Feb 2023 00:54:15 +0000 Subject: [PATCH 22/32] Bump version --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 96a6e6416..a730def7c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "poetry.core.masonry.api" [tool.poetry] name = "iambic" -version = "0.1.53" +version = "0.1.54" description = "The python package used to generate, parse, and execute noqform yaml templates." authors = ["Noq Software "] readme = "README.md" From 4c79cc30b808407d0aced1e974eb148bd762b92d Mon Sep 17 00:00:00 2001 From: Will-NOQ <102608210+Will-NOQ@users.noreply.github.com> Date: Mon, 27 Feb 2023 20:06:15 -0600 Subject: [PATCH 23/32] Added a hidden git aware flag on plan. --- iambic/main.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/iambic/main.py b/iambic/main.py index 309d229fc..63591163d 100644 --- a/iambic/main.py +++ b/iambic/main.py @@ -280,8 +280,13 @@ def run_git_apply( default=os.getenv("IAMBIC_REPO_DIR"), help="The repo directory containing the templates. Example: ~/iambic-templates", ) -def plan(templates: list, plan_output: str, repo_dir: str): - if plan_output: +@click.option( + "--git-aware", + is_flag=True, + hidden=True, +) +def plan(templates: list, plan_output: str, repo_dir: str, git_aware: bool): + if git_aware: run_git_plan(plan_output, repo_dir=repo_dir) else: if not templates: From 7066e29c52c0a3b24ffcad57f6b072555d73c662 Mon Sep 17 00:00:00 2001 From: Steven Moy Date: Mon, 27 Feb 2023 18:29:33 -0800 Subject: [PATCH 24/32] Point to new registry --- Dockerfile | 2 +- Makefile | 2 +- docker-compose-cicd.yaml | 2 +- iambic/github/templates/iambic-git-apply-via-pr-comments.yml | 2 +- iambic/github/templates/iambic-git-plan.yml | 2 +- install.sh | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/Dockerfile b/Dockerfile index f912d2806..d865e7dc7 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM public.ecr.aws/s2p9s3r8/iambic_container_base:1.0 as runtime-layer +FROM public.ecr.aws/o4z3c2v2/iambic_container_base:1.0 as runtime-layer # ######## REFERENCE YOUR OWN HANDLER HERE ######################## # CMD [ "main.app" ]``` diff --git a/Makefile b/Makefile index 31fc4714b..49224cf63 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,5 @@ BUILD_VERSION := $(shell python build_utils/tag_and_build_container.py print-current-version) -IAMBIC_PUBLIC_ECR_ALIAS := "s2p9s3r8" +IAMBIC_PUBLIC_ECR_ALIAS := "o4z3c2v2" .PHONY: prepare_for_dist prepare_for_dist: diff --git a/docker-compose-cicd.yaml b/docker-compose-cicd.yaml index 71db79cf2..bc015bd68 100644 --- a/docker-compose-cicd.yaml +++ b/docker-compose-cicd.yaml @@ -3,7 +3,7 @@ services: iambic-cicd: # NOTE: This compose file loads the image from ECR, not from what you are running # locally. - image: public.ecr.aws/s2p9s3r8/iambic:latest + image: public.ecr.aws/o4z3c2v2/iambic:latest environment: - IAMBIC_CONFIG=arn:aws:secretsmanager:us-west-2:759357822767:secret:dev/iambic-full - IAMBIC_CONFIG_ASSUME_ROLE=arn:aws:iam::759357822767:role/IambicSpokeRole diff --git a/iambic/github/templates/iambic-git-apply-via-pr-comments.yml b/iambic/github/templates/iambic-git-apply-via-pr-comments.yml index 6bd8f9595..b9c9bd2f6 100644 --- a/iambic/github/templates/iambic-git-apply-via-pr-comments.yml +++ b/iambic/github/templates/iambic-git-apply-via-pr-comments.yml @@ -35,7 +35,7 @@ jobs: docker run -e AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID -e AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY -e AWS_SESSION_TOKEN=$AWS_SESSION_TOKEN -e AWS_DEFAULT_REGION=$AWS_DEFAULT_REGION -e AWS_REGION=$AWS_REGION -v $(pwd)/github_context/:/root/github_context/ -v $(pwd)/data/:/root/data/ --entrypoint /bin/bash ${IMAGE} -c "python -m iambic.cicd.github" env: GITHUB_CONTEXT: ${{ toJSON(github) }} - IMAGE: "public.ecr.aws/s2p9s3r8/iambic:latest" + IMAGE: "public.ecr.aws/o4z3c2v2/iambic:latest" - name: Archive proosed_changes.yaml artifacts uses: actions/upload-artifact@v3 with: diff --git a/iambic/github/templates/iambic-git-plan.yml b/iambic/github/templates/iambic-git-plan.yml index 68cc3e988..95b2731a2 100644 --- a/iambic/github/templates/iambic-git-plan.yml +++ b/iambic/github/templates/iambic-git-plan.yml @@ -35,7 +35,7 @@ jobs: docker run -e AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID -e AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY -e AWS_SESSION_TOKEN=$AWS_SESSION_TOKEN -e AWS_DEFAULT_REGION=$AWS_DEFAULT_REGION -e AWS_REGION=$AWS_REGION -v $(pwd)/github_context/:/root/github_context/ -v $(pwd)/data/:/root/data/ --entrypoint /bin/bash ${IMAGE} -c "python -m iambic.cicd.github" env: GITHUB_CONTEXT: ${{ toJSON(github) }} - IMAGE: "public.ecr.aws/s2p9s3r8/iambic:latest" + IMAGE: "public.ecr.aws/o4z3c2v2/iambic:latest" - name: Archive proosed_changes.yaml artifacts uses: actions/upload-artifact@v3 with: diff --git a/install.sh b/install.sh index e870eb433..d21fb8993 100644 --- a/install.sh +++ b/install.sh @@ -13,7 +13,7 @@ fi SHELL_NAME=$(ps -p $$ | tail -1 | awk '{print $NF}') echo "Detected shell: ${SHELL_NAME}" IAMBIC_GIT_REPO_PATH="${IAMBIC_GIT_REPO_PATH:-${HOME}/iambic-templates}" -ECR_PATH="public.ecr.aws/s2p9s3r8/iambic:latest" +ECR_PATH="public.ecr.aws/o4z3c2v2/iambic:latest" echo "Installing iambic..." echo "We are creating an iambic git repository in the directory ${IAMBIC_GIT_REPO_PATH}. If you want to change this directory, please edit the DEFAULT_IAMBIC_GIT_REPO variable in the install.sh script." From a020285e9b1ee6d51fa76a69ff84382139edf940 Mon Sep 17 00:00:00 2001 From: Steven Moy Date: Mon, 27 Feb 2023 18:35:31 -0800 Subject: [PATCH 25/32] Update registry --- deployment/github_app/main.tf | 52 +++++++++++++------ deployment/github_app/vars.tf | 33 +++++++++--- .../docs/getting_started/github/github.mdx | 1 + 3 files changed, 63 insertions(+), 23 deletions(-) diff --git a/deployment/github_app/main.tf b/deployment/github_app/main.tf index 84012d2ed..119fdfe26 100644 --- a/deployment/github_app/main.tf +++ b/deployment/github_app/main.tf @@ -5,7 +5,6 @@ terraform { version = "~> 4.16" } } - required_version = ">= 1.2.0" } @@ -18,33 +17,47 @@ data "aws_caller_identity" "current" {} locals { account_id = data.aws_caller_identity.current.account_id ecr_image_tag = "latest" - iambic_public_repo = "public.ecr.aws/s2p9s3r8/iambic" - iambic_image_tag = "latest" } -resource "aws_ecr_repository" "iambic_private_ecr" { - name = "iambic_private_ecr" - image_tag_mutability = "MUTABLE" +resource "null_resource" "iambic_public_repo" { - image_scanning_configuration { - scan_on_push = true + triggers = { + always_run = "${timestamp()}" } -/* + depends_on = [ + aws_ecr_repository.iambic_private_ecr, + ] + provisioner "local-exec" { command = < Date: Mon, 27 Feb 2023 18:42:06 -0800 Subject: [PATCH 26/32] Update registry --- .github/workflows/build-container.yml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build-container.yml b/.github/workflows/build-container.yml index 9df4b1a2c..a9512e45b 100644 --- a/.github/workflows/build-container.yml +++ b/.github/workflows/build-container.yml @@ -50,10 +50,12 @@ jobs: - name: Configure AWS Credentials uses: aws-actions/configure-aws-credentials@v1 with: - role-to-assume: arn:aws:iam::759357822767:role/iambic_image_builder + role-to-assume: arn:aws:iam::242345320040:role/iambic_image_builder aws-region: us-east-1 - name: build container id: build-container run: | - aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws/l1s5s8m2 - . build-env/bin/activate && make build_docker upload_docker \ No newline at end of file + aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws/o4z3c2v2 + . build-env/bin/activate && make build_docker upload_docker + docker logout public.ecr.aws/o4z3c2v2 + docker buildx prune --filter=until=96h -f From 050eb0cd985616638e35ac1c24af071806c9b731 Mon Sep 17 00:00:00 2001 From: Steven Moy Date: Mon, 27 Feb 2023 18:45:16 -0800 Subject: [PATCH 27/32] Update registry --- Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 49224cf63..879bc1594 100644 --- a/Makefile +++ b/Makefile @@ -7,7 +7,7 @@ prepare_for_dist: .PHONY: auth_to_ecr auth_to_ecr: - bash -c "AWS_PROFILE=development/iambic_image_builder aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws/s2p9s3r" + bash -c "AWS_PROFILE=iambic_open_source/iambic_image_builder aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws/o4z3c2v2" docker_buildx := docker buildx build \ --platform=linux/amd64 \ @@ -63,4 +63,4 @@ build_docker_base_image: .PHONY: upload_docker_base_image upload_docker_base_image: @echo "--> Uploading Iambic Docker base container image" - $(docker_base_image_buildx) --push . \ No newline at end of file + $(docker_base_image_buildx) --push . From 7b226dc7571e12f9907c66e49f53868e55eaccd4 Mon Sep 17 00:00:00 2001 From: Version Auto Bump Date: Tue, 28 Feb 2023 02:51:49 +0000 Subject: [PATCH 28/32] Bump version --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index a730def7c..3d06f6614 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "poetry.core.masonry.api" [tool.poetry] name = "iambic" -version = "0.1.54" +version = "0.1.55" description = "The python package used to generate, parse, and execute noqform yaml templates." authors = ["Noq Software "] readme = "README.md" From 8aed835ed18554598cafc380eac7043244301e19 Mon Sep 17 00:00:00 2001 From: Steven Moy Date: Mon, 27 Feb 2023 19:24:54 -0800 Subject: [PATCH 29/32] Disbale itest image build --- .github/workflows/run-test.yml | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/.github/workflows/run-test.yml b/.github/workflows/run-test.yml index 7e0c273b2..ca5fd5de2 100644 --- a/.github/workflows/run-test.yml +++ b/.github/workflows/run-test.yml @@ -36,11 +36,12 @@ jobs: with: role-to-assume: arn:aws:iam::442632209887:role/iambic_image_builder aws-region: us-east-1 - - name: build-itest-image - id: build-itest-image - run: | - aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws/l1s5s8m2 - . env/bin/activate && make -f Makefile.itest build_docker_itest upload_docker_itest + # Disable image builder for now since we are not using it + #- name: build-itest-image + # id: build-itest-image + # run: | + # aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws/l1s5s8m2 + # . env/bin/activate && make -f Makefile.itest build_docker_itest upload_docker_itest - name: Configure AWS Credentials uses: aws-actions/configure-aws-credentials@v1 with: From 974497a699a569feb11efcd3d38bd4769d3b0b1c Mon Sep 17 00:00:00 2001 From: Version Auto Bump Date: Tue, 28 Feb 2023 13:30:28 +0000 Subject: [PATCH 30/32] Bump version --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 3d06f6614..757b24ff6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "poetry.core.masonry.api" [tool.poetry] name = "iambic" -version = "0.1.55" +version = "0.1.56" description = "The python package used to generate, parse, and execute noqform yaml templates." authors = ["Noq Software "] readme = "README.md" From cec2e5787a8166b2ed96e48125a61d165b780a61 Mon Sep 17 00:00:00 2001 From: Curtis Castrapel Date: Tue, 28 Feb 2023 05:59:38 -0800 Subject: [PATCH 31/32] EN-1798: Unbound parenthesis fix --- iambic/core/git.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/iambic/core/git.py b/iambic/core/git.py index 4411498b9..26b3f9c32 100644 --- a/iambic/core/git.py +++ b/iambic/core/git.py @@ -8,10 +8,11 @@ from deepdiff import DeepDiff from git import Repo from git.exc import GitCommandError +from pydantic import BaseModel as PydanticBaseModel + from iambic.config.templates import TEMPLATES from iambic.core.logger import log from iambic.core.utils import NOQ_TEMPLATE_REGEX, file_regex_search, yaml -from pydantic import BaseModel as PydanticBaseModel if TYPE_CHECKING: from iambic.config.dynamic_config import Config @@ -266,7 +267,9 @@ def create_templates_for_modified_files( account_regex = ( rf"({aws_account.account_id}|{aws_account.account_name})" ) - if re.search(account_regex, deleted_exclude_accounts_str): + if re.search( + re.escape(account_regex), deleted_exclude_accounts_str + ): log.debug( "Resource on account not marked deletion.", account=account_regex, @@ -297,7 +300,7 @@ def create_templates_for_modified_files( This means marking prod for deletion as it has been implicitly deleted. """ for account in main_template.included_accounts: - if re.search(account, deleted_exclude_accounts_str): + if re.search(re.escape(account), deleted_exclude_accounts_str): log.debug( "Resource on account not marked deletion.", account=account, @@ -335,7 +338,7 @@ def create_templates_for_modified_files( """ for account in template.excluded_accounts: if main_template_excluded_accounts_str and re.search( - account, main_template_excluded_accounts_str + re.escape(account), main_template_excluded_accounts_str ): # The account was already excluded so add it to the template_excluded_accounts log.debug( @@ -345,7 +348,7 @@ def create_templates_for_modified_files( ) template_excluded_accounts.append(account) elif ( - re.search(account, main_template_included_accounts_str) + re.search(re.escape(account), main_template_included_accounts_str) or "*" in main_template.included_accounts ): # The account was previously included so mark it for deletion From ab703653b1ef04ce002372b7843f04b766345122 Mon Sep 17 00:00:00 2001 From: Version Auto Bump Date: Tue, 28 Feb 2023 14:18:49 +0000 Subject: [PATCH 32/32] Bump version --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 757b24ff6..2355123bd 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "poetry.core.masonry.api" [tool.poetry] name = "iambic" -version = "0.1.56" +version = "0.1.57" description = "The python package used to generate, parse, and execute noqform yaml templates." authors = ["Noq Software "] readme = "README.md"