Merge branch 'main' of github.com:noqdev/iambic

* 'main' of github.com:noqdev/iambic: (22 commits)
  Bump version
  Change sbom to json
  Bump version
  Bump version
  Trivy without exit code
  Use load_templates which support wrapping yaml problems and validation problems
  Bump version
  Clean up action files
  Publish job
  Wrap ValidationError with file path information
  Upload artifact
  give security-events write
  skip aws.mdx
  logout docker
  give write access to content
  NT
  Test temp GHA
  Trivy Scan and SBOM support
  Bump version
  Implement traceback in the git plan/apply workflow
  ...

.github/workflows/build-container.yml

@@ -31,7 +31,8 @@ jobs:
       - name: bootstrap
         run: |
           python3.10 -m venv build-env
-          . build-env/bin/activate && pip install poetry
+          . build-env/bin/activate && pip install poetry setuptools pip --upgrade
+          curl -sfL https://raw.githubusercontent.com/aquasecurity/trivy/main/contrib/install.sh | sudo sh -s -- -b /usr/local/bin v0.37.3
       - name: set git identity
         run: |
           git config user.name "Version Auto Bump"
@@ -55,7 +56,20 @@ jobs:
       - name: build container
         id: build-container
         run: |
+          docker logout ghcr.io
           aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws/o4z3c2v2
-          . build-env/bin/activate && make build_docker upload_docker
+          . build-env/bin/activate && make build_docker
+          make trivy_scan
+          make trivy_sbom
+          make upload_docker
           docker logout public.ecr.aws/o4z3c2v2
           docker buildx prune --filter=until=96h -f
+      - uses: actions/upload-artifact@v3
+        with:
+          name: trivy-sbom
+          path: iambic.sbom.json
+      # Uncomment after OSS (Requires for GH Advanced Security):
+      # - name: Upload Trivy scan results to GitHub Security tab
+      #   uses: github/codeql-action/upload-sarif@v2
+      #   with:
+      #     sarif_file: 'iambic.sbom.json'

.github/workflows/detect-secrets.yml

@@ -18,4 +18,4 @@ jobs:
           path: ./
           base: ${{ github.event.repository.default_branch }}
           head: HEAD
-          extra_args: --debug --only-verified --max_depth=1000
+          extra_args: --debug --only-verified --max_depth=1000

.github/workflows/publish-release.yml

@@ -0,0 +1,30 @@
+name: Publish Release
+on:
+  push:
+    tags:
+      - 'v*'
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    permissions:
+      id-token: write
+      contents: write
+      security-events: write
+      statuses: write
+      discussions: write
+    steps:
+    - name: bootstrap
+      run: |
+        curl -sfL https://raw.githubusercontent.com/aquasecurity/trivy/main/contrib/install.sh | sudo sh -s -- -b /usr/local/bin v0.37.3
+    - uses: actions/checkout@v3
+    - name: scan container
+      id: scan-container
+      run: |
+        docker logout ghcr.io
+        make trivy_scan
+        make trivy_sbom
+    - name: Release
+      uses: softprops/action-gh-release@v1
+      if: startsWith(github.ref, 'refs/tags/')
+      with:
+        files: iambic.sbom.json

Dockerfile

@@ -40,6 +40,12 @@ COPY --chown=iambic:iambic poetry.lock ${FUNCTION_DIR}/poetry.lock
 COPY --chown=iambic:iambic pyproject.toml ${FUNCTION_DIR}/pyproject.toml
 COPY --chown=iambic:iambic README.md ${FUNCTION_DIR}/README.md
 
+# ######## REFERENCE YOUR OWN HANDLER HERE ########################
+# CMD [ "main.app" ]```
+######## ############################### ########################
+# Install Requirements
+# Install the function's dependencies
+RUN pip3 install poetry awslambdaric argh watchdog setuptools pip --upgrade
 WORKDIR ${FUNCTION_DIR}
 
 RUN pip install poetry \

Dockerfile.awslambdaric

@@ -53,7 +53,7 @@ FROM base-layer as build-layer
 ######## ########################### ########################
 
 # install lambda runtime interface client for python
-RUN pip3 install awslambdaric --target "${FUNCTION_DIR}"
+RUN pip3 install awslambdaric setuptools pip --target "${FUNCTION_DIR}" --upgrade
 
 
 FROM base-layer as runtime-layer

Makefile

@@ -28,6 +28,14 @@ upload_docker:
 	@echo "--> Uploading Iambic Docker image"
 	$(docker_buildx) --push .
 
+.PHONY: trivy_scan
+trivy_scan:
+	trivy image --output iambic.trivy.scan.txt --skip-files /app/docs/web/docs/getting_started/aws/aws.mdx --secret-config trivy-secret.yaml --severity HIGH,CRITICAL public.ecr.aws/${IAMBIC_PUBLIC_ECR_ALIAS}/iambic:latest
+
+.PHONY: trivy_sbom
+trivy_sbom:
+	trivy image --format spdx-json --output iambic.sbom.json public.ecr.aws/${IAMBIC_PUBLIC_ECR_ALIAS}/iambic:latest
+
 .PHONY: create_manifest
 create_manifest:
 	docker manifest create public.ecr.aws/${IAMBIC_PUBLIC_ECR_ALIAS}/iambic public.ecr.aws/${IAMBIC_PUBLIC_ECR_ALIAS}/iambic:latest

iambic/core/git.py

@@ -12,6 +12,7 @@
 
 from iambic.config.templates import TEMPLATES
 from iambic.core.logger import log
+from iambic.core.parser import load_templates
 from iambic.core.utils import NOQ_TEMPLATE_REGEX, file_regex_search, yaml
 
 if TYPE_CHECKING:
@@ -232,8 +233,9 @@ def create_templates_for_modified_files(
 
         main_template = template_cls(file_path=git_diff.path, **main_template_dict)
 
-        template_dict = yaml.load(open(git_diff.path))
-        template = template_cls(file_path=git_diff.path, **template_dict)
+        # template_dict = yaml.load(open(git_diff.path))
+        # template = template_cls(file_path=git_diff.path, **template_dict)
+        template = load_templates([git_diff.path])[0]
 
         # EN-1634 dealing with providers that have no concept of multi-accounts
         # a hack to just ignore template that does not have included_accounts attribute

iambic/core/parser.py

@@ -1,10 +1,12 @@
 from __future__ import annotations
 
+from pydantic import ValidationError
+from ruamel.yaml.scanner import ScannerError
+
 from iambic.config.templates import TEMPLATES
 from iambic.core.logger import log
 from iambic.core.models import BaseTemplate
 from iambic.core.utils import transform_comments, yaml
-from pydantic import ValidationError
 
 
 def load_templates(
@@ -13,10 +15,10 @@ def load_templates(
     templates = []
 
     for template_path in template_paths:
-        template_dict = transform_comments(yaml.load(open(template_path)))
-        if template_dict["template_type"] in ["NOQ::Core::Config"]:
-            continue
         try:
+            template_dict = transform_comments(yaml.load(open(template_path)))
+            if template_dict["template_type"] in ["NOQ::Core::Config"]:
+                continue
             template_cls = TEMPLATES.template_map[template_dict["template_type"]]
             template_cls.update_forward_refs()
             templates.append(template_cls(file_path=template_path, **template_dict))
@@ -28,11 +30,13 @@ def load_templates(
             )
             # We should allow to continue to allow unknown template type; otherwise,
             # we cannot support forward or backward compatibility during version changes.
-        except ValidationError as err:
+        except (ValidationError, ScannerError) as err:
             log.critical(
                 "Invalid template structure", file_path=template_path, error=repr(err)
             )
             if raise_validation_err:
-                raise
+                raise ValueError(
+                    f"{template_path} template has validation error"
+                ) from err
 
     return templates

iambic/plugins/v0_1_0/example/iambic_plugin.py

@@ -1,10 +1,14 @@
 from __future__ import annotations
 
+from pydantic import BaseModel
+
 from iambic.core.iambic_plugin import ProviderPlugin
 from iambic.plugins.v0_1_0 import PLUGIN_VERSION
 from iambic.plugins.v0_1_0.example.handlers import import_example_resources, load
+from iambic.plugins.v0_1_0.example.local_database.models import (
+    ExampleLocalDatabaseTemplate,
+)
 from iambic.plugins.v0_1_0.example.local_file.models import ExampleLocalFileTemplate
-from pydantic import BaseModel
 
 
 class ExampleConfig(BaseModel):
@@ -18,7 +22,5 @@ class ExampleConfig(BaseModel):
     requires_secret=True,
     async_import_callable=import_example_resources,
     async_load_callable=load,
-    templates=[
-        ExampleLocalFileTemplate,
-    ],
+    templates=[ExampleLocalFileTemplate, ExampleLocalDatabaseTemplate],
 )

iambic/plugins/v0_1_0/example/local_database/models.py

@@ -0,0 +1,57 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+from pydantic import Field
+
+from iambic.core.context import ExecutionContext
+from iambic.core.models import (
+    BaseModel,
+    BaseTemplate,
+    ExpiryModel,
+    TemplateChangeDetails,
+)
+
+EXAMPLE_LOCAL_DATABASE_TEMPLATE_TYPE = "NOQ::Example::LocalDatabase"
+
+if TYPE_CHECKING:
+    from iambic.plugins.v0_1_0.example.iambic_plugin import ExampleConfig
+
+
+class ExampleLocalDatabaseTemplateProperties(BaseModel):
+    name: str = Field(..., description="name of Local Database")
+
+    @property
+    def resource_type(self) -> str:
+        return "example:local_database:properties"
+
+    @property
+    def resource_id(self) -> str:
+        return self.name
+
+
+class ExampleLocalDatabaseTemplate(BaseTemplate, ExpiryModel):
+    template_type = EXAMPLE_LOCAL_DATABASE_TEMPLATE_TYPE
+    properties: ExampleLocalDatabaseTemplateProperties = Field(
+        ..., description="Properties for Example Local Database Template"
+    )
+    name: str = Field(..., description="name of Local Database")
+
+    @property
+    def resource_type(self) -> str:
+        return "example:local_database"
+
+    @property
+    def resource_id(self) -> str:
+        return self.name
+
+    async def apply(
+        self, config: ExampleConfig, context: ExecutionContext
+    ) -> TemplateChangeDetails:
+        template_changes = TemplateChangeDetails(
+            resource_id=self.resource_id,
+            resource_type=self.template_type,
+            template_path=self.file_path,
+        )
+        template_changes.proposed_changes = []
+        return template_changes

iambic/plugins/v0_1_0/example/local_file/models.py

@@ -15,7 +15,7 @@
 EXAMPLE_LOCAL_FILE_TEMPLATE_TYPE = "NOQ::Example::LocalFile"
 
 if TYPE_CHECKING:
-    from iambic.plugins.v0_1_0.okta.iambic_plugin import ExampleConfig
+    from iambic.plugins.v0_1_0.example.iambic_plugin import ExampleConfig
 
 
 class ExampleLocalFileTemplateProperties(BaseModel):

iambic/plugins/v0_1_0/github/github.py

@@ -8,6 +8,7 @@
 import sys
 import tempfile
 import time
+import traceback
 from enum import Enum
 from typing import Any, Callable
 from urllib.parse import urlparse
@@ -322,7 +323,9 @@ def handle_iambic_git_apply(
     if pull_request.mergeable_state != MERGEABLE_STATE_CLEAN:
         # TODO log error and also make a comment to PR
         pull_request.create_issue_comment(
-            "mergeable_state is {0}".format(pull_request.mergeable_state)
+            "Mergable state is {0}. This probably means that the necessary approvals have not been granted for the request.".format(
+                pull_request.mergeable_state
+            )
         )
         return HandleIssueCommentReturnCode.MERGEABLE_STATE_NOT_CLEAN
 
@@ -375,10 +378,11 @@ def handle_iambic_git_apply(
         return HandleIssueCommentReturnCode.MERGED
 
     except Exception as e:
-        log.error("fault", exception=str(e))
+        captured_traceback = traceback.format_exc()
+        log.error("fault", exception=captured_traceback)
         pull_request.create_issue_comment(
-            "exception during git-apply is {0} \n {1}".format(
-                pull_request.mergeable_state, e
+            "exception during apply is {0} \n ```{1}```".format(
+                pull_request.mergeable_state, captured_traceback
             )
         )
         raise e
@@ -416,10 +420,11 @@ def handle_iambic_git_plan(
         copy_data_to_data_directory()
         return HandleIssueCommentReturnCode.PLANNED
     except Exception as e:
-        log.error("fault", exception=str(e))
+        captured_traceback = traceback.format_exc()
+        log.error("fault", exception=captured_traceback)
         pull_request.create_issue_comment(
-            "exception during git-plan is {0} \n {1}".format(
-                pull_request.mergeable_state, e
+            "exception during plan is {0} \n ```{1}```".format(
+                pull_request.mergeable_state, captured_traceback
             )
         )
         raise e
@@ -440,10 +445,11 @@ def handle_pull_request(github_client: github.Github, context: dict[str, Any]) -
     try:
         pull_request.create_issue_comment("iambic git-plan")
     except Exception as e:
-        log.error("fault", exception=str(e))
+        captured_traceback = traceback.format_exc()
+        log.error("fault", exception=captured_traceback)
         pull_request.create_issue_comment(
-            "exception during pull-request is {0} \n {1}".format(
-                pull_request.mergeable_state, e
+            "exception during pull-request is {0} \n ```{1}```".format(
+                pull_request.mergeable_state, captured_traceback
             )
         )
         raise e
@@ -557,7 +563,9 @@ def _handle_expire(repo_url: str, default_branch: str) -> None:
             log.info("handle_expire ran", **log_params)
 
             default_branch = get_remote_default_branch(repo)
-            repo.remotes.origin.push(refspec=f"HEAD:{default_branch}").raise_if_error()  # FIXME
+            repo.remotes.origin.push(
+                refspec=f"HEAD:{default_branch}"
+            ).raise_if_error()  # FIXME
         else:
             log.info("handle_expire no changes")
     except Exception as e:
@@ -582,6 +590,8 @@ def _handle_expire(repo_url: str, default_branch: str) -> None:
 COMMENT_DISPATCH_MAP: dict[str, Callable] = {
     "iambic git-apply": handle_iambic_git_apply,
     "iambic git-plan": handle_iambic_git_plan,
+    "iambic apply": handle_iambic_git_apply,
+    "iambic plan": handle_iambic_git_plan,
 }
 
 if __name__ == "__main__":

iambic/plugins/v0_1_0/github/github_app.py

@@ -17,7 +17,6 @@
 import jwt
 from botocore.exceptions import ClientError
 
-from iambic.core.git import get_remote_default_branch
 import iambic.core.utils
 import iambic.plugins.v0_1_0.github.github
 from iambic.core.logger import log
@@ -297,6 +296,8 @@ def handle_workflow_run(
 COMMENT_DISPATCH_MAP: dict[str, Callable] = {
     "iambic git-apply": handle_iambic_git_apply,
     "iambic git-plan": handle_iambic_git_plan,
+    "iambic apply": handle_iambic_git_apply,
+    "iambic plan": handle_iambic_git_plan,
 }
 
 WORKFLOW_DISPATCH_MAP: dict[str, Callable] = {

pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "poetry.core.masonry.api"
 
 [tool.poetry]
 name = "iambic"
-version = "0.1.57"
+version = "0.1.63"
 description = "The python package used to generate, parse, and execute noqform yaml templates."
 authors = ["Noq Software <[email protected]>"]
 readme = "README.md"