From 18dd03910cc4e08fa29d3ed2c29df9726569c6ee Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Anton=20Gr=C3=BCbel?= Date: Tue, 15 Aug 2023 12:08:45 +0200 Subject: [PATCH] chore: Remove ignores from import-linter (#5437) * move doc generator code * remove obsolete secrets consts file * change danger token to normal GH token * reduce permissions for danger job * change type hint --- .github/workflows/pr-test.yml | 5 +- .../graph_builder/graph_components/blocks.py | 29 +--------- checkov/{common/util => }/docs_generator.py | 0 checkov/main.py | 2 +- checkov/secrets/consts.py | 3 -- .../graph_builder/graph_components/blocks.py | 54 ++++++++++++++++++- pyproject.toml | 6 --- tests/common/utils/test_docs_generator.py | 4 +- tests/terraform/util/test_doc_generator.py | 2 +- 9 files changed, 63 insertions(+), 42 deletions(-) rename checkov/{common/util => }/docs_generator.py (100%) delete mode 100644 checkov/secrets/consts.py diff --git a/.github/workflows/pr-test.yml b/.github/workflows/pr-test.yml index 4e962b6e731..3393543355d 100644 --- a/.github/workflows/pr-test.yml +++ b/.github/workflows/pr-test.yml @@ -13,6 +13,9 @@ jobs: danger-check: runs-on: [ self-hosted, public, linux, x64 ] + permissions: + contents: read + pull-requests: read steps: - name: Checkout code uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3 @@ -22,7 +25,7 @@ jobs: node-version: "16" - name: Install and run DangerJS env: - GITHUB_TOKEN: ${{ secrets.DANGER_GITHUB_TOKEN }} + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | npm install -g danger danger ci --verbose --failOnErrors diff --git a/checkov/common/graph/graph_builder/graph_components/blocks.py b/checkov/common/graph/graph_builder/graph_components/blocks.py index 24a203188ea..ad7f8472538 100644 --- a/checkov/common/graph/graph_builder/graph_components/blocks.py +++ b/checkov/common/graph/graph_builder/graph_components/blocks.py @@ -1,13 +1,12 @@ from __future__ import annotations from collections.abc import Collection -from typing import Union, Dict, Any, List, cast +from typing import Union, Dict, Any, List from checkov.common.graph.graph_builder.graph_components.attribute_names import CustomAttributes from checkov.common.graph.graph_builder.utils import calculate_hash, join_trimmed_strings from checkov.common.graph.graph_builder.variable_rendering.breadcrumb_metadata import BreadcrumbMetadata from checkov.common.util.data_structures_utils import pickle_deepcopy -from checkov.terraform.graph_builder.graph_components.block_types import BlockType class Block: @@ -85,10 +84,6 @@ def get_attribute_dict(self, add_hash: bool = True) -> Dict[str, Any]: base_attributes = self.get_base_attributes() self.get_origin_attributes(base_attributes) - if hasattr(self, "module_dependency") and hasattr(self, "module_dependency_num"): - base_attributes[CustomAttributes.MODULE_DEPENDENCY] = self.module_dependency - base_attributes[CustomAttributes.MODULE_DEPENDENCY_NUM] = self.module_dependency_num - if self.changed_attributes: # add changed attributes only for calculating the hash base_attributes["changed_attributes"] = sorted(self.changed_attributes.keys()) @@ -97,29 +92,9 @@ def get_attribute_dict(self, add_hash: bool = True) -> Dict[str, Any]: sorted_breadcrumbs = dict(sorted(self.breadcrumbs.items())) base_attributes[CustomAttributes.RENDERING_BREADCRUMBS] = sorted_breadcrumbs - if hasattr(self, 'foreach_attrs'): - base_attributes[CustomAttributes.FOREACH_ATTRS] = self.foreach_attrs - - if hasattr(self, 'source_module_object'): - base_attributes[CustomAttributes.SOURCE_MODULE_OBJECT] = self.source_module_object - if add_hash: base_attributes[CustomAttributes.HASH] = calculate_hash(base_attributes) - if self.block_type == BlockType.DATA: - base_attributes[CustomAttributes.RESOURCE_TYPE] = f'data.{self.id.split(".")[0]}' - - if self.block_type == BlockType.MODULE: - # since module names are user defined we are just setting 'module' as resource type for easier searching - base_attributes[CustomAttributes.RESOURCE_TYPE] = "module" - - if self.block_type == BlockType.PROVIDER: - # provider_name is always a string, base_attributes needs better typing pipenv run mypy - provider_name = cast(str, base_attributes[CustomAttributes.BLOCK_NAME]) - provider_type = provider_name.split(".")[0] - # ex: provider.aws - base_attributes[CustomAttributes.RESOURCE_TYPE] = f"provider.{provider_type}" - if "changed_attributes" in base_attributes: # removed changed attributes if it was added previously for calculating hash. del base_attributes["changed_attributes"] @@ -251,7 +226,7 @@ def _should_set_changed_attributes(change_origin_id: int | None, attribute_at_de def get_export_data(self) -> Dict[str, Union[bool, str]]: return {"type": self.block_type, "name": self.name, "path": self.path} - def get_base_attributes(self) -> Dict[str, Union[str, List[str], Dict[str, Any]]]: + def get_base_attributes(self) -> Dict[str, Any]: return { CustomAttributes.BLOCK_NAME: self.name, CustomAttributes.BLOCK_TYPE: self.block_type, diff --git a/checkov/common/util/docs_generator.py b/checkov/docs_generator.py similarity index 100% rename from checkov/common/util/docs_generator.py rename to checkov/docs_generator.py diff --git a/checkov/main.py b/checkov/main.py index 0266844e919..07c1caf88b9 100755 --- a/checkov/main.py +++ b/checkov/main.py @@ -49,12 +49,12 @@ from checkov.common.util.banner import banner as checkov_banner, tool as checkov_tool from checkov.common.util.config_utils import get_default_config_paths from checkov.common.util.consts import CHECKOV_RUN_SCA_PACKAGE_SCAN_V2 -from checkov.common.util.docs_generator import print_checks from checkov.common.util.ext_argument_parser import ExtArgumentParser from checkov.common.util.runner_dependency_handler import RunnerDependencyHandler from checkov.common.util.type_forcers import convert_str_to_bool from checkov.contributor_metrics import report_contributor_metrics from checkov.dockerfile.runner import Runner as dockerfile_runner +from checkov.docs_generator import print_checks from checkov.github.runner import Runner as github_configuration_runner from checkov.github_actions.runner import Runner as github_actions_runner from checkov.gitlab.runner import Runner as gitlab_configuration_runner diff --git a/checkov/secrets/consts.py b/checkov/secrets/consts.py deleted file mode 100644 index 420c7cbe85c..00000000000 --- a/checkov/secrets/consts.py +++ /dev/null @@ -1,3 +0,0 @@ -from __future__ import annotations - -from checkov.common.secrets.consts import ValidationStatus # noqa # TODO: added for reimport purposes, remove after migration diff --git a/checkov/terraform/graph_builder/graph_components/blocks.py b/checkov/terraform/graph_builder/graph_components/blocks.py index 0b3401efd0a..38462813e0e 100644 --- a/checkov/terraform/graph_builder/graph_components/blocks.py +++ b/checkov/terraform/graph_builder/graph_components/blocks.py @@ -1,10 +1,12 @@ from __future__ import annotations import os -from typing import Union, Dict, Any, List, Optional, Set, TYPE_CHECKING +from typing import Union, Dict, Any, List, Optional, Set, TYPE_CHECKING, cast import dpath import re +from checkov.common.graph.graph_builder import CustomAttributes +from checkov.common.graph.graph_builder.utils import calculate_hash from checkov.common.runners.base_runner import strtobool from checkov.common.typing import TFDefinitionKeyType from checkov.terraform.graph_builder.utils import INTERPOLATION_EXPR @@ -83,6 +85,56 @@ def __eq__(self, other: object) -> bool: self.block_type == other.block_type and self.attributes == other.attributes and \ self.id == other.id and self.has_dynamic_block == other.has_dynamic_block and self.source == other.source + def get_attribute_dict(self, add_hash: bool = True) -> dict[str, Any]: + """ + :return: map of all the block's native attributes (from the source file), + combined with the attributes generated by the module builder. + If the attributes are not a primitive type, they are converted to strings. + """ + base_attributes = self.get_base_attributes() + self.get_origin_attributes(base_attributes) + + if hasattr(self, "module_dependency") and hasattr(self, "module_dependency_num"): + base_attributes[CustomAttributes.MODULE_DEPENDENCY] = self.module_dependency + base_attributes[CustomAttributes.MODULE_DEPENDENCY_NUM] = self.module_dependency_num + + if self.changed_attributes: + # add changed attributes only for calculating the hash + base_attributes["changed_attributes"] = sorted(self.changed_attributes.keys()) + + if self.breadcrumbs: + sorted_breadcrumbs = dict(sorted(self.breadcrumbs.items())) + base_attributes[CustomAttributes.RENDERING_BREADCRUMBS] = sorted_breadcrumbs + + if hasattr(self, 'foreach_attrs'): + base_attributes[CustomAttributes.FOREACH_ATTRS] = self.foreach_attrs + + if hasattr(self, 'source_module_object'): + base_attributes[CustomAttributes.SOURCE_MODULE_OBJECT] = self.source_module_object + + if add_hash: + base_attributes[CustomAttributes.HASH] = calculate_hash(base_attributes) + + if self.block_type == BlockType.DATA: + base_attributes[CustomAttributes.RESOURCE_TYPE] = f'data.{self.id.split(".")[0]}' + + if self.block_type == BlockType.MODULE: + # since module names are user defined we are just setting 'module' as resource type for easier searching + base_attributes[CustomAttributes.RESOURCE_TYPE] = "module" + + if self.block_type == BlockType.PROVIDER: + # provider_name is always a string, base_attributes needs better typing pipenv run mypy + provider_name = cast(str, base_attributes[CustomAttributes.BLOCK_NAME]) + provider_type = provider_name.split(".")[0] + # ex: provider.aws + base_attributes[CustomAttributes.RESOURCE_TYPE] = f"provider.{provider_type}" + + if "changed_attributes" in base_attributes: + # removed changed attributes if it was added previously for calculating hash. + del base_attributes["changed_attributes"] + + return base_attributes + def add_module_connection(self, attribute_key: str, vertex_id: int) -> None: self.module_connections.setdefault(attribute_key, []).append(vertex_id) diff --git a/pyproject.toml b/pyproject.toml index 26c74785889..56825ecf0e6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -52,11 +52,6 @@ ignore_imports = [ "checkov.common.bridgecrew.integration_features.features.policies_3d_integration -> checkov.policies_3d.*", # considering what to do "checkov.common.runners.base_post_runner -> checkov.policies_3d.checks_infra.base_check", # considering what to do - "checkov.common.util.docs_generator -> checkov.*.registry", # move to a different place - "checkov.common.util.docs_generator -> checkov.*.checks.registry", # move to a different place - "checkov.common.util.docs_generator -> checkov.*.checks.*.registry", # move to a different place - "checkov.common.util.docs_generator -> checkov.secrets.runner", # move to a different place - "checkov.common.runners.runner_registry -> checkov.terraform.context_parsers.registry", # move runner_registry to a different place "checkov.common.runners.runner_registry -> checkov.terraform.parser", # move runner_registry to a different place @@ -69,7 +64,6 @@ ignore_imports = [ "checkov.common.checks_infra.solvers.connections_solvers.base_connection_solver -> checkov.terraform.graph_builder.graph_components.block_types", # move to checkov.common.graph.graph_builder.graph_components? "checkov.common.runners.runner_registry -> checkov.sca_image.runner", # old IR, needs to be removed (argo_workflows, bitbucket_pipelines) - "checkov.common.graph.graph_builder.graph_components.blocks -> checkov.terraform.graph_builder.graph_components.block_types", # override get_attribute_dict() inside TerraformBlock ] [[tool.importlinter.contracts]] diff --git a/tests/common/utils/test_docs_generator.py b/tests/common/utils/test_docs_generator.py index 86dfee36ff2..33e36e06056 100644 --- a/tests/common/utils/test_docs_generator.py +++ b/tests/common/utils/test_docs_generator.py @@ -2,7 +2,7 @@ from pytest_mock import MockerFixture -from checkov.common.util.docs_generator import get_compare_key, get_check_link +from checkov.docs_generator import get_compare_key, get_check_link class TestOutputSorting(unittest.TestCase): @@ -84,7 +84,7 @@ def test_get_check_link_for_markdown(mocker: MockerFixture): # given abs_path = "/path/to/checkov/checkov/terraform/checks/resource/aws/LambdaXrayEnabled.py" - mocker.patch("checkov.common.util.docs_generator.CREATE_MARKDOWN_HYPERLINKS", return_value=True) + mocker.patch("checkov.docs_generator.CREATE_MARKDOWN_HYPERLINKS", return_value=True) # when link = get_check_link(absolute_path=abs_path) diff --git a/tests/terraform/util/test_doc_generator.py b/tests/terraform/util/test_doc_generator.py index 7963d131863..cfe8faf72e3 100644 --- a/tests/terraform/util/test_doc_generator.py +++ b/tests/terraform/util/test_doc_generator.py @@ -7,7 +7,7 @@ from checkov.common.checks_infra.registry import get_graph_checks_registry from checkov.common.bridgecrew.check_type import CheckType -from checkov.common.util.docs_generator import get_checks +from checkov.docs_generator import get_checks def test_get_checks_returned_check_number():