diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 8018ebf7c4d..e804ab178bb 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -32,3 +32,9 @@ repos: args: [] additional_dependencies: - vistir<0.7.0 # can be removed, when v4.0.0 of pipenv-setup comes out + - repo: https://github.com/seddonym/import-linter # checks the import dependencies between each other + rev: v1.10.0 + hooks: + - id: import-linter + language_version: python3.9 + args: ["--show-timings"] diff --git a/checkov/common/bridgecrew/platform_integration.py b/checkov/common/bridgecrew/platform_integration.py index 2d0a68e1207..2f85a93680b 100644 --- a/checkov/common/bridgecrew/platform_integration.py +++ b/checkov/common/bridgecrew/platform_integration.py @@ -45,13 +45,13 @@ get_user_agent_header, get_default_post_headers, get_prisma_get_headers, get_prisma_auth_header, \ get_auth_error_message, normalize_bc_url from checkov.common.util.type_forcers import convert_prisma_policy_filter_to_dict, convert_str_to_bool -from checkov.secrets.coordinator import EnrichedSecret from checkov.version import version as checkov_version if TYPE_CHECKING: import argparse from checkov.common.bridgecrew.bc_source import SourceType from checkov.common.output.report import Report + from checkov.secrets.coordinator import EnrichedSecret from mypy_boto3_s3.client import S3Client from requests import Response from typing_extensions import TypeGuard diff --git a/checkov/common/output/secrets_record.py b/checkov/common/output/secrets_record.py index f39f19e6a7c..0dfa6ed3e3a 100644 --- a/checkov/common/output/secrets_record.py +++ b/checkov/common/output/secrets_record.py @@ -7,13 +7,12 @@ from termcolor import colored from checkov.common.models.enums import CheckResult -from checkov.secrets.consts import ValidationStatus +from checkov.common.secrets.consts import ValidationStatus, GIT_HISTORY_NOT_BEEN_REMOVED from checkov.common.bridgecrew.severities import Severity from checkov.common.output.record import Record from checkov.common.typing import _CheckResult -from checkov.secrets.git_types import GIT_HISTORY_NOT_BEEN_REMOVED COMMIT_ADDED_STR = 'Commit Added' COMMIT_REMOVED_STR = 'Commit Removed' diff --git a/checkov/common/runners/runner_registry.py b/checkov/common/runners/runner_registry.py index eaf79e9a390..40565e52c50 100644 --- a/checkov/common/runners/runner_registry.py +++ b/checkov/common/runners/runner_registry.py @@ -39,13 +39,13 @@ from checkov.common.util import data_structures_utils from checkov.common.util.banner import tool as tool_name from checkov.common.util.json_utils import CustomJSONEncoder +from checkov.common.util.parser_utils import strip_terraform_module_referrer from checkov.common.util.secrets_omitter import SecretsOmitter from checkov.common.util.type_forcers import convert_csv_string_arg_to_list, force_list from checkov.sca_image.runner import Runner as image_runner -from checkov.secrets.consts import SECRET_VALIDATION_STATUSES +from checkov.common.secrets.consts import SECRET_VALIDATION_STATUSES from checkov.terraform.context_parsers.registry import parser_registry from checkov.terraform.parser import Parser -from checkov.terraform.runner import Runner as tf_runner if TYPE_CHECKING: from checkov.common.output.baseline import Baseline @@ -672,7 +672,7 @@ def get_enriched_resources( for repo_root, parse_results in repo_definitions.items(): for full_file_path, definition in parse_results['tf_definitions'].items(): definitions_context = parser_registry.enrich_definitions_context((full_file_path, definition)) - abs_scanned_file, _ = tf_runner._strip_module_referrer(full_file_path) + abs_scanned_file, _ = strip_terraform_module_referrer(file_path=full_file_path) scanned_file = os.path.relpath(abs_scanned_file, repo_root) for block_type, block_value in definition.items(): if block_type in CHECK_BLOCK_TYPES: diff --git a/checkov/common/secrets/__init__.py b/checkov/common/secrets/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/checkov/common/secrets/consts.py b/checkov/common/secrets/consts.py new file mode 100644 index 00000000000..075a36d2517 --- /dev/null +++ b/checkov/common/secrets/consts.py @@ -0,0 +1,36 @@ +from __future__ import annotations + +from enum import Enum + +GIT_HISTORY_NOT_BEEN_REMOVED = "not-removed" + + +class ValidationStatus(Enum): + PRIVILEGED = "Privileged" + VALID = "Valid" + INVALID = "Invalid" + UNKNOWN = "Unknown" + UNAVAILABLE = "Unavailable" + + def __str__(self) -> str: + # needed, because of a Python 3.11 change + return self.value + + +SECRET_VALIDATION_STATUSES = { + ValidationStatus.VALID.value, + ValidationStatus.PRIVILEGED.value, + ValidationStatus.INVALID.value, + ValidationStatus.UNKNOWN.value, + ValidationStatus.UNAVAILABLE.value, +} + + +class VerifySecretsResult(Enum): + INSUFFICIENT_PARAMS = "INSUFFICIENT_PARAMS" + FAILURE = "FAILURE" + SUCCESS = "SUCESS" + + def __str__(self) -> str: + # needed, because of a Python 3.11 change + return self.value diff --git a/checkov/common/util/parser_utils.py b/checkov/common/util/parser_utils.py index 4adb844c093..4ea1aa0826a 100644 --- a/checkov/common/util/parser_utils.py +++ b/checkov/common/util/parser_utils.py @@ -422,3 +422,21 @@ def get_module_name(file_path: TFDefinitionKeyType) -> str | None: def get_abs_path(file_path: TFDefinitionKeyType) -> str: return file_path[:get_current_module_index(file_path)] if isinstance(file_path, str) else str(file_path.file_path) + + +def strip_terraform_module_referrer(file_path: str) -> tuple[str, str | None]: + """ + For file paths containing module referrer information (e.g.: "module/module.tf[main.tf#0]"), this + returns a tuple containing the file path (e.g., "module/module.tf") and referrer (e.g., "main.tf#0"). + If the file path does not contain a referred, the tuple will contain the original file path and None. + """ + if file_path.endswith(TERRAFORM_NESTED_MODULE_PATH_ENDING) and TERRAFORM_NESTED_MODULE_PATH_PREFIX in file_path: + return ( + file_path[: file_path.index(TERRAFORM_NESTED_MODULE_PATH_PREFIX)], + file_path[ + file_path.index(TERRAFORM_NESTED_MODULE_PATH_PREFIX) + + TERRAFORM_NESTED_MODULE_PATH_SEPARATOR_LENGTH : -TERRAFORM_NESTED_MODULE_PATH_SEPARATOR_LENGTH + ], + ) + else: + return file_path, None diff --git a/checkov/runner_filter.py b/checkov/runner_filter.py index a03775ee7ee..4a7254fc113 100644 --- a/checkov/runner_filter.py +++ b/checkov/runner_filter.py @@ -8,7 +8,7 @@ import re from checkov.common.bridgecrew.check_type import CheckType -from checkov.secrets.consts import ValidationStatus +from checkov.common.secrets.consts import ValidationStatus from checkov.common.bridgecrew.code_categories import CodeCategoryMapping, CodeCategoryConfiguration, CodeCategoryType from checkov.common.bridgecrew.severities import Severity, Severities diff --git a/checkov/secrets/consts.py b/checkov/secrets/consts.py index dbfa5064671..420c7cbe85c 100644 --- a/checkov/secrets/consts.py +++ b/checkov/secrets/consts.py @@ -1,31 +1,3 @@ from __future__ import annotations -from enum import Enum - -class ValidationStatus(Enum): - PRIVILEGED = 'Privileged' - VALID = 'Valid' - INVALID = 'Invalid' - UNKNOWN = 'Unknown' - UNAVAILABLE = 'Unavailable' - - def __str__(self) -> str: - # needed, because of a Python 3.11 change - return self.value - - -SECRET_VALIDATION_STATUSES = [ValidationStatus.VALID.value, - ValidationStatus.PRIVILEGED.value, - ValidationStatus.INVALID.value, - ValidationStatus.UNKNOWN.value, - ValidationStatus.UNAVAILABLE.value] - - -class VerifySecretsResult(Enum): - INSUFFICIENT_PARAMS = 'INSUFFICIENT_PARAMS' - FAILURE = 'FAILURE' - SUCCESS = 'SUCESS' - - def __str__(self) -> str: - # needed, because of a Python 3.11 change - return self.value +from checkov.common.secrets.consts import ValidationStatus # noqa # TODO: added for reimport purposes, remove after migration diff --git a/checkov/secrets/git_history_store.py b/checkov/secrets/git_history_store.py index 5b9f20afec4..24f6732124d 100644 --- a/checkov/secrets/git_history_store.py +++ b/checkov/secrets/git_history_store.py @@ -7,8 +7,9 @@ from checkov.common.util.data_structures_utils import pickle_deepcopy from checkov.common.util.secrets import omit_secret_value_from_line +from checkov.common.secrets.consts import GIT_HISTORY_NOT_BEEN_REMOVED from checkov.secrets.git_types import EnrichedPotentialSecretMetadata, EnrichedPotentialSecret, Commit, ADDED, REMOVED, \ - GIT_HISTORY_OPTIONS, CommitDiff, GIT_HISTORY_NOT_BEEN_REMOVED + GIT_HISTORY_OPTIONS, CommitDiff if TYPE_CHECKING: from detect_secrets.core.potential_secret import PotentialSecret diff --git a/checkov/secrets/git_types.py b/checkov/secrets/git_types.py index 89985eaa67c..fb7056cc8cd 100644 --- a/checkov/secrets/git_types.py +++ b/checkov/secrets/git_types.py @@ -10,7 +10,6 @@ PROHIBITED_FILES = ('Pipfile.lock', 'yarn.lock', 'package-lock.json', 'requirements.txt', 'go.sum') -GIT_HISTORY_NOT_BEEN_REMOVED = 'not-removed' ADDED = 'added' REMOVED = 'removed' GIT_HISTORY_OPTIONS = {ADDED, REMOVED} diff --git a/checkov/secrets/runner.py b/checkov/secrets/runner.py index f6dd4719ae8..74eba7f995d 100644 --- a/checkov/secrets/runner.py +++ b/checkov/secrets/runner.py @@ -37,7 +37,7 @@ from checkov.common.util.dockerfile import is_dockerfile from checkov.common.util.secrets import omit_secret_value_from_line from checkov.runner_filter import RunnerFilter -from checkov.secrets.consts import ValidationStatus, VerifySecretsResult +from checkov.common.secrets.consts import ValidationStatus, VerifySecretsResult from checkov.secrets.coordinator import EnrichedSecret, SecretsCoordinator from checkov.secrets.plugins.load_detectors import get_runnable_plugins from checkov.secrets.git_history_store import GitHistorySecretStore diff --git a/checkov/secrets/scan_git_history.py b/checkov/secrets/scan_git_history.py index 2688d035d02..49ab4fb7f0e 100644 --- a/checkov/secrets/scan_git_history.py +++ b/checkov/secrets/scan_git_history.py @@ -6,13 +6,14 @@ import platform from typing import TYPE_CHECKING, Optional, List, Tuple +from detect_secrets.core import scan + from checkov.common.util.stopit import ThreadingTimeout, SignalTimeout, TimeoutException from checkov.common.util.decorators import time_it from checkov.common.parallelizer.parallel_runner import parallel_runner -from detect_secrets.core import scan - +from checkov.common.secrets.consts import GIT_HISTORY_NOT_BEEN_REMOVED from checkov.secrets.git_history_store import GitHistorySecretStore, RawStore, RENAME_STR, FILE_RESULTS_STR -from checkov.secrets.git_types import Commit, CommitMetadata, GIT_HISTORY_NOT_BEEN_REMOVED, PROHIBITED_FILES +from checkov.secrets.git_types import Commit, CommitMetadata, PROHIBITED_FILES if TYPE_CHECKING: from detect_secrets import SecretsCollection diff --git a/checkov/terraform/runner.py b/checkov/terraform/runner.py index b419abd5e39..3535cdac2ad 100644 --- a/checkov/terraform/runner.py +++ b/checkov/terraform/runner.py @@ -4,7 +4,7 @@ import logging import os import platform -from typing import Dict, Optional, Tuple, Any, Set, TYPE_CHECKING +from typing import Dict, Optional, Any, Set, TYPE_CHECKING import dpath import igraph @@ -25,9 +25,8 @@ from checkov.common.util.consts import RESOLVED_MODULE_ENTRY_NAME from checkov.common.util.data_structures_utils import pickle_deepcopy from checkov.common.util.parser_utils import get_module_from_full_path, get_abs_path, \ - get_tf_definition_key_from_module_dependency, TERRAFORM_NESTED_MODULE_PATH_PREFIX, \ - TERRAFORM_NESTED_MODULE_PATH_ENDING, TERRAFORM_NESTED_MODULE_PATH_SEPARATOR_LENGTH, \ - TERRAFORM_NESTED_MODULE_INDEX_SEPARATOR, get_module_name + get_tf_definition_key_from_module_dependency, TERRAFORM_NESTED_MODULE_INDEX_SEPARATOR, get_module_name, \ + strip_terraform_module_referrer from checkov.common.util.secrets import omit_secret_value_from_checks, omit_secret_value_from_graph_checks from checkov.common.variables.context import EvaluationContext from checkov.runner_filter import RunnerFilter @@ -377,7 +376,7 @@ def check_tf_definition( abs_scanned_file = get_abs_path(full_file_path) abs_referrer = None else: - abs_scanned_file, abs_referrer = self._strip_module_referrer(full_file_path) + abs_scanned_file, abs_referrer = strip_terraform_module_referrer(file_path=full_file_path) scanned_file = f"/{os.path.relpath(abs_scanned_file, root_folder)}" logging.debug(f"Scanning file: {scanned_file}") self.run_all_blocks(definition, self.context, full_file_path, root_folder, report, @@ -507,7 +506,7 @@ def run_block( entity_context_path) results = registry.scan(scanned_file, entity, skipped_checks, runner_filter) if isinstance(full_file_path, str): - absolute_scanned_file_path, _ = self._strip_module_referrer(file_path=full_file_path) + absolute_scanned_file_path, _ = strip_terraform_module_referrer(file_path=full_file_path) if isinstance(full_file_path, TFDefinitionKey): absolute_scanned_file_path = get_abs_path(full_file_path) # This duplicates a call at the start of scan, but adding this here seems better than kludging with some tuple return type @@ -605,7 +604,7 @@ def push_skipped_checks_down_old( # where referrer could be a path, or path1->path2, etc for definition in definition_context: - _, mod_ref = Runner._strip_module_referrer(definition) + _, mod_ref = strip_terraform_module_referrer(file_path=definition) if mod_ref is None: continue @@ -677,19 +676,6 @@ def push_skipped_checks_down( # append the skipped checks from the module to the other resources. resource_config["skipped_checks"] += skipped_checks - @staticmethod - def _strip_module_referrer(file_path: str) -> Tuple[str, Optional[str]]: - """ - For file paths containing module referrer information (e.g.: "module/module.tf[main.tf#0]"), this - returns a tuple containing the file path (e.g., "module/module.tf") and referrer (e.g., "main.tf#0"). - If the file path does not contain a referred, the tuple will contain the original file path and None. - """ - if file_path.endswith(TERRAFORM_NESTED_MODULE_PATH_ENDING) and TERRAFORM_NESTED_MODULE_PATH_PREFIX in file_path: - return file_path[:file_path.index(TERRAFORM_NESTED_MODULE_PATH_PREFIX)], \ - file_path[file_path.index(TERRAFORM_NESTED_MODULE_PATH_PREFIX) + TERRAFORM_NESTED_MODULE_PATH_SEPARATOR_LENGTH: -TERRAFORM_NESTED_MODULE_PATH_SEPARATOR_LENGTH] - else: - return file_path, None - def _find_id_for_referrer(self, full_file_path: str) -> Optional[str]: cached_referrer = self.referrer_cache.get(full_file_path) if cached_referrer: diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 00000000000..26c74785889 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,127 @@ +[tool.black] +line-length = 120 + +[tool.importlinter] +root_package = "checkov" + +[[tool.importlinter.contracts]] +name = "common forbidden to import other modules" +type = "forbidden" +source_modules = [ + "checkov.common", +] +forbidden_modules = [ + "checkov.ansible", + "checkov.argo_workflows", + "checkov.arm", + "checkov.azure_pipelines", + "checkov.bicep", + "checkov.bitbucket", + "checkov.bitbucket_pipelines", + "checkov.circleci_pipelines", + "checkov.cloudformation", + "checkov.dockerfile", + "checkov.example_runner", + "checkov.github", + "checkov.github_actions", + "checkov.gitlab", + "checkov.gitlab_ci", + "checkov.helm", + "checkov.json_doc", + "checkov.kubernetes", + "checkov.kustomize", + "checkov.openapi", + "checkov.policies_3d", + "checkov.sca_image", + "checkov.sca_package", + "checkov.sca_package_2", + "checkov.secrets", + "checkov.serverless", + "checkov.terraform", + "checkov.terraform_json", + "checkov.yaml_doc", +] +ignore_imports = [ + "checkov.common.util.json_utils -> checkov.terraform.modules.module_objects", # needed for JSON serialization + + "checkov.common.bridgecrew.platform_integration -> checkov.secrets.coordinator", # type hint + "checkov.common.graph.graph_manager -> checkov.terraform.parser", # type hint + "checkov.common.util.parser_utils -> checkov.terraform.modules.module_objects", # type hint + "checkov.common.typing -> checkov.terraform.modules.module_objects", # type hint + + "checkov.common.bridgecrew.integration_features.features.policies_3d_integration -> checkov.policies_3d.*", # considering what to do + "checkov.common.runners.base_post_runner -> checkov.policies_3d.checks_infra.base_check", # considering what to do + + "checkov.common.util.docs_generator -> checkov.*.registry", # move to a different place + "checkov.common.util.docs_generator -> checkov.*.checks.registry", # move to a different place + "checkov.common.util.docs_generator -> checkov.*.checks.*.registry", # move to a different place + "checkov.common.util.docs_generator -> checkov.secrets.runner", # move to a different place + + "checkov.common.runners.runner_registry -> checkov.terraform.context_parsers.registry", # move runner_registry to a different place + "checkov.common.runners.runner_registry -> checkov.terraform.parser", # move runner_registry to a different place + + "checkov.common.output.report -> checkov.policies_3d.output", # move to checkov.common.output + "checkov.common.output.report -> checkov.sca_package.output", # move to checkov.common.output + "checkov.common.output.report -> checkov.sca_package_2.output", # move to checkov.common.output + + "checkov.common.checks_infra.solvers.connections_solvers.connection_exists_solver -> checkov.terraform.graph_builder.graph_components.block_types", # move to checkov.common.graph.graph_builder.graph_components? + "checkov.common.checks_infra.solvers.attribute_solvers.base_attribute_solver -> checkov.terraform.graph_builder.graph_components.block_types", # move to checkov.common.graph.graph_builder.graph_components? + "checkov.common.checks_infra.solvers.connections_solvers.base_connection_solver -> checkov.terraform.graph_builder.graph_components.block_types", # move to checkov.common.graph.graph_builder.graph_components? + + "checkov.common.runners.runner_registry -> checkov.sca_image.runner", # old IR, needs to be removed (argo_workflows, bitbucket_pipelines) + "checkov.common.graph.graph_builder.graph_components.blocks -> checkov.terraform.graph_builder.graph_components.block_types", # override get_attribute_dict() inside TerraformBlock +] + +[[tool.importlinter.contracts]] +name = "kubernetes forbidden to import its children" +type = "forbidden" +source_modules = [ + "checkov.kubernetes", +] +forbidden_modules = [ + "checkov.helm", + "checkov.kustomize", +] + +[[tool.importlinter.contracts]] +name = "terraform forbidden to import its children" +type = "forbidden" +source_modules = [ + "checkov.terraform", +] +forbidden_modules = [ + "checkov.terraform_json", +] + +[[tool.importlinter.contracts]] +name = "object runners forbidden to import its children" +type = "forbidden" +source_modules = [ + "checkov.json_doc", + "checkov.yaml_doc", +] +forbidden_modules = [ + "checkov.ansible", + "checkov.argo_workflows", + "checkov.azure_pipelines", + "checkov.bitbucket", + "checkov.bitbucket_pipelines", + "checkov.circleci_pipelines", + "checkov.example_runner", + "checkov.github", + "checkov.github_actions", + "checkov.gitlab", + "checkov.gitlab_ci", + "checkov.openapi", +] + +[[tool.importlinter.contracts]] +# this one is a bit special, because 'bicep' is not a real child of 'arm' but it leverages the checks written for 'arm' +name = "bicep forbidden to import arm" +type = "forbidden" +source_modules = [ + "checkov.bicep", +] +forbidden_modules = [ + "checkov.arm", +] diff --git a/tests/common/output/conftest.py b/tests/common/output/conftest.py index 63560f8a193..c8667801721 100644 --- a/tests/common/output/conftest.py +++ b/tests/common/output/conftest.py @@ -2,7 +2,7 @@ from typing import Any import pytest -from checkov.secrets.consts import ValidationStatus +from checkov.common.secrets.consts import ValidationStatus from checkov.common.bridgecrew.check_type import CheckType from checkov.common.models.enums import CheckResult diff --git a/tests/secrets/test_secrets_verification.py b/tests/secrets/test_secrets_verification.py index 7613fdce588..70533800340 100644 --- a/tests/secrets/test_secrets_verification.py +++ b/tests/secrets/test_secrets_verification.py @@ -6,7 +6,7 @@ import responses from checkov.common.bridgecrew.check_type import CheckType -from checkov.secrets.consts import VerifySecretsResult +from checkov.common.secrets.consts import VerifySecretsResult @mock.patch.dict(os.environ, {"CKV_VALIDATE_SECRETS": "true"})