chore: add import-linter to enforce module independency (#5386)

add import-linter to enforce module independency

.pre-commit-config.yaml

@@ -32,3 +32,9 @@ repos:
         args: []
         additional_dependencies:
           - vistir<0.7.0  # can be removed, when v4.0.0 of pipenv-setup comes out
+  - repo: https://github.com/seddonym/import-linter  # checks the import dependencies between each other
+    rev: v1.10.0
+    hooks:
+      - id: import-linter
+        language_version: python3.9
+        args: ["--show-timings"]

checkov/common/bridgecrew/platform_integration.py

@@ -45,13 +45,13 @@
     get_user_agent_header, get_default_post_headers, get_prisma_get_headers, get_prisma_auth_header, \
     get_auth_error_message, normalize_bc_url
 from checkov.common.util.type_forcers import convert_prisma_policy_filter_to_dict, convert_str_to_bool
-from checkov.secrets.coordinator import EnrichedSecret
 from checkov.version import version as checkov_version
 
 if TYPE_CHECKING:
     import argparse
     from checkov.common.bridgecrew.bc_source import SourceType
     from checkov.common.output.report import Report
+    from checkov.secrets.coordinator import EnrichedSecret
     from mypy_boto3_s3.client import S3Client
     from requests import Response
     from typing_extensions import TypeGuard

checkov/common/output/secrets_record.py

@@ -7,13 +7,12 @@
 from termcolor import colored
 
 from checkov.common.models.enums import CheckResult
-from checkov.secrets.consts import ValidationStatus
+from checkov.common.secrets.consts import ValidationStatus, GIT_HISTORY_NOT_BEEN_REMOVED
 
 from checkov.common.bridgecrew.severities import Severity
 
 from checkov.common.output.record import Record
 from checkov.common.typing import _CheckResult
-from checkov.secrets.git_types import GIT_HISTORY_NOT_BEEN_REMOVED
 
 COMMIT_ADDED_STR = 'Commit Added'
 COMMIT_REMOVED_STR = 'Commit Removed'

checkov/common/runners/runner_registry.py

@@ -39,13 +39,13 @@
 from checkov.common.util import data_structures_utils
 from checkov.common.util.banner import tool as tool_name
 from checkov.common.util.json_utils import CustomJSONEncoder
+from checkov.common.util.parser_utils import strip_terraform_module_referrer
 from checkov.common.util.secrets_omitter import SecretsOmitter
 from checkov.common.util.type_forcers import convert_csv_string_arg_to_list, force_list
 from checkov.sca_image.runner import Runner as image_runner
-from checkov.secrets.consts import SECRET_VALIDATION_STATUSES
+from checkov.common.secrets.consts import SECRET_VALIDATION_STATUSES
 from checkov.terraform.context_parsers.registry import parser_registry
 from checkov.terraform.parser import Parser
-from checkov.terraform.runner import Runner as tf_runner
 
 if TYPE_CHECKING:
     from checkov.common.output.baseline import Baseline
@@ -672,7 +672,7 @@ def get_enriched_resources(
         for repo_root, parse_results in repo_definitions.items():
             for full_file_path, definition in parse_results['tf_definitions'].items():
                 definitions_context = parser_registry.enrich_definitions_context((full_file_path, definition))
-                abs_scanned_file, _ = tf_runner._strip_module_referrer(full_file_path)
+                abs_scanned_file, _ = strip_terraform_module_referrer(file_path=full_file_path)
                 scanned_file = os.path.relpath(abs_scanned_file, repo_root)
                 for block_type, block_value in definition.items():
                     if block_type in CHECK_BLOCK_TYPES:

checkov/common/secrets/consts.py

@@ -0,0 +1,36 @@
+from __future__ import annotations
+
+from enum import Enum
+
+GIT_HISTORY_NOT_BEEN_REMOVED = "not-removed"
+
+
+class ValidationStatus(Enum):
+    PRIVILEGED = "Privileged"
+    VALID = "Valid"
+    INVALID = "Invalid"
+    UNKNOWN = "Unknown"
+    UNAVAILABLE = "Unavailable"
+
+    def __str__(self) -> str:
+        # needed, because of a Python 3.11 change
+        return self.value
+
+
+SECRET_VALIDATION_STATUSES = {
+    ValidationStatus.VALID.value,
+    ValidationStatus.PRIVILEGED.value,
+    ValidationStatus.INVALID.value,
+    ValidationStatus.UNKNOWN.value,
+    ValidationStatus.UNAVAILABLE.value,
+}
+
+
+class VerifySecretsResult(Enum):
+    INSUFFICIENT_PARAMS = "INSUFFICIENT_PARAMS"
+    FAILURE = "FAILURE"
+    SUCCESS = "SUCESS"
+
+    def __str__(self) -> str:
+        # needed, because of a Python 3.11 change
+        return self.value

checkov/common/util/parser_utils.py

@@ -422,3 +422,21 @@ def get_module_name(file_path: TFDefinitionKeyType) -> str | None:
 
 def get_abs_path(file_path: TFDefinitionKeyType) -> str:
     return file_path[:get_current_module_index(file_path)] if isinstance(file_path, str) else str(file_path.file_path)
+
+
+def strip_terraform_module_referrer(file_path: str) -> tuple[str, str | None]:
+    """
+    For file paths containing module referrer information (e.g.: "module/module.tf[main.tf#0]"), this
+    returns a tuple containing the file path (e.g., "module/module.tf") and referrer (e.g., "main.tf#0").
+    If the file path does not contain a referred, the tuple will contain the original file path and None.
+    """
+    if file_path.endswith(TERRAFORM_NESTED_MODULE_PATH_ENDING) and TERRAFORM_NESTED_MODULE_PATH_PREFIX in file_path:
+        return (
+            file_path[: file_path.index(TERRAFORM_NESTED_MODULE_PATH_PREFIX)],
+            file_path[
+                file_path.index(TERRAFORM_NESTED_MODULE_PATH_PREFIX)
+                + TERRAFORM_NESTED_MODULE_PATH_SEPARATOR_LENGTH : -TERRAFORM_NESTED_MODULE_PATH_SEPARATOR_LENGTH
+            ],
+        )
+    else:
+        return file_path, None

checkov/runner_filter.py

@@ -8,7 +8,7 @@
 import re
 
 from checkov.common.bridgecrew.check_type import CheckType
-from checkov.secrets.consts import ValidationStatus
+from checkov.common.secrets.consts import ValidationStatus
 
 from checkov.common.bridgecrew.code_categories import CodeCategoryMapping, CodeCategoryConfiguration, CodeCategoryType
 from checkov.common.bridgecrew.severities import Severity, Severities

checkov/secrets/consts.py

@@ -1,31 +1,3 @@
 from __future__ import annotations
-from enum import Enum
 
-
-class ValidationStatus(Enum):
-    PRIVILEGED = 'Privileged'
-    VALID = 'Valid'
-    INVALID = 'Invalid'
-    UNKNOWN = 'Unknown'
-    UNAVAILABLE = 'Unavailable'
-
-    def __str__(self) -> str:
-        # needed, because of a Python 3.11 change
-        return self.value
-
-
-SECRET_VALIDATION_STATUSES = [ValidationStatus.VALID.value,
-                              ValidationStatus.PRIVILEGED.value,
-                              ValidationStatus.INVALID.value,
-                              ValidationStatus.UNKNOWN.value,
-                              ValidationStatus.UNAVAILABLE.value]
-
-
-class VerifySecretsResult(Enum):
-    INSUFFICIENT_PARAMS = 'INSUFFICIENT_PARAMS'
-    FAILURE = 'FAILURE'
-    SUCCESS = 'SUCESS'
-
-    def __str__(self) -> str:
-        # needed, because of a Python 3.11 change
-        return self.value
+from checkov.common.secrets.consts import ValidationStatus  # noqa  # TODO: added for reimport purposes, remove after migration

checkov/secrets/git_history_store.py

@@ -7,8 +7,9 @@
 
 from checkov.common.util.data_structures_utils import pickle_deepcopy
 from checkov.common.util.secrets import omit_secret_value_from_line
+from checkov.common.secrets.consts import GIT_HISTORY_NOT_BEEN_REMOVED
 from checkov.secrets.git_types import EnrichedPotentialSecretMetadata, EnrichedPotentialSecret, Commit, ADDED, REMOVED, \
-    GIT_HISTORY_OPTIONS, CommitDiff, GIT_HISTORY_NOT_BEEN_REMOVED
+    GIT_HISTORY_OPTIONS, CommitDiff
 
 if TYPE_CHECKING:
     from detect_secrets.core.potential_secret import PotentialSecret

checkov/secrets/git_types.py

@@ -10,7 +10,6 @@
 PROHIBITED_FILES = ('Pipfile.lock', 'yarn.lock', 'package-lock.json', 'requirements.txt', 'go.sum')
 
 
-GIT_HISTORY_NOT_BEEN_REMOVED = 'not-removed'
 ADDED = 'added'
 REMOVED = 'removed'
 GIT_HISTORY_OPTIONS = {ADDED, REMOVED}

checkov/secrets/runner.py

@@ -37,7 +37,7 @@
 from checkov.common.util.dockerfile import is_dockerfile
 from checkov.common.util.secrets import omit_secret_value_from_line
 from checkov.runner_filter import RunnerFilter
-from checkov.secrets.consts import ValidationStatus, VerifySecretsResult
+from checkov.common.secrets.consts import ValidationStatus, VerifySecretsResult
 from checkov.secrets.coordinator import EnrichedSecret, SecretsCoordinator
 from checkov.secrets.plugins.load_detectors import get_runnable_plugins
 from checkov.secrets.git_history_store import GitHistorySecretStore

checkov/secrets/scan_git_history.py

@@ -6,13 +6,14 @@
 import platform
 from typing import TYPE_CHECKING, Optional, List, Tuple
 
+from detect_secrets.core import scan
+
 from checkov.common.util.stopit import ThreadingTimeout, SignalTimeout, TimeoutException
 from checkov.common.util.decorators import time_it
 from checkov.common.parallelizer.parallel_runner import parallel_runner
-from detect_secrets.core import scan
-
+from checkov.common.secrets.consts import GIT_HISTORY_NOT_BEEN_REMOVED
 from checkov.secrets.git_history_store import GitHistorySecretStore, RawStore, RENAME_STR, FILE_RESULTS_STR
-from checkov.secrets.git_types import Commit, CommitMetadata, GIT_HISTORY_NOT_BEEN_REMOVED, PROHIBITED_FILES
+from checkov.secrets.git_types import Commit, CommitMetadata, PROHIBITED_FILES
 
 if TYPE_CHECKING:
     from detect_secrets import SecretsCollection

checkov/terraform/runner.py

@@ -4,7 +4,7 @@
 import logging
 import os
 import platform
-from typing import Dict, Optional, Tuple, Any, Set, TYPE_CHECKING
+from typing import Dict, Optional, Any, Set, TYPE_CHECKING
 
 import dpath
 import igraph
@@ -25,9 +25,8 @@
 from checkov.common.util.consts import RESOLVED_MODULE_ENTRY_NAME
 from checkov.common.util.data_structures_utils import pickle_deepcopy
 from checkov.common.util.parser_utils import get_module_from_full_path, get_abs_path, \
-    get_tf_definition_key_from_module_dependency, TERRAFORM_NESTED_MODULE_PATH_PREFIX, \
-    TERRAFORM_NESTED_MODULE_PATH_ENDING, TERRAFORM_NESTED_MODULE_PATH_SEPARATOR_LENGTH, \
-    TERRAFORM_NESTED_MODULE_INDEX_SEPARATOR, get_module_name
+    get_tf_definition_key_from_module_dependency, TERRAFORM_NESTED_MODULE_INDEX_SEPARATOR, get_module_name, \
+    strip_terraform_module_referrer
 from checkov.common.util.secrets import omit_secret_value_from_checks, omit_secret_value_from_graph_checks
 from checkov.common.variables.context import EvaluationContext
 from checkov.runner_filter import RunnerFilter
@@ -377,7 +376,7 @@ def check_tf_definition(
                 abs_scanned_file = get_abs_path(full_file_path)
                 abs_referrer = None
             else:
-                abs_scanned_file, abs_referrer = self._strip_module_referrer(full_file_path)
+                abs_scanned_file, abs_referrer = strip_terraform_module_referrer(file_path=full_file_path)
             scanned_file = f"/{os.path.relpath(abs_scanned_file, root_folder)}"
             logging.debug(f"Scanning file: {scanned_file}")
             self.run_all_blocks(definition, self.context, full_file_path, root_folder, report,
@@ -507,7 +506,7 @@ def run_block(
                                                                                       entity_context_path)
             results = registry.scan(scanned_file, entity, skipped_checks, runner_filter)
             if isinstance(full_file_path, str):
-                absolute_scanned_file_path, _ = self._strip_module_referrer(file_path=full_file_path)
+                absolute_scanned_file_path, _ = strip_terraform_module_referrer(file_path=full_file_path)
             if isinstance(full_file_path, TFDefinitionKey):
                 absolute_scanned_file_path = get_abs_path(full_file_path)
             # This duplicates a call at the start of scan, but adding this here seems better than kludging with some tuple return type
@@ -605,7 +604,7 @@ def push_skipped_checks_down_old(
         # where referrer could be a path, or path1->path2, etc
 
         for definition in definition_context:
-            _, mod_ref = Runner._strip_module_referrer(definition)
+            _, mod_ref = strip_terraform_module_referrer(file_path=definition)
             if mod_ref is None:
                 continue
 
@@ -677,19 +676,6 @@ def push_skipped_checks_down(
                             # append the skipped checks from the module to the other resources.
                             resource_config["skipped_checks"] += skipped_checks
 
-    @staticmethod
-    def _strip_module_referrer(file_path: str) -> Tuple[str, Optional[str]]:
-        """
-        For file paths containing module referrer information (e.g.: "module/module.tf[main.tf#0]"), this
-        returns a tuple containing the file path (e.g., "module/module.tf") and referrer (e.g., "main.tf#0").
-        If the file path does not contain a referred, the tuple will contain the original file path and None.
-        """
-        if file_path.endswith(TERRAFORM_NESTED_MODULE_PATH_ENDING) and TERRAFORM_NESTED_MODULE_PATH_PREFIX in file_path:
-            return file_path[:file_path.index(TERRAFORM_NESTED_MODULE_PATH_PREFIX)], \
-                file_path[file_path.index(TERRAFORM_NESTED_MODULE_PATH_PREFIX) + TERRAFORM_NESTED_MODULE_PATH_SEPARATOR_LENGTH: -TERRAFORM_NESTED_MODULE_PATH_SEPARATOR_LENGTH]
-        else:
-            return file_path, None
-
     def _find_id_for_referrer(self, full_file_path: str) -> Optional[str]:
         cached_referrer = self.referrer_cache.get(full_file_path)
         if cached_referrer: