From f6751f273d8827635851b4617ffffbeaf1e4666d Mon Sep 17 00:00:00 2001 From: Barak Fatal <35402131+bo156@users.noreply.github.com> Date: Wed, 16 Aug 2023 10:27:52 +0300 Subject: [PATCH] feat(terraform): Remove old tf parser (#5420) * Removed old parser usage and fixed all tests besides module_dependency * Removed unneeded Module fields like module_dependency_map * removed TFBlock fields like module_dependency and module_dependency_num * Added TODO to make sure I don't forget to remove TFDefinitionKeyType * Fixed tests with local related paths * Fixed runner_registry usage of tf-definitions * Fixed the last failing test to match the new parser * Fixed all mypy errors but 1 in runner registry * Mypy works, but tests not * Middle of debugging tests * Fixed remaining test_runner tests * Fixed tests which specifically used str instead of TFDefinitionKey * Deleted test which used old definitions * Minimized usage of TFDefinitionKeyType * flake8 * hopefully fix import linter * fixed type is not subscriptable --- checkov/common/graph/graph_manager.py | 4 +- checkov/common/runners/runner_registry.py | 14 +- checkov/common/typing.py | 1 + checkov/common/util/env_vars_config.py | 1 - checkov/common/util/parser_utils.py | 106 +-- .../terraform/context_parsers/base_parser.py | 5 +- checkov/terraform/context_parsers/registry.py | 20 +- .../graph_builder/graph_components/blocks.py | 17 +- .../graph_builder/graph_components/module.py | 64 +- .../graph_builder/graph_to_tf_definitions.py | 18 +- .../terraform/graph_builder/local_graph.py | 175 ++--- checkov/terraform/graph_manager.py | 26 +- checkov/terraform/modules/module_utils.py | 105 +-- checkov/terraform/parser.py | 668 ------------------ checkov/terraform/plan_runner.py | 1 + checkov/terraform/runner.py | 111 +-- checkov/terraform/tf_parser.py | 61 +- pyproject.toml | 6 +- .../test_runner_registry_plan_enrichment.py | 2 +- .../context_parsers/test_locals_parser.py | 18 +- .../test_variable_context_parser.py | 24 +- .../test_variable_context_parser2.py | 23 +- .../db_connector/test_graph_connector.py | 6 +- .../graph/graph_builder/test_local_graph.py | 244 ++----- .../test_terraform_graph_parser.py | 28 +- .../graph/runner/persistent_data.json | 215 ------ .../graph/runner/test_graph_builder.py | 18 +- .../test_foreach_renderer.py | 2 +- .../test_render_scenario.py | 12 +- .../expected.json | 224 ++++-- tests/terraform/parser/test_module.py | 5 +- .../parser/test_new_parser_modules.py | 20 +- .../parser/test_parse_file_vs_dir.py | 6 +- .../terraform/parser/test_parser_internals.py | 2 +- tests/terraform/parser/test_parser_modules.py | 145 ---- tests/terraform/runner/test_runner.py | 144 +--- 36 files changed, 508 insertions(+), 2033 deletions(-) delete mode 100644 checkov/terraform/parser.py delete mode 100644 tests/terraform/graph/runner/persistent_data.json delete mode 100644 tests/terraform/parser/test_parser_modules.py diff --git a/checkov/common/graph/graph_manager.py b/checkov/common/graph/graph_manager.py index 20136347531..a7a6a6a7dcd 100644 --- a/checkov/common/graph/graph_manager.py +++ b/checkov/common/graph/graph_manager.py @@ -5,7 +5,7 @@ if TYPE_CHECKING: from checkov.common.graph.graph_builder.local_graph import LocalGraph # noqa - from checkov.terraform.parser import Parser + from checkov.terraform.tf_parser import TFParser from checkov.common.typing import LibraryGraph, LibraryGraphConnector _LocalGraph = TypeVar("_LocalGraph", bound="LocalGraph[Any]") @@ -13,7 +13,7 @@ class GraphManager(Generic[_LocalGraph, _Definitions]): - def __init__(self, db_connector: LibraryGraphConnector, parser: Parser | None, source: str = "") -> None: + def __init__(self, db_connector: LibraryGraphConnector, parser: TFParser | None, source: str = "") -> None: self.db_connector = db_connector self.source = source self.parser = parser diff --git a/checkov/common/runners/runner_registry.py b/checkov/common/runners/runner_registry.py index 40565e52c50..621b6cb7854 100644 --- a/checkov/common/runners/runner_registry.py +++ b/checkov/common/runners/runner_registry.py @@ -39,13 +39,12 @@ from checkov.common.util import data_structures_utils from checkov.common.util.banner import tool as tool_name from checkov.common.util.json_utils import CustomJSONEncoder -from checkov.common.util.parser_utils import strip_terraform_module_referrer from checkov.common.util.secrets_omitter import SecretsOmitter from checkov.common.util.type_forcers import convert_csv_string_arg_to_list, force_list from checkov.sca_image.runner import Runner as image_runner from checkov.common.secrets.consts import SECRET_VALIDATION_STATUSES from checkov.terraform.context_parsers.registry import parser_registry -from checkov.terraform.parser import Parser +from checkov.terraform.tf_parser import TFParser if TYPE_CHECKING: from checkov.common.output.baseline import Baseline @@ -655,14 +654,14 @@ def enrich_report_with_guidelines(scan_report: Report) -> None: def get_enriched_resources( repo_roots: list[str | Path], download_external_modules: bool ) -> dict[str, dict[str, Any]]: + from checkov.terraform.modules.module_objects import TFDefinitionKey + repo_definitions = {} for repo_root in repo_roots: - tf_definitions: dict[str, Any] = {} parsing_errors: dict[str, Exception] = {} repo_root = os.path.abspath(repo_root) - Parser().parse_directory( + tf_definitions: dict[TFDefinitionKey, dict[str, list[dict[str, Any]]]] = TFParser().parse_directory( directory=repo_root, # assume plan file is in the repo-root - out_definitions=tf_definitions, out_parsing_errors=parsing_errors, download_external_modules=download_external_modules, ) @@ -670,9 +669,10 @@ def get_enriched_resources( enriched_resources = {} for repo_root, parse_results in repo_definitions.items(): - for full_file_path, definition in parse_results['tf_definitions'].items(): + definitions = cast("dict[TFDefinitionKey, dict[str, list[dict[str, Any]]]]", parse_results['tf_definitions']) + for full_file_path, definition in definitions.items(): definitions_context = parser_registry.enrich_definitions_context((full_file_path, definition)) - abs_scanned_file, _ = strip_terraform_module_referrer(file_path=full_file_path) + abs_scanned_file = full_file_path.file_path scanned_file = os.path.relpath(abs_scanned_file, repo_root) for block_type, block_value in definition.items(): if block_type in CHECK_BLOCK_TYPES: diff --git a/checkov/common/typing.py b/checkov/common/typing.py index 99d6ce0f76c..7cb20ec95e9 100644 --- a/checkov/common/typing.py +++ b/checkov/common/typing.py @@ -24,6 +24,7 @@ ResourceAttributesToOmit: TypeAlias = Dict[_Resource, _Attributes] LibraryGraph: TypeAlias = "Union[DiGraph, Graph]" LibraryGraphConnector: TypeAlias = "Union[DBConnector[DiGraph], DBConnector[Graph]]" +# TODO Remove this type and only use TFDefinitionKey TFDefinitionKeyType: TypeAlias = "Union[str, TFDefinitionKey]" diff --git a/checkov/common/util/env_vars_config.py b/checkov/common/util/env_vars_config.py index 272dde419c1..6a3bfca29b9 100644 --- a/checkov/common/util/env_vars_config.py +++ b/checkov/common/util/env_vars_config.py @@ -45,7 +45,6 @@ def __init__(self) -> None: self.IGNORE_HIDDEN_DIRECTORIES = convert_str_to_bool(os.getenv("CKV_IGNORE_HIDDEN_DIRECTORIES", True)) self.MAX_FILE_SIZE = force_int(os.getenv("CHECKOV_MAX_FILE_SIZE", 5_000_000)) # 5 MB is default limit self.MAX_IAC_FILE_SIZE = force_int(os.getenv("CHECKOV_MAX_IAC_FILE_SIZE", 50_000_000)) # 50 MB is default limit - self.NEW_TF_PARSER = convert_str_to_bool(os.getenv("CHECKOV_NEW_TF_PARSER", True)) self.NO_OUTPUT = convert_str_to_bool(os.getenv("CHECKOV_NO_OUTPUT", False)) self.OPENAI_MAX_FINDINGS = force_int(os.getenv("CKV_OPENAI_MAX_FINDINGS", 5)) self.OPENAI_MAX_TOKENS = force_int(os.getenv("CKV_OPENAI_MAX_TOKENS", 512)) diff --git a/checkov/common/util/parser_utils.py b/checkov/common/util/parser_utils.py index 4ea1aa0826a..1ff925904f8 100644 --- a/checkov/common/util/parser_utils.py +++ b/checkov/common/util/parser_utils.py @@ -1,17 +1,13 @@ from __future__ import annotations import json -import os import re from dataclasses import dataclass from enum import Enum -from typing import Any, List, Optional, Tuple +from typing import Any, List import hcl2 -from checkov.common.runners.base_runner import strtobool -from checkov.common.typing import TFDefinitionKeyType - _FUNCTION_NAME_CHARS = frozenset("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789") _ARG_VAR_PATTERN = re.compile(r"[a-zA-Z_]+(\.[a-zA-Z_]+)+") @@ -340,103 +336,3 @@ def to_string(value: Any) -> str: elif value is False: return "false" return str(value) - - -def get_current_module_index(full_path: str) -> Optional[int]: - hcl_index = None - tf_index = None - if TERRAFORM_NESTED_MODULE_PATH_PREFIX not in full_path and TERRAFORM_NESTED_MODULE_INDEX_SEPARATOR not in full_path: - return len(full_path) - if f'.hcl{TERRAFORM_NESTED_MODULE_PATH_PREFIX}' in full_path: - hcl_index = full_path.index(f'.hcl{TERRAFORM_NESTED_MODULE_PATH_PREFIX}') + 4 # len('.hcl') - elif f'.hcl{TERRAFORM_NESTED_MODULE_INDEX_SEPARATOR}' in full_path: - hcl_index = full_path.index(f'.hcl{TERRAFORM_NESTED_MODULE_INDEX_SEPARATOR}') + 4 # len('.hcl') - if f'.tf{TERRAFORM_NESTED_MODULE_PATH_PREFIX}' in full_path: - tf_index = full_path.index(f'.tf{TERRAFORM_NESTED_MODULE_PATH_PREFIX}') + 3 # len('.tf') - elif f'.tf{TERRAFORM_NESTED_MODULE_INDEX_SEPARATOR}' in full_path: - tf_index = full_path.index(f'.tf{TERRAFORM_NESTED_MODULE_INDEX_SEPARATOR}') + 3 # len('.tf') - if hcl_index and tf_index: - # returning the index of the first file - return min(hcl_index, tf_index) - if hcl_index: - return hcl_index - return tf_index - - -def is_nested(full_path: TFDefinitionKeyType | None) -> bool: - from checkov.terraform.modules.module_objects import TFDefinitionKey - if isinstance(full_path, str): - return TERRAFORM_NESTED_MODULE_PATH_PREFIX in full_path - if isinstance(full_path, TFDefinitionKey): - return full_path.tf_source_modules is not None - return False - - -def get_tf_definition_key(nested_module: str, module_name: str, module_index: Any, nested_key: str = '') -> str: - return f"{nested_module}{TERRAFORM_NESTED_MODULE_PATH_PREFIX}{module_name}{TERRAFORM_NESTED_MODULE_INDEX_SEPARATOR}{module_index}{nested_key}{TERRAFORM_NESTED_MODULE_PATH_ENDING}" - - -def get_tf_definition_key_from_module_dependency( - path: str, module_dependency: str | None, module_dependency_num: str | None -) -> str: - if not module_dependency: - return path - if not is_nested(module_dependency): - return f"{path}{TERRAFORM_NESTED_MODULE_PATH_PREFIX}{module_dependency}{TERRAFORM_NESTED_MODULE_INDEX_SEPARATOR}{module_dependency_num}{TERRAFORM_NESTED_MODULE_PATH_ENDING}" - module_index = get_current_module_index(module_dependency) - return f"{path}{TERRAFORM_NESTED_MODULE_PATH_PREFIX}{module_dependency[:module_index]}{TERRAFORM_NESTED_MODULE_INDEX_SEPARATOR}{module_dependency_num}{module_dependency[module_index:]}{TERRAFORM_NESTED_MODULE_PATH_ENDING}" - - -def get_module_from_full_path(file_path: TFDefinitionKeyType | None) -> Tuple[TFDefinitionKeyType | None, str | None]: - from checkov.terraform.modules.module_objects import TFDefinitionKey - if not file_path or not is_nested(file_path): - return None, None - if isinstance(file_path, TFDefinitionKey): - if file_path.tf_source_modules is None: - return None, None - if strtobool(os.getenv('ENABLE_DEFINITION_KEY', 'False')): - return TFDefinitionKey(file_path=file_path.tf_source_modules.path, tf_source_modules=file_path.tf_source_modules.nested_tf_module), None - return file_path.tf_source_modules.path, None - tmp_path = file_path[file_path.index(TERRAFORM_NESTED_MODULE_PATH_PREFIX) + TERRAFORM_NESTED_MODULE_PATH_SEPARATOR_LENGTH: -TERRAFORM_NESTED_MODULE_PATH_SEPARATOR_LENGTH] - if is_nested(tmp_path): - module = get_abs_path(tmp_path) + tmp_path[tmp_path.index(TERRAFORM_NESTED_MODULE_PATH_PREFIX):] - index = tmp_path[tmp_path.index(TERRAFORM_NESTED_MODULE_INDEX_SEPARATOR) + TERRAFORM_NESTED_MODULE_PATH_SEPARATOR_LENGTH:tmp_path.index(TERRAFORM_NESTED_MODULE_PATH_PREFIX)] - else: - module = get_abs_path(tmp_path) - index = tmp_path[tmp_path.index(TERRAFORM_NESTED_MODULE_INDEX_SEPARATOR) + TERRAFORM_NESTED_MODULE_PATH_SEPARATOR_LENGTH:] - return module, index - - -def get_module_name(file_path: TFDefinitionKeyType) -> str | None: - from checkov.terraform.modules.module_objects import TFDefinitionKey - if isinstance(file_path, TFDefinitionKey): - if not file_path.tf_source_modules: - return None - module_name = file_path.tf_source_modules.name - if file_path.tf_source_modules.foreach_idx: - foreach_or_count = '"' if isinstance(file_path.tf_source_modules.foreach_idx, str) else '' - module_name = f'{module_name}[{foreach_or_count}{file_path.tf_source_modules.foreach_idx}{foreach_or_count}]' - return module_name - return None - - -def get_abs_path(file_path: TFDefinitionKeyType) -> str: - return file_path[:get_current_module_index(file_path)] if isinstance(file_path, str) else str(file_path.file_path) - - -def strip_terraform_module_referrer(file_path: str) -> tuple[str, str | None]: - """ - For file paths containing module referrer information (e.g.: "module/module.tf[main.tf#0]"), this - returns a tuple containing the file path (e.g., "module/module.tf") and referrer (e.g., "main.tf#0"). - If the file path does not contain a referred, the tuple will contain the original file path and None. - """ - if file_path.endswith(TERRAFORM_NESTED_MODULE_PATH_ENDING) and TERRAFORM_NESTED_MODULE_PATH_PREFIX in file_path: - return ( - file_path[: file_path.index(TERRAFORM_NESTED_MODULE_PATH_PREFIX)], - file_path[ - file_path.index(TERRAFORM_NESTED_MODULE_PATH_PREFIX) - + TERRAFORM_NESTED_MODULE_PATH_SEPARATOR_LENGTH : -TERRAFORM_NESTED_MODULE_PATH_SEPARATOR_LENGTH - ], - ) - else: - return file_path, None diff --git a/checkov/terraform/context_parsers/base_parser.py b/checkov/terraform/context_parsers/base_parser.py index 16703c0689c..3d933a9c0d8 100644 --- a/checkov/terraform/context_parsers/base_parser.py +++ b/checkov/terraform/context_parsers/base_parser.py @@ -14,8 +14,7 @@ from checkov.common.comment.enum import COMMENT_REGEX from checkov.common.models.enums import ContextCategories from checkov.common.resource_code_logger_filter import add_resource_code_filter_to_logger -from checkov.common.typing import TFDefinitionKeyType -from checkov.common.util.parser_utils import get_abs_path +from checkov.terraform import TFDefinitionKey, get_abs_path from checkov.terraform.context_parsers.registry import parser_registry OPEN_CURLY = "{" @@ -155,7 +154,7 @@ def _compute_definition_end_line(self, start_line_num: int) -> int: return end_line_num def run( - self, tf_file: TFDefinitionKeyType, definition_blocks: List[Dict[str, Any]], collect_skip_comments: bool = True + self, tf_file: TFDefinitionKey, definition_blocks: List[Dict[str, Any]], collect_skip_comments: bool = True ) -> Dict[str, Any]: # TF files for loaded modules have this formation: [#] # Chop off everything after the file name for our purposes here diff --git a/checkov/terraform/context_parsers/registry.py b/checkov/terraform/context_parsers/registry.py index 27d403cf9ad..435922be70e 100644 --- a/checkov/terraform/context_parsers/registry.py +++ b/checkov/terraform/context_parsers/registry.py @@ -1,14 +1,11 @@ from __future__ import annotations import logging -import os from typing import Dict, TYPE_CHECKING, Tuple, List, Any import dpath from checkov.common.resource_code_logger_filter import add_resource_code_filter_to_logger -from checkov.common.runners.base_runner import strtobool -from checkov.common.typing import TFDefinitionKeyType from checkov.terraform.modules.module_objects import TFDefinitionKey if TYPE_CHECKING: @@ -17,7 +14,7 @@ class ParserRegistry: context_parsers: Dict[str, "BaseContextParser"] = {} # noqa: CCE003 - definitions_context: Dict[TFDefinitionKeyType, Dict[str, Dict[str, Any]]] = {} # noqa: CCE003 + definitions_context: Dict[TFDefinitionKey, Dict[str, Dict[str, Any]]] = {} # noqa: CCE003 def __init__(self) -> None: self.logger = logging.getLogger(__name__) @@ -30,25 +27,20 @@ def reset_definitions_context(self) -> None: self.definitions_context = {} def enrich_definitions_context( - self, definitions: Tuple[str, Dict[str, List[Dict[str, Any]]]], collect_skip_comments: bool = True - ) -> Dict[TFDefinitionKeyType, Dict[str, Dict[str, Any]]]: + self, definitions: Tuple[TFDefinitionKey, Dict[str, List[Dict[str, Any]]]], collect_skip_comments: bool = True + ) -> Dict[TFDefinitionKey, Dict[str, Dict[str, Any]]]: supported_definitions = [parser_type for parser_type in self.context_parsers.keys()] (tf_definition_key, definition_blocks_types) = definitions - enable_definition_key = strtobool(os.getenv('ENABLE_DEFINITION_KEY', 'False')) - if isinstance(tf_definition_key, TFDefinitionKey): - tf_file: TFDefinitionKeyType = tf_definition_key.file_path if not enable_definition_key else tf_definition_key - else: - tf_file = tf_definition_key if not enable_definition_key \ - else TFDefinitionKey(file_path=tf_definition_key, tf_source_modules=None) if definition_blocks_types: definition_blocks_types = {x: definition_blocks_types[x] for x in definition_blocks_types.keys()} for definition_type in definition_blocks_types.keys(): if definition_type in supported_definitions: - dpath.new(self.definitions_context, [tf_file, definition_type], {}) + dpath.new(self.definitions_context, [tf_definition_key, definition_type], {}) context_parser = self.context_parsers[definition_type] definition_blocks = definition_blocks_types[definition_type] - self.definitions_context[tf_file][definition_type] = context_parser.run(tf_file, definition_blocks, collect_skip_comments) + self.definitions_context[tf_definition_key][definition_type] = \ + context_parser.run(tf_definition_key, definition_blocks, collect_skip_comments) return self.definitions_context diff --git a/checkov/terraform/graph_builder/graph_components/blocks.py b/checkov/terraform/graph_builder/graph_components/blocks.py index 38462813e0e..f0bbfd4b6b3 100644 --- a/checkov/terraform/graph_builder/graph_components/blocks.py +++ b/checkov/terraform/graph_builder/graph_components/blocks.py @@ -1,13 +1,11 @@ from __future__ import annotations -import os from typing import Union, Dict, Any, List, Optional, Set, TYPE_CHECKING, cast import dpath import re from checkov.common.graph.graph_builder import CustomAttributes from checkov.common.graph.graph_builder.utils import calculate_hash -from checkov.common.runners.base_runner import strtobool from checkov.common.typing import TFDefinitionKeyType from checkov.terraform.graph_builder.utils import INTERPOLATION_EXPR from checkov.common.graph.graph_builder.graph_components.blocks import Block @@ -21,8 +19,6 @@ class TerraformBlock(Block): __slots__ = ( "module_connections", - "module_dependency", - "module_dependency_num", "source_module", "has_dynamic_block", "dynamic_attributes", @@ -62,8 +58,6 @@ def __init__( has_dynamic_block=has_dynamic_block, dynamic_attributes=dynamic_attributes, ) - self.module_dependency: TFDefinitionKeyType | None = "" - self.module_dependency_num: str | None = "" if path: self.path = path # type:ignore[assignment] # Block class would need to be a Generic type to make this pass if attributes.get(RESOLVED_MODULE_ENTRY_NAME): @@ -72,10 +66,9 @@ def __init__( self.module_connections: Dict[str, List[int]] = {} self.source_module: Set[int] = set() self.has_dynamic_block = has_dynamic_block - if strtobool(os.getenv('CHECKOV_NEW_TF_PARSER', 'True')): - self.source_module_object: Optional[TFModule] = None - self.for_each_index: Optional[Any] = None - self.foreach_attrs: list[str] | None = None + self.source_module_object: Optional[TFModule] = None + self.for_each_index: Optional[Any] = None + self.foreach_attrs: list[str] | None = None def __eq__(self, other: object) -> bool: if not isinstance(other, TerraformBlock): @@ -256,8 +249,6 @@ def to_dict(self) -> dict[str, Any]: 'config': self.config, 'id': self.id, 'module_connections': self.module_connections, - 'module_dependency': self.module_dependency, - 'module_dependency_num': self.module_dependency_num, 'name': self.name, 'path': self.path, 'source': self.source, @@ -274,7 +265,5 @@ def from_dict(data: dict[str, Any]) -> TerraformBlock: tf_block.breadcrumbs = data.get('breadcrumbs', {}) tf_block.module_connections = data.get('module_connections', {}) - tf_block.module_dependency = data.get('module_dependency', '') tf_block.source_module = data.get('source_module', set()) - tf_block.module_dependency_num = data.get('module_dependency_num', '') return tf_block diff --git a/checkov/terraform/graph_builder/graph_components/module.py b/checkov/terraform/graph_builder/graph_components/module.py index 817ef6b377e..162c3a8d6cc 100644 --- a/checkov/terraform/graph_builder/graph_components/module.py +++ b/checkov/terraform/graph_builder/graph_components/module.py @@ -2,36 +2,28 @@ import json import os -from typing import List, Dict, Any, Set, Callable, Tuple, TYPE_CHECKING, Optional, cast +from typing import List, Dict, Any, Set, Callable, Tuple, TYPE_CHECKING, cast -from checkov.common.runners.base_runner import strtobool +from checkov.common.typing import TFDefinitionKeyType from checkov.common.util.data_structures_utils import pickle_deepcopy -from checkov.common.util.parser_utils import get_abs_path, get_module_from_full_path from checkov.terraform.graph_builder.graph_components.block_types import BlockType from checkov.terraform.graph_builder.graph_components.blocks import TerraformBlock from checkov.terraform.parser_functions import handle_dynamic_values -from checkov.terraform.modules import TFDefinitionKey from hcl2 import START_LINE, END_LINE if TYPE_CHECKING: from typing_extensions import TypeAlias -_AddBlockTypeCallable: TypeAlias = "Callable[[Module, list[dict[str, dict[str, Any]]], str | TFDefinitionKey], None]" +_AddBlockTypeCallable: TypeAlias = "Callable[[Module, list[dict[str, dict[str, Any]]], TFDefinitionKeyType], None]" class Module: def __init__( self, source_dir: str, - module_address_map: Dict[Tuple[str, str], str], external_modules_source_map: Dict[Tuple[str, str], str], - module_dependency_map: Optional[Dict[str, List[List[str]]]] = None, - dep_index_mapping: Optional[Dict[Tuple[str, str], List[str]]] = None, ) -> None: # when adding a new field be sure to add it to the equality function below - self.dep_index_mapping = dep_index_mapping - self.module_dependency_map = module_dependency_map - self.module_address_map = module_address_map self.external_modules_source_map = external_modules_source_map self.path = "" self.blocks: List[TerraformBlock] = [] @@ -41,15 +33,12 @@ def __init__( self.resources_types: Set[str] = set() self.source_dir = source_dir self.render_dynamic_blocks_env_var = os.getenv('CHECKOV_RENDER_DYNAMIC_MODULES', 'True') - self.use_new_tf_parser = strtobool(os.getenv('CHECKOV_NEW_TF_PARSER', 'True')) def __eq__(self, other: object) -> bool: if not isinstance(other, Module): return False - return self.dep_index_mapping == other.dep_index_mapping and \ - self.module_dependency_map == other.module_dependency_map and self.module_address_map == other.module_address_map and \ - self.external_modules_source_map == other.external_modules_source_map and \ + return self.external_modules_source_map == other.external_modules_source_map and \ self.path == other.path and \ self.customer_name == other.customer_name and \ self.account_id == other.account_id and \ @@ -60,9 +49,6 @@ def __eq__(self, other: object) -> bool: def to_dict(self) -> dict[str, Any]: return { - 'dep_index_mapping': self.dep_index_mapping, - 'module_dependency_map': self.module_dependency_map, - 'module_address_map': self.module_address_map, 'external_modules_source_map': self.external_modules_source_map, 'path': self.path, 'customer_name': self.customer_name, @@ -71,17 +57,14 @@ def to_dict(self) -> dict[str, Any]: 'resources_types': self.resources_types, 'source_dir': self.source_dir, 'render_dynamic_blocks_env_var': self.render_dynamic_blocks_env_var, - 'use_new_tf_parser': self.use_new_tf_parser, 'blocks': [block.to_dict() for block in self.blocks] } @staticmethod def from_dict(module_dict: dict[str, Any]) -> Module: module = Module(source_dir=module_dict.get('source_dir', ''), - module_address_map=module_dict.get('module_address_map', {}), - external_modules_source_map=module_dict.get('external_modules_source_map', {}), - module_dependency_map=module_dict.get('module_dependency_map'), - dep_index_mapping=module_dict.get('dep_index_mapping')) + external_modules_source_map=module_dict.get('external_modules_source_map', {}) + ) module.blocks = [TerraformBlock.from_dict(block) for block in module_dict.get('blocks', [])] module.path = module_dict.get('path', '') module.customer_name = module_dict.get('customer_name', '') @@ -90,31 +73,26 @@ def from_dict(module_dict: dict[str, Any]) -> Module: module.resources_types = module_dict.get('resources_types', set()) module.source_dir = module_dict.get('source_dir', '') module.render_dynamic_blocks_env_var = module_dict.get('render_dynamic_blocks_env_var', '') - module.use_new_tf_parser = module_dict.get('use_new_tf_parser', False) return module def add_blocks( - self, block_type: str, blocks: List[Dict[str, Dict[str, Any]]], path: str | TFDefinitionKey, source: str + self, block_type: str, blocks: List[Dict[str, Dict[str, Any]]], path: TFDefinitionKeyType, source: str ) -> None: self.source = source if block_type in self._block_type_to_func: self._block_type_to_func[block_type](self, blocks, path) def _add_to_blocks(self, block: TerraformBlock) -> None: - if self.use_new_tf_parser: - if isinstance(block.path, str): - block.source_module_object = None - block.path = block.path - else: - block.source_module_object = block.path.tf_source_modules - block.path = block.path.file_path + if isinstance(block.path, str): + block.source_module_object = None + block.path = block.path else: - block.module_dependency, block.module_dependency_num = get_module_from_full_path(block.path) - block.path = get_abs_path(block.path) + block.source_module_object = block.path.tf_source_modules + block.path = block.path.file_path self.blocks.append(block) return - def _add_provider(self, blocks: List[Dict[str, Dict[str, Any]]], path: str | TFDefinitionKey) -> None: + def _add_provider(self, blocks: List[Dict[str, Dict[str, Any]]], path: TFDefinitionKeyType) -> None: for provider_dict in blocks: for name in provider_dict: attributes = provider_dict[name] @@ -135,7 +113,7 @@ def _add_provider(self, blocks: List[Dict[str, Dict[str, Any]]], path: str | TFD ) self._add_to_blocks(provider_block) - def _add_variable(self, blocks: List[Dict[str, Dict[str, Any]]], path: str | TFDefinitionKey) -> None: + def _add_variable(self, blocks: List[Dict[str, Dict[str, Any]]], path: TFDefinitionKeyType) -> None: for variable_dict in blocks: for name in variable_dict: attributes = variable_dict[name] @@ -149,7 +127,7 @@ def _add_variable(self, blocks: List[Dict[str, Dict[str, Any]]], path: str | TFD ) self._add_to_blocks(variable_block) - def _add_locals(self, blocks: List[Dict[str, Dict[str, Any]]], path: str | TFDefinitionKey) -> None: + def _add_locals(self, blocks: List[Dict[str, Dict[str, Any]]], path: TFDefinitionKeyType) -> None: for blocks_section in blocks: for name in blocks_section: if name in (START_LINE, END_LINE): @@ -166,7 +144,7 @@ def _add_locals(self, blocks: List[Dict[str, Dict[str, Any]]], path: str | TFDef ) self._add_to_blocks(local_block) - def _add_output(self, blocks: List[Dict[str, Dict[str, Any]]], path: str | TFDefinitionKey) -> None: + def _add_output(self, blocks: List[Dict[str, Dict[str, Any]]], path: TFDefinitionKeyType) -> None: for output_dict in blocks: for name, attributes in output_dict.items(): if isinstance(attributes, dict): @@ -180,7 +158,7 @@ def _add_output(self, blocks: List[Dict[str, Dict[str, Any]]], path: str | TFDef ) self._add_to_blocks(output_block) - def _add_module(self, blocks: List[Dict[str, Dict[str, Any]]], path: str | TFDefinitionKey) -> None: + def _add_module(self, blocks: List[Dict[str, Dict[str, Any]]], path: TFDefinitionKeyType) -> None: for module_dict in blocks: for name, attributes in module_dict.items(): if isinstance(attributes, dict): @@ -194,7 +172,7 @@ def _add_module(self, blocks: List[Dict[str, Dict[str, Any]]], path: str | TFDef ) self._add_to_blocks(module_block) - def _add_resource(self, blocks: List[Dict[str, Dict[str, Any]]], path: str | TFDefinitionKey) -> None: + def _add_resource(self, blocks: List[Dict[str, Dict[str, Any]]], path: TFDefinitionKeyType) -> None: for resource_dict in blocks: for resource_type, resources in resource_dict.items(): self.resources_types.add(resource_type) @@ -234,7 +212,7 @@ def clean_bad_characters(resource_conf: dict[str, Any]) -> dict[str, Any]: except json.JSONDecodeError: return resource_conf - def _add_data(self, blocks: List[Dict[str, Dict[str, Any]]], path: str | TFDefinitionKey) -> None: + def _add_data(self, blocks: List[Dict[str, Dict[str, Any]]], path: TFDefinitionKeyType) -> None: for data_dict in blocks: for data_type in data_dict: for name in data_dict[data_type]: @@ -250,7 +228,7 @@ def _add_data(self, blocks: List[Dict[str, Dict[str, Any]]], path: str | TFDefin ) self._add_to_blocks(data_block) - def _add_terraform_block(self, blocks: List[Dict[str, Dict[str, Any]]], path: str | TFDefinitionKey) -> None: + def _add_terraform_block(self, blocks: List[Dict[str, Dict[str, Any]]], path: TFDefinitionKeyType) -> None: for terraform_dict in blocks: terraform_block = TerraformBlock( block_type=BlockType.TERRAFORM, @@ -262,7 +240,7 @@ def _add_terraform_block(self, blocks: List[Dict[str, Dict[str, Any]]], path: st ) self._add_to_blocks(terraform_block) - def _add_tf_var(self, blocks: list[Dict[str, Dict[str, Any]]], path: str | TFDefinitionKey) -> None: + def _add_tf_var(self, blocks: list[Dict[str, Dict[str, Any]]], path: TFDefinitionKeyType) -> None: for block in blocks: for tf_var_name, attributes in block.items(): tfvar_block = TerraformBlock( diff --git a/checkov/terraform/graph_builder/graph_to_tf_definitions.py b/checkov/terraform/graph_builder/graph_to_tf_definitions.py index cbd6719356d..5d1fe323de4 100644 --- a/checkov/terraform/graph_builder/graph_to_tf_definitions.py +++ b/checkov/terraform/graph_builder/graph_to_tf_definitions.py @@ -4,9 +4,6 @@ from typing import List, Dict, Any, Tuple from checkov.common.graph.graph_builder import CustomAttributes -from checkov.common.runners.base_runner import strtobool -from checkov.common.typing import TFDefinitionKeyType -from checkov.common.util.parser_utils import get_tf_definition_key_from_module_dependency from checkov.terraform.modules.module_objects import TFDefinitionKey from checkov.terraform.graph_builder.graph_components.block_types import BlockType from checkov.terraform.graph_builder.graph_components.blocks import TerraformBlock @@ -14,9 +11,8 @@ def convert_graph_vertices_to_tf_definitions( vertices: List[TerraformBlock], root_folder: str -) -> Tuple[Dict[TFDefinitionKeyType, Dict[str, Any]], Dict[str, Dict[str, Any]]]: - use_new_tf_parser = strtobool(os.getenv('CHECKOV_NEW_TF_PARSER', 'True')) - tf_definitions: Dict[TFDefinitionKeyType, Dict[str, Any]] = {} +) -> Tuple[Dict[TFDefinitionKey, Dict[str, Any]], Dict[str, Dict[str, Any]]]: + tf_definitions: Dict[TFDefinitionKey, Dict[str, Any]] = {} breadcrumbs: Dict[str, Dict[str, Any]] = {} for vertex in vertices: block_path = vertex.path @@ -27,13 +23,9 @@ def convert_graph_vertices_to_tf_definitions( if block_type == BlockType.TF_VARIABLE: continue - tf_path: TFDefinitionKeyType = TFDefinitionKey(file_path=block_path) if use_new_tf_parser else block_path - if vertex.module_dependency or hasattr(vertex, "source_module_object") and vertex.source_module_object: - if use_new_tf_parser: - tf_path = TFDefinitionKey(file_path=block_path, tf_source_modules=vertex.source_module_object) - else: - module_dependency = str(vertex.module_dependency) if vertex.module_dependency else None - tf_path = get_tf_definition_key_from_module_dependency(block_path, module_dependency, vertex.module_dependency_num) + tf_path = TFDefinitionKey(file_path=block_path) + if vertex.source_module_object: + tf_path = TFDefinitionKey(file_path=block_path, tf_source_modules=vertex.source_module_object) tf_definitions.setdefault(tf_path, {}).setdefault(block_type, []).append(vertex.config) relative_block_path = f"/{os.path.relpath(block_path, root_folder)}" add_breadcrumbs(vertex, breadcrumbs, relative_block_path) diff --git a/checkov/terraform/graph_builder/local_graph.py b/checkov/terraform/graph_builder/local_graph.py index 5d4dc76f1e5..0ce7ae8b635 100644 --- a/checkov/terraform/graph_builder/local_graph.py +++ b/checkov/terraform/graph_builder/local_graph.py @@ -5,7 +5,7 @@ from collections import defaultdict from functools import partial from pathlib import Path -from typing import List, Optional, Union, Any, Dict, Set, Tuple, overload, TYPE_CHECKING +from typing import List, Optional, Union, Any, Dict, overload from typing_extensions import TypedDict @@ -17,12 +17,10 @@ from checkov.common.graph.graph_builder.utils import calculate_hash, join_trimmed_strings, filter_sub_keys from checkov.common.runners.base_runner import strtobool from checkov.common.util.data_structures_utils import pickle_deepcopy -from checkov.common.util.parser_utils import get_abs_path, get_tf_definition_key_from_module_dependency from checkov.common.util.type_forcers import force_int from checkov.terraform.graph_builder.foreach.builder import ForeachBuilder from checkov.terraform.graph_builder.variable_rendering.vertex_reference import TerraformVertexReference from checkov.terraform.modules.module_objects import TFModule -from checkov.terraform.checks.utils.dependency_path_handler import unify_dependency_path from checkov.terraform.context_parsers.registry import parser_registry from checkov.terraform.graph_builder.graph_components.block_types import BlockType from checkov.terraform.graph_builder.graph_components.blocks import TerraformBlock @@ -36,8 +34,6 @@ from checkov.terraform.graph_builder.utils import is_local_path from checkov.terraform.graph_builder.variable_rendering.renderer import TerraformVariableRenderer -if TYPE_CHECKING: - from checkov.common.typing import TFDefinitionKeyType MODULE_RESERVED_ATTRIBUTES = ("source", "version") CROSS_VARIABLE_EDGE_PREFIX = '[cross-variable] ' @@ -62,7 +58,6 @@ def __init__(self, module: Module) -> None: self.vertices_by_module_dependency: Dict[TFModule | None, Dict[str, List[int]]] = defaultdict(partial(defaultdict, list)) # type:ignore[arg-type] self.enable_foreach_handling = strtobool(os.getenv('CHECKOV_ENABLE_FOREACH_HANDLING', 'True')) self.enable_modules_foreach_handling = strtobool(os.getenv('CHECKOV_ENABLE_MODULES_FOREACH_HANDLING', 'True')) - self.use_new_tf_parser = strtobool(os.getenv('CHECKOV_NEW_TF_PARSER', 'True')) self.foreach_blocks: Dict[str, List[int]] = {BlockType.RESOURCE: [], BlockType.MODULE: []} def build_graph(self, render_variables: bool) -> None: @@ -121,13 +116,8 @@ def _add_block_data_to_graph(self, idx: int, block: TerraformBlock) -> None: # map between file paths and module vertices indexes from that file self.map_path_to_module.setdefault(block.path, []).append(idx) - if self.use_new_tf_parser: - self.vertices_by_module_dependency[block.source_module_object][block.block_type].append(idx) - self.vertices_by_module_dependency_by_name[block.source_module_object][block.block_type][block.name].append(idx) - else: - # mypy: fixed, when the old parser is removed - self.vertices_by_module_dependency[(block.module_dependency, block.module_dependency_num)][block.block_type].append(idx) # type:ignore[index] - self.vertices_by_module_dependency_by_name[(block.module_dependency, block.module_dependency_num)][block.block_type][block.name].append(idx) # type:ignore[index] + self.vertices_by_module_dependency[block.source_module_object][block.block_type].append(idx) + self.vertices_by_module_dependency_by_name[block.source_module_object][block.block_type][block.name].append(idx) self.in_edges[idx] = [] self.out_edges[idx] = [] @@ -162,61 +152,21 @@ def get_module_vertices_mapping(self) -> None: For each vertex, if it's originated in a module import, add to the vertex the index of the matching module vertex as 'source_module' """ - if self.use_new_tf_parser: - for vertex in self.vertices: - if not vertex.source_module_object: + for vertex in self.vertices: + if not vertex.source_module_object: + continue + for idx in self.vertices_by_block_type[BlockType.MODULE]: + if vertex.source_module_object.name != self.vertices[idx].name: continue - for idx in self.vertices_by_block_type[BlockType.MODULE]: - if vertex.source_module_object.name != self.vertices[idx].name: - continue - if vertex.source_module_object.path != self.vertices[idx].path: - continue - if vertex.source_module_object.nested_tf_module != self.vertices[idx].source_module_object: - continue - if vertex.source_module_object.foreach_idx != self.vertices[idx].for_each_index: - continue - vertex.source_module.add(idx) - break - return - - if not self.module.module_dependency_map: - # no need to proceed further - return - - block_dirs_to_modules: Dict[Tuple[str, str | None], Dict[str, Set[int]]] = defaultdict(dict) - for dir_name, paths_to_modules in self.module.module_dependency_map.items(): - # for each directory, find the module vertex that imported it - for path_to_module in paths_to_modules: - if not path_to_module: + if vertex.source_module_object.path != self.vertices[idx].path: continue - path_to_module_str = unify_dependency_path(path_to_module) - if block_dirs_to_modules.get((dir_name, path_to_module_str)): + if vertex.source_module_object.nested_tf_module != self.vertices[idx].source_module_object: continue - module_file = get_abs_path(path_to_module[-1]) - module_list = self.map_path_to_module.get(module_file, []) - for module_index in module_list: - module_vertex = self.vertices[module_index] - if get_path_with_nested_modules(module_vertex) == path_to_module_str: - module_vertex_dir = self.get_dirname(module_vertex.path) - module_source = module_vertex.attributes.get("source", [""])[0] - module_version = module_vertex.attributes.get("version", ["latest"])[0] - dest_module_path = self._get_dest_module_path( - curr_module_dir=module_vertex_dir, - dest_module_source=module_source, - dest_module_version=module_version - ) - if dest_module_path == dir_name: - module_dependency_num = self.module.module_address_map.get((module_vertex.path, module_vertex.name)) - if module_dependency_num: - block_dirs_to_modules[(dir_name, path_to_module_str)].setdefault(module_dependency_num, set()).add(module_index) - - for vertex in self.vertices: - # match the right module vertex according to the vertex path directory - module_dependency_nums = block_dirs_to_modules.get((self.get_dirname(vertex.path), vertex.module_dependency)) # type:ignore[arg-type] # will be fixed when removing terraform/checks from mypy exclusion - if module_dependency_nums: - module_indices = module_dependency_nums.get(vertex.module_dependency_num) # type:ignore[arg-type] # vertex.module_dependency_num can be None, which is ok - if module_indices: - vertex.source_module = module_indices + if vertex.source_module_object.foreach_idx != self.vertices[idx].for_each_index: + continue + vertex.source_module.add(idx) + break + return def _build_edges(self) -> None: logging.info("Creating edges") @@ -247,41 +197,35 @@ def _build_edges_for_vertex(self, origin_node_index: int, vertex: TerraformBlock sub_values = [remove_index_pattern_from_str(sub_value) for sub_value in vertex_reference.sub_parts] for i in range(len(sub_values)): reference_name = join_trimmed_strings(char_to_join=".", str_lst=sub_values, num_to_trim=i) - source_module_object = None - if self.use_new_tf_parser: - source_module_object = vertex.source_module_object + source_module_object = vertex.source_module_object if referenced_modules is not None: for module in referenced_modules: referenced_module_idx = module.get("idx") referenced_module_path = module.get("path") - referenced_module_object = module.get("source_module_object") - if not self.use_new_tf_parser: - source_module_object = referenced_module_object if source_module_object else None if referenced_module_path is None: dest_node_index = -1 else: dest_node_index = self._find_vertex_index_relative_to_path( vertex_reference.block_type, reference_name, referenced_module_path, - vertex.module_dependency, - vertex.module_dependency_num, referenced_module_idx, + referenced_module_idx, source_module_object=source_module_object ) self._create_edge_from_reference(attribute_key, origin_node_index, dest_node_index, sub_values, vertex_reference, cross_variable_edges) - if vertex.module_dependency or hasattr(vertex, "source_module_object"): + if vertex.source_module_object: dest_node_index = self._find_vertex_index_relative_to_path( - vertex_reference.block_type, reference_name, vertex.path, vertex.module_dependency, - vertex.module_dependency_num, source_module_object=source_module_object + vertex_reference.block_type, reference_name, vertex.path, + source_module_object=source_module_object ) if dest_node_index == -1: dest_node_index = self._find_vertex_index_relative_to_path( - vertex_reference.block_type, reference_name, vertex.path, vertex.path, - vertex.module_dependency_num, source_module_object=source_module_object + vertex_reference.block_type, reference_name, vertex.path, + source_module_object=source_module_object ) else: dest_node_index = self._find_vertex_index_relative_to_path( - vertex_reference.block_type, reference_name, vertex.path, vertex.module_dependency, - vertex.module_dependency_num, source_module_object=source_module_object + vertex_reference.block_type, reference_name, vertex.path, + source_module_object=source_module_object ) if dest_node_index > -1 and origin_node_index > -1: self._create_edge_from_reference(attribute_key, origin_node_index, dest_node_index, sub_values, @@ -332,22 +276,12 @@ def _create_edge_from_reference(self, attribute_key: Any, origin_node_index: int cross_variable_edges) def _get_target_variables(self, vertex: TerraformBlock, dest_module_path: str) -> list[int]: - if self.use_new_tf_parser: - target_path = get_vertex_as_tf_module(vertex) - return [ - index - for index in self.vertices_by_module_dependency.get(target_path, {}).get(BlockType.VARIABLE, []) - if self.get_dirname(self.vertices[index].path) == dest_module_path - ] - else: - target_path_old = get_path_with_nested_modules(vertex) - return [ - index - for index in self.vertices_by_module_dependency.get( # type:ignore[call-overload] # fixed, when the old parser is removed - (target_path_old, self.module.module_address_map.get((vertex.path, vertex.name))), {}).get( - BlockType.VARIABLE, []) - if self.get_dirname(self.vertices[index].path) == dest_module_path - ] + target_path = get_vertex_as_tf_module(vertex) + return [ + index + for index in self.vertices_by_module_dependency.get(target_path, {}).get(BlockType.VARIABLE, []) + if self.get_dirname(self.vertices[index].path) == dest_module_path + ] def _build_cross_variable_edges(self) -> None: aliases = self._get_aliases() @@ -436,25 +370,15 @@ def _find_vertex_index_relative_to_path( block_type: str, name: str, block_path: str, - module_path: TFDefinitionKeyType | None, - module_num: str | None, relative_module_idx: Optional[int] = None, source_module_object: Optional[TFModule] = None, ) -> int: relative_vertices: list[int] = [] - if self.use_new_tf_parser: - if relative_module_idx is None: - module_dependency_by_name_key = source_module_object - else: - vertex = self.vertices[relative_module_idx] - module_dependency_by_name_key = vertex.source_module_object - elif relative_module_idx is not None: - # This part of the code is very inefficient for large graphs - # It's better to avoid using it by setting `self.use_new_tf_parser` - module_dependency_by_name_key = next(k for k, v in self.vertices_by_module_dependency.items() if - v.get(BlockType.MODULE, []).__contains__(relative_module_idx)) + if relative_module_idx is None: + module_dependency_by_name_key = source_module_object else: - module_dependency_by_name_key = (module_path, module_num) # type:ignore[assignment] # fixed, when the old parser is removed + vertex = self.vertices[relative_module_idx] + module_dependency_by_name_key = vertex.source_module_object # important to use this specific map for big graph performance possible_vertices = self.vertices_by_module_dependency_by_name.get(module_dependency_by_name_key, {}).get(block_type, {}).get(name, []) @@ -650,24 +574,15 @@ def _update_nested_modules_address(self) -> None: vertex_context[CustomAttributes.TF_RESOURCE_ADDRESS] = address def _should_add_edge(self, vertex: TerraformBlock, dest_module_path: str, module_node: TerraformBlock) -> bool: - if self.use_new_tf_parser: - if not vertex.source_module_object: - return False + if not vertex.source_module_object: + return False - return (self.get_dirname(vertex.path) == dest_module_path) and \ - ( - vertex.source_module_object == module_node.source_module_object # The vertex is in the same file - or self.get_abspath(vertex.source_module_object.path) - == self.get_abspath(module_node.path) # The vertex is in the correct dependency path) - ) - else: - return (self.get_dirname(vertex.path) == dest_module_path) and ( - vertex.module_dependency == module_node.module_dependency # The vertex is in the same file - or ( - vertex.module_dependency is not None - and self.get_abspath(vertex.module_dependency) == self.get_abspath(module_node.path) # type:ignore[arg-type] # old flow, will be removed - ) # The vertex is in the correct dependency path - ) + return (self.get_dirname(vertex.path) == dest_module_path) and \ + ( + vertex.source_module_object == module_node.source_module_object # The vertex is in the same file + or self.get_abspath(vertex.source_module_object.path) + == self.get_abspath(module_node.path) # The vertex is in the correct dependency path) + ) def to_list(data: Any) -> list[Any] | dict[str, Any]: @@ -756,11 +671,5 @@ def update_list_attribute( return config -def get_path_with_nested_modules(block: TerraformBlock) -> str: - if not block.module_dependency: - return block.path - return get_tf_definition_key_from_module_dependency(block.path, block.module_dependency, block.module_dependency_num) # type:ignore[arg-type] # will be fixed when removing terraform/checks from mypy exclusion - - def get_vertex_as_tf_module(block: TerraformBlock) -> TFModule: return TFModule(block.path, block.name, block.source_module_object) diff --git a/checkov/terraform/graph_manager.py b/checkov/terraform/graph_manager.py index 9f8e5136327..f4fd9c9f65a 100644 --- a/checkov/terraform/graph_manager.py +++ b/checkov/terraform/graph_manager.py @@ -1,26 +1,24 @@ from __future__ import annotations import logging -import os from typing import Type, Any, TYPE_CHECKING -from checkov.common.runners.base_runner import strtobool from checkov.common.util.consts import DEFAULT_EXTERNAL_MODULES_DIR from checkov.terraform.graph_builder.local_graph import TerraformLocalGraph -from checkov.terraform.parser import Parser from checkov.common.graph.graph_manager import GraphManager from checkov.terraform.tf_parser import TFParser if TYPE_CHECKING: + from checkov.terraform.modules.module_objects import TFDefinitionKey from checkov.common.typing import LibraryGraphConnector -class TerraformGraphManager(GraphManager[TerraformLocalGraph, "dict[str, dict[str, Any]]"]): +class TerraformGraphManager(GraphManager[TerraformLocalGraph, "dict[TFDefinitionKey, dict[str, Any]]"]): def __init__(self, db_connector: LibraryGraphConnector, source: str = "") -> None: - self.parser: Parser # just to make sure it won't be None + self.parser: TFParser # just to make sure it won't be None - parser = TFParser() if strtobool(os.getenv('CHECKOV_NEW_TF_PARSER', 'True')) else Parser() + parser = TFParser() super().__init__(db_connector=db_connector, parser=parser, source=source) def build_multi_graph_from_source_directory( @@ -34,7 +32,7 @@ def build_multi_graph_from_source_directory( external_modules_download_path: str = DEFAULT_EXTERNAL_MODULES_DIR, vars_files: list[str] | None = None, create_graph: bool = True, - ) -> list[tuple[TerraformLocalGraph | None, dict[str, dict[str, Any]]]]: + ) -> list[tuple[TerraformLocalGraph | None, list[dict[TFDefinitionKey, dict[str, Any]]]]]: logging.info("Parsing HCL files in source dir to multi graph") modules_with_definitions = self.parser.parse_multi_graph_hcl_module( source_dir=source_dir, @@ -47,7 +45,7 @@ def build_multi_graph_from_source_directory( create_graph=create_graph, ) - graphs: list[tuple[TerraformLocalGraph | None, dict[str, dict[str, Any]]]] = [] + graphs: list[tuple[TerraformLocalGraph | None, list[dict[TFDefinitionKey, dict[str, Any]]]]] = [] for module, tf_definitions in modules_with_definitions: if create_graph and module: logging.info("Building graph from parsed module") @@ -68,7 +66,7 @@ def build_graph_from_source_directory( external_modules_download_path: str = DEFAULT_EXTERNAL_MODULES_DIR, vars_files: list[str] | None = None, create_graph: bool = True, - ) -> tuple[TerraformLocalGraph | None, dict[str, dict[str, Any]]]: + ) -> tuple[TerraformLocalGraph | None, dict[TFDefinitionKey, dict[str, Any]]]: logging.info("Parsing HCL files in source dir to graph") module, tf_definitions = self.parser.parse_hcl_module( source_dir=source_dir, @@ -89,20 +87,22 @@ def build_graph_from_source_directory( return local_graph, tf_definitions - def build_graph_from_definitions(self, definitions: dict[str, dict[str, Any]], render_variables: bool = True) -> TerraformLocalGraph: + def build_graph_from_definitions(self, definitions: dict[TFDefinitionKey, dict[str, Any]], + render_variables: bool = True) -> TerraformLocalGraph: module, _ = self.parser.parse_hcl_module_from_tf_definitions(definitions, "", self.source) local_graph = TerraformLocalGraph(module) local_graph.build_graph(render_variables=render_variables) return local_graph - def build_multi_graph_from_definitions(self, definitions: dict[str, dict[str, Any]], render_variables: bool = True) -> list[TerraformLocalGraph]: + def build_multi_graph_from_definitions(self, definitions: dict[TFDefinitionKey, dict[str, Any]], + render_variables: bool = True) -> list[TerraformLocalGraph]: module, tf_definitions = self.parser.parse_hcl_module_from_tf_definitions(definitions, "", self.source) dirs_to_definitions = self.parser.create_definition_by_dirs(tf_definitions) graphs: list[TerraformLocalGraph] = [] - for source_path, definitions in dirs_to_definitions.items(): - module, parsed_tf_definitions = self.parser.parse_hcl_module_from_multi_tf_definitions(definitions, source_path, self.source) + for source_path, dir_definitions in dirs_to_definitions.items(): + module, parsed_tf_definitions = self.parser.parse_hcl_module_from_multi_tf_definitions(dir_definitions, source_path, self.source) local_graph = TerraformLocalGraph(module) local_graph.build_graph(render_variables=render_variables) graphs.append(local_graph) diff --git a/checkov/terraform/modules/module_utils.py b/checkov/terraform/modules/module_utils.py index 481b7a0bb08..5e8a3791d41 100644 --- a/checkov/terraform/modules/module_utils.py +++ b/checkov/terraform/modules/module_utils.py @@ -1,25 +1,22 @@ from __future__ import annotations -import itertools import json import logging import os -from collections import defaultdict from collections.abc import Sequence -from pathlib import Path -from typing import Any, Optional, TYPE_CHECKING, TypeVar, cast +from typing import Any, TYPE_CHECKING, TypeVar, cast, Tuple -import hcl2 from lark import Tree import re +from checkov.common.typing import TFDefinitionKeyType from checkov.common.util.consts import DEFAULT_EXTERNAL_MODULES_DIR from checkov.common.util.data_structures_utils import pickle_deepcopy from checkov.common.util.json_utils import CustomJSONEncoder, object_hook +from checkov.terraform.modules.module_objects import TFDefinitionKey from checkov.terraform.checks.utils.dependency_path_handler import unify_dependency_path from checkov.terraform.graph_builder.utils import remove_module_dependency_in_path -from checkov.common.util.parser_utils import TERRAFORM_NESTED_MODULE_PATH_PREFIX, TERRAFORM_NESTED_MODULE_PATH_ENDING, \ - is_nested, get_abs_path, get_module_from_full_path +from checkov.common.util.parser_utils import TERRAFORM_NESTED_MODULE_PATH_PREFIX, TERRAFORM_NESTED_MODULE_PATH_ENDING if TYPE_CHECKING: from typing_extensions import TypeAlias @@ -53,36 +50,6 @@ def validate_malformed_definitions(raw_data: _Hcl2Payload) -> _Hcl2Payload: } -def load_or_die_quietly( - file: str | Path | os.DirEntry[str], parsing_errors: dict[str, Exception], clean_definitions: bool = True -) -> Optional[_Hcl2Payload]: - """ -Load JSON or HCL, depending on filename. - :return: None if the file can't be loaded - """ - - file_path = os.fspath(file) - file_name = os.path.basename(file_path) - - try: - logging.debug(f"Parsing {file_path}") - - with open(file_path, "r", encoding="utf-8-sig") as f: - if file_name.endswith(".json"): - return cast("_Hcl2Payload", json.load(f)) - else: - raw_data = hcl2.load(f) - non_malformed_definitions = validate_malformed_definitions(raw_data) - if clean_definitions: - return clean_bad_definitions(non_malformed_definitions) - else: - return non_malformed_definitions - except Exception as e: - logging.debug(f'failed while parsing file {file_path}', exc_info=True) - parsing_errors[file_path] = e - return None - - def clean_bad_definitions(tf_definition_list: _Hcl2Payload) -> _Hcl2Payload: return { block_type: [ @@ -204,43 +171,6 @@ def get_next_vertices(evaluated_files: list[str], unevaluated_files: list[str]) return next_level, unevaluated -def get_module_dependency_map_support_nested_modules( - tf_definitions: dict[str, Any] -) -> tuple[dict[str, list[list[str]]], dict[str, Any], dict[tuple[str, str | None], Any]]: - module_dependency_map: dict[str, list[list[str]]] = defaultdict(list) - dep_index_mapping = defaultdict(list) - for tf_definition_key in tf_definitions.keys(): - if not is_nested(tf_definition_key): - dir_name = os.path.dirname(tf_definition_key) - module_dependency_map[dir_name].append([]) - continue - modules_list, path = get_nested_modules_data_as_list(tf_definition_key) - dir_name = os.path.dirname(path) - module_dependency_map[dir_name].append([m for m, i in modules_list if m]) - dep_index_mapping[(path, modules_list[-1][0])].append(modules_list[-1][1]) - - for key, dir_list in module_dependency_map.items(): - dir_list.sort() - module_dependency_map[key] = list(dir_list for dir_list, _ in itertools.groupby(dir_list)) - return module_dependency_map, tf_definitions, dep_index_mapping - - -def get_nested_modules_data_as_list(file_path: str) -> tuple[list[tuple[str | None, str | None]], str]: - from checkov.terraform.modules.module_objects import TFDefinitionKey - path = get_abs_path(file_path) - module_path: str | None = file_path - modules_list = [] - - while is_nested(module_path): - module, index = get_module_from_full_path(module_path) - if isinstance(module, TFDefinitionKey): - module = module.file_path - modules_list.append((module, index)) - module_path = module - modules_list.reverse() - return modules_list, path - - def clean_parser_types(conf: _Conf) -> _Conf: if not conf: return conf @@ -298,3 +228,30 @@ def clean_parser_types_lst(values: list[Any]) -> list[Any]: def serialize_definitions(tf_definitions: _Conf) -> _Conf: return cast("_Conf", json.loads(json.dumps(tf_definitions, cls=CustomJSONEncoder), object_hook=object_hook)) + + +def get_module_from_full_path(file_path: TFDefinitionKey | None) -> Tuple[TFDefinitionKey | None, None]: + if not file_path or not is_nested(file_path): + return None, None + if file_path.tf_source_modules is None: + return None, None + return TFDefinitionKey(file_path=file_path.tf_source_modules.path, tf_source_modules=file_path.tf_source_modules.nested_tf_module), None + + +def get_module_name(file_path: TFDefinitionKey) -> str | None: + if not file_path.tf_source_modules: + return None + module_name = file_path.tf_source_modules.name + if file_path.tf_source_modules.foreach_idx: + foreach_or_count = '"' if isinstance(file_path.tf_source_modules.foreach_idx, str) else '' + module_name = f'{module_name}[{foreach_or_count}{file_path.tf_source_modules.foreach_idx}{foreach_or_count}]' + return module_name + + +def is_nested(full_path: TFDefinitionKey | None) -> bool: + return full_path.tf_source_modules is not None if full_path is not None else False + + +def get_abs_path(file_path: TFDefinitionKeyType) -> str: + # file_path might be str for terraform-plan + return file_path if isinstance(file_path, str) else str(file_path.file_path) diff --git a/checkov/terraform/parser.py b/checkov/terraform/parser.py deleted file mode 100644 index 603b0b25ce0..00000000000 --- a/checkov/terraform/parser.py +++ /dev/null @@ -1,668 +0,0 @@ -from __future__ import annotations - -import logging -import os -from typing import Optional, Dict, Mapping, Set, Tuple, Callable, Any, List, TYPE_CHECKING - -import deep_merge - -from checkov.common.runners.base_runner import filter_ignored_paths, IGNORE_HIDDEN_DIRECTORY_ENV, strtobool -from checkov.common.util.consts import DEFAULT_EXTERNAL_MODULES_DIR, RESOLVED_MODULE_ENTRY_NAME -from checkov.common.util.data_structures_utils import pickle_deepcopy -from checkov.common.util.type_forcers import force_list -from checkov.common.variables.context import EvaluationContext -from checkov.terraform.graph_builder.graph_components.block_types import BlockType -from checkov.terraform.graph_builder.graph_components.module import Module -from checkov.terraform.module_loading.content import ModuleContent -from checkov.terraform.module_loading.module_finder import load_tf_modules -from checkov.terraform.module_loading.registry import module_loader_registry as default_ml_registry, \ - ModuleLoaderRegistry -from checkov.common.util.parser_utils import get_tf_definition_key_from_module_dependency, \ - TERRAFORM_NESTED_MODULE_PATH_ENDING, is_acceptable_module_param -from checkov.terraform.modules.module_utils import load_or_die_quietly, safe_index, \ - remove_module_dependency_from_path, get_module_dependency_map, get_module_dependency_map_support_nested_modules, \ - clean_parser_types, serialize_definitions - -if TYPE_CHECKING: - from checkov.terraform import TFDefinitionKey - - -def _filter_ignored_paths(root: str, paths: list[str], excluded_paths: list[str] | None) -> None: - filter_ignored_paths(root, paths, excluded_paths) - for path in force_list(paths): - if path == default_ml_registry.external_modules_folder_name: - paths.remove(path) - - -class Parser: - def __init__(self, module_class: type[Module] = Module) -> None: - self.module_class = module_class - self._parsed_directories: set[str] = set() - self.external_modules_source_map: Dict[Tuple[str, str], str] = {} - self.module_address_map: Dict[Tuple[str, str], str] = {} - self.loaded_files_map = {} - - # This ensures that we don't try to double-load modules - # Tuple is , , (see _load_modules) - self._loaded_modules: Set[Tuple[str, int, str]] = set() - self.external_variables_data = [] - self.enable_nested_modules = strtobool(os.getenv('CHECKOV_ENABLE_NESTED_MODULES', 'True')) - - def _init(self, directory: str, out_definitions: Optional[Dict], - out_evaluations_context: Dict[str, Dict[str, EvaluationContext]], - out_parsing_errors: Dict[str, Exception], - env_vars: Mapping[str, str], - download_external_modules: bool, - external_modules_download_path: str, - excluded_paths: Optional[List[str]] = None, - tf_var_files: Optional[List[str]] = None): - self.directory = directory - self.out_definitions = out_definitions - self.out_evaluations_context = out_evaluations_context - self.out_parsing_errors = out_parsing_errors - self.env_vars = env_vars - self.download_external_modules = download_external_modules - self.external_modules_download_path = external_modules_download_path - self.external_modules_source_map = {} - self.module_address_map = {} - self.tf_var_files = tf_var_files - self.dirname_cache = {} - - if self.out_evaluations_context is None: - self.out_evaluations_context = {} - if self.out_parsing_errors is None: - self.out_parsing_errors = {} - if self.env_vars is None: - self.env_vars = dict(os.environ) - self.excluded_paths = excluded_paths - self.visited_definition_keys = set() - self.module_to_resolved = {} - self.keys_to_remove = set() - - def _check_process_dir(self, directory: str) -> bool: - if directory not in self._parsed_directories: - self._parsed_directories.add(directory) - return True - else: - return False - - def parse_directory(self, directory: str, out_definitions: Optional[Dict], - out_evaluations_context: Dict[str, Dict[str, EvaluationContext]] = None, - out_parsing_errors: Dict[str, Exception] = None, - env_vars: Mapping[str, str] = None, - download_external_modules: bool = False, - external_modules_download_path: str = DEFAULT_EXTERNAL_MODULES_DIR, - excluded_paths: Optional[List[str]] = None, - vars_files: Optional[List[str]] = None, - external_modules_content_cache: Optional[Dict[str, ModuleContent]] = None): - self._init(directory, out_definitions, out_evaluations_context, out_parsing_errors, env_vars, - download_external_modules, external_modules_download_path, excluded_paths) - self._parsed_directories.clear() - default_ml_registry.root_dir = directory - default_ml_registry.download_external_modules = download_external_modules - default_ml_registry.external_modules_folder_name = external_modules_download_path - default_ml_registry.module_content_cache = external_modules_content_cache if external_modules_content_cache else {} - load_tf_modules(directory) - self._parse_directory(dir_filter=lambda d: self._check_process_dir(d), vars_files=vars_files) - if self.enable_nested_modules: - self._update_resolved_modules() - - def parse_file(self, file: str, parsing_errors: Optional[Dict[str, Exception]] = None) -> Optional[Dict[str, Any]]: - if file.endswith(".tf") or file.endswith(".tf.json") or file.endswith(".hcl"): - parse_result = load_or_die_quietly(file, parsing_errors) - if parse_result: - parse_result = serialize_definitions(parse_result) - parse_result = clean_parser_types(parse_result) - return parse_result - - return None - - def _parse_directory(self, include_sub_dirs: bool = True, - module_loader_registry: ModuleLoaderRegistry = default_ml_registry, - dir_filter: Callable[[str], bool] = lambda _: True, - vars_files: Optional[List[str]] = None) -> None: - """ - Load and resolve configuration files starting in the given directory, merging the - resulting data into `tf_definitions`. This loads data according to the Terraform Code Organization - specification (https://www.terraform.io/docs/configuration/index.html#code-organization), starting - in the given directory and possibly moving out from there. - - The resulting data dictionary generally follows the layout of HCL parsing with a couple distinctions: - - Data is broken out by file from which the data was loaded. So: : - - Loaded modules will also be keyed by referrer info: [#]: - - Module block will included a "__resolved__" key with a list of the file/referrer names under - which data for the file was loaded. For example: "__resolved__": ["main.tf#0"]. The values will - correspond to the file names mentioned in the first bullet. - - All variables that can be resolved will be resolved. - - - :param include_sub_dirs: If true, subdirectories will be walked. - - :param module_loader_registry: Registry used for resolving modules. This allows customization of how - much resolution is performed (and easier testing) by using a manually - constructed registry rather than the default. - :param dir_filter: Determines whether or not a directory should be processed. Returning - True will allow processing. The argument will be the absolute path of - the directory. - """ - keys_referenced_as_modules: Set[str] = set() - - if include_sub_dirs: - for sub_dir, d_names, _ in os.walk(self.directory): - # filter subdirectories for future iterations (we filter files while iterating the directory) - _filter_ignored_paths(sub_dir, d_names, self.excluded_paths) - if dir_filter(os.path.abspath(sub_dir)): - self._internal_dir_load(sub_dir, module_loader_registry, dir_filter, - keys_referenced_as_modules, vars_files=vars_files, - root_dir=self.directory, excluded_paths=self.excluded_paths) - else: - self._internal_dir_load(self.directory, module_loader_registry, dir_filter, - keys_referenced_as_modules, vars_files=vars_files) - - # Ensure anything that was referenced as a module is removed - for key in keys_referenced_as_modules: - if key in self.out_definitions: - del self.out_definitions[key] - - def _internal_dir_load(self, directory: str, - module_loader_registry: ModuleLoaderRegistry, - dir_filter: Callable[[str], bool], - keys_referenced_as_modules: Set[str], - specified_vars: Optional[Mapping[str, str]] = None, - vars_files: Optional[List[str]] = None, - root_dir: Optional[str] = None, - excluded_paths: Optional[List[str]] = None, - nested_modules_data=None): - """ - See `parse_directory` docs. - :param directory: Directory in which .tf and .tfvars files will be loaded. - :param module_loader_registry: Registry used for resolving modules. This allows customization of how - much resolution is performed (and easier testing) by using a manually - constructed registry rather than the default. - :param dir_filter: Determines whether or not a directory should be processed. Returning - True will allow processing. The argument will be the absolute path of - the directory. - :param specified_vars: Specifically defined variable values, overriding values from any other source. - """ - - # Stage 1: Look for applicable files in the directory: - # https://www.terraform.io/docs/configuration/index.html#code-organization - # Load the raw data for non-variable files, but perform no processing other than loading - # variable default values. - # Variable files are also flagged for later processing. - var_value_and_file_map: Dict[str, Tuple[Any, str]] = {} - hcl_tfvars: Optional[os.DirEntry] = None - json_tfvars: Optional[os.DirEntry] = None - auto_vars_files: List[os.DirEntry] = [] # *.auto.tfvars / *.auto.tfvars.json - explicit_var_files: List[os.DirEntry] = [] # files passed with --var-file; only process the ones that are in this directory - - dir_contents = list(os.scandir(directory)) - if excluded_paths or IGNORE_HIDDEN_DIRECTORY_ENV: - filter_ignored_paths(directory, dir_contents, excluded_paths) - - tf_files_to_load = [] - for file in dir_contents: - # Ignore directories and hidden files - try: - if not file.is_file(): - continue - except OSError: - # Skip files that can't be accessed - continue - - # Variable files - # See: https://www.terraform.io/docs/configuration/variables.html#variable-definitions-tfvars-files - if file.name == "terraform.tfvars.json": - json_tfvars = file - elif file.name == "terraform.tfvars": - hcl_tfvars = file - elif file.name.endswith(".auto.tfvars.json") or file.name.endswith(".auto.tfvars"): - auto_vars_files.append(file) - elif vars_files and file.path in vars_files: - explicit_var_files.append(file) - - # Resource files - elif file.name.endswith(".tf") or file.name.endswith('.hcl'): # TODO: add support for .tf.json - tf_files_to_load.append(file) - - files_to_data = self._load_files(tf_files_to_load) - - for file, data in sorted(files_to_data, key=lambda x: x[0]): - if not data: - continue - self.out_definitions[file] = data - - # Load variable defaults - # (see https://www.terraform.io/docs/configuration/variables.html#declaring-an-input-variable) - var_blocks = data.get("variable") - if var_blocks and isinstance(var_blocks, list): - for var_block in var_blocks: - if not isinstance(var_block, dict): - continue - for var_name, var_definition in var_block.items(): - if not isinstance(var_definition, dict): - continue - - default_value = var_definition.get("default") - if default_value is not None and isinstance(default_value, list): - self.external_variables_data.append((var_name, default_value[0], file)) - var_value_and_file_map[var_name] = default_value[0], file - - # Stage 2: Load vars in proper order: - # https://www.terraform.io/docs/configuration/variables.html#variable-definition-precedence - # Defaults are loaded in stage 1. - # Then loading in this order with later taking precedence: - # - Environment variables - # - The terraform.tfvars file, if present. - # - The terraform.tfvars.json file, if present. - # - Any *.auto.tfvars or *.auto.tfvars.json files, processed in lexical order of - # their filenames. - # Overriding everything else, variables form `specified_vars`, which are considered - # directly set. - for key, value in self.env_vars.items(): # env vars - if not key.startswith("TF_VAR_"): - continue - var_value_and_file_map[key[7:]] = value, f"env:{key}" - self.external_variables_data.append((key[7:], value, f"env:{key}")) - if hcl_tfvars: # terraform.tfvars - data = load_or_die_quietly(hcl_tfvars, self.out_parsing_errors, clean_definitions=False) - if data: - var_value_and_file_map.update({k: (safe_index(v, 0), hcl_tfvars.path) for k, v in data.items()}) - self.external_variables_data.extend([(k, safe_index(v, 0), hcl_tfvars.path) for k, v in data.items()]) - if json_tfvars: # terraform.tfvars.json - data = load_or_die_quietly(json_tfvars, self.out_parsing_errors) - if data: - var_value_and_file_map.update({k: (v, json_tfvars.path) for k, v in data.items()}) - self.external_variables_data.extend([(k, v, json_tfvars.path) for k, v in data.items()]) - - auto_var_files_to_data = self._load_files(auto_vars_files) - for var_file, data in sorted(auto_var_files_to_data, key=lambda x: x[0]): - if data: - var_value_and_file_map.update({k: (v, var_file) for k, v in data.items()}) - self.external_variables_data.extend([(k, v, var_file) for k, v in data.items()]) - - explicit_var_files_to_data = self._load_files(explicit_var_files) - # it's possible that os.scandir returned the var files in a different order than they were specified - for var_file, data in sorted(explicit_var_files_to_data, key=lambda x: vars_files.index(x[0])): - if data: - var_value_and_file_map.update({k: (v, var_file) for k, v in data.items()}) - self.external_variables_data.extend([(k, v, var_file) for k, v in data.items()]) - - if specified_vars: # specified - var_value_and_file_map.update({k: (v, "manual specification") for k, v in specified_vars.items()}) - self.external_variables_data.extend([(k, v, "manual specification") for k, v in specified_vars.items()]) - - # Stage 4: Load modules - # This stage needs to be done in a loop (again... alas, no DAG) because modules might not - # be loadable until other modules are loaded. This happens when parameters to one module - # depend on the output of another. For such cases, the base module must be loaded, then - # a parameter resolution pass needs to happen, then the second module can be loaded. - # - # One gotcha is that we need to make sure we load all modules at some point, even if their - # parameters don't resolve. So, if we hit a spot where resolution doesn't change anything - # and there are still modules to be loaded, they will be forced on the next pass. - force_final_module_load = False - for i in range(0, 10): # circuit breaker - no more than 10 loops - logging.debug("Module load loop %d", i) - - # Stage 4a: Load eligible modules - # Add directory to self._parsed_directories to avoid loading it as sub dir - if self.enable_nested_modules: - dir_filter(directory) - has_more_modules = self._load_modules(directory, module_loader_registry, dir_filter, - keys_referenced_as_modules, force_final_module_load, - nested_modules_data=nested_modules_data) - - # Stage 4b: Variable resolution round 2 - now with (possibly more) modules - made_var_changes = False - if not has_more_modules: - break # nothing more to do - elif not made_var_changes: - # If there are more modules to load but no variables were resolved, then to a final module - # load, forcing things through without complete resolution. - force_final_module_load = True - - def _load_files(self, files: list[os.DirEntry]): - def _load_file(file: os.DirEntry): - parsing_errors = {} - result = load_or_die_quietly(file, parsing_errors) - # the exceptions type can un-pickleable - for path, e in parsing_errors.items(): - parsing_errors[path] = e - - return (file.path, result), parsing_errors - - files_to_data = [] - files_to_parse = [] - for file in files: - data = self.loaded_files_map.get(file.path) - if data: - files_to_data.append((file.path, data)) - else: - files_to_parse.append(file) - - results = [_load_file(f) for f in files_to_parse] - for result, parsing_errors in results: - self.out_parsing_errors.update(parsing_errors) - files_to_data.append(result) - if result[0] not in self.loaded_files_map: - self.loaded_files_map[result[0]] = result[1] - return files_to_data - - def _load_modules(self, root_dir: str, module_loader_registry: ModuleLoaderRegistry, - dir_filter: Callable[[str], bool], - keys_referenced_as_modules: Set[str], ignore_unresolved_params: bool = False, - nested_modules_data=None) -> bool: - """ - Load modules which have not already been loaded and can be loaded (don't have unresolved parameters). - - :param ignore_unresolved_params: If true, not-yet-loaded modules will be loaded even if they are - passed parameters that are not fully resolved. - :return: True if there were modules that were not loaded due to unresolved - parameters. - """ - all_module_definitions = {} - all_module_evaluations_context = {} - skipped_a_module = False - for file in list(self.out_definitions.keys()): - # Don't process a file in a directory other than the directory we're processing. For example, - # if we're down dealing with //something.tf, we don't want to rescan files - # up in . - if self.get_dirname(file) != root_dir: - continue - # Don't process a file reference which has already been processed - if file.endswith(TERRAFORM_NESTED_MODULE_PATH_ENDING): - continue - - file_data = self.out_definitions.get(file) - if file_data is None: - continue - module_calls = file_data.get("module") - if not module_calls or not isinstance(module_calls, list): - continue - - for module_index, module_call in enumerate(module_calls): - if not isinstance(module_call, dict): - continue - - # There should only be one module reference per outer dict, but... safety first - for module_call_name, module_call_data in module_call.items(): - if not isinstance(module_call_data, dict): - continue - - if self.enable_nested_modules: - file_key = self.get_file_key_with_nested_data(file, nested_modules_data) - current_nested_data = (file_key, module_index, module_call_name) - - resolved_loc_list = [] - if current_nested_data in self.module_to_resolved: - resolved_loc_list = self.module_to_resolved[current_nested_data] - self.module_to_resolved[current_nested_data] = resolved_loc_list - - module_address = (file, module_index, module_call_name) - if not self.enable_nested_modules: - if module_address in self._loaded_modules: - continue - - # Variables being passed to module, "source" and "version" are reserved - specified_vars = {k: v[0] if isinstance(v, list) else v for k, v in module_call_data.items() - if k != "source" and k != "version"} - - if not ignore_unresolved_params: - has_unresolved_params = False - for k, v in specified_vars.items(): - if not is_acceptable_module_param(v) or not is_acceptable_module_param(k): - has_unresolved_params = True - break - if has_unresolved_params: - skipped_a_module = True - continue - self._loaded_modules.add(module_address) - - source = module_call_data.get("source") - if not source or not isinstance(source, list): - continue - source = source[0] - if not isinstance(source, str): - logging.debug(f"Skipping loading of {module_call_name} as source is not a string, it is: {source}") - continue - elif source in ['./', '.']: - logging.debug(f"Skipping loading of {module_call_name} as source is the current dir") - continue - - # Special handling for local sources to make sure we aren't double-parsing - if source.startswith("./") or source.startswith("../"): - source = os.path.normpath( - os.path.join(os.path.dirname(remove_module_dependency_from_path(file)), source)) - - version = module_call_data.get("version", "latest") - if version and isinstance(version, list): - version = version[0] - try: - content = module_loader_registry.load(root_dir, source, version) - if not content.loaded(): - logging.info(f'Got no content for {source}:{version}') - continue - - new_nested_modules_data = {'module_index': module_index, 'file': file, - 'nested_modules_data': nested_modules_data} - - self._internal_dir_load(directory=content.path(), - module_loader_registry=module_loader_registry, - dir_filter=dir_filter, specified_vars=specified_vars, - keys_referenced_as_modules=keys_referenced_as_modules, - nested_modules_data=new_nested_modules_data) - - module_definitions = { - path: definition - for path, definition in self.out_definitions.items() - if self.get_dirname(path) == content.path() - } - - if not module_definitions: - continue - - # NOTE: Modules are put into the main TF definitions structure "as normal" with the - # notable exception of the file name. For loaded modules referrer information is - # appended to the file name to create this format: - # [#] - # For example: - # /the/path/module/my_module.tf[/the/path/main.tf#0] - # The referrer and index allow a module allow a module to be loaded multiple - # times with differing data. - # - # In addition, the referring block will have a "__resolved__" key added with a - # list pointing to the location of the module data that was resolved. For example: - # "__resolved__": ["/the/path/module/my_module.tf[/the/path/main.tf#0]"] - - if not self.enable_nested_modules: - resolved_loc_list = module_call_data.get(RESOLVED_MODULE_ENTRY_NAME) - if resolved_loc_list is None: - resolved_loc_list = [] - module_call_data[RESOLVED_MODULE_ENTRY_NAME] = resolved_loc_list - - # NOTE: Modules can load other modules, so only append referrer information where it - # has not already been added. - - keys = list(module_definitions.keys()) - for key in keys: - if key.endswith(TERRAFORM_NESTED_MODULE_PATH_ENDING) or file.endswith(TERRAFORM_NESTED_MODULE_PATH_ENDING): - continue - keys_referenced_as_modules.add(key) - if self.enable_nested_modules: - new_key = self.get_new_nested_module_key(key, file, module_index, nested_modules_data) - if new_key in self.visited_definition_keys: - del module_definitions[key] - del self.out_definitions[key] - continue - else: - new_key = get_tf_definition_key_from_module_dependency(key, file, module_index) - module_definitions[new_key] = module_definitions[key] - del module_definitions[key] - del self.out_definitions[key] - self.keys_to_remove.add(key) - - if self.enable_nested_modules: - self.visited_definition_keys.add(new_key) - if new_key not in resolved_loc_list: - resolved_loc_list.append(new_key) - if (file, module_call_name) not in self.module_address_map: - self.module_address_map[(file, module_call_name)] = str(module_index) - resolved_loc_list.sort() # For testing, need predictable ordering - - if all_module_definitions: - deep_merge.merge(all_module_definitions, module_definitions) - else: - all_module_definitions = module_definitions - - self.external_modules_source_map[(source, version)] = content.path() - except Exception as e: - logging.warning("Unable to load module (source=\"%s\" version=\"%s\"): %s", - source, version, e) - - if all_module_definitions: - deep_merge.merge(self.out_definitions, all_module_definitions) - if all_module_evaluations_context: - deep_merge.merge(self.out_evaluations_context, all_module_evaluations_context) - return skipped_a_module - - def parse_hcl_module( - self, - source_dir: str, - source: str, - download_external_modules: bool = False, - external_modules_download_path: str = DEFAULT_EXTERNAL_MODULES_DIR, - parsing_errors: dict[str, Exception] | None = None, - excluded_paths: list[str] | None = None, - vars_files: list[str] | None = None, - external_modules_content_cache: dict[str, ModuleContent] | None = None, - create_graph: bool = True, - ) -> tuple[Module | None, dict[str, dict[str, Any]]]: - tf_definitions: dict[str, dict[str, Any]] = {} - self.parse_directory(directory=source_dir, out_definitions=tf_definitions, out_evaluations_context={}, - out_parsing_errors=parsing_errors if parsing_errors is not None else {}, - download_external_modules=download_external_modules, - external_modules_download_path=external_modules_download_path, excluded_paths=excluded_paths, - vars_files=vars_files, external_modules_content_cache=external_modules_content_cache) - tf_definitions = clean_parser_types(tf_definitions) - tf_definitions = serialize_definitions(tf_definitions) - - module = None - if create_graph: - module, tf_definitions = self.parse_hcl_module_from_tf_definitions(tf_definitions, source_dir, source) - - return module, tf_definitions - - def parse_multi_graph_hcl_module( - self, - source_dir: str, - source: str, - download_external_modules: bool = False, - external_modules_download_path: str = DEFAULT_EXTERNAL_MODULES_DIR, - parsing_errors: dict[str, Exception] | None = None, - excluded_paths: list[str] | None = None, - vars_files: list[str] | None = None, - external_modules_content_cache: dict[str, ModuleContent | None] | None = None, - create_graph: bool = True, - ) -> list[tuple[Module, list[dict[TFDefinitionKey, dict[str, Any]]]]]: - # just added for typing purposes, it is only usde via the new parser - return [] - - def _remove_unused_path_recursive(self, path): - self.out_definitions.pop(path, None) - for key in list(self.module_to_resolved.keys()): - file_key, module_index, module_name = key - if path == file_key: - for resolved_path in self.module_to_resolved[key]: - self._remove_unused_path_recursive(resolved_path) - self.module_to_resolved.pop(key, None) - - def _update_resolved_modules(self): - for key in list(self.module_to_resolved.keys()): - file_key, module_index, module_name = key - if file_key in self.keys_to_remove: - for path in self.module_to_resolved[key]: - self._remove_unused_path_recursive(path) - self.module_to_resolved.pop(key, None) - - for key, resolved_list in self.module_to_resolved.items(): - file_key, module_index, module_name = key - if file_key not in self.out_definitions: - continue - self.out_definitions[file_key]['module'][module_index][module_name][RESOLVED_MODULE_ENTRY_NAME] = resolved_list - - def parse_hcl_module_from_tf_definitions( - self, - tf_definitions: Dict[str, Dict[str, Any]], - source_dir: str, - source: str, - ) -> Tuple[Module, Dict[str, Dict[str, Any]]]: - if self.enable_nested_modules: - module_dependency_map, tf_definitions, dep_index_mapping = get_module_dependency_map_support_nested_modules(tf_definitions) - else: - module_dependency_map, tf_definitions, dep_index_mapping = get_module_dependency_map(tf_definitions) - module = self.get_new_module( - source_dir=source_dir, - module_dependency_map=module_dependency_map, - module_address_map=self.module_address_map, - external_modules_source_map=self.external_modules_source_map, - dep_index_mapping=dep_index_mapping, - ) - self.add_tfvars(module, source) - copy_of_tf_definitions = pickle_deepcopy(tf_definitions) - for file_path, blocks in copy_of_tf_definitions.items(): - for block_type in blocks: - try: - module.add_blocks(block_type, blocks[block_type], file_path, source) - except Exception as e: - logging.warning(f'Failed to add block {blocks[block_type]}. Error:') - logging.warning(e, exc_info=False) - return module, tf_definitions - - def get_file_key_with_nested_data(self, file, nested_data): - if not nested_data: - return file - nested_str = self.get_file_key_with_nested_data(nested_data.get("file"), nested_data.get('nested_modules_data')) - nested_module_index = nested_data.get('module_index') - return get_tf_definition_key_from_module_dependency(file, nested_str, nested_module_index) - - def get_new_nested_module_key(self, key, file, module_index, nested_data) -> str: - if not nested_data: - return get_tf_definition_key_from_module_dependency(key, file, module_index) - visited_key_to_add = get_tf_definition_key_from_module_dependency(key, file, module_index) - self.visited_definition_keys.add(visited_key_to_add) - nested_key = self.get_new_nested_module_key('', nested_data.get('file'), - nested_data.get('module_index'), - nested_data.get('nested_modules_data')) - return get_tf_definition_key_from_module_dependency(key, f"{file}{nested_key}", module_index) - - def add_tfvars(self, module: Module, source: str) -> None: - if not self.external_variables_data: - return - for (var_name, default, path) in self.external_variables_data: - if ".tfvars" in path: - block = {var_name: {"default": default}} - module.add_blocks(BlockType.TF_VARIABLE, block, path, source) - - def get_dirname(self, path: str) -> str: - dirname_path = self.dirname_cache.get(path) - if not dirname_path: - dirname_path = os.path.dirname(path) - self.dirname_cache[path] = dirname_path - return dirname_path - - @staticmethod - def get_new_module( - source_dir: str, - module_dependency_map: dict[str, list[list[str]]], - module_address_map: dict[tuple[str, str], str], - external_modules_source_map: dict[tuple[str, str], str], - dep_index_mapping: dict[tuple[str, str], list[str]], - ) -> Module: - return Module( - source_dir=source_dir, - module_dependency_map=module_dependency_map, - module_address_map=module_address_map, - external_modules_source_map=external_modules_source_map, - dep_index_mapping=dep_index_mapping - ) diff --git a/checkov/terraform/plan_runner.py b/checkov/terraform/plan_runner.py index 495aa526402..bd0892250d5 100644 --- a/checkov/terraform/plan_runner.py +++ b/checkov/terraform/plan_runner.py @@ -198,6 +198,7 @@ def check_tf_definition(self, report, root_folder, runner_filter, collect_skip_c @staticmethod def _get_file_path(full_file_path: TFDefinitionKeyType, root_folder: str | pathlib.Path) -> tuple[str, str]: if isinstance(full_file_path, TFDefinitionKey): + # It might be str for terraform-plan files full_file_path = full_file_path.file_path if platform.system() == "Windows": temp = os.path.split(full_file_path)[0] diff --git a/checkov/terraform/runner.py b/checkov/terraform/runner.py index 335e29f66a3..b77bb822a22 100644 --- a/checkov/terraform/runner.py +++ b/checkov/terraform/runner.py @@ -24,9 +24,7 @@ from checkov.common.util import data_structures_utils from checkov.common.util.consts import RESOLVED_MODULE_ENTRY_NAME from checkov.common.util.data_structures_utils import pickle_deepcopy -from checkov.common.util.parser_utils import get_module_from_full_path, get_abs_path, \ - get_tf_definition_key_from_module_dependency, get_module_name, \ - strip_terraform_module_referrer +from checkov.terraform import get_module_from_full_path, get_module_name, get_abs_path from checkov.common.util.secrets import omit_secret_value_from_checks, omit_secret_value_from_graph_checks from checkov.common.variables.context import EvaluationContext from checkov.runner_filter import RunnerFilter @@ -43,7 +41,6 @@ from checkov.terraform.graph_builder.local_graph import TerraformLocalGraph from checkov.terraform.graph_manager import TerraformGraphManager from checkov.terraform.image_referencer.manager import TerraformImageReferencerManager -from checkov.terraform.parser import Parser from checkov.terraform.tf_parser import TFParser from checkov.terraform.tag_providers import get_resource_tags from checkov.common.runners.base_runner import strtobool @@ -51,7 +48,7 @@ if TYPE_CHECKING: from networkx import DiGraph from checkov.common.images.image_referencer import Image - from checkov.common.typing import TFDefinitionKeyType, LibraryGraphConnector, _SkippedCheck + from checkov.common.typing import LibraryGraphConnector, _SkippedCheck # Allow the evaluation of empty variables dpath.options.ALLOW_EMPTY_STRING_KEYS = True @@ -64,7 +61,7 @@ class Runner(ImageReferencerMixin[None], BaseRunner[TerraformGraphManager]): def __init__( self, - parser: Parser | TFParser | None = None, + parser: TFParser | None = None, db_connector: LibraryGraphConnector | None = None, external_registries: list[BaseRegistry] | None = None, source: str = GraphSource.TERRAFORM, @@ -74,9 +71,9 @@ def __init__( super().__init__(file_extensions=['.tf', '.hcl']) self.external_registries = [] if external_registries is None else external_registries self.graph_class = graph_class - self.parser = parser or TFParser() if strtobool(os.getenv('CHECKOV_NEW_TF_PARSER', 'True')) else Parser() - self.definitions: dict[TFDefinitionKeyType, dict[str, Any]] | None = None - self.context = None + self.parser = parser or TFParser() + self.definitions: dict[TFDefinitionKey, dict[str, Any]] | None = None + self.context: dict[TFDefinitionKey, dict[str, Any]] | None = None self.breadcrumbs = None self.evaluations_context: Dict[str, Dict[str, EvaluationContext]] = {} self.graph_manager: TerraformGraphManager = graph_manager if graph_manager is not None else TerraformGraphManager( @@ -261,13 +258,8 @@ def get_graph_checks_report(self, root_folder: str, runner_filter: RunnerFilter, root_folder = os.path.split(full_file_path)[0] resource_id = ".".join(entity_context['definition_path']) resource = resource_id - module_dependency = entity.get(CustomAttributes.MODULE_DEPENDENCY) - module_dependency_num = entity.get(CustomAttributes.MODULE_DEPENDENCY_NUM) definition_context_file_path = full_file_path - if module_dependency and module_dependency_num: - resource = entity.get(CustomAttributes.TF_RESOURCE_ADDRESS, resource_id) - definition_context_file_path = get_tf_definition_key_from_module_dependency(full_file_path, module_dependency, module_dependency_num) - elif entity.get(CustomAttributes.TF_RESOURCE_ADDRESS) and entity.get(CustomAttributes.TF_RESOURCE_ADDRESS) != resource_id: + if entity.get(CustomAttributes.TF_RESOURCE_ADDRESS) and entity.get(CustomAttributes.TF_RESOURCE_ADDRESS) != resource_id: # for plan resources resource = entity[CustomAttributes.TF_RESOURCE_ADDRESS] entity_config = self.get_graph_resource_entity_config(entity) @@ -310,17 +302,13 @@ def get_graph_checks_report(self, root_folder: str, runner_filter: RunnerFilter, def get_entity_context_and_evaluations(self, entity: dict[str, Any]) -> dict[str, Any] | None: block_type = entity[CustomAttributes.BLOCK_TYPE] full_file_path = entity[CustomAttributes.FILE_PATH] - # TODO Barak delete MODULE_DEPENDENCY, MODULE_DEPENDENCY_NUM - if entity.get(CustomAttributes.MODULE_DEPENDENCY): - full_file_path = get_tf_definition_key_from_module_dependency(full_file_path, entity[CustomAttributes.MODULE_DEPENDENCY], entity[CustomAttributes.MODULE_DEPENDENCY_NUM]) - if strtobool(os.getenv('ENABLE_DEFINITION_KEY', 'False')): - full_file_path = TFDefinitionKey(file_path=entity.get(CustomAttributes.FILE_PATH), tf_source_modules=entity.get(CustomAttributes.SOURCE_MODULE_OBJECT)) + full_file_path = TFDefinitionKey(file_path=entity.get(CustomAttributes.FILE_PATH), tf_source_modules=entity.get(CustomAttributes.SOURCE_MODULE_OBJECT)) definition_path = entity[CustomAttributes.BLOCK_NAME].split('.') entity_context_path = [block_type] + definition_path try: - entity_context = self.context[full_file_path] # type: ignore + entity_context = self.context[full_file_path] for k in entity_context_path: if k in entity_context: entity_context = entity_context[k] @@ -351,8 +339,7 @@ def check_tf_definition( self.push_skipped_checks_down_from_modules(self.context) for full_file_path, definition in self.definitions.items(): self.pbar.set_additional_data({'Current File Scanned': os.path.relpath( - full_file_path.file_path if isinstance(full_file_path, TFDefinitionKey) else full_file_path, - root_folder)}) + full_file_path.file_path)}) abs_scanned_file = get_abs_path(full_file_path) abs_referrer = None scanned_file = f"/{os.path.relpath(abs_scanned_file, root_folder)}" @@ -366,7 +353,7 @@ def run_all_blocks( self, definition: dict[str, list[dict[str, Any]]], definitions_context: dict[str, dict[str, Any]], - full_file_path: TFDefinitionKeyType, + full_file_path: TFDefinitionKey, root_folder: str, report: Report, scanned_file: str, @@ -386,7 +373,7 @@ def run_block( self, entities: list[dict[str, Any]], definition_context: dict[str, dict[str, Any]], - full_file_path: TFDefinitionKeyType, + full_file_path: TFDefinitionKey, root_folder: str, report: Report, scanned_file: str, @@ -433,10 +420,7 @@ def run_block( else: entity_context_path = entity_context_path_header + block_type + definition_path # Entity can exist only once per dir, for file as well - if not strtobool(os.getenv('ENABLE_DEFINITION_KEY', 'False')): - context_path = full_file_path.file_path if isinstance(full_file_path, TFDefinitionKey) else full_file_path - else: - context_path = full_file_path if isinstance(full_file_path, TFDefinitionKey) else TFDefinitionKey(file_path=full_file_path, tf_source_modules=None) + context_path = full_file_path try: entity_context = data_structures_utils.get_inner_dict( definition_context[context_path], @@ -458,10 +442,7 @@ def run_block( entity_evaluations = BaseVariableEvaluation.reduce_entity_evaluations(variables_evaluations, entity_context_path) results = registry.scan(scanned_file, entity, skipped_checks, runner_filter) - if isinstance(full_file_path, str): - absolute_scanned_file_path, _ = strip_terraform_module_referrer(file_path=full_file_path) - if isinstance(full_file_path, TFDefinitionKey): - absolute_scanned_file_path = get_abs_path(full_file_path) + absolute_scanned_file_path = get_abs_path(full_file_path) # This duplicates a call at the start of scan, but adding this here seems better than kludging with some tuple return type tags = get_resource_tags(entity_type, entity_config) if results: @@ -493,7 +474,7 @@ def run_block( bc_category=check.bc_category, benchmarks=check.benchmarks, details=check.details, - definition_context_file_path=full_file_path + definition_context_file_path=full_file_path.file_path ) if CHECKOV_CREATE_GRAPH: entity_key = entity_id @@ -530,63 +511,14 @@ def parse_file(file: str) -> tuple[str, dict[str, Any], dict[str, Exception]] | if result: file, parse_result, file_parsing_errors = result if parse_result is not None: - if isinstance(self.parser, Parser): - self.definitions[file] = parse_result - if isinstance(self.parser, TFParser): - self.definitions[TFDefinitionKey(file_path=file)] = parse_result + self.definitions[TFDefinitionKey(file_path=file)] = parse_result if file_parsing_errors: parsing_errors.update(file_parsing_errors) - @staticmethod - def push_skipped_checks_down_old( - definition_context: dict[str, dict[str, Any]], module_path: str, skipped_checks: list[_SkippedCheck] - ) -> None: - # this method pushes the skipped_checks down the 1 level to all resource types. - - if skipped_checks is None: - return - - if len(skipped_checks) == 0: - return - - # iterate over definitions to find those that reference the module path - # definition is in the format [#] - # where referrer could be a path, or path1->path2, etc - - for definition in definition_context: - _, mod_ref = strip_terraform_module_referrer(file_path=definition) - if mod_ref is None: - continue - - if module_path not in mod_ref: - continue - - for block_type, block_configs in definition_context[definition].items(): - # skip if type is not a Terraform resource - if block_type not in CHECK_BLOCK_TYPES: - continue - - if block_type == "module": - # modules don't have a type, just a name - for resource_config in block_configs.values(): - # append the skipped checks also from a module to another module - resource_config["skipped_checks"] += skipped_checks - else: - # there may be multiple resource types - aws_bucket, etc - for resource_configs in block_configs.values(): - # there may be multiple names for each resource type - for resource_config in resource_configs.values(): - # append the skipped checks from the module to the other resources. - resource_config["skipped_checks"] += skipped_checks - - def push_skipped_checks_down_from_modules(self, definition_context: dict[str, dict[str, Any]]) -> None: + def push_skipped_checks_down_from_modules(self, definition_context: dict[TFDefinitionKey, dict[str, Any]]) -> None: module_context_parser = parser_registry.context_parsers[BlockType.MODULE] for tf_definition_key, definition in self.definitions.items(): - if not strtobool(os.getenv('ENABLE_DEFINITION_KEY', 'False')): - full_file_path = tf_definition_key.file_path if isinstance(tf_definition_key, TFDefinitionKey) else tf_definition_key - else: - full_file_path = tf_definition_key if isinstance(tf_definition_key, TFDefinitionKey)\ - else TFDefinitionKey(file_path=tf_definition_key, tf_source_modules=None) + full_file_path = tf_definition_key definition_modules_context = definition_context.get(full_file_path, {}).get(BlockType.MODULE, {}) for entity in definition.get(BlockType.MODULE, []): module_name = module_context_parser.get_entity_context_path(entity)[0] @@ -596,15 +528,14 @@ def push_skipped_checks_down_from_modules(self, definition_context: dict[str, di def push_skipped_checks_down( self, - definition_context: dict[str, dict[str, Any]], + definition_context: dict[TFDefinitionKey, dict[str, Any]], skipped_checks: list[_SkippedCheck], - resolved_paths: list[TFDefinitionKeyType], + resolved_paths: list[TFDefinitionKey], ) -> None: # this method pushes the skipped_checks down the 1 level to all resource types. if not skipped_checks or not resolved_paths: return - resolved_file_paths = [path.file_path if isinstance(path, TFDefinitionKey) else path for path in resolved_paths] - for ind, definition in enumerate(resolved_file_paths): + for ind, definition in enumerate(resolved_paths): for block_type, block_configs in definition_context.get(definition, {}).items(): # skip if type is not a Terraform resource if block_type not in CHECK_BLOCK_TYPES: diff --git a/checkov/terraform/tf_parser.py b/checkov/terraform/tf_parser.py index 14188f4bd72..e0e944092d5 100644 --- a/checkov/terraform/tf_parser.py +++ b/checkov/terraform/tf_parser.py @@ -1,5 +1,6 @@ from __future__ import annotations +import json import logging import os from collections import defaultdict @@ -7,13 +8,14 @@ from typing import Optional, Dict, Mapping, Set, Tuple, Callable, Any, List, cast, TYPE_CHECKING import deep_merge +import hcl2 from checkov.common.runners.base_runner import filter_ignored_paths, IGNORE_HIDDEN_DIRECTORY_ENV -from checkov.common.typing import TFDefinitionKeyType from checkov.common.util.consts import DEFAULT_EXTERNAL_MODULES_DIR, RESOLVED_MODULE_ENTRY_NAME from checkov.common.util.data_structures_utils import pickle_deepcopy from checkov.common.util.type_forcers import force_list from checkov.common.variables.context import EvaluationContext +from checkov.terraform import validate_malformed_definitions, clean_bad_definitions from checkov.terraform.graph_builder.graph_components.block_types import BlockType from checkov.terraform.graph_builder.graph_components.module import Module from checkov.terraform.module_loading.content import ModuleContent @@ -21,9 +23,9 @@ from checkov.terraform.module_loading.registry import module_loader_registry as default_ml_registry, \ ModuleLoaderRegistry from checkov.common.util.parser_utils import is_acceptable_module_param -from checkov.terraform.modules.module_utils import load_or_die_quietly, safe_index, \ +from checkov.terraform.modules.module_utils import safe_index, \ remove_module_dependency_from_path, \ - clean_parser_types, serialize_definitions + clean_parser_types, serialize_definitions, _Hcl2Payload from checkov.terraform.modules.module_objects import TFModule, TFDefinitionKey if TYPE_CHECKING: @@ -47,7 +49,7 @@ def __init__(self, module_class: type[Module] = Module) -> None: self.external_variables_data: list[tuple[str, Any, str]] = [] def _init(self, directory: str, - out_evaluations_context: Dict[str, Dict[str, EvaluationContext]] | None, + out_evaluations_context: Dict[TFDefinitionKey, Dict[str, EvaluationContext]] | None, out_parsing_errors: Dict[str, Exception] | None, env_vars: Mapping[str, str] | None, download_external_modules: bool, @@ -80,7 +82,7 @@ def _check_process_dir(self, directory: str) -> bool: def parse_directory( self, directory: str, - out_evaluations_context: Dict[str, Dict[str, EvaluationContext]] | None = None, + out_evaluations_context: Dict[TFDefinitionKey, Dict[str, EvaluationContext]] | None = None, out_parsing_errors: Dict[str, Exception] | None = None, env_vars: Mapping[str, str] | None = None, download_external_modules: bool = False, @@ -435,7 +437,6 @@ def parse_hcl_module_from_tf_definitions( ) -> Tuple[Module, Dict[TFDefinitionKey, Dict[str, Any]]]: module = self.get_new_module( source_dir=source_dir, - module_address_map=self.module_address_map, external_modules_source_map=self.external_modules_source_map, ) self.add_tfvars(module, source) @@ -457,7 +458,6 @@ def parse_hcl_module_from_multi_tf_definitions( ) -> tuple[Module, list[dict[TFDefinitionKey, dict[str, Any]]]]: module = self.get_new_module( source_dir=source_dir, - module_address_map=self.module_address_map, external_modules_source_map=self.external_modules_source_map, ) self.add_tfvars_with_source_dir(module, source, source_dir) @@ -509,20 +509,19 @@ def add_tfvars_with_source_dir(self, module: Module, source: str, source_dir: st block = [{var_name: {"default": default}}] module.add_blocks(BlockType.TF_VARIABLE, block, path, source) - def get_dirname(self, path: TFDefinitionKeyType) -> str: - if isinstance(path, TFDefinitionKey): - path = path.file_path - dirname_path = self.dirname_cache.get(path) + def get_dirname(self, path: TFDefinitionKey) -> str: + file_path = path.file_path + dirname_path = self.dirname_cache.get(file_path) if not dirname_path: - dirname_path = os.path.dirname(path) - self.dirname_cache[path] = dirname_path + dirname_path = os.path.dirname(file_path) + self.dirname_cache[file_path] = dirname_path return dirname_path def should_loaded_file(self, file: TFDefinitionKey, root_dir: str) -> bool: return not self.get_dirname(file) != root_dir def get_module_source( - self, module_call_data: dict[str, Any], module_call_name: str, file: TFDefinitionKeyType + self, module_call_data: dict[str, Any], module_call_name: str, file: TFDefinitionKey ) -> Optional[str]: source = module_call_data.get("source") if not source or not isinstance(source, list): @@ -532,7 +531,7 @@ def get_module_source( return None if source.startswith("./") or source.startswith("../"): - file_to_load = file.file_path if isinstance(file, TFDefinitionKey) else file + file_to_load = file.file_path source = os.path.normpath(os.path.join(os.path.dirname(remove_module_dependency_from_path(file_to_load)), source)) return source @@ -653,12 +652,10 @@ def get_content_path(module_loader_registry: ModuleLoaderRegistry, root_dir: str @staticmethod def get_new_module( source_dir: str, - module_address_map: dict[tuple[str, str], str], external_modules_source_map: dict[tuple[str, str], str], ) -> Module: return Module( source_dir=source_dir, - module_address_map=module_address_map, external_modules_source_map=external_modules_source_map, ) @@ -675,3 +672,33 @@ def get_tf_definition_object_from_module_dependency( if not is_nested_object(module_dependency): return TFDefinitionKey(path.file_path, TFModule(path=module_dependency.file_path, name=module_dependency_name)) return TFDefinitionKey(path.file_path, TFModule(path=module_dependency.file_path, name=module_dependency_name, nested_tf_module=module_dependency.tf_source_modules)) + + +def load_or_die_quietly( + file: str | Path | os.DirEntry[str], parsing_errors: dict[str, Exception], clean_definitions: bool = True +) -> Optional[_Hcl2Payload]: + """ +Load JSON or HCL, depending on filename. + :return: None if the file can't be loaded + """ + + file_path = os.fspath(file) + file_name = os.path.basename(file_path) + + try: + logging.debug(f"Parsing {file_path}") + + with open(file_path, "r", encoding="utf-8-sig") as f: + if file_name.endswith(".json"): + return cast("_Hcl2Payload", json.load(f)) + else: + raw_data = hcl2.load(f) + non_malformed_definitions = validate_malformed_definitions(raw_data) + if clean_definitions: + return clean_bad_definitions(non_malformed_definitions) + else: + return non_malformed_definitions + except Exception as e: + logging.debug(f'failed while parsing file {file_path}', exc_info=True) + parsing_errors[file_path] = e + return None diff --git a/pyproject.toml b/pyproject.toml index 56825ecf0e6..54e3e041e81 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -45,15 +45,15 @@ ignore_imports = [ "checkov.common.util.json_utils -> checkov.terraform.modules.module_objects", # needed for JSON serialization "checkov.common.bridgecrew.platform_integration -> checkov.secrets.coordinator", # type hint - "checkov.common.graph.graph_manager -> checkov.terraform.parser", # type hint - "checkov.common.util.parser_utils -> checkov.terraform.modules.module_objects", # type hint + "checkov.common.graph.graph_manager -> checkov.terraform.tf_parser", # type hint "checkov.common.typing -> checkov.terraform.modules.module_objects", # type hint "checkov.common.bridgecrew.integration_features.features.policies_3d_integration -> checkov.policies_3d.*", # considering what to do "checkov.common.runners.base_post_runner -> checkov.policies_3d.checks_infra.base_check", # considering what to do "checkov.common.runners.runner_registry -> checkov.terraform.context_parsers.registry", # move runner_registry to a different place - "checkov.common.runners.runner_registry -> checkov.terraform.parser", # move runner_registry to a different place + "checkov.common.runners.runner_registry -> checkov.terraform.tf_parser", # move runner_registry to a different place + "checkov.common.runners.runner_registry -> checkov.terraform.modules.module_objects", # Should fix usage of get_enriched_resources "checkov.common.output.report -> checkov.policies_3d.output", # move to checkov.common.output "checkov.common.output.report -> checkov.sca_package.output", # move to checkov.common.output diff --git a/tests/common/runner_registry/test_runner_registry_plan_enrichment.py b/tests/common/runner_registry/test_runner_registry_plan_enrichment.py index 6ac0b380549..61f5695d025 100644 --- a/tests/common/runner_registry/test_runner_registry_plan_enrichment.py +++ b/tests/common/runner_registry/test_runner_registry_plan_enrichment.py @@ -180,7 +180,7 @@ def _load_tf_modules(*args, **kwargs): ) } - mocker.patch("checkov.terraform.parser.load_tf_modules", side_effect=_load_tf_modules) + mocker.patch("checkov.terraform.tf_parser.load_tf_modules", side_effect=_load_tf_modules) # when report = runner_registry.run(repo_root_for_plan_enrichment=[repo_root], files=[str(valid_plan_path)])[0] diff --git a/tests/terraform/context_parsers/test_locals_parser.py b/tests/terraform/context_parsers/test_locals_parser.py index 3e912c1316c..c0b688b4045 100644 --- a/tests/terraform/context_parsers/test_locals_parser.py +++ b/tests/terraform/context_parsers/test_locals_parser.py @@ -1,5 +1,7 @@ import unittest -from checkov.terraform.parser import Parser + +from checkov.terraform import TFDefinitionKey +from checkov.terraform.tf_parser import TFParser from checkov.terraform.context_parsers.registry import parser_registry import os @@ -8,11 +10,9 @@ class TestLocalsContextParser(unittest.TestCase): def setup_dir(self, rel_path): test_root_dir = os.path.dirname(os.path.realpath(__file__)) + rel_path - tf_definitions = {} parsing_errors = {} definitions_context = {} - Parser().parse_directory(directory=test_root_dir, - out_definitions=tf_definitions, + tf_definitions = TFParser().parse_directory(directory=test_root_dir, out_parsing_errors=parsing_errors) for definition in tf_definitions.items(): definitions_context = parser_registry.enrich_definitions_context(definition) @@ -20,7 +20,9 @@ def setup_dir(self, rel_path): def test_assignments_exists(self): definitions_context = self.setup_dir('/../evaluation/resources/default_evaluation/') - assignments = definitions_context[os.path.dirname(os.path.realpath(__file__)) + '/../evaluation/resources/default_evaluation/main.tf']['locals']['assignments'] + file_path = os.path.dirname(os.path.realpath(__file__)) + '/../evaluation/resources/default_evaluation/main.tf' + key = TFDefinitionKey(file_path=file_path, tf_source_modules=None) + assignments = definitions_context[key]['locals']['assignments'] self.assertIsNotNone(assignments) expected_assignments = {'dummy_with_dash': '${format("-%s",var.dummy_1)}', 'dummy_with_comma': '${format(":%s",var.dummy_1)}', 'bucket_name': '${var.bucket_name}'} @@ -30,9 +32,9 @@ def test_assignments_exists(self): def test_assignment_value(self): definitions_context = self.setup_dir('/../evaluation/resources/locals_evaluation/') - assignments = definitions_context[ - os.path.dirname(os.path.realpath(__file__)) + '/../evaluation/resources/locals_evaluation/main.tf'][ - 'locals'].get('assignments') + file_path = os.path.dirname(os.path.realpath(__file__)) + '/../evaluation/resources/locals_evaluation/main.tf' + key = TFDefinitionKey(file_path=file_path, tf_source_modules=None) + assignments = definitions_context[key]['locals'].get('assignments') self.assertIsNotNone(assignments) self.assertEqual(1, len(assignments.items())) for k, v in assignments.items(): diff --git a/tests/terraform/context_parsers/test_variable_context_parser.py b/tests/terraform/context_parsers/test_variable_context_parser.py index c01bfd1de21..bc067f27c98 100644 --- a/tests/terraform/context_parsers/test_variable_context_parser.py +++ b/tests/terraform/context_parsers/test_variable_context_parser.py @@ -1,5 +1,7 @@ import unittest -from checkov.terraform.parser import Parser + +from checkov.terraform import TFDefinitionKey +from checkov.terraform.tf_parser import TFParser from checkov.terraform.context_parsers.registry import parser_registry import os @@ -7,32 +9,32 @@ class TestVariableContextParser(unittest.TestCase): def setUp(self): test_root_dir = os.path.dirname(os.path.realpath(__file__)) + '/../evaluation/resources/default_evaluation/' - tf_definitions = {} parsing_errors = {} - Parser().parse_directory(directory=test_root_dir, - out_definitions=tf_definitions, + tf_definitions = TFParser().parse_directory(directory=test_root_dir, out_parsing_errors=parsing_errors) for definition in tf_definitions.items(): definitions_context = parser_registry.enrich_definitions_context(definition) self.definitions_context = definitions_context def test_assignments_exists(self): + file_path = os.path.dirname(os.path.realpath(__file__))\ + + '/../evaluation/resources/default_evaluation/variables.tf' + key = TFDefinitionKey(file_path=file_path, tf_source_modules=None) self.assertIsNotNone( - self.definitions_context[os.path.dirname( - os.path.realpath(__file__)) + '/../evaluation/resources/default_evaluation/variables.tf'][ + self.definitions_context[key][ 'variable'].get( 'assignments')) def test_assignment_value(self): + file_path = os.path.dirname(os.path.realpath(__file__)) + '/../evaluation/resources/default_evaluation/variables.tf' + key = TFDefinitionKey(file_path=file_path, tf_source_modules=None) self.assertFalse( - self.definitions_context[os.path.dirname( - os.path.realpath(__file__)) + '/../evaluation/resources/default_evaluation/variables.tf'][ - 'variable'].get( + self.definitions_context[key]['variable'].get( 'assignments').get('user_exists') ) + self.assertEqual( - self.definitions_context[os.path.dirname( - os.path.realpath(__file__)) + '/../evaluation/resources/default_evaluation/variables.tf'][ + self.definitions_context[key][ 'variable'].get( 'assignments').get('app_client_id'), 'Temp') diff --git a/tests/terraform/context_parsers/test_variable_context_parser2.py b/tests/terraform/context_parsers/test_variable_context_parser2.py index 0fea3dafa45..5a16d42d70a 100644 --- a/tests/terraform/context_parsers/test_variable_context_parser2.py +++ b/tests/terraform/context_parsers/test_variable_context_parser2.py @@ -1,5 +1,7 @@ import unittest -from checkov.terraform.parser import Parser + +from checkov.terraform import TFDefinitionKey +from checkov.terraform.tf_parser import TFParser from checkov.terraform.context_parsers.registry import parser_registry import os @@ -7,10 +9,8 @@ class TestVariableContextParser(unittest.TestCase): def setUp(self): test_root_dir = os.path.dirname(os.path.realpath(__file__)) + '/../evaluation/resources/default_evaluation/' - tf_definitions = {} parsing_errors = {} - Parser().parse_directory(directory=test_root_dir, - out_definitions=tf_definitions, + tf_definitions = TFParser().parse_directory(directory=test_root_dir, out_evaluations_context={}, out_parsing_errors=parsing_errors) for definition in tf_definitions.items(): @@ -18,22 +18,25 @@ def setUp(self): self.definitions_context = definitions_context def test_assignments_exists(self): + file_path = os.path.dirname(os.path.realpath(__file__))\ + + '/../evaluation/resources/default_evaluation/variables.tf' + key = TFDefinitionKey(file_path=file_path, tf_source_modules=None) self.assertIsNotNone( - self.definitions_context[os.path.dirname( - os.path.realpath(__file__)) + '/../evaluation/resources/default_evaluation/variables.tf'][ + self.definitions_context[key][ 'variable'].get( 'assignments')) def test_assignment_value(self): + file_path = os.path.dirname(os.path.realpath(__file__)) + '/../evaluation/resources/default_evaluation/variables.tf' + key = TFDefinitionKey(file_path=file_path, tf_source_modules=None) self.assertFalse( - self.definitions_context[os.path.dirname( - os.path.realpath(__file__)) + '/../evaluation/resources/default_evaluation/variables.tf'][ + self.definitions_context[key][ 'variable'].get( 'assignments').get('user_exists') ) + self.assertEqual( - self.definitions_context[os.path.dirname( - os.path.realpath(__file__)) + '/../evaluation/resources/default_evaluation/variables.tf'][ + self.definitions_context[key][ 'variable'].get( 'assignments').get('app_client_id'), 'Temp') diff --git a/tests/terraform/graph/db_connector/test_graph_connector.py b/tests/terraform/graph/db_connector/test_graph_connector.py index db829dd4819..c0732e39928 100644 --- a/tests/terraform/graph/db_connector/test_graph_connector.py +++ b/tests/terraform/graph/db_connector/test_graph_connector.py @@ -4,7 +4,7 @@ from checkov.common.graph.db_connectors.networkx.networkx_db_connector import NetworkxConnector from checkov.common.graph.db_connectors.igraph.igraph_db_connector import IgraphConnector from checkov.terraform.graph_builder.local_graph import TerraformLocalGraph -from checkov.terraform.parser import Parser +from checkov.terraform.tf_parser import TFParser TEST_DIRNAME = os.path.dirname(os.path.realpath(__file__)) @@ -12,7 +12,7 @@ class TestGraphConnector(TestCase): def test_creating_networkx_graph(self): resources_dir = os.path.realpath(os.path.join(TEST_DIRNAME, '../resources/encryption')) - hcl_config_parser = Parser() + hcl_config_parser = TFParser() module, _ = hcl_config_parser.parse_hcl_module(resources_dir, 'AWS') local_graph = TerraformLocalGraph(module) local_graph._create_vertices() @@ -21,7 +21,7 @@ def test_creating_networkx_graph(self): def test_creating_igraph_graph(self): resources_dir = os.path.realpath(os.path.join(TEST_DIRNAME, '../resources/encryption')) - hcl_config_parser = Parser() + hcl_config_parser = TFParser() module, _ = hcl_config_parser.parse_hcl_module(resources_dir, 'AWS') local_graph = TerraformLocalGraph(module) local_graph._create_vertices() diff --git a/tests/terraform/graph/graph_builder/test_local_graph.py b/tests/terraform/graph/graph_builder/test_local_graph.py index 963b72955f1..82ca433249e 100644 --- a/tests/terraform/graph/graph_builder/test_local_graph.py +++ b/tests/terraform/graph/graph_builder/test_local_graph.py @@ -11,11 +11,11 @@ from checkov.common.graph.graph_builder.graph_components.attribute_names import CustomAttributes from checkov.common.util.parser_utils import TERRAFORM_NESTED_MODULE_PATH_PREFIX, TERRAFORM_NESTED_MODULE_PATH_ENDING, \ TERRAFORM_NESTED_MODULE_INDEX_SEPARATOR +from checkov.terraform import TFModule from checkov.terraform.graph_builder.graph_components.block_types import BlockType from checkov.terraform.graph_builder.graph_components.blocks import TerraformBlock from checkov.terraform.graph_builder.graph_components.generic_resource_encryption import ENCRYPTION_BY_RESOURCE_TYPE from checkov.terraform.graph_builder.graph_to_tf_definitions import convert_graph_vertices_to_tf_definitions -from checkov.terraform.parser import Parser from checkov.terraform.graph_builder.local_graph import TerraformLocalGraph from checkov.terraform.graph_manager import TerraformGraphManager from checkov.terraform.tf_parser import TFParser @@ -60,7 +60,7 @@ def test_single_edge_with_same_label(self): def test_set_variables_values_from_modules(self): resources_dir = os.path.realpath(os.path.join(TEST_DIRNAME, '../resources/variable_rendering/render_from_module_vpc')) - hcl_config_parser = Parser() + hcl_config_parser = TFParser() module, _ = hcl_config_parser.parse_hcl_module(resources_dir, source=self.source) local_graph = TerraformLocalGraph(module) local_graph._create_vertices() @@ -215,7 +215,7 @@ def test_set_variables_values_from_modules_with_new_tf_parser(self): def test_encryption_aws(self): resources_dir = os.path.realpath(os.path.join(TEST_DIRNAME, '../resources/encryption')) - hcl_config_parser = Parser() + hcl_config_parser = TFParser() module, _ = hcl_config_parser.parse_hcl_module(resources_dir, self.source) local_graph = TerraformLocalGraph(module) local_graph._create_vertices() @@ -242,7 +242,7 @@ def test_encryption_aws(self): def test_vertices_from_local_graph(self): resources_dir = os.path.realpath(os.path.join(TEST_DIRNAME, '../resources/variable_rendering/render_from_module_vpc')) - hcl_config_parser = Parser() + hcl_config_parser = TFParser() module, _ = hcl_config_parser.parse_hcl_module(resources_dir, self.source) local_graph = TerraformLocalGraph(module) local_graph._create_vertices() @@ -250,59 +250,19 @@ def test_vertices_from_local_graph(self): self.assertIsNotNone(tf_definitions) self.assertIsNotNone(breadcrumbs) - def test_module_dependencies(self): - resources_dir = os.path.realpath(os.path.join(TEST_DIRNAME, '../resources/modules/stacks')) - hcl_config_parser = Parser() - module, _ = hcl_config_parser.parse_hcl_module(resources_dir, self.source) - self.assertEqual(module.module_dependency_map[f'{resources_dir}/prod'], [[]]) - self.assertEqual(module.module_dependency_map[f'{resources_dir}/stage'], [[]]) - self.assertEqual(module.module_dependency_map[f'{resources_dir}/test'], [[]]) - self.assertEqual(module.module_dependency_map[f'{resources_dir}/prod/sub-prod'], [[f'{resources_dir}/prod/main.tf']]) - expected_inner_modules = [ - [ - f'{resources_dir}/prod/main.tf', - f'{resources_dir}/prod/sub-prod/main.tf{TERRAFORM_NESTED_MODULE_PATH_PREFIX}{resources_dir}/prod/main.tf{TERRAFORM_NESTED_MODULE_INDEX_SEPARATOR}0{TERRAFORM_NESTED_MODULE_PATH_ENDING}', - ], - [ - f'{resources_dir}/stage/main.tf' - ], - [ - f'{resources_dir}/test/main.tf' - ], - ] - self.assertEqual(module.module_dependency_map[f'{os.path.dirname(resources_dir)}/s3_inner_modules'], expected_inner_modules) - resources_dir_no_stacks = resources_dir.replace('/stacks', '') - expected_inner_modules = [ - [ - f'{resources_dir}/prod/main.tf', - f'{resources_dir}/prod/sub-prod/main.tf{TERRAFORM_NESTED_MODULE_PATH_PREFIX}{resources_dir}/prod/main.tf{TERRAFORM_NESTED_MODULE_INDEX_SEPARATOR}0{TERRAFORM_NESTED_MODULE_PATH_ENDING}', - f'{resources_dir_no_stacks}/s3_inner_modules/main.tf{TERRAFORM_NESTED_MODULE_PATH_PREFIX}{resources_dir}/prod/sub-prod/main.tf{TERRAFORM_NESTED_MODULE_INDEX_SEPARATOR}0{TERRAFORM_NESTED_MODULE_PATH_PREFIX}{resources_dir}/prod/main.tf{TERRAFORM_NESTED_MODULE_INDEX_SEPARATOR}0{TERRAFORM_NESTED_MODULE_PATH_ENDING}{TERRAFORM_NESTED_MODULE_PATH_ENDING}', - ], - [ - f'{resources_dir}/stage/main.tf', - f'{resources_dir_no_stacks}/s3_inner_modules/main.tf{TERRAFORM_NESTED_MODULE_PATH_PREFIX}{resources_dir}/stage/main.tf{TERRAFORM_NESTED_MODULE_INDEX_SEPARATOR}0{TERRAFORM_NESTED_MODULE_PATH_ENDING}', - ], - [ - f'{resources_dir}/test/main.tf', - f'{resources_dir_no_stacks}/s3_inner_modules/main.tf{TERRAFORM_NESTED_MODULE_PATH_PREFIX}{resources_dir}/test/main.tf{TERRAFORM_NESTED_MODULE_INDEX_SEPARATOR}0{TERRAFORM_NESTED_MODULE_PATH_ENDING}', - ], - ] - self.assertEqual(module.module_dependency_map[f'{os.path.dirname(resources_dir)}/s3_inner_modules/inner'], expected_inner_modules) - def test_blocks_from_local_graph_module(self): resources_dir = os.path.realpath(os.path.join(TEST_DIRNAME, '../resources/modules/stacks')) - hcl_config_parser = Parser() + hcl_config_parser = TFParser() module, _ = hcl_config_parser.parse_hcl_module(resources_dir, self.source) self.assertEqual(len(list(filter(lambda block: block.block_type == BlockType.RESOURCE and block.name == 'aws_s3_bucket.inner_s3', module.blocks))), 3) self.assertEqual(len(list(filter(lambda block: block.block_type == BlockType.MODULE and block.name == 'inner_module_call', module.blocks))), 3) self.assertEqual(len(list(filter(lambda block: block.block_type == BlockType.MODULE and block.name == 's3', module.blocks))), 3) self.assertEqual(len(list(filter(lambda block: block.block_type == BlockType.MODULE and block.name == 'sub-module', module.blocks))), 1) - @mock.patch.dict(os.environ, {"CHECKOV_NEW_TF_PARSER": "False"}) def test_vertices_from_local_graph_module(self): parent_dir = Path(TEST_DIRNAME).parent resources_dir = str(parent_dir / "resources/modules/stacks") - hcl_config_parser = Parser() + hcl_config_parser = TFParser() module, _ = hcl_config_parser.parse_hcl_module(resources_dir, self.source) local_graph = TerraformLocalGraph(module) local_graph.build_graph(render_variables=True) @@ -325,8 +285,8 @@ def test_vertices_from_local_graph_module(self): for vertex in local_graph.vertices if vertex.name == "aws_s3_bucket.inner_s3" and vertex.source_module == {8} ) - self.assertDictEqual( - { + + expected_vertex_1_breadcrumbs = { "versioning.enabled": [ { "type": "module", @@ -343,30 +303,40 @@ def test_vertices_from_local_graph_module(self): ], "source_module_": [ { - "type": "module", - "name": "sub-module", - "path": str(parent_dir / "resources/modules/stacks/prod/main.tf"), - "idx": 12 - }, + 'type': 'module', + 'name': 'sub-module', + 'path': str(parent_dir / 'resources/modules/stacks/prod/main.tf'), + 'idx': 12, + 'source_module_object': None + } + , { - "type": "module", - "name": "s3", - "path": str(parent_dir / "resources/modules/stacks/prod/sub-prod/main.tf"), - "idx": 13 - }, + 'type': 'module', + 'name': 's3', + 'path': str(parent_dir / 'resources/modules/stacks/prod/sub-prod/main.tf'), + 'idx': 13, + 'source_module_object': TFModule(path=str(parent_dir / 'resources/modules/stacks/prod/main.tf'), + name='sub-module', foreach_idx=None, nested_tf_module=None) + } + , { - "type": "module", - "name": "inner_module_call", - "path": str(parent_dir / "resources/modules/s3_inner_modules/main.tf"), - "idx": 6 - }, + 'type': 'module', + 'name': 'inner_module_call', + 'path': str(parent_dir / 'resources/modules/s3_inner_modules/main.tf'), + 'idx': 6, + 'source_module_object': TFModule(path=str(parent_dir / 'resources/modules/stacks/prod/sub-prod/main.tf'), + name='s3', foreach_idx=None, + nested_tf_module=TFModule(path=str(parent_dir / 'resources/modules/stacks/prod/main.tf'), + name='sub-module', foreach_idx=None, + nested_tf_module=None) + ) + } ], - }, - bucket_vertex_1.breadcrumbs, - ) + } + + self.assertDictEqual(expected_vertex_1_breadcrumbs, bucket_vertex_1.breadcrumbs) - self.assertDictEqual( - { + expected_vertex_2_breadcrumbs = { "versioning.enabled": [ { "type": "module", @@ -383,24 +353,25 @@ def test_vertices_from_local_graph_module(self): ], "source_module_": [ { - "type": "module", - "name": "s3", - "path": str(parent_dir / "resources/modules/stacks/stage/main.tf"), - "idx": 14 + 'type': 'module', + 'name': 's3', + 'path': str(parent_dir / 'resources/modules/stacks/stage/main.tf'), + 'idx': 14, + 'source_module_object': None }, { - "type": "module", - "name": "inner_module_call", - "path": str(parent_dir / "resources/modules/s3_inner_modules/main.tf"), - "idx": 7 - }, + 'type': 'module', + 'name': 'inner_module_call', + 'path': str(parent_dir / 'resources/modules/s3_inner_modules/main.tf'), + 'idx': 7, + 'source_module_object': TFModule(path=str(parent_dir / 'resources/modules/stacks/stage/main.tf'), + name='s3', foreach_idx=None, nested_tf_module=None) + } ], - }, - bucket_vertex_2.breadcrumbs, - ) + } + self.assertDictEqual(expected_vertex_2_breadcrumbs, bucket_vertex_2.breadcrumbs) - self.assertDictEqual( - { + expected_vertex_3_breadcrumbs = { "versioning.enabled": [ { "type": "module", @@ -417,86 +388,25 @@ def test_vertices_from_local_graph_module(self): ], "source_module_": [ { - "type": "module", - "name": "s3", - "path": str(parent_dir / "resources/modules/stacks/test/main.tf"), - "idx": 15 + 'type': 'module', + 'name': 's3', + 'path': str(parent_dir / 'resources/modules/stacks/test/main.tf'), + 'idx': 15, + 'source_module_object': None }, { - "type": "module", - "name": "inner_module_call", - "path": str(parent_dir / "resources/modules/s3_inner_modules/main.tf"), - "idx": 8 - }, + 'type': 'module', + 'name': 'inner_module_call', + 'path': str(parent_dir / 'resources/modules/s3_inner_modules/main.tf'), + 'idx': 8, + 'source_module_object': TFModule(path=str(parent_dir / 'resources/modules/stacks/test/main.tf'), + name='s3', foreach_idx=None, nested_tf_module=None) + } ], - }, - bucket_vertex_3.breadcrumbs, - ) + } + self.assertDictEqual(expected_vertex_3_breadcrumbs, bucket_vertex_3.breadcrumbs) - @mock.patch.dict(os.environ, {"CHECKOV_NEW_TF_PARSER": "False"}) def test_variables_same_name_different_modules(self): - resources_dir = os.path.realpath(os.path.join(TEST_DIRNAME, '../resources/modules/same_var_names')) - hcl_config_parser = Parser() - module, _ = hcl_config_parser.parse_hcl_module(resources_dir, self.source) - local_graph = TerraformLocalGraph(module) - local_graph.build_graph(render_variables=True) - print(local_graph.edges) - self.assertEqual(12, len(local_graph.edges)) - self.assertEqual(13, len(local_graph.vertices)) - - module_variable_edges = [ - e for e in local_graph.edges - if local_graph.vertices[e.dest].block_type == "module" and local_graph.vertices[e.dest].path.endswith( - 'same_var_names/module2/main.tf') - ] - - # Check they point to 2 different modules - self.assertEqual(2, len(module_variable_edges)) - self.assertNotEqual(local_graph.vertices[module_variable_edges[0].origin], - local_graph.vertices[module_variable_edges[1].origin]) - - - module_variable_edges = [ - e for e in local_graph.edges - if local_graph.vertices[e.dest].block_type == "module" and local_graph.vertices[e.dest].path.endswith('same_var_names/module1/main.tf') - ] - - # Check they point to 2 different modules - self.assertEqual(2, len(module_variable_edges)) - self.assertNotEqual(local_graph.vertices[module_variable_edges[0].origin], local_graph.vertices[module_variable_edges[1].origin]) - - def test_variables_same_name_different_modules_with_new_tf_parser(self): - resources_dir = os.path.realpath(os.path.join(TEST_DIRNAME, '../resources/modules/same_var_names')) - hcl_config_parser = TFParser() - module, _ = hcl_config_parser.parse_hcl_module(resources_dir, self.source) - local_graph = TerraformLocalGraph(module) - local_graph.build_graph(render_variables=True) - print(local_graph.edges) - self.assertEqual(12, len(local_graph.edges)) - self.assertEqual(13, len(local_graph.vertices)) - - module_variable_edges = [ - e for e in local_graph.edges - if local_graph.vertices[e.dest].block_type == "module" and local_graph.vertices[e.dest].path.endswith( - 'same_var_names/module2/main.tf') - ] - - # Check they point to 2 different modules - self.assertEqual(2, len(module_variable_edges)) - self.assertNotEqual(local_graph.vertices[module_variable_edges[0].origin], - local_graph.vertices[module_variable_edges[1].origin]) - - - module_variable_edges = [ - e for e in local_graph.edges - if local_graph.vertices[e.dest].block_type == "module" and local_graph.vertices[e.dest].path.endswith('same_var_names/module1/main.tf') - ] - - # Check they point to 2 different modules - self.assertEqual(2, len(module_variable_edges)) - self.assertNotEqual(local_graph.vertices[module_variable_edges[0].origin], local_graph.vertices[module_variable_edges[1].origin]) - - def test_variables_same_name_different_modules_with_new_tf_parser(self): resources_dir = os.path.realpath(os.path.join(TEST_DIRNAME, '../resources/modules/same_var_names')) hcl_config_parser = TFParser() module, _ = hcl_config_parser.parse_hcl_module(resources_dir, self.source) @@ -527,23 +437,3 @@ def test_variables_same_name_different_modules_with_new_tf_parser(self): self.assertEqual(2, len(module_variable_edges)) self.assertNotEqual(local_graph.vertices[module_variable_edges[0].origin], local_graph.vertices[module_variable_edges[1].origin]) - @mock.patch.dict(os.environ, {"CHECKOV_NEW_TF_PARSER": "False"}) - def test_nested_modules_instances(self): - resources_dir = os.path.realpath(os.path.join(TEST_DIRNAME, '../resources/modules/nested_modules_instances')) - hcl_config_parser = Parser() - module, _ = hcl_config_parser.parse_hcl_module(resources_dir, self.source) - local_graph = TerraformLocalGraph(module) - local_graph.build_graph(render_variables=True) - - vertices = [vertex.to_dict() for vertex in local_graph.vertices] - edges = [edge.to_dict() for edge in local_graph.edges] - - with open(os.path.realpath(os.path.join(TEST_DIRNAME, '../resources/modules/nested_modules_instances/expected_local_graph.json')), 'r') as f: - expected = json.load(f) - - self.assertCountEqual( - json.loads(json.dumps(vertices).replace(resources_dir, '')), - json.loads(json.dumps(expected.get('vertices')).replace(resources_dir, '')), - - ) - self.assertCountEqual(edges, expected.get('edges')) diff --git a/tests/terraform/graph/graph_builder/test_terraform_graph_parser.py b/tests/terraform/graph/graph_builder/test_terraform_graph_parser.py index 633f890a695..87b8f070f76 100644 --- a/tests/terraform/graph/graph_builder/test_terraform_graph_parser.py +++ b/tests/terraform/graph/graph_builder/test_terraform_graph_parser.py @@ -2,9 +2,10 @@ from lark import Tree +from checkov.terraform import TFDefinitionKey from checkov.terraform.modules.module_utils import clean_parser_types -from checkov.terraform.parser import Parser -from unittest import TestCase, mock +from checkov.terraform.tf_parser import TFParser +from unittest import TestCase TEST_DIRNAME = os.path.dirname(os.path.realpath(__file__)) @@ -44,14 +45,16 @@ def test_hcl_parsing_consistent_old_new(self): cur_dir = os.path.dirname(os.path.realpath(__file__)) tf_dir = f'{cur_dir}/../resources/tf_parsing_comparison/tf_regular' old_tf_dir = f'{cur_dir}/../resources/tf_parsing_comparison/tf_old' - _, tf_definitions = Parser().parse_hcl_module(tf_dir, 'AWS') - _, old_tf_definitions = Parser().parse_hcl_module(old_tf_dir, 'AWS') - self.assertDictEqual(tf_definitions[f'{tf_dir}/main.tf'], old_tf_definitions[f'{old_tf_dir}/main.tf']) + _, tf_definitions = TFParser().parse_hcl_module(tf_dir, 'AWS') + _, old_tf_definitions = TFParser().parse_hcl_module(old_tf_dir, 'AWS') + definition_value = list(tf_definitions.values())[0] + old_definition_value = list(tf_definitions.values())[0] + self.assertDictEqual(definition_value, old_definition_value) def test_hcl_parsing_old_booleans_correctness(self): cur_dir = os.path.dirname(os.path.realpath(__file__)) tf_dir = f'{cur_dir}/../resources/tf_parsing_comparison/tf_regular' - _, tf_definitions = Parser().parse_hcl_module(tf_dir, 'AWS') + _, tf_definitions = TFParser().parse_hcl_module(tf_dir, 'AWS') expected = [ { "aws_cloudtrail": { @@ -155,7 +158,8 @@ def test_hcl_parsing_old_booleans_correctness(self): } }, ] - tf_definitions_resources = tf_definitions[f'{tf_dir}/main.tf']['resource'] + definition_key = TFDefinitionKey(file_path=os.path.join(tf_dir, "main.tf"), tf_source_modules=None) + tf_definitions_resources = tf_definitions[definition_key]['resource'] for index in range(len(tf_definitions_resources)): self.assertDictEqual( tf_definitions_resources[index], @@ -165,21 +169,21 @@ def test_hcl_parsing_old_booleans_correctness(self): def test_hcl_parsing_sorting(self): source_dir = os.path.realpath(os.path.join(TEST_DIRNAME, '../resources/tf_parsing_comparison/modifications_diff')) - config_parser = Parser() + config_parser = TFParser() _, tf_definitions = config_parser.parse_hcl_module(source_dir, 'AWS') expected = ['https://www.googleapis.com/auth/devstorage.read_only', 'https://www.googleapis.com/auth/logging.write', 'https://www.googleapis.com/auth/monitoring.write', 'https://www.googleapis.com/auth/service.management.readonly', 'https://www.googleapis.com/auth/servicecontrol', 'https://www.googleapis.com/auth/trace.append'] - result_resource = tf_definitions[source_dir + '/main.tf']['resource'][0]['google_compute_instance']['tfer--test3']['service_account'][0]['scopes'][0] + defintion_key = TFDefinitionKey(file_path=os.path.join(source_dir, "main.tf"), tf_source_modules=None) + result_resource = tf_definitions[defintion_key]['resource'][0]['google_compute_instance']['tfer--test3']['service_account'][0]['scopes'][0] self.assertListEqual(result_resource, expected) def test_build_graph_with_linked_modules(self): source_dir = os.path.realpath(os.path.join(TEST_DIRNAME, '../resources/nested_modules_double_call')) - config_parser = Parser() - definitions = {} + config_parser = TFParser() - config_parser.parse_directory(source_dir, definitions) + definitions = config_parser.parse_directory(source_dir) assert len(definitions.keys()) == 13 assert '/Users/arosenfeld/Desktop/dev/checkov/tests/terraform/graph/resources/nested_modules_double_call/main.tf' not in definitions assert '/Users/arosenfeld/Desktop/dev/checkov/tests/terraform/graph/resources/nested_modules_double_call/third/main.tf[/Users/arosenfeld/Desktop/dev/checkov/tests/terraform/graph/resources/nested_modules_double_call/main.tf#0]' not in definitions diff --git a/tests/terraform/graph/runner/persistent_data.json b/tests/terraform/graph/runner/persistent_data.json deleted file mode 100644 index 0272eb1a04d..00000000000 --- a/tests/terraform/graph/runner/persistent_data.json +++ /dev/null @@ -1,215 +0,0 @@ -{ - "tf_definitions": { - "/Users/nkor/dev/checkov_v2/tests/graph/terraform/resources/graph_files_test/more_vars.tf": { - "variable": [ - { - "encryption": { - "default": [ - "AES256" - ] - } - } - ] - }, - "/Users/nkor/dev/checkov_v2/tests/graph/terraform/resources/graph_files_test/pass_s3.tf": { - "resource": [ - { - "aws_s3_bucket": { - "bucket_with_versioning": { - "server_side_encryption_configuration": [ - { - "rule": [ - { - "apply_server_side_encryption_by_default": [ - { - "sse_algorithm": [ - "AES256" - ] - } - ] - } - ] - } - ], - "versioning": [ - { - "enabled": [ - true - ] - } - ] - } - } - } - ] - }, - "/Users/nkor/dev/checkov_v2/tests/graph/terraform/resources/graph_files_test/variables.tf": { - "variable": [ - { - "versioning": { - "default": [ - true - ] - } - } - ] - } - }, - "definitions_context": { - "/Users/nkor/dev/checkov_v2/tests/graph/terraform/resources/graph_files_test/more_vars.tf": { - "variable": { - "encryption": { - "start_line": 1, - "end_line": 3, - "code_lines": [ - [ - 1, - "variable \"encryption\" {\n" - ], - [ - 2, - " default = \"AES256\"\n" - ], - [ - 3, - "}" - ] - ], - "skipped_checks": [] - }, - "assignments": { - "encryption": "AES256" - } - } - }, - "/Users/nkor/dev/checkov_v2/tests/graph/terraform/resources/graph_files_test/pass_s3.tf": { - "resource": { - "aws_s3_bucket": { - "bucket_with_versioning": { - "start_line": 1, - "end_line": 13, - "code_lines": [ - [ - 1, - "resource \"aws_s3_bucket\" \"bucket_with_versioning\" {\n" - ], - [ - 2, - " versioning {\n" - ], - [ - 3, - " enabled = var.versioning\n" - ], - [ - 4, - " }\n" - ], - [ - 5, - "\n" - ], - [ - 6, - " server_side_encryption_configuration {\n" - ], - [ - 7, - " rule {\n" - ], - [ - 8, - " apply_server_side_encryption_by_default {\n" - ], - [ - 9, - " sse_algorithm = var.encryption\n" - ], - [ - 10, - " }\n" - ], - [ - 11, - " }\n" - ], - [ - 12, - " }\n" - ], - [ - 13, - "}" - ] - ], - "skipped_checks": [] - } - } - } - }, - "/Users/nkor/dev/checkov_v2/tests/graph/terraform/resources/graph_files_test/variables.tf": { - "variable": { - "versioning": { - "start_line": 1, - "end_line": 3, - "code_lines": [ - [ - 1, - "variable \"versioning\" {\n" - ], - [ - 2, - " default = true\n" - ], - [ - 3, - "}" - ] - ], - "skipped_checks": [] - }, - "assignments": { - "versioning": true - } - } - } - }, - "breadcrumbs": { - "/pass_s3.tf": { - "aws_s3_bucket.bucket_with_versioning": { - "server_side_encryption_configuration.rule.apply_server_side_encryption_by_default.sse_algorithm": [ - { - "type": "variable", - "name": "encryption", - "path": "/Users/nkor/dev/checkov_v2/tests/graph/terraform/resources/graph_files_test/more_vars.tf", - "module_connection": false - } - ], - "server_side_encryption_configuration.rule.apply_server_side_encryption_by_default": [ - { - "type": "variable", - "name": "encryption", - "path": "/Users/nkor/dev/checkov_v2/tests/graph/terraform/resources/graph_files_test/more_vars.tf", - "module_connection": false - } - ], - "server_side_encryption_configuration.rule": [ - { - "type": "variable", - "name": "encryption", - "path": "/Users/nkor/dev/checkov_v2/tests/graph/terraform/resources/graph_files_test/more_vars.tf", - "module_connection": false - } - ], - "versioning.enabled": [ - { - "type": "variable", - "name": "versioning", - "path": "/Users/nkor/dev/checkov_v2/tests/graph/terraform/resources/graph_files_test/variables.tf", - "module_connection": false - } - ] - } - } - } -} \ No newline at end of file diff --git a/tests/terraform/graph/runner/test_graph_builder.py b/tests/terraform/graph/runner/test_graph_builder.py index 12f8f3878fb..17924779507 100644 --- a/tests/terraform/graph/runner/test_graph_builder.py +++ b/tests/terraform/graph/runner/test_graph_builder.py @@ -23,7 +23,7 @@ def test_build_graph(self): tf_definitions = runner.definitions self.assertEqual(5, len(report.failed_checks)) for file, definitions in tf_definitions.items(): - if file.endswith('pass_s3.tf'): + if file.file_path.endswith('pass_s3.tf'): s3_bucket_config = definitions['resource'][0]['aws_s3_bucket']['bucket_with_versioning'] # Evaluation succeeded for included vars self.assertTrue(s3_bucket_config['versioning'][0]['enabled'][0]) @@ -53,22 +53,6 @@ def test_run_clean(self): self.assertEqual(5, len(report.passed_checks)) self.assertEqual(0, len(report.skipped_checks)) - def test_run_persistent_data(self): - resources_path = os.path.join(os.path.dirname(os.path.dirname(__file__)), "resources", "graph_files_test") - runner = Runner(db_connector=self.db_connector()) - data_path = os.path.join(os.path.dirname(__file__), "persistent_data.json") - with open(data_path) as f: - data = json.load(f) - tf_definitions = data["tf_definitions"] - breadcrumbs = data["breadcrumbs"] - definitions_context = data["definitions_context"] - runner.set_external_data(tf_definitions, definitions_context, breadcrumbs) - report = runner.run(root_folder=resources_path) - # note that we dont count graph violations in this case - self.assertEqual(len(report.failed_checks), 0) - self.assertEqual(len(report.passed_checks), 1) - self.assertEqual(len(report.skipped_checks), 0) - def test_module_and_variables(self): resources_path = os.path.join(os.path.dirname(os.path.dirname(__file__)), "resources", "modules-and-vars") runner = Runner(db_connector=self.db_connector()) diff --git a/tests/terraform/graph/variable_rendering/test_foreach_renderer.py b/tests/terraform/graph/variable_rendering/test_foreach_renderer.py index 9be1935bea0..b3ce81ffa8c 100644 --- a/tests/terraform/graph/variable_rendering/test_foreach_renderer.py +++ b/tests/terraform/graph/variable_rendering/test_foreach_renderer.py @@ -185,7 +185,7 @@ def test_tf_definitions_and_breadcrumbs(): expected_data = load_expected_data('expected_data_foreach.json') tf_definitions_to_check = {} for path, res in tf_definitions.items(): - path_list = path.split('/')[-2:] + path_list = path.file_path.split('/')[-2:] real_path = os.path.join(path_list[0], path_list[1]) tf_definitions_to_check[real_path] = tf_definitions[path] assert_object_equal(tf_definitions_to_check, expected_data['tf_definitions']) diff --git a/tests/terraform/graph/variable_rendering/test_render_scenario.py b/tests/terraform/graph/variable_rendering/test_render_scenario.py index 279cd518b01..0b98fd87b2f 100644 --- a/tests/terraform/graph/variable_rendering/test_render_scenario.py +++ b/tests/terraform/graph/variable_rendering/test_render_scenario.py @@ -6,7 +6,7 @@ import jmespath -from checkov.common.util.json_utils import object_hook +from checkov.common.util.json_utils import object_hook, CustomJSONEncoder from checkov.common.util.parser_utils import TERRAFORM_NESTED_MODULE_PATH_PREFIX, TERRAFORM_NESTED_MODULE_PATH_ENDING, \ TERRAFORM_NESTED_MODULE_INDEX_SEPARATOR, TERRAFORM_NESTED_MODULE_PATH_SEPARATOR_LENGTH from checkov.terraform.modules.module_objects import TFDefinitionKey @@ -85,14 +85,14 @@ def test_nested_modules_instances_enable(self): dir_name = 'nested_modules_instances_enable' resources_dir = os.path.realpath(os.path.join(TEST_DIRNAME, '../../parser/resources/parser_scenarios', dir_name)) - from checkov.terraform.parser import Parser - parser = Parser() - tf_definitions = {} - parser.parse_directory(directory=resources_dir, out_definitions=tf_definitions) + from checkov.terraform.tf_parser import TFParser + parser = TFParser() + tf_definitions = parser.parse_directory(directory=resources_dir) with open(f'{resources_dir}/expected.json') as fp: expected = json.load(fp) - result, expected = json.dumps(tf_definitions, sort_keys=True), json.dumps(expected, sort_keys=True) + result, expected = json.dumps(tf_definitions, sort_keys=True, cls=CustomJSONEncoder), \ + json.dumps(expected, sort_keys=True, cls=CustomJSONEncoder) result = result.replace(resources_dir, '') expected = expected.replace(resources_dir, '') assert result == expected diff --git a/tests/terraform/parser/resources/parser_scenarios/nested_modules_instances_enable/expected.json b/tests/terraform/parser/resources/parser_scenarios/nested_modules_instances_enable/expected.json index 33f7e7107c6..117e4d820e7 100644 --- a/tests/terraform/parser/resources/parser_scenarios/nested_modules_instances_enable/expected.json +++ b/tests/terraform/parser/resources/parser_scenarios/nested_modules_instances_enable/expected.json @@ -1,30 +1,46 @@ { - "/tf_module/main.tf": { - "provider": [ - {"aws": {"region": ["us-west-2"], "__start_line__": 1, "__end_line__": 3}} - ], + "{\"file_path\": \"/tf_module/main.tf\", \"tf_source_modules\": null}": { "module": [ { "s3_module": { - "source": ["./module"], - "bucket": ["${aws_s3_bucket.example.id}"], - "__start_line__": 5, "__end_line__": 9, "__resolved__": [ - "/tf_module/module/main.tf([{/tf_module/main.tf#*#0}])", - "/tf_module/module/variable.tf([{/tf_module/main.tf#*#0}])" + "{\"file_path\": \"/tf_module/module/main.tf\", \"tf_source_modules\": {\"path\": \"/tf_module/main.tf\", \"name\": \"s3_module\", \"foreach_idx\": null, \"nested_tf_module\": null}}", + "{\"file_path\": \"/tf_module/module/variable.tf\", \"tf_source_modules\": {\"path\": \"/tf_module/main.tf\", \"name\": \"s3_module\", \"foreach_idx\": null, \"nested_tf_module\": null}}" + ], + "__start_line__": 5, + "bucket": [ + "${aws_s3_bucket.example.id}" + ], + "source": [ + "./module" ] } }, { "s3_module2": { - "source": ["./module"], - "bucket": ["${aws_s3_bucket.example2.id}"], - "__start_line__": 11, "__end_line__": 15, "__resolved__": [ - "/tf_module/module/main.tf([{/tf_module/main.tf#*#1}])", - "/tf_module/module/variable.tf([{/tf_module/main.tf#*#1}])" + "{\"file_path\": \"/tf_module/module/main.tf\", \"tf_source_modules\": {\"path\": \"/tf_module/main.tf\", \"name\": \"s3_module2\", \"foreach_idx\": null, \"nested_tf_module\": null}}", + "{\"file_path\": \"/tf_module/module/variable.tf\", \"tf_source_modules\": {\"path\": \"/tf_module/main.tf\", \"name\": \"s3_module2\", \"foreach_idx\": null, \"nested_tf_module\": null}}" + ], + "__start_line__": 11, + "bucket": [ + "${aws_s3_bucket.example2.id}" + ], + "source": [ + "./module" + ] + } + } + ], + "provider": [ + { + "aws": { + "__end_line__": 3, + "__start_line__": 1, + "region": [ + "us-west-2" ] } } @@ -33,113 +49,189 @@ { "aws_s3_bucket": { "example": { - "bucket": ["example"], + "__end_line__": 19, "__start_line__": 17, - "__end_line__": 19 + "bucket": [ + "example" + ] } } }, { "aws_s3_bucket": { "example2": { - "bucket": ["example"], + "__end_line__": 23, "__start_line__": 21, - "__end_line__": 23 + "bucket": [ + "example" + ] } } } ] }, - "/tf_module/module/module2/main.tf([{/tf_module/module/main.tf#*#0([{/tf_module/main.tf#*#0}])}])": { + "{\"file_path\": \"/tf_module/module/main.tf\", \"tf_source_modules\": {\"path\": \"/tf_module/main.tf\", \"name\": \"s3_module\", \"foreach_idx\": null, \"nested_tf_module\": null}}": { + "module": [ + { + "inner_s3_module": { + "__end_line__": 4, + "__resolved__": [ + "{\"file_path\": \"/tf_module/module/module2/main.tf\", \"tf_source_modules\": {\"path\": \"/tf_module/module/main.tf\", \"name\": \"inner_s3_module\", \"foreach_idx\": null, \"nested_tf_module\": {\"path\": \"/tf_module/main.tf\", \"name\": \"s3_module\", \"foreach_idx\": null, \"nested_tf_module\": null}}}", + "{\"file_path\": \"/tf_module/module/module2/variable.tf\", \"tf_source_modules\": {\"path\": \"/tf_module/module/main.tf\", \"name\": \"inner_s3_module\", \"foreach_idx\": null, \"nested_tf_module\": {\"path\": \"/tf_module/main.tf\", \"name\": \"s3_module\", \"foreach_idx\": null, \"nested_tf_module\": null}}}" + ], + "__start_line__": 1, + "bucket2": [ + "${var.bucket}" + ], + "source": [ + "./module2" + ] + } + } + ] + }, + "{\"file_path\": \"/tf_module/module/main.tf\", \"tf_source_modules\": {\"path\": \"/tf_module/main.tf\", \"name\": \"s3_module2\", \"foreach_idx\": null, \"nested_tf_module\": null}}": { + "module": [ + { + "inner_s3_module": { + "__end_line__": 4, + "__resolved__": [ + "{\"file_path\": \"/tf_module/module/module2/main.tf\", \"tf_source_modules\": {\"path\": \"/tf_module/module/main.tf\", \"name\": \"inner_s3_module\", \"foreach_idx\": null, \"nested_tf_module\": {\"path\": \"/tf_module/main.tf\", \"name\": \"s3_module2\", \"foreach_idx\": null, \"nested_tf_module\": null}}}", + "{\"file_path\": \"/tf_module/module/module2/variable.tf\", \"tf_source_modules\": {\"path\": \"/tf_module/module/main.tf\", \"name\": \"inner_s3_module\", \"foreach_idx\": null, \"nested_tf_module\": {\"path\": \"/tf_module/main.tf\", \"name\": \"s3_module2\", \"foreach_idx\": null, \"nested_tf_module\": null}}}" + ], + "__start_line__": 1, + "bucket2": [ + "${var.bucket}" + ], + "source": [ + "./module2" + ] + } + } + ] + }, + "{\"file_path\": \"/tf_module/module/module2/main.tf\", \"tf_source_modules\": {\"path\": \"/tf_module/module/main.tf\", \"name\": \"inner_s3_module\", \"foreach_idx\": null, \"nested_tf_module\": {\"path\": \"/tf_module/main.tf\", \"name\": \"s3_module\", \"foreach_idx\": null, \"nested_tf_module\": null}}}": { "locals": [ - {"bucket2": ["${var.bucket2}"], "__start_line__": 1, "__end_line__": 3} + { + "__end_line__": 3, + "__start_line__": 1, + "bucket2": [ + "${var.bucket2}" + ] + } ], "resource": [ { "aws_s3_bucket_public_access_block": { "var_bucket": { - "bucket": ["${local.bucket2}"], - "block_public_acls": [true], - "block_public_policy": [true], - "ignore_public_acls": [true], - "restrict_public_buckets": [true], + "__end_line__": 11, "__start_line__": 5, - "__end_line__": 11 + "block_public_acls": [ + true + ], + "block_public_policy": [ + true + ], + "bucket": [ + "${local.bucket2}" + ], + "ignore_public_acls": [ + true + ], + "restrict_public_buckets": [ + true + ] } } } ] }, - "/tf_module/module/module2/variable.tf([{/tf_module/module/main.tf#*#0([{/tf_module/main.tf#*#0}])}])": { - "variable": [ - {"bucket2": {"type": ["${string}"], "__start_line__": 1, "__end_line__": 3}} - ] - }, - "/tf_module/module/module2/main.tf([{/tf_module/module/main.tf#*#0([{/tf_module/main.tf#*#1}])}])": { + "{\"file_path\": \"/tf_module/module/module2/main.tf\", \"tf_source_modules\": {\"path\": \"/tf_module/module/main.tf\", \"name\": \"inner_s3_module\", \"foreach_idx\": null, \"nested_tf_module\": {\"path\": \"/tf_module/main.tf\", \"name\": \"s3_module2\", \"foreach_idx\": null, \"nested_tf_module\": null}}}": { "locals": [ - {"bucket2": ["${var.bucket2}"], "__start_line__": 1, "__end_line__": 3} + { + "__end_line__": 3, + "__start_line__": 1, + "bucket2": [ + "${var.bucket2}" + ] + } ], "resource": [ { "aws_s3_bucket_public_access_block": { "var_bucket": { - "bucket": ["${local.bucket2}"], - "block_public_acls": [true], - "block_public_policy": [true], - "ignore_public_acls": [true], - "restrict_public_buckets": [true], + "__end_line__": 11, "__start_line__": 5, - "__end_line__": 11 + "block_public_acls": [ + true + ], + "block_public_policy": [ + true + ], + "bucket": [ + "${local.bucket2}" + ], + "ignore_public_acls": [ + true + ], + "restrict_public_buckets": [ + true + ] } } } ] }, - "/tf_module/module/module2/variable.tf([{/tf_module/module/main.tf#*#0([{/tf_module/main.tf#*#1}])}])": { + "{\"file_path\": \"/tf_module/module/module2/variable.tf\", \"tf_source_modules\": {\"path\": \"/tf_module/module/main.tf\", \"name\": \"inner_s3_module\", \"foreach_idx\": null, \"nested_tf_module\": {\"path\": \"/tf_module/main.tf\", \"name\": \"s3_module\", \"foreach_idx\": null, \"nested_tf_module\": null}}}": { "variable": [ - {"bucket2": {"type": ["${string}"], "__start_line__": 1, "__end_line__": 3}} - ] - }, - "/tf_module/module/main.tf([{/tf_module/main.tf#*#0}])": { - "module": [ { - "inner_s3_module": { - "source": ["./module2"], - "bucket2": ["${var.bucket}"], + "bucket2": { + "__end_line__": 3, "__start_line__": 1, - "__end_line__": 4, - "__resolved__": [ - "/tf_module/module/module2/main.tf([{/tf_module/module/main.tf#*#0([{/tf_module/main.tf#*#0}])}])", - "/tf_module/module/module2/variable.tf([{/tf_module/module/main.tf#*#0([{/tf_module/main.tf#*#0}])}])" + "type": [ + "${string}" ] } } ] }, - "/tf_module/module/variable.tf([{/tf_module/main.tf#*#0}])": { + "{\"file_path\": \"/tf_module/module/module2/variable.tf\", \"tf_source_modules\": {\"path\": \"/tf_module/module/main.tf\", \"name\": \"inner_s3_module\", \"foreach_idx\": null, \"nested_tf_module\": {\"path\": \"/tf_module/main.tf\", \"name\": \"s3_module2\", \"foreach_idx\": null, \"nested_tf_module\": null}}}": { "variable": [ - {"bucket": {"type": ["${string}"], "__start_line__": 1, "__end_line__": 3}} + { + "bucket2": { + "__end_line__": 3, + "__start_line__": 1, + "type": [ + "${string}" + ] + } + } ] }, - "/tf_module/module/main.tf([{/tf_module/main.tf#*#1}])": { - "module": [ + "{\"file_path\": \"/tf_module/module/variable.tf\", \"tf_source_modules\": {\"path\": \"/tf_module/main.tf\", \"name\": \"s3_module\", \"foreach_idx\": null, \"nested_tf_module\": null}}": { + "variable": [ { - "inner_s3_module": { - "source": ["./module2"], - "bucket2": ["${var.bucket}"], + "bucket": { + "__end_line__": 3, "__start_line__": 1, - "__end_line__": 4, - "__resolved__": [ - "/tf_module/module/module2/main.tf([{/tf_module/module/main.tf#*#0([{/tf_module/main.tf#*#1}])}])", - "/tf_module/module/module2/variable.tf([{/tf_module/module/main.tf#*#0([{/tf_module/main.tf#*#1}])}])" + "type": [ + "${string}" ] } } ] }, - "/tf_module/module/variable.tf([{/tf_module/main.tf#*#1}])": { + "{\"file_path\": \"/tf_module/module/variable.tf\", \"tf_source_modules\": {\"path\": \"/tf_module/main.tf\", \"name\": \"s3_module2\", \"foreach_idx\": null, \"nested_tf_module\": null}}": { "variable": [ - {"bucket": {"type": ["${string}"], "__start_line__": 1, "__end_line__": 3}} + { + "bucket": { + "__end_line__": 3, + "__start_line__": 1, + "type": [ + "${string}" + ] + } + } ] } -} +} \ No newline at end of file diff --git a/tests/terraform/parser/test_module.py b/tests/terraform/parser/test_module.py index acbadb3e39b..25a5ed1e128 100644 --- a/tests/terraform/parser/test_module.py +++ b/tests/terraform/parser/test_module.py @@ -6,7 +6,6 @@ from checkov.terraform.modules.module_utils import validate_malformed_definitions, clean_bad_definitions, \ clean_parser_types, serialize_definitions -from checkov.terraform.parser import Parser from checkov.terraform.tf_parser import TFParser from checkov.common.util.consts import DEFAULT_EXTERNAL_MODULES_DIR @@ -33,7 +32,7 @@ def test_module_double_slash_cleanup(self): definitions = { '/mock/path/to.tf': clean_bad_definitions(non_malformed_definitions) } - module, _ = Parser().parse_hcl_module_from_tf_definitions(definitions, '', 'terraform') + module, _ = TFParser().parse_hcl_module_from_tf_definitions(definitions, '', 'terraform') print(module) self.assertEqual(1, len(module.blocks)) self.assertEqual('ingress.annotations.kubernetes\\.io/ingress\\.class', module.blocks[0].attributes['set.name']) @@ -55,7 +54,7 @@ def test_module_double_slash_cleanup_string(self): definitions = { '/mock/path/to.tf': clean_bad_definitions(non_malformed_definitions) } - module, _ = Parser().parse_hcl_module_from_tf_definitions(definitions, '', 'terraform') + module, _ = TFParser().parse_hcl_module_from_tf_definitions(definitions, '', 'terraform') print(module) self.assertEqual(1, len(module.blocks)) self.assertEqual('ingress.annotations.kubernetes\\.io/ingress\\.class', module.blocks[0].attributes['set.name']) diff --git a/tests/terraform/parser/test_new_parser_modules.py b/tests/terraform/parser/test_new_parser_modules.py index a91feaa29d0..fa1dd70784a 100644 --- a/tests/terraform/parser/test_new_parser_modules.py +++ b/tests/terraform/parser/test_new_parser_modules.py @@ -10,7 +10,6 @@ from checkov.terraform.modules.module_objects import TFDefinitionKey, TFModule from checkov.terraform.graph_builder.local_graph import TerraformLocalGraph from checkov.terraform.tf_parser import TFParser -from checkov.terraform.parser import Parser @pytest.fixture @@ -182,7 +181,7 @@ def test_load_nested_dup_module(self): assert module2_key0_nest0 in o_definitions assert module2_key1_nest0 in o_definitions - def test_new_tf_parser(self): + def test_tf_parser(self): parser = TFParser() directory = os.path.join(self.resources_dir, "parser_dup_nested") module, tf_definitions = parser.parse_hcl_module(source_dir=directory, source='terraform') @@ -198,23 +197,6 @@ def test_new_tf_parser(self): assert module assert tf_definitions - @mock.patch.dict(os.environ, {"CHECKOV_NEW_TF_PARSER": "False"}) - def test_old_parser(self): - parser = Parser() - directory = os.path.join(self.resources_dir, "parser_dup_nested") - module, tf_definitions = parser.parse_hcl_module(source_dir=directory, source='terraform') - - local_graph = TerraformLocalGraph(module) - local_graph.build_graph(render_variables=True) - - for i, vertex in enumerate(local_graph.vertices): - assert vertex.source_module == self.expected_source_modules[i] - - assert len(local_graph.edges) == 20 - - assert module - assert tf_definitions - def test_parser_with_tvars(self): parser = TFParser() directory = os.path.join(self.resources_dir, "parser_tfvars") diff --git a/tests/terraform/parser/test_parse_file_vs_dir.py b/tests/terraform/parser/test_parse_file_vs_dir.py index b5508d9cfda..185236cc9a2 100644 --- a/tests/terraform/parser/test_parse_file_vs_dir.py +++ b/tests/terraform/parser/test_parse_file_vs_dir.py @@ -1,17 +1,17 @@ import os import unittest -from checkov.terraform.parser import Parser +from checkov.terraform.tf_parser import TFParser class TestFileVsDirParser(unittest.TestCase): def test_file_dir_parser_results_match(self): - parser = Parser() + parser = TFParser() current_dir = os.path.dirname(os.path.realpath(__file__)) file_path = current_dir + '/resources/parse_file_vs_dir/main.tf' dir_path = current_dir + '/resources/parse_file_vs_dir' - tf_definitions_file = parser.parse_file(file_path) + tf_definitions_file = parser.parse_file(file_path, {}) _, tf_definitions_dir = parser.parse_hcl_module(dir_path, 'terraform') self.assertDictEqual(tf_definitions_file, tf_definitions_dir.get(list(tf_definitions_dir.keys())[0])) diff --git a/tests/terraform/parser/test_parser_internals.py b/tests/terraform/parser/test_parser_internals.py index 41d3f37fcab..0ebe085070a 100644 --- a/tests/terraform/parser/test_parser_internals.py +++ b/tests/terraform/parser/test_parser_internals.py @@ -1,7 +1,7 @@ from pathlib import Path from checkov.common.util.parser_utils import eval_string -from checkov.terraform.parser import load_or_die_quietly +from checkov.terraform.tf_parser import load_or_die_quietly def test_eval_string_to_list(): diff --git a/tests/terraform/parser/test_parser_modules.py b/tests/terraform/parser/test_parser_modules.py deleted file mode 100644 index ad6441855ab..00000000000 --- a/tests/terraform/parser/test_parser_modules.py +++ /dev/null @@ -1,145 +0,0 @@ -import os -import shutil -import unittest -from pathlib import Path -from unittest import mock - -import pytest - -from checkov.common.util.consts import DEFAULT_EXTERNAL_MODULES_DIR -from checkov.common.util.parser_utils import TERRAFORM_NESTED_MODULE_PATH_PREFIX, TERRAFORM_NESTED_MODULE_PATH_ENDING, \ - TERRAFORM_NESTED_MODULE_INDEX_SEPARATOR -from checkov.terraform.parser import Parser - - -@pytest.fixture -def tmp_path(request, tmp_path: Path): - # https://pytest.org/en/latest/how-to/unittest.html#mixing-pytest-fixtures-into-unittest-testcase-subclasses-using-marks - request.cls.tmp_path = tmp_path - - -@pytest.mark.usefixtures("tmp_path") -class TestParserInternals(unittest.TestCase): - - def setUp(self) -> None: - from checkov.terraform.module_loading.registry import ModuleLoaderRegistry - - # needs to be reset, because the cache belongs to the class not instance - ModuleLoaderRegistry.module_content_cache = {} - - self.resources_dir = os.path.realpath(os.path.join(os.path.dirname(os.path.realpath(__file__)), - "./resources")) - self.external_module_path = '' - - def tearDown(self) -> None: - if os.path.exists(self.external_module_path): - shutil.rmtree(self.external_module_path) - - def test_load_registry_module(self): - parser = Parser() - directory = os.path.join(self.resources_dir, "registry_security_group") - self.external_module_path = os.path.join(directory, DEFAULT_EXTERNAL_MODULES_DIR) - out_definitions = {} - parser.parse_directory(directory=directory, out_definitions=out_definitions, - out_evaluations_context={}, - download_external_modules=True, - external_modules_download_path=DEFAULT_EXTERNAL_MODULES_DIR) - - external_aws_modules_path = os.path.join(self.external_module_path, 'github.com/terraform-aws-modules/terraform-aws-security-group/v3.18.0') - assert os.path.exists(external_aws_modules_path) - - def test_load_inner_registry_module_with_nested_modules(self): - parser = Parser() - directory = os.path.join(self.resources_dir, "registry_security_group_inner_module") - self.external_module_path = os.path.join(self.tmp_path, DEFAULT_EXTERNAL_MODULES_DIR) - out_definitions = {} - parser.parse_directory(directory=directory, out_definitions=out_definitions, - out_evaluations_context={}, - download_external_modules=True, - external_modules_download_path=self.external_module_path) - self.assertEqual(11, len(list(out_definitions.keys()))) - expected_remote_module_path = f'{self.external_module_path}/github.com/terraform-aws-modules/terraform-aws-security-group/v4.0.0' - expected_inner_remote_module_path = f'{expected_remote_module_path}/modules/http-80' - expected_main_file = os.path.join(directory, 'main.tf') - expected_inner_main_file = os.path.join(directory, expected_inner_remote_module_path, 'main.tf') - expected_file_names = [ - expected_main_file, - os.path.join(directory, expected_inner_remote_module_path, f'auto_values.tf{TERRAFORM_NESTED_MODULE_PATH_PREFIX}{expected_main_file}{TERRAFORM_NESTED_MODULE_INDEX_SEPARATOR}0{TERRAFORM_NESTED_MODULE_PATH_ENDING}'), - os.path.join(directory, expected_inner_remote_module_path, f'main.tf{TERRAFORM_NESTED_MODULE_PATH_PREFIX}{expected_main_file}{TERRAFORM_NESTED_MODULE_INDEX_SEPARATOR}0{TERRAFORM_NESTED_MODULE_PATH_ENDING}'), - os.path.join(directory, expected_inner_remote_module_path, f'outputs.tf{TERRAFORM_NESTED_MODULE_PATH_PREFIX}{expected_main_file}{TERRAFORM_NESTED_MODULE_INDEX_SEPARATOR}0{TERRAFORM_NESTED_MODULE_PATH_ENDING}'), - os.path.join(directory, expected_inner_remote_module_path, f'variables.tf{TERRAFORM_NESTED_MODULE_PATH_PREFIX}{expected_main_file}{TERRAFORM_NESTED_MODULE_INDEX_SEPARATOR}0{TERRAFORM_NESTED_MODULE_PATH_ENDING}'), - os.path.join(directory, expected_inner_remote_module_path, f'versions.tf{TERRAFORM_NESTED_MODULE_PATH_PREFIX}{expected_main_file}{TERRAFORM_NESTED_MODULE_INDEX_SEPARATOR}0{TERRAFORM_NESTED_MODULE_PATH_ENDING}'), - - os.path.join(directory, expected_remote_module_path, f'main.tf{TERRAFORM_NESTED_MODULE_PATH_PREFIX}{expected_inner_main_file}{TERRAFORM_NESTED_MODULE_INDEX_SEPARATOR}0{TERRAFORM_NESTED_MODULE_PATH_ENDING}'), - os.path.join(directory, expected_remote_module_path, f'outputs.tf{TERRAFORM_NESTED_MODULE_PATH_PREFIX}{expected_inner_main_file}{TERRAFORM_NESTED_MODULE_INDEX_SEPARATOR}0{TERRAFORM_NESTED_MODULE_PATH_ENDING}'), - os.path.join(directory, expected_remote_module_path, f'rules.tf{TERRAFORM_NESTED_MODULE_PATH_PREFIX}{expected_inner_main_file}{TERRAFORM_NESTED_MODULE_INDEX_SEPARATOR}0{TERRAFORM_NESTED_MODULE_PATH_ENDING}'), - os.path.join(directory, expected_remote_module_path, f'variables.tf{TERRAFORM_NESTED_MODULE_PATH_PREFIX}{expected_inner_main_file}{TERRAFORM_NESTED_MODULE_INDEX_SEPARATOR}0{TERRAFORM_NESTED_MODULE_PATH_ENDING}'), - os.path.join(directory, expected_remote_module_path, f'versions.tf{TERRAFORM_NESTED_MODULE_PATH_PREFIX}{expected_inner_main_file}{TERRAFORM_NESTED_MODULE_INDEX_SEPARATOR}0{TERRAFORM_NESTED_MODULE_PATH_ENDING}'), - ] - - for expected_file_name in expected_file_names: - if not any(definition for definition in out_definitions.keys() if definition.startswith(expected_file_name[:-3])): - self.fail(f"expected file {expected_file_name} to be in out_definitions") - - def test_invalid_module_sources(self): - parser = Parser() - directory = os.path.join(self.resources_dir, "failing_module_address") - self.external_module_path = os.path.join(directory, DEFAULT_EXTERNAL_MODULES_DIR) - out_definitions = {} - parser.parse_directory(directory=directory, out_definitions=out_definitions, - out_evaluations_context={}, - download_external_modules=True, - external_modules_download_path=DEFAULT_EXTERNAL_MODULES_DIR) - # check that only the original file was parsed successfully without getting bad external modules - self.assertEqual(1, len(list(out_definitions.keys()))) - - def test_malformed_output_blocks(self): - parser = Parser() - directory = os.path.join(self.resources_dir, "malformed_outputs") - self.external_module_path = os.path.join(directory, DEFAULT_EXTERNAL_MODULES_DIR) - out_definitions = {} - parser.parse_directory(directory=directory, out_definitions=out_definitions, - out_evaluations_context={}, - download_external_modules=True, - external_modules_download_path=DEFAULT_EXTERNAL_MODULES_DIR) - file_path, entity_definitions = next(iter(out_definitions.items())) - self.assertEqual(2, len(list(out_definitions[file_path]['output']))) - - def test_load_local_module(self): - # given - parser = Parser() - directory = os.path.join(self.resources_dir, "local_module") - out_definitions = {} - - # when - parser.parse_directory( - directory=directory, out_definitions=out_definitions, out_evaluations_context={} - ) - - # then - self.assertEqual(len(out_definitions), 3) # root file + 2x module file - self.assertEqual(len(parser.loaded_files_map), 2) # root file + 1x module file - - def test_load_nested_dup_module(self): - parser = Parser() - directory = os.path.join(self.resources_dir, "parser_dup_nested") - out_definitions = {} - parser.parse_directory(directory=directory, out_evaluations_context={}, out_definitions=out_definitions) - - self.assertEqual(len(out_definitions), 7) - self.assertEqual(len(parser.loaded_files_map), 3) - - def test_load_local_nested_module(self): - # given - parser = Parser() - directory = os.path.join(self.resources_dir, "parser_nested_modules") - out_definitions = {} - - # when - parser.parse_directory( - directory=directory, out_definitions=out_definitions, out_evaluations_context={} - ) - - # then - self.assertEqual(len(out_definitions), 5) # root file + 2x module file - self.assertEqual(len(parser.loaded_files_map), 5) # root file + 1x module file diff --git a/tests/terraform/runner/test_runner.py b/tests/terraform/runner/test_runner.py index e162222a6f5..c3103e2ec41 100644 --- a/tests/terraform/runner/test_runner.py +++ b/tests/terraform/runner/test_runner.py @@ -30,7 +30,7 @@ from checkov.terraform.checks.resource.base_resource_check import BaseResourceCheck from checkov.terraform.context_parsers.registry import parser_registry from checkov.terraform.graph_manager import TerraformGraphManager -from checkov.terraform.parser import Parser +from checkov.terraform.tf_parser import TFParser from checkov.terraform.runner import Runner from checkov.terraform.checks.resource.registry import resource_registry from checkov.terraform.checks.module.registry import module_registry @@ -42,24 +42,20 @@ @parameterized_class([ - {"db_connector": NetworkxConnector, "use_new_tf_parser": "True", "tf_split_graph": "True", "graph": "NETWORKX"}, - {"db_connector": NetworkxConnector, "use_new_tf_parser": "True", "tf_split_graph": "False", "graph": "NETWORKX"}, - {"db_connector": NetworkxConnector, "use_new_tf_parser": "False", "tf_split_graph": "False", "graph": "NETWORKX"}, - {"db_connector": IgraphConnector, "use_new_tf_parser": "True", "tf_split_graph": "True", "graph": "IGRAPH"}, - {"db_connector": IgraphConnector, "use_new_tf_parser": "True", "tf_split_graph": "False", "graph": "IGRAPH"}, - {"db_connector": IgraphConnector, "use_new_tf_parser": "False", "tf_split_graph": "False", "graph": "IGRAPH"} + {"db_connector": NetworkxConnector, "tf_split_graph": "True", "graph": "NETWORKX"}, + {"db_connector": NetworkxConnector, "tf_split_graph": "False", "graph": "NETWORKX"}, + {"db_connector": IgraphConnector, "tf_split_graph": "True", "graph": "IGRAPH"}, + {"db_connector": IgraphConnector, "tf_split_graph": "False", "graph": "IGRAPH"}, ]) class TestRunnerValid(unittest.TestCase): def setUp(self) -> None: self.orig_checks = resource_registry.checks self.db_connector = self.db_connector os.environ["CHECKOV_GRAPH_FRAMEWORK"] = self.graph - os.environ["CHECKOV_NEW_TF_PARSER"] = self.use_new_tf_parser os.environ["TF_SPLIT_GRAPH"] = self.tf_split_graph def tearDown(self): del os.environ["CHECKOV_GRAPH_FRAMEWORK"] - del os.environ["CHECKOV_NEW_TF_PARSER"] del os.environ["TF_SPLIT_GRAPH"] def test_registry_has_type(self): @@ -551,7 +547,6 @@ def scan_module_conf(self, conf): self.assertEqual(len(result.passed_checks), 1) self.assertIn('some-module', map(lambda record: record.resource, result.passed_checks)) - @mock.patch.dict(os.environ, {"CHECKOV_NEW_TF_PARSER": "False"}) @mock.patch.dict(os.environ, {"TF_SPLIT_GRAPH": "False"}) @mock.patch.dict(os.environ, {"CHECKOV_ENABLE_FOREACH_HANDLING": "False"}) def test_terraform_multiple_module_versions(self): @@ -861,117 +856,11 @@ def test_external_definitions_context(self): }, }, } - tf_definitions = { - f"{current_dir}/resources/valid_tf_only_passed_checks/example.tf": { - "resource": [ - { - "aws_s3_bucket": { - "foo-bucket": { - "region": ["${var.region}"], - "bucket": ["${local.bucket_name}"], - "force_destroy": [True], - "versioning": [{"enabled": [True], "mfa_delete": [True]}], - "logging": [ - {"target_bucket": ["${aws_s3_bucket.log_bucket.id}"], "target_prefix": ["log/"]} - ], - "server_side_encryption_configuration": [ - { - "rule": [ - { - "apply_server_side_encryption_by_default": [ - { - "kms_master_key_id": ["${aws_kms_key.mykey.arn}"], - "sse_algorithm": ["aws:kms"], - } - ] - } - ] - } - ], - "acl": ["private"], - "tags": [ - '${merge\n (\n var.common_tags,\n map(\n "name", "VM Virtual Machine",\n "group", "foo"\n )\n )\n }' - ], - } - } - } - ], - "data": [{"aws_caller_identity": {"current": {}}}], - "provider": [ - { - "kubernetes": { - "version": ["1.10.0"], - "host": ["${module.aks_cluster.kube_config[0].host}"], - "client_certificate": [ - "${base64decode(module.aks_cluster.kube_config[0].client_certificate)}" - ], - "client_key": ["${base64decode(module.aks_cluster.kube_config[0].client_key)}"], - "cluster_ca_certificate": [ - "${base64decode(module.aks_cluster.kube_config[0].cluster_ca_certificate)}" - ], - } - } - ], - "module": [ - { - "new_relic": { - "source": ["s3::https://s3.amazonaws.com/my-artifacts/new-relic-k8s-0.2.5.zip"], - "kubernetes_host": ["${module.aks_cluster.kube_config[0].host}"], - "kubernetes_client_certificate": [ - "${base64decode(module.aks_cluster.kube_config[0].client_certificate)}" - ], - "kubernetes_client_key": ["${base64decode(module.aks_cluster.kube_config[0].client_key)}"], - "kubernetes_cluster_ca_certificate": [ - "${base64decode(module.aks_cluster.kube_config[0].cluster_ca_certificate)}" - ], - "cluster_name": ["${module.naming_conventions.aks_name}"], - "new_relic_license": ['${data.vault_generic_secret.new_relic_license.data["license"]}'], - "cluster_ca_bundle_b64": ["${module.aks_cluster.kube_config[0].cluster_ca_certificate}"], - "module_depends_on": [["${null_resource.delay_aks_deployments}"]], - } - } - ], - }, - f"{current_dir}/resources/valid_tf_only_passed_checks/example_skip_acl.tf": { - "resource": [ - { - "aws_s3_bucket": { - "foo-bucket": { - "region": ["${var.region}"], - "bucket": ["${local.bucket_name}"], - "force_destroy": [True], - "tags": [{"Name": "foo-${data.aws_caller_identity.current.account_id}"}], - "versioning": [{"enabled": [True]}], - "logging": [ - {"target_bucket": ["${aws_s3_bucket.log_bucket.id}"], "target_prefix": ["log/"]} - ], - "server_side_encryption_configuration": [ - { - "rule": [ - { - "apply_server_side_encryption_by_default": [ - { - "kms_master_key_id": ["${aws_kms_key.mykey.arn}"], - "sse_algorithm": ["aws:kms"], - } - ] - } - ] - } - ], - "acl": ["public-read"], - } - } - } - ], - "data": [{"aws_caller_identity": {"current": {}}}], - }, - } + runner = Runner(db_connector=self.db_connector()) - parser = Parser() - runner.definitions = tf_definitions - runner.set_external_data(tf_definitions, external_definitions_context, breadcrumbs={}) - parser.parse_directory(tf_dir_path, tf_definitions) + parser = TFParser() + tf_definitions = parser.parse_directory(tf_dir_path) + runner.set_external_data(tf_definitions, external_definitions_context, breadcrumbs={}) # type: ignore report = Report('terraform') runner.check_tf_definition(root_folder=tf_dir_path, report=report, runner_filter=RunnerFilter()) self.assertGreaterEqual(len(report.passed_checks), 1) @@ -1319,21 +1208,6 @@ def test_wrong_check_imports(self): assert len(check_imports) == 0, f"Wrong imports were added: {check_imports}" - def test_entity_context_fetching(self): - runner = Runner(db_connector=self.db_connector()) - runner.context = {'/mock/os/terraform-aws-vpc/example/examplea/module.vpc.tf': {'module': {'module': {'vpc': {'start_line': 1, 'end_line': 7, 'code_lines': [(1, 'module "vpc" {\n'), (2, ' source = "../../"\n'), (3, ' cidr = var.cidr\n'), (4, ' zone = var.zone\n'), (5, ' common_tags = var.common_tags\n'), (6, ' account_name = var.account_name\n'), (7, '}\n')], 'skipped_checks': []}}}}, '/mock/os/terraform-aws-vpc/example/examplea/provider.aws.tf': {'provider': {'aws': {'default': {'start_line': 1, 'end_line': 3, 'code_lines': [(1, 'provider "aws" {\n'), (2, ' region = "eu-west-2"\n'), (3, '}\n')], 'skipped_checks': []}}}}, '/mock/os/terraform-aws-vpc/example/examplea/variables.tf': {'variable': {'cidr': {'start_line': 1, 'end_line': 3, 'code_lines': [(1, 'variable "cidr" {\n'), (2, ' type = string\n'), (3, '}\n')], 'skipped_checks': []}, 'zone': {'start_line': 5, 'end_line': 7, 'code_lines': [(5, 'variable "zone" {\n'), (6, ' type = list(any)\n'), (7, '}\n')], 'skipped_checks': []}, 'account_name': {'start_line': 9, 'end_line': 11, 'code_lines': [(9, 'variable "account_name" {\n'), (10, ' type = string\n'), (11, '}\n')], 'skipped_checks': []}, 'common_tags': {'start_line': 13, 'end_line': 15, 'code_lines': [(13, 'variable "common_tags" {\n'), (14, ' type = map(any)\n'), (15, '}\n')], 'skipped_checks': []}}}, f'/mock/os/terraform-aws-vpc/aws_eip.nateip.tf{TERRAFORM_NESTED_MODULE_PATH_PREFIX}/mock/os/terraform-aws-vpc/example/examplea/module.vpc.tf{TERRAFORM_NESTED_MODULE_INDEX_SEPARATOR}0{TERRAFORM_NESTED_MODULE_PATH_ENDING}': {'resource': {'aws_eip': {'nateip': {'start_line': 1, 'end_line': 4, 'code_lines': [(1, 'resource "aws_eip" "nateip" {\n'), (2, ' count = var.subnets\n'), (3, ' tags = var.common_tags\n'), (4, '}\n')], 'skipped_checks': []}}}}, f'/mock/os/terraform-aws-vpc/aws_internet_gateway.gw.tf{TERRAFORM_NESTED_MODULE_PATH_PREFIX}/mock/os/terraform-aws-vpc/example/examplea/module.vpc.tf{TERRAFORM_NESTED_MODULE_INDEX_SEPARATOR}0{TERRAFORM_NESTED_MODULE_PATH_ENDING}': {'resource': {'aws_internet_gateway': {'gw': {'start_line': 1, 'end_line': 6, 'code_lines': [(1, 'resource "aws_internet_gateway" "gw" {\n'), (2, ' vpc_id = aws_vpc.main.id\n'), (3, '\n'), (4, ' tags = merge(var.common_tags,\n'), (5, ' tomap({ "Name" = "${upper(var.account_name)}-IGW" }))\n'), (6, '}\n')], 'skipped_checks': []}}}}, f'/mock/os/terraform-aws-vpc/aws_nat_gateway.natgateway.tf{TERRAFORM_NESTED_MODULE_PATH_PREFIX}/mock/os/terraform-aws-vpc/example/examplea/module.vpc.tf{TERRAFORM_NESTED_MODULE_INDEX_SEPARATOR}0{TERRAFORM_NESTED_MODULE_PATH_ENDING}': {'resource': {'aws_nat_gateway': {'natgateway': {'start_line': 1, 'end_line': 8, 'code_lines': [(1, 'resource "aws_nat_gateway" "natgateway" {\n'), (2, ' count = var.subnets\n'), (3, ' allocation_id = element(aws_eip.nateip.*.id, count.index)\n'), (4, ' depends_on = [aws_internet_gateway.gw]\n'), (5, ' subnet_id = element(aws_subnet.public.*.id, count.index)\n'), (6, ' tags = merge(var.common_tags,\n'), (7, ' tomap({ "Name" = "${upper(var.account_name)}-AZ${count.index + 1}" }))\n'), (8, '}\n')], 'skipped_checks': []}}}}, f'/mock/os/terraform-aws-vpc/aws_network_acl.NetworkAclPrivate.tf{TERRAFORM_NESTED_MODULE_PATH_PREFIX}/mock/os/terraform-aws-vpc/example/examplea/module.vpc.tf{TERRAFORM_NESTED_MODULE_INDEX_SEPARATOR}0{TERRAFORM_NESTED_MODULE_PATH_ENDING}': {'resource': {'aws_network_acl': {'networkaclprivate': {'start_line': 1, 'end_line': 25, 'code_lines': [(1, 'resource "aws_network_acl" "networkaclprivate" {\n'), (2, ' vpc_id = aws_vpc.main.id\n'), (3, ' subnet_ids = aws_subnet.private.*.id\n'), (4, '\n'), (5, ' egress {\n'), (6, ' rule_no = 100\n'), (7, ' action = "allow"\n'), (8, ' cidr_block = "0.0.0.0/0"\n'), (9, ' from_port = 0\n'), (10, ' to_port = 0\n'), (11, ' protocol = "all"\n'), (12, ' }\n'), (13, '\n'), (14, ' ingress {\n'), (15, ' rule_no = 100\n'), (16, ' action = "allow"\n'), (17, ' cidr_block = "0.0.0.0/0"\n'), (18, ' from_port = 0\n'), (19, ' to_port = 0\n'), (20, ' protocol = "all"\n'), (21, ' }\n'), (22, '\n'), (23, ' tags = merge(var.common_tags,\n'), (24, ' tomap({ "Name" = "${var.account_name}-NetworkAcl-Private" }))\n'), (25, '}\n')], 'skipped_checks': []}}}}, f'/mock/os/terraform-aws-vpc/aws_network_acl.NetworkAclPublic.tf{TERRAFORM_NESTED_MODULE_PATH_PREFIX}/mock/os/terraform-aws-vpc/example/examplea/module.vpc.tf{TERRAFORM_NESTED_MODULE_INDEX_SEPARATOR}0{TERRAFORM_NESTED_MODULE_PATH_ENDING}': {'resource': {'aws_network_acl': {'networkaclpublic': {'start_line': 1, 'end_line': 25, 'code_lines': [(1, 'resource "aws_network_acl" "networkaclpublic" {\n'), (2, ' vpc_id = aws_vpc.main.id\n'), (3, ' subnet_ids = aws_subnet.public.*.id\n'), (4, '\n'), (5, ' egress {\n'), (6, ' rule_no = 100\n'), (7, ' action = "allow"\n'), (8, ' cidr_block = "0.0.0.0/0"\n'), (9, ' from_port = 0\n'), (10, ' to_port = 0\n'), (11, ' protocol = "all"\n'), (12, ' }\n'), (13, '\n'), (14, ' ingress {\n'), (15, ' rule_no = 100\n'), (16, ' action = "allow"\n'), (17, ' cidr_block = "0.0.0.0/0"\n'), (18, ' from_port = 0\n'), (19, ' to_port = 0\n'), (20, ' protocol = "all"\n'), (21, ' }\n'), (22, '\n'), (23, ' tags = merge(var.common_tags,\n'), (24, ' tomap({ "Name" = "${var.account_name}-NetworkAcl-Public" }))\n'), (25, '}\n')], 'skipped_checks': []}}}}, f'/mock/os/terraform-aws-vpc/aws_route_table.private.tf{TERRAFORM_NESTED_MODULE_PATH_PREFIX}/mock/os/terraform-aws-vpc/example/examplea/module.vpc.tf{TERRAFORM_NESTED_MODULE_INDEX_SEPARATOR}0{TERRAFORM_NESTED_MODULE_PATH_ENDING}': {'resource': {'aws_route_table': {'private': {'start_line': 1, 'end_line': 8, 'code_lines': [(1, 'resource "aws_route_table" "private" {\n'), (2, ' vpc_id = aws_vpc.main.id\n'), (3, '\n'), (4, ' propagating_vgws = [aws_vpn_gateway.vpn_gw.id]\n'), (5, '\n'), (6, ' tags = merge(var.common_tags,\n'), (7, ' tomap({ "Name" = "${var.account_name}-Private-${element(aws_subnet.private.*.id, 0)}" }))\n'), (8, '}\n')], 'skipped_checks': []}}, 'aws_route': {'private': {'start_line': 10, 'end_line': 14, 'code_lines': [(10, 'resource "aws_route" "private" {\n'), (11, ' route_table_id = aws_route_table.private.id\n'), (12, ' destination_cidr_block = "0.0.0.0/0"\n'), (13, ' nat_gateway_id = element(aws_nat_gateway.natgateway.*.id, 0)\n'), (14, '}\n')], 'skipped_checks': []}}}}, f'/mock/os/terraform-aws-vpc/aws_route_table.public.tf{TERRAFORM_NESTED_MODULE_PATH_PREFIX}/mock/os/terraform-aws-vpc/example/examplea/module.vpc.tf{TERRAFORM_NESTED_MODULE_INDEX_SEPARATOR}0{TERRAFORM_NESTED_MODULE_PATH_ENDING}': {'resource': {'aws_route_table': {'public': {'start_line': 1, 'end_line': 6, 'code_lines': [(1, 'resource "aws_route_table" "public" {\n'), (2, ' vpc_id = aws_vpc.main.id\n'), (3, '\n'), (4, ' tags = merge(var.common_tags,\n'), (5, ' tomap({ "Name" = "${upper(var.account_name)}-Public" }))\n'), (6, '}\n')], 'skipped_checks': []}}, 'aws_route': {'public': {'start_line': 8, 'end_line': 12, 'code_lines': [(8, 'resource "aws_route" "public" {\n'), (9, ' route_table_id = aws_route_table.public.id\n'), (10, ' destination_cidr_block = "0.0.0.0/0"\n'), (11, ' gateway_id = aws_internet_gateway.gw.id\n'), (12, '}\n')], 'skipped_checks': []}}}}, f'/mock/os/terraform-aws-vpc/aws_route_table_association.private.tf{TERRAFORM_NESTED_MODULE_PATH_PREFIX}/mock/os/terraform-aws-vpc/example/examplea/module.vpc.tf{TERRAFORM_NESTED_MODULE_INDEX_SEPARATOR}0{TERRAFORM_NESTED_MODULE_PATH_ENDING}': {'resource': {'aws_route_table_association': {'private': {'start_line': 1, 'end_line': 5, 'code_lines': [(1, 'resource "aws_route_table_association" "private" {\n'), (2, ' count = var.subnets\n'), (3, ' subnet_id = element(aws_subnet.private.*.id, count.index)\n'), (4, ' route_table_id = aws_route_table.private.id\n'), (5, '}\n')], 'skipped_checks': []}}}}, f'/mock/os/terraform-aws-vpc/aws_route_table_association.public.tf{TERRAFORM_NESTED_MODULE_PATH_PREFIX}/mock/os/terraform-aws-vpc/example/examplea/module.vpc.tf{TERRAFORM_NESTED_MODULE_INDEX_SEPARATOR}0{TERRAFORM_NESTED_MODULE_PATH_ENDING}': {'resource': {'aws_route_table_association': {'public': {'start_line': 1, 'end_line': 5, 'code_lines': [(1, 'resource "aws_route_table_association" "public" {\n'), (2, ' count = var.subnets\n'), (3, ' subnet_id = element(aws_subnet.public.*.id, count.index)\n'), (4, ' route_table_id = aws_route_table.public.id\n'), (5, '}\n')], 'skipped_checks': []}}}}, f'/mock/os/terraform-aws-vpc/aws_subnet.private.tf{TERRAFORM_NESTED_MODULE_PATH_PREFIX}/mock/os/terraform-aws-vpc/example/examplea/module.vpc.tf{TERRAFORM_NESTED_MODULE_INDEX_SEPARATOR}0{TERRAFORM_NESTED_MODULE_PATH_ENDING}': {'resource': {'aws_subnet': {'private': {'start_line': 1, 'end_line': 10, 'code_lines': [(1, 'resource "aws_subnet" "private" {\n'), (2, ' count = var.subnets\n'), (3, ' vpc_id = aws_vpc.main.id\n'), (4, ' cidr_block = local.private_cidrs[count.index]\n'), (5, ' availability_zone = data.aws_availability_zones.available.names[count.index]\n'), (6, '\n'), (7, ' tags = merge(var.common_tags,\n'), (8, ' tomap({ "Type" = "Private" }),\n'), (9, ' tomap({ "Name" = "${upper(var.account_name)}-Private-${var.zone[count.index]}" }))\n'), (10, '}\n')], 'skipped_checks': []}}}}, f'/mock/os/terraform-aws-vpc/aws_subnet.public.tf{TERRAFORM_NESTED_MODULE_PATH_PREFIX}/mock/os/terraform-aws-vpc/example/examplea/module.vpc.tf{TERRAFORM_NESTED_MODULE_INDEX_SEPARATOR}0{TERRAFORM_NESTED_MODULE_PATH_ENDING}': {'resource': {'aws_subnet': {'public': {'start_line': 1, 'end_line': 10, 'code_lines': [(1, 'resource "aws_subnet" "public" {\n'), (2, ' count = var.subnets\n'), (3, ' vpc_id = aws_vpc.main.id\n'), (4, ' cidr_block = local.public_cidrs[count.index]\n'), (5, ' availability_zone = data.aws_availability_zones.available.names[count.index]\n'), (6, '\n'), (7, ' tags = merge(var.common_tags,\n'), (8, ' tomap({ "Type" = "Public" }),\n'), (9, ' tomap({ "Name" = "${upper(var.account_name)}-Public-${var.zone[count.index]}" }))\n'), (10, '}\n')], 'skipped_checks': []}}}}, f'/mock/os/terraform-aws-vpc/aws_vpc.main.tf{TERRAFORM_NESTED_MODULE_PATH_PREFIX}/mock/os/terraform-aws-vpc/example/examplea/module.vpc.tf{TERRAFORM_NESTED_MODULE_INDEX_SEPARATOR}0{TERRAFORM_NESTED_MODULE_PATH_ENDING}': {'locals': {'start_line': 10, 'end_line': 12, 'code_lines': [(10, 'locals {\n'), (11, ' tags = merge(var.common_tags, tomap({ "Name" = upper(var.account_name) }))\n'), (12, '}\n')], 'assignments': {'tags': "merge([],tomap({'Name':'upper(test)'}))"}, 'skipped_checks': []}, 'resource': {'aws_vpc': {'main': {'start_line': 1, 'end_line': 7, 'code_lines': [(1, 'resource "aws_vpc" "main" {\n'), (2, ' cidr_block = var.cidr\n'), (3, ' enable_dns_support = true\n'), (4, ' enable_dns_hostnames = true\n'), (5, '\n'), (6, ' tags = local.tags\n'), (7, '}\n')], 'skipped_checks': []}}}}, f'/mock/os/terraform-aws-vpc/aws_vpn_gateway.vpn_gw.tf{TERRAFORM_NESTED_MODULE_PATH_PREFIX}/mock/os/terraform-aws-vpc/example/examplea/module.vpc.tf{TERRAFORM_NESTED_MODULE_INDEX_SEPARATOR}0{TERRAFORM_NESTED_MODULE_PATH_ENDING}': {'resource': {'aws_vpn_gateway': {'vpn_gw': {'start_line': 1, 'end_line': 6, 'code_lines': [(1, 'resource "aws_vpn_gateway" "vpn_gw" {\n'), (2, ' vpc_id = aws_vpc.main.id\n'), (3, '\n'), (4, ' tags = merge(var.common_tags,\n'), (5, ' tomap({ "Name" = "${upper(var.account_name)}-VGW" }))\n'), (6, '}\n')], 'skipped_checks': []}}}}, f'/mock/os/terraform-aws-vpc/data.aws_availability_zones.tf{TERRAFORM_NESTED_MODULE_PATH_PREFIX}/mock/os/terraform-aws-vpc/example/examplea/module.vpc.tf{TERRAFORM_NESTED_MODULE_INDEX_SEPARATOR}0{TERRAFORM_NESTED_MODULE_PATH_ENDING}': {'data': {'aws_availability_zones': {'available': {'start_line': 1, 'end_line': 0, 'code_lines': [], 'skipped_checks': []}}}}, f'/mock/os/terraform-aws-vpc/variables.tf{TERRAFORM_NESTED_MODULE_PATH_PREFIX}/mock/os/terraform-aws-vpc/example/examplea/module.vpc.tf{TERRAFORM_NESTED_MODULE_INDEX_SEPARATOR}0{TERRAFORM_NESTED_MODULE_PATH_ENDING}': {'locals': {'start_line': 27, 'end_line': 30, 'code_lines': [(27, 'locals {\n'), (28, ' public_cidrs = [cidrsubnet(var.cidr, 3, 0), cidrsubnet(var.cidr, 3, 1), cidrsubnet(var.cidr, 3, 2)]\n'), (29, ' private_cidrs = [cidrsubnet(var.cidr, 3, 3), cidrsubnet(var.cidr, 3, 4), cidrsubnet(var.cidr, 3, 5)]\n'), (30, '}\n')], 'skipped_checks': []}, 'variable': {'account_name': {'start_line': 1, 'end_line': 4, 'code_lines': [(1, 'variable "account_name" {\n'), (2, ' type = string\n'), (3, ' description = "The Name of the Account"\n'), (4, '}\n')], 'skipped_checks': []}, 'cidr': {'start_line': 6, 'end_line': 9, 'code_lines': [(6, 'variable "cidr" {\n'), (7, ' type = string\n'), (8, ' description = "The range to be associated with the VPC and cleaved into the subnets"\n'), (9, '}\n')], 'skipped_checks': []}, 'common_tags': {'start_line': 11, 'end_line': 14, 'code_lines': [(11, 'variable "common_tags" {\n'), (12, ' type = map(any)\n'), (13, ' description = "A tagging scheme"\n'), (14, '}\n')], 'skipped_checks': []}, 'zone': {'start_line': 16, 'end_line': 19, 'code_lines': [(16, 'variable "zone" {\n'), (17, ' type = list(any)\n'), (18, ' description = "Availability zone names"\n'), (19, '}\n')], 'skipped_checks': []}, 'subnets': {'start_line': 21, 'end_line': 25, 'code_lines': [(21, 'variable "subnets" {\n'), (22, ' type = number\n'), (23, ' default = 3\n'), (24, ' description = "The number of subnets required, less than or equal to the number of availability zones"\n'), (25, '}\n')], 'skipped_checks': []}, 'assignments': {'subnets': 3}}}} - entity_with_non_found_path = {'block_name_': 'aws_vpc.main', 'block_type_': 'resource', 'file_path_': '/mock/os/terraform-aws-vpc/aws_vpc.main.tf', 'config_': {'aws_vpc': {'main': {'cidr_block': ['10.0.0.0/21'], 'enable_dns_hostnames': [True], 'enable_dns_support': [True], 'tags': ["merge([],tomap({'Name':'upper(test)'}))"]}}}, 'label_': 'BlockType.RESOURCE: aws_vpc.main', 'id_': 'aws_vpc.main', 'source_': 'Terraform', 'cidr_block': '10.0.0.0/21', 'enable_dns_hostnames': True, 'enable_dns_support': True, 'tags': "merge([],tomap({'Name':'upper(test)'}))", 'resource_type': 'aws_vpc', 'rendering_breadcrumbs_': {'cidr_block': [{'type': 'module', 'name': 'vpc', 'path': '/mock/os/terraform-aws-vpc/example/examplea/module.vpc.tf', 'module_connection': False}, {'type': 'variable', 'name': 'cidr', 'path': '/mock/os/terraform-aws-vpc/variables.tf', 'module_connection': False}, {'type': 'locals', 'name': 'private_cidrs', 'path': '/mock/os/terraform-aws-vpc/variables.tf', 'module_connection': False}, {'type': 'locals', 'name': 'public_cidrs', 'path': '/mock/os/terraform-aws-vpc/variables.tf', 'module_connection': False}, {'type': 'locals', 'name': 'private_cidrs', 'path': '/mock/os/terraform-aws-vpc/variables.tf', 'module_connection': False}, {'type': 'locals', 'name': 'public_cidrs', 'path': '/mock/os/terraform-aws-vpc/variables.tf', 'module_connection': False}, {'type': 'output', 'name': 'private_cidrs', 'path': '/mock/os/terraform-aws-vpc/outputs.tf', 'module_connection': False}], 'source_module_': [{'type': 'module', 'name': 'vpc', 'path': '/mock/os/terraform-aws-vpc/example/examplea/module.vpc.tf'}], 'tags': [{'type': 'module', 'name': 'vpc', 'path': '/mock/os/terraform-aws-vpc/example/examplea/module.vpc.tf', 'module_connection': False}, {'type': 'variable', 'name': 'account_name', 'path': '/mock/os/terraform-aws-vpc/variables.tf', 'module_connection': False}, {'type': 'locals', 'name': 'tags', 'path': '/mock/os/terraform-aws-vpc/aws_vpc.main.tf', 'module_connection': False}]}, 'hash': 'bac3bb7d21610be9ad786c1e9b5a2b3f6f13e60699fa935b32bb1f9f10a792e4', 'module_dependency_': '/mock/os/terraform-aws-vpc/example/examplea/module.vpc.tf', 'module_dependency_num_': '0'} - entity_context = runner.get_entity_context_and_evaluations(entity_with_non_found_path) - - assert entity_context is not None - assert entity_context['start_line'] == 1 and entity_context['end_line'] == 7 - - entity_with_found_path = {'block_name_': 'aws_vpc.main', 'block_type_': 'resource', 'file_path_': f'/mock/os/terraform-aws-vpc/aws_vpc.main.tf{TERRAFORM_NESTED_MODULE_PATH_PREFIX}/mock/os/terraform-aws-vpc/example/examplea/module.vpc.tf{TERRAFORM_NESTED_MODULE_INDEX_SEPARATOR}0{TERRAFORM_NESTED_MODULE_PATH_ENDING}', 'config_': {'aws_vpc': {'main': {'cidr_block': ['10.0.0.0/21'], 'enable_dns_hostnames': [True], 'enable_dns_support': [True], 'tags': ["merge([],tomap({'Name':'upper(test)'}))"]}}}, 'label_': 'BlockType.RESOURCE: aws_vpc.main', 'id_': 'aws_vpc.main', 'source_': 'Terraform', 'cidr_block': '10.0.0.0/21', 'enable_dns_hostnames': True, 'enable_dns_support': True, 'tags': "merge([],tomap({'Name':'upper(test)'}))", 'resource_type': 'aws_vpc', 'rendering_breadcrumbs_': {'cidr_block': [{'type': 'module', 'name': 'vpc', 'path': '/mock/os/terraform-aws-vpc/example/examplea/module.vpc.tf', 'module_connection': False}, {'type': 'variable', 'name': 'cidr', 'path': '/mock/os/terraform-aws-vpc/variables.tf', 'module_connection': False}, {'type': 'locals', 'name': 'private_cidrs', 'path': '/mock/os/terraform-aws-vpc/variables.tf', 'module_connection': False}, {'type': 'locals', 'name': 'public_cidrs', 'path': '/mock/os/terraform-aws-vpc/variables.tf', 'module_connection': False}, {'type': 'locals', 'name': 'private_cidrs', 'path': '/mock/os/terraform-aws-vpc/variables.tf', 'module_connection': False}, {'type': 'locals', 'name': 'public_cidrs', 'path': '/mock/os/terraform-aws-vpc/variables.tf', 'module_connection': False}, {'type': 'output', 'name': 'private_cidrs', 'path': '/mock/os/terraform-aws-vpc/outputs.tf', 'module_connection': False}], 'source_module_': [{'type': 'module', 'name': 'vpc', 'path': '/mock/os/terraform-aws-vpc/example/examplea/module.vpc.tf'}], 'tags': [{'type': 'module', 'name': 'vpc', 'path': '/mock/os/terraform-aws-vpc/example/examplea/module.vpc.tf', 'module_connection': False}, {'type': 'variable', 'name': 'account_name', 'path': '/mock/os/terraform-aws-vpc/variables.tf', 'module_connection': False}, {'type': 'locals', 'name': 'tags', 'path': '/mock/os/terraform-aws-vpc/aws_vpc.main.tf', 'module_connection': False}]}, 'hash': 'bac3bb7d21610be9ad786c1e9b5a2b3f6f13e60699fa935b32bb1f9f10a792e4'} - entity_context = runner.get_entity_context_and_evaluations(entity_with_found_path) - - assert entity_context is not None - assert entity_context['start_line'] == 1 and entity_context['end_line'] == 7 - def test_resource_ids_nested_modules(self): resources_path = os.path.join( os.path.dirname(os.path.realpath(__file__)), "resources", "resource_ids_nested_modules")