Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat(general): Remove code upload for on-prem integrations #5624

Merged
merged 8 commits into from
Oct 12, 2023
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 7 additions & 2 deletions checkov/common/bridgecrew/platform_integration.py
Original file line number Diff line number Diff line change
Expand Up @@ -134,6 +134,7 @@ def __init__(self) -> None:
self.persist_graphs_timeout = int(os.getenv('BC_PERSIST_GRAPHS_TIMEOUT', 60))
self.ca_certificate: str | None = None
self.no_cert_verify: bool = False
self.on_prem: bool = False

def set_bc_api_url(self, new_url: str) -> None:
self.bc_api_url = normalize_bc_url(new_url)
Expand Down Expand Up @@ -483,9 +484,9 @@ def persist_scan_results(self, scan_reports: list[Report]) -> None:
# just process reports with actual results in it
self.scan_reports = [scan_report for scan_report in scan_reports if not scan_report.is_empty(full=True)]

reduced_scan_reports = reduce_scan_reports(self.scan_reports)
reduced_scan_reports = reduce_scan_reports(self.scan_reports, self.on_prem)
checks_metadata_paths = enrich_and_persist_checks_metadata(self.scan_reports, self.s3_client, self.bucket,
self.repo_path)
self.repo_path, self.on_prem)
dpath.merge(reduced_scan_reports, checks_metadata_paths)
persist_checks_results(reduced_scan_reports, self.s3_client, self.bucket, self.repo_path)

Expand Down Expand Up @@ -1128,5 +1129,9 @@ def get_sso_prismacloud_url(self, report_url: str) -> str:

return report_url

def setup_on_prem(self) -> None:
if self.customer_run_config_response:
self.on_prem = self.customer_run_config_response.get('onPrem', False)


bc_integration = BcPlatformIntegration()
12 changes: 8 additions & 4 deletions checkov/common/bridgecrew/wrapper.py
Original file line number Diff line number Diff line change
Expand Up @@ -58,18 +58,20 @@ def _put_json_object(s3_client: S3Client, json_obj: Any, bucket: str, object_pat
raise


def _extract_checks_metadata(report: Report, full_repo_object_key: str) -> dict[str, dict[str, Any]]:
def _extract_checks_metadata(report: Report, full_repo_object_key: str, on_prem: bool) -> dict[str, dict[str, Any]]:
metadata: dict[str, dict[str, Any]] = defaultdict(dict)
for check in itertools.chain(report.passed_checks, report.failed_checks, report.skipped_checks):
metadata_key = f'{check.file_path}:{check.resource}'
check_meta = {k: getattr(check, k, "") for k in check_metadata_keys}
check_meta['file_object_path'] = full_repo_object_key + check.file_path
if on_prem:
check_meta['code_block'] = []
metadata[metadata_key][check.check_id] = check_meta

return metadata


def reduce_scan_reports(scan_reports: list[Report]) -> dict[str, _ReducedScanReport]:
def reduce_scan_reports(scan_reports: list[Report], on_prem: Optional[bool] = False) -> dict[str, _ReducedScanReport]:
"""
Transform checkov reports objects into compact dictionaries
:param scan_reports: List of checkov output reports
Expand All @@ -79,6 +81,8 @@ def reduce_scan_reports(scan_reports: list[Report]) -> dict[str, _ReducedScanRep
for report in scan_reports:
check_type = report.check_type
reduced_keys = secrets_check_reduced_keys if check_type == CheckType.SECRETS else check_reduced_keys
if on_prem:
reduced_keys = tuple(k for k in reduced_keys if k != 'code_block')
reduced_scan_reports[check_type] = \
{
"checks": {
Expand Down Expand Up @@ -136,7 +140,7 @@ def persist_logs_stream(logs_stream: StringIO, s3_client: S3Client, bucket: str,


def enrich_and_persist_checks_metadata(
scan_reports: list[Report], s3_client: S3Client, bucket: str, full_repo_object_key: str
scan_reports: list[Report], s3_client: S3Client, bucket: str, full_repo_object_key: str, on_prem: bool
) -> dict[str, dict[str, str]]:
"""
Save checks metadata into bridgecrew's platform
Expand All @@ -145,7 +149,7 @@ def enrich_and_persist_checks_metadata(
checks_metadata_paths: dict[str, dict[str, str]] = {}
for scan_report in scan_reports:
check_type = scan_report.check_type
checks_metadata_object = _extract_checks_metadata(scan_report, full_repo_object_key)
checks_metadata_object = _extract_checks_metadata(scan_report, full_repo_object_key, on_prem)
checks_metadata_object_path = f'{full_repo_object_key}/{checkov_results_prefix}/{check_type}/checks_metadata.json'
dpath.new(checks_metadata_paths, f"{check_type}/checks_metadata_path", checks_metadata_object_path)
_put_json_object(s3_client, checks_metadata_object, bucket, checks_metadata_object_path)
Expand Down
51 changes: 33 additions & 18 deletions checkov/main.py
Original file line number Diff line number Diff line change
Expand Up @@ -385,7 +385,7 @@ def run(self, banner: str = checkov_banner, tool: str = checkov_tool, source_typ
logger.error('Please try setting the environment variable LOG_LEVEL=DEBUG and re-running the command, and provide the output to support', exc_info=True)
self.exit_run()
else:
if self.config.support:
if bc_integration.support_flag_enabled:
logger.warning("--bc-api-key argument is required when using --support")
logger.debug('No API key found. Scanning locally only.')
self.config.include_all_checkov_policies = True
Expand All @@ -411,6 +411,21 @@ def run(self, banner: str = checkov_banner, tool: str = checkov_tool, source_typ
'(but note that this will not include any custom platform configurations or policy metadata).',
file=sys.stderr)
self.exit_run()
bc_integration.setup_on_prem()
if bc_integration.on_prem:
# disable --support for on-premises integrations
if bc_integration.support_flag_enabled:
logger.warning("--support flag is not supported for on-premises integrations")
bc_integration.support_flag_enabled = False
# disable sca_package, sca_image for on-premises integrations
if not outer_registry:
removed_check_types = []
for runner in list(runner_registry.runners):
if runner.check_type in [CheckType.SCA_IMAGE, CheckType.SCA_PACKAGE]:
removed_check_types.append(runner.check_type)
runner_registry.runners.remove(runner)
if removed_check_types:
logger.warning(f"Following runners won't run as they are not supported for on-premises integrations: {removed_check_types}")

bc_integration.get_prisma_build_policies(self.config.policy_metadata_filter)

Expand Down Expand Up @@ -530,8 +545,8 @@ def run(self, banner: str = checkov_banner, tool: str = checkov_tool, source_typ
self.parser.error("--branch argument is required when using --docker-image")
return None
files = [os.path.abspath(self.config.dockerfile_path)]
runner = sca_image_runner()
result = runner.run(
sca_runner = sca_image_runner()
result = sca_runner.run(
root_folder='',
image_id=self.config.docker_image,
dockerfile_path=self.config.dockerfile_path,
Expand All @@ -549,7 +564,7 @@ def run(self, banner: str = checkov_banner, tool: str = checkov_tool, source_typ
if not self.config.skip_results_upload:
bc_integration.persist_repository(os.path.dirname(self.config.dockerfile_path), files=files)
bc_integration.persist_scan_results(self.scan_reports)
bc_integration.persist_image_scan_results(runner.raw_report, self.config.dockerfile_path,
bc_integration.persist_image_scan_results(sca_runner.raw_report, self.config.dockerfile_path,
self.config.docker_image,
self.config.branch)

Expand Down Expand Up @@ -630,7 +645,7 @@ def run(self, banner: str = checkov_banner, tool: str = checkov_tool, source_typ
raise

finally:
if self.config.support:
if bc_integration.support_flag_enabled:
bc_integration.persist_logs_stream(logs_stream)

def exit_run(self) -> None:
Expand Down Expand Up @@ -664,19 +679,19 @@ def upload_results(
) -> None:
"""Upload scan results and other relevant files"""

bc_integration.persist_repository(
root_dir=root_folder,
files=files,
excluded_paths=excluded_paths,
included_paths=included_paths,
)
if git_configuration_folders:
bc_integration.persist_git_configuration(os.getcwd(), git_configuration_folders)
if sca_supported_ir_report:
scan_reports_to_upload = [report for report in self.scan_reports if report.check_type != 'sca_image']
scan_reports_to_upload.append(sca_supported_ir_report)
else:
scan_reports_to_upload = self.scan_reports
scan_reports_to_upload = self.scan_reports
if not bc_integration.on_prem:
bc_integration.persist_repository(
root_dir=root_folder,
files=files,
excluded_paths=excluded_paths,
included_paths=included_paths,
)
if git_configuration_folders:
bc_integration.persist_git_configuration(os.getcwd(), git_configuration_folders)
if sca_supported_ir_report:
scan_reports_to_upload = [report for report in self.scan_reports if report.check_type != 'sca_image']
scan_reports_to_upload.append(sca_supported_ir_report)
bc_integration.persist_scan_results(scan_reports_to_upload)
bc_integration.persist_run_metadata(self.run_metadata)
if bc_integration.enable_persist_graphs:
Expand Down