Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat(secrets): Improve the entropy keyword combinator secret scanner #5307

Merged
merged 4 commits into from
Jul 9, 2023
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ def __init__(self) -> None:
super().__init__(name=name, id=id, categories=categories, supported_resources=supported_resources)

def get_inspected_key(self) -> str:
return "Properties/LogPublishingOptions/AUDIT_LOGS/Enabled" # checkov:skip=CKV_SECRET_6 false positive
return "Properties/LogPublishingOptions/AUDIT_LOGS/Enabled"


check = ElasticsearchDomainAuditLogging()
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@ def scan_resource_conf(self, conf):
if metadata_options['HttpEndpoint'] == "disabled":
return CheckResult.PASSED
if 'HttpTokens' in metadata_options.keys():
if metadata_options['HttpTokens'] == "required": # checkov:skip=CKV_SECRET_6 false positive
if metadata_options['HttpTokens'] == "required":
return CheckResult.PASSED
return CheckResult.FAILED

Expand Down
2 changes: 1 addition & 1 deletion checkov/github/schemas/organization.py
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@ def __init__(self) -> None:
"node_id": {
"type": "string",
"examples": [
"MDEyOk9yZ2FuaXphdGlvbjE\u003d" # checkov:skip=CKV_SECRET_6 false positive
"MDEyOk9yZ2FuaXphdGlvbjE\u003d"
]
},
"url": {
Expand Down
14 changes: 6 additions & 8 deletions checkov/secrets/plugins/detector_utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -108,11 +108,11 @@
flags=re.IGNORECASE,
)

ALLOW_LIST = ('secretsmanager', "secretName") # can add more keys like that
ALLOW_LIST = ('secretsmanager', "secretName", "secret_name", "creation_token") # can add more keys like that
ALLOW_LIST_REGEX = r'|'.join(ALLOW_LIST)
# Support for suffix of function name i.e "secretsmanager:GetSecretValue"
CAMEL_CASE_NAMES = r'[A-Z]([A-Z0-9]*[a-z][a-z0-9]*[A-Z]|[a-z0-9]*[A-Z][A-Z0-9]*[a-z])[A-Za-z0-9]*'
FUNCTION_CALL_AFTER_KEYWORD_REGEX = re.compile(r'({allowlist}):\s*{suffix}'.format(
FUNCTION_CALL_AFTER_KEYWORD_REGEX = re.compile(r'({allowlist})\s*(:|=)\s*{suffix}'.format(
allowlist=ALLOW_LIST_REGEX,
suffix=AFFIX_REGEX,
))
Expand Down Expand Up @@ -178,13 +178,11 @@ def remove_fp_secrets_in_keys(detected_secrets: set[PotentialSecret], line: str)
formatted_line = line.replace('"', '').replace("'", '')
secrets_to_remove = set()
for detected_secret in detected_secrets:
if detected_secret.secret_value and formatted_line.startswith(
detected_secret.secret_value):
# Found keyword prefix as potential secret
# Found keyword prefix as potential secret
if detected_secret.secret_value and formatted_line.startswith(detected_secret.secret_value):
secrets_to_remove.add(detected_secret)
if detected_secret.secret_value and formatted_line and \
FUNCTION_CALL_AFTER_KEYWORD_REGEX.search(formatted_line):
# found a function name at the end of the line
# found a function name at the end of the line
if detected_secret.secret_value and formatted_line and FUNCTION_CALL_AFTER_KEYWORD_REGEX.search(formatted_line):
secrets_to_remove.add(detected_secret)
detected_secrets -= secrets_to_remove

Expand Down
10 changes: 7 additions & 3 deletions checkov/secrets/plugins/entropy_keyword_combinator.py
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@
MAX_LINE_LENGTH = 10000
MAX_KEYWORD_LIMIT = 500
ENTROPY_KEYWORD_COMBINATOR_LIMIT = 3
ENTROPY_KEYWORD_LIMIT = 4.5
ENTROPY_KEYWORD_LIMIT = 4.8

DENY_LIST_REGEX = r'|'.join(DENYLIST)
# Support for suffix after keyword i.e. password_secure = "value"
Expand Down Expand Up @@ -121,6 +121,7 @@ class EntropyKeywordCombinator(BasePlugin):
def __init__(self, limit: float = ENTROPY_KEYWORD_LIMIT, max_line_length: int = MAX_LINE_LENGTH) -> None:
iac_limit = ENTROPY_KEYWORD_COMBINATOR_LIMIT
self.high_entropy_scanners_iac = (Base64HighEntropyString(limit=iac_limit), HexHighEntropyString(limit=iac_limit))
self.entropy_scanners_non_iac_with_keyword = (Base64HighEntropyString(limit=iac_limit + 0.3), HexHighEntropyString(limit=iac_limit + 0.3))
self.high_entropy_scanners = (Base64HighEntropyString(limit=limit), HexHighEntropyString(limit=limit))
self.keyword_scanner = KeywordDetector()
self.max_line_length = max_line_length
Expand Down Expand Up @@ -153,15 +154,18 @@ def analyze_line(
if single_line_parser:
# Getting last detected one as only 1 violation available for line
secret_value, quoted_secret = EntropyKeywordCombinator.receive_last_secret_detected(keyword_on_key)
old_line = line
line = quoted_secret if quoted_secret else line
return single_line_parser.detect_secret(
detected_secrets = single_line_parser.detect_secret(
scanners=self.high_entropy_scanners_iac,
filename=filename,
raw_context=raw_context,
line=line,
line_number=line_number,
kwargs=kwargs
)
remove_fp_secrets_in_keys(detected_secrets, old_line)
return detected_secrets
else:
# preprocess line before detecting secrets - add quotes on potential secrets to allow triggering
# entropy detector
Expand Down Expand Up @@ -207,7 +211,7 @@ def analyze_line(
else:
return detect_secret(
# If we found a keyword (i.e. db_pass = ), lower the threshold to the iac threshold
scanners=self.high_entropy_scanners if not keyword_on_key else self.high_entropy_scanners_iac,
scanners=self.high_entropy_scanners if not keyword_on_key else self.entropy_scanners_non_iac_with_keyword,
filename=filename,
line=line,
line_number=line_number,
Expand Down
8 changes: 4 additions & 4 deletions checkov/secrets/runner.py
Original file line number Diff line number Diff line change
Expand Up @@ -60,14 +60,14 @@
'Base64 High Entropy String': 'CKV_SECRET_6',
'IBM Cloud IAM Key': 'CKV_SECRET_7',
'IBM COS HMAC Credentials': 'CKV_SECRET_8',
'JSON Web Token': 'CKV_SECRET_9', # checkov:skip=CKV_SECRET_6 false positive
'JSON Web Token': 'CKV_SECRET_9',
'Secret Keyword': 'CKV_SECRET_10',
'Mailchimp Access Key': 'CKV_SECRET_11',
'NPM tokens': 'CKV_SECRET_12', # checkov:skip=CKV_SECRET_6 false positive
'NPM tokens': 'CKV_SECRET_12',
'Private Key': 'CKV_SECRET_13',
'Slack Token': 'CKV_SECRET_14', # checkov:skip=CKV_SECRET_6 false positive
'Slack Token': 'CKV_SECRET_14',
'SoftLayer Credentials': 'CKV_SECRET_15',
'Square OAuth Secret': 'CKV_SECRET_16', # checkov:skip=CKV_SECRET_6 false positive
'Square OAuth Secret': 'CKV_SECRET_16',
'Stripe Access Key': 'CKV_SECRET_17',
'Twilio API Key': 'CKV_SECRET_18',
'Hex High Entropy String': 'CKV_SECRET_19'
Expand Down
2 changes: 1 addition & 1 deletion checkov/serverless/parsers/parser.py
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@

logger = logging.getLogger(__name__)

IAM_ROLE_STATEMENTS_TOKEN = 'iamRoleStatements' # nosec # checkov:skip=CKV_SECRET_6 false positive
IAM_ROLE_STATEMENTS_TOKEN = 'iamRoleStatements' # nosec
CFN_RESOURCES_TOKEN = 'resources' # nosec
PROVIDER_TOKEN = 'provider' # nosec
FUNCTIONS_TOKEN = 'functions' # nosec
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@ def test_summary(self):
runner = Runner()
current_dir = os.path.dirname(os.path.realpath(__file__))

test_files_dir = current_dir + "/example_IAMPolicyAttachedToGroupOrRoles" # checkov:skip=CKV_SECRET_6 false positive
test_files_dir = current_dir + "/example_IAMPolicyAttachedToGroupOrRoles"
report = runner.run(root_folder=test_files_dir,runner_filter=RunnerFilter(checks=[check.id]))
summary = report.get_summary()

Expand Down
2 changes: 1 addition & 1 deletion tests/common/utils/test_http_utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@ def get_report_url() -> str:
@mock.patch.dict(os.environ, {"REQUEST_MAX_TRIES": "5", "SLEEP_BETWEEN_REQUEST_TRIES": "0.01"})
def test_request_wrapper_all_fail_with_connection_error_for_get_scan_result(mock_bc_integration):
# given
mock_url = mock_bc_integration.bc_api_url + "/api/v1/vulnerabilities/scan-results/2e97f5afea42664309f492a1e2083b43479c2936" # checkov:skip=CKV_SECRET_6 false positive
mock_url = mock_bc_integration.bc_api_url + "/api/v1/vulnerabilities/scan-results/2e97f5afea42664309f492a1e2083b43479c2936"
responses.add(
method=responses.GET,
url=mock_url,
Expand Down
24 changes: 24 additions & 0 deletions tests/secrets/sanity/iac_fp/a.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,24 @@
def a():
test_files_dir = current_dir + "/example_LustreFSEncryptedWithCMK"

mock_url = mock_bc_integration.bc_api_url + "/api/v1/vulnerabilities/scan-results/2e97f5afea42664309f492a1e2083b43479c2936"

return "Properties/LogPublishingOptions/AUDIT_LOGS/Enabled"

metadata_options['HttpTokens'] == "required"

"MDEyOk9yZ2FuaXphdGlvbjE\u003d"

IAM_ROLE_STATEMENTS_TOKEN = 'iamRoleStatements'

a = {
'JSON Web Token': 'CKV_SECRET_9',
'NPM tokens': 'CKV_SECRET_12',
'Slack Token': 'CKV_SECRET_14',
'SoftLayer Credentials': 'CKV_SECRET_15',
'Square OAuth Secret': 'CKV_SECRET_16'
}

self.go("GCPMySQLdbInstancePoint_In_TimeRecoveryBackupIsEnabled")

a = {'s3_origin_config': [{'origin_access_identity': ['origin-access-identity/cloudfront/ABCDEFG1234567']}]}
3 changes: 3 additions & 0 deletions tests/secrets/sanity/non_iac_fp/main.tf
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
secret_name = "example_secret_name"

creation_token = "my-product"
24 changes: 24 additions & 0 deletions tests/secrets/test_runner.py
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,30 @@ def test_sanity_check_secrets(self):
self.assertEqual(report.skipped_checks, [])
report.print_console()

def test_fp_sanity_check_secrets_non_iac(self):
current_dir = os.path.dirname(os.path.realpath(__file__))
valid_dir_path = current_dir + "/sanity/iac_fp"
runner = Runner()
report = runner.run(root_folder=valid_dir_path, external_checks_dir=None,
runner_filter=RunnerFilter(framework=['secrets'], checks=['CKV_SECRET_6'], enable_secret_scan_all_files=True))
self.assertEqual(len(report.failed_checks), 0)
self.assertEqual(report.parsing_errors, [])
self.assertEqual(report.passed_checks, [])
self.assertEqual(report.skipped_checks, [])
report.print_console()

def test_fp_sanity_check_secrets_iac(self):
current_dir = os.path.dirname(os.path.realpath(__file__))
valid_dir_path = current_dir + "/sanity/non_iac_fp"
runner = Runner()
report = runner.run(root_folder=valid_dir_path, external_checks_dir=None,
runner_filter=RunnerFilter(framework=['secrets'], checks=['CKV_SECRET_6'], enable_secret_scan_all_files=True))
self.assertEqual(len(report.failed_checks), 0)
self.assertEqual(report.parsing_errors, [])
self.assertEqual(report.passed_checks, [])
self.assertEqual(report.skipped_checks, [])
report.print_console()

def test_sanity_check_non_secrets(self):
current_dir = os.path.dirname(os.path.realpath(__file__))
valid_dir_path = current_dir + "/sanity/non_secrets"
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ class TestCloudfrontDistributionEncryption(unittest.TestCase):
def test_failure(self):
resource_conf = {'origin': [
{'domain_name': ['${aws_s3_bucket.b.bucket_regional_domain_name}'], 'origin_id': ['${local.s3_origin_id}'],
's3_origin_config': [{'origin_access_identity': ['origin-access-identity/cloudfront/ABCDEFG1234567']}]}], # checkov:skip=CKV_SECRET_6 false positive
's3_origin_config': [{'origin_access_identity': ['origin-access-identity/cloudfront/ABCDEFG1234567']}]}],
'enabled': [True], 'is_ipv6_enabled': [True], 'comment': ['Some comment'],
'default_root_object': ['index.html'], 'logging_config': [
{'include_cookies': [False], 'bucket': ['mylogs.s3.amazonaws.com'], 'prefix': ['myprefix']}],
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ def test(self):
runner = Runner()
current_dir = os.path.dirname(os.path.realpath(__file__))

test_files_dir = current_dir + "/example_DLMEventsCrossRegionEncryptionWithCMK" # checkov:skip=CKV_SECRET_6 false positive
test_files_dir = current_dir + "/example_DLMEventsCrossRegionEncryptionWithCMK"
report = runner.run(root_folder=test_files_dir, runner_filter=RunnerFilter(checks=[check.id]))
summary = report.get_summary()

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ def test(self):
runner = Runner()
current_dir = os.path.dirname(os.path.realpath(__file__))

test_files_dir = current_dir + "/example_DLMScheduleCrossRegionEncryption" # checkov:skip=CKV_SECRET_6 false positive
test_files_dir = current_dir + "/example_DLMScheduleCrossRegionEncryption"
report = runner.run(root_folder=test_files_dir, runner_filter=RunnerFilter(checks=[check.id]))
summary = report.get_summary()

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ def test(self):
runner = Runner()
current_dir = os.path.dirname(os.path.realpath(__file__))

test_files_dir = current_dir + "/example_DLMScheduleCrossRegionEncryptionWithCMK" # checkov:skip=CKV_SECRET_6 false positive
test_files_dir = current_dir + "/example_DLMScheduleCrossRegionEncryptionWithCMK"
report = runner.run(root_folder=test_files_dir, runner_filter=RunnerFilter(checks=[check.id]))
summary = report.get_summary()

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@ def test(self):
runner = Runner()
current_dir = os.path.dirname(os.path.realpath(__file__))

test_files_dir = current_dir + "/example_EBSVolumeEncryptedWithCMK" # checkov:skip=CKV_SECRET_6 false positive
test_files_dir = current_dir + "/example_EBSVolumeEncryptedWithCMK"
report = runner.run(
root_folder=test_files_dir, runner_filter=RunnerFilter(checks=[check.id])
)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@
class TestECSClusterLoggingEncryptedWithCMK(unittest.TestCase):
def test(self):
# given
test_files_dir = Path(__file__).parent / "example_ECSClusterLoggingEncryptedWithCMK" # checkov:skip=CKV_SECRET_6 false positive
test_files_dir = Path(__file__).parent / "example_ECSClusterLoggingEncryptedWithCMK"

# when
report = Runner().run(root_folder=str(test_files_dir), runner_filter=RunnerFilter(checks=[check.id]))
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ def test(self):
runner = Runner()
current_dir = os.path.dirname(os.path.realpath(__file__))

test_files_dir = current_dir + "/example_EMRClusterConfEncryptsLocalDisk" # checkov:skip=CKV_SECRET_6 false positive
test_files_dir = current_dir + "/example_EMRClusterConfEncryptsLocalDisk"
report = runner.run(root_folder=test_files_dir, runner_filter=RunnerFilter(checks=[check.id]))
summary = report.get_summary()

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ def test(self):
runner = Runner()
current_dir = os.path.dirname(os.path.realpath(__file__))

test_files_dir = current_dir + "/example_FSXOpenZFSFileSystemEncryptedWithCMK" # checkov:skip=CKV_SECRET_6 false positive
test_files_dir = current_dir + "/example_FSXOpenZFSFileSystemEncryptedWithCMK"
report = runner.run(
root_folder=test_files_dir, runner_filter=RunnerFilter(checks=[check.id])
)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ def test(self):
runner = Runner()
current_dir = os.path.dirname(os.path.realpath(__file__))

test_files_dir = current_dir + "/example_FSXWindowsFSEncryptedWithCMK" # checkov:skip=CKV_SECRET_6 false positive
test_files_dir = current_dir + "/example_FSXWindowsFSEncryptedWithCMK"
report = runner.run(
root_folder=test_files_dir, runner_filter=RunnerFilter(checks=[check.id])
)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ def test(self):
runner = Runner()
current_dir = os.path.dirname(os.path.realpath(__file__))

test_files_dir = current_dir + "/example_ImagebuilderDistributionConfigurationEncryptedWithCMK" # checkov:skip=CKV_SECRET_6 false positive
test_files_dir = current_dir + "/example_ImagebuilderDistributionConfigurationEncryptedWithCMK"
report = runner.run(
root_folder=test_files_dir, runner_filter=RunnerFilter(checks=[check.id])
)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ def test(self):
runner = Runner()
current_dir = os.path.dirname(os.path.realpath(__file__))

test_files_dir = current_dir + "/example_LustreFSEncryptedWithCMK" # checkov:skip=CKV_SECRET_6 false positive
test_files_dir = current_dir + "/example_LustreFSEncryptedWithCMK"
report = runner.run(
root_folder=test_files_dir, runner_filter=RunnerFilter(checks=[check.id])
)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ def test(self):
runner = Runner()
current_dir = os.path.dirname(os.path.realpath(__file__))

test_files_dir = current_dir + "/example_RDSClusterActivityStreamEncryptedWithCMK" # checkov:skip=CKV_SECRET_6 false positive
test_files_dir = current_dir + "/example_RDSClusterActivityStreamEncryptedWithCMK"
report = runner.run(
root_folder=test_files_dir, runner_filter=RunnerFilter(checks=[check.id])
)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@
class TestRDSInstanceAutoBackupEncryptionWithCMK(unittest.TestCase):
def test(self):
# given
test_files_dir = Path(__file__).parent / "example_RDSInstanceAutoBackupEncryptionWithCMK" # checkov:skip=CKV_SECRET_6 false positive
test_files_dir = Path(__file__).parent / "example_RDSInstanceAutoBackupEncryptionWithCMK"

# when
report = Runner().run(root_folder=str(test_files_dir), runner_filter=RunnerFilter(checks=[check.id]))
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ def test(self):
runner = Runner()
current_dir = os.path.dirname(os.path.realpath(__file__))

test_files_dir = current_dir + "/example_RDSInstancePerfInsightsEncryptionWithCMK" # checkov:skip=CKV_SECRET_6 false positive
test_files_dir = current_dir + "/example_RDSInstancePerfInsightsEncryptionWithCMK"
report = runner.run(root_folder=test_files_dir, runner_filter=RunnerFilter(checks=[check.id]))
summary = report.get_summary()

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ def test(self):
runner = Runner()
current_dir = os.path.dirname(os.path.realpath(__file__))

test_files_dir = current_dir + "/example_RedshiftSnapshotCopyGrantEncryptedWithCMK" # checkov:skip=CKV_SECRET_6 false positive
test_files_dir = current_dir + "/example_RedshiftSnapshotCopyGrantEncryptedWithCMK"
report = runner.run(
root_folder=test_files_dir, runner_filter=RunnerFilter(checks=[check.id])
)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ def test(self):
runner = Runner()
current_dir = os.path.dirname(os.path.realpath(__file__))

test_files_dir = current_dir + "/example_S3BucketObjectEncryptedWithCMK" # checkov:skip=CKV_SECRET_6 false positive
test_files_dir = current_dir + "/example_S3BucketObjectEncryptedWithCMK"
report = runner.run(
root_folder=test_files_dir, runner_filter=RunnerFilter(checks=[check.id])
)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ def test(self):
runner = Runner()
current_dir = os.path.dirname(os.path.realpath(__file__))

test_files_dir = current_dir + "/example_S3ObjectCopyEncryptedWithCMK" # checkov:skip=CKV_SECRET_6 false positive
test_files_dir = current_dir + "/example_S3ObjectCopyEncryptedWithCMK"
report = runner.run(
root_folder=test_files_dir, runner_filter=RunnerFilter(checks=[check.id])
)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@
class TestBigQueryTableEncryptedWithCMK(unittest.TestCase):
def test(self):
# given
test_files_dir = Path(__file__).parent / "example_BigQueryTableEncryptedWithCMK" # checkov:skip=CKV_SECRET_6 false positive
test_files_dir = Path(__file__).parent / "example_BigQueryTableEncryptedWithCMK"

# when
report = Runner().run(root_folder=str(test_files_dir), runner_filter=RunnerFilter(checks=[check.id]))
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@
class TestCloudPubSubEncryptedWithCMK(unittest.TestCase):
def test(self):
# given
test_files_dir = Path(__file__).parent / "example_CloudPubSubEncryptedWithCMK" # checkov:skip=CKV_SECRET_6 false positive
test_files_dir = Path(__file__).parent / "example_CloudPubSubEncryptedWithCMK"

# when
report = Runner().run(root_folder=str(test_files_dir), runner_filter=RunnerFilter(checks=[check.id]))
Expand Down
Loading