Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

PEM slight changes #371

Merged
merged 4 commits into from
Jul 3, 2023
Merged

PEM slight changes #371

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
627666c
PEM slight changes
babenek Jun 29, 2023
ee5ac09
Merge branch 'main' into auxiliary
babenek Jun 29, 2023
f6b88c3
[no ci] PEM END PATTERN
babenek Jul 3, 2023
42d920e
Update credsweeper/common/constants.py
babenek Jul 3, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions credsweeper/common/constants.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -140,3 +140,6 @@ class DiffRowType(Enum):

# default value for config and ValuePemPatternCheck
DEFAULT_PEM_PATTERN_LEN = 5

# PEM x509 patterns
PEM_BEGIN_PATTERN = "-----BEGIN"
6 changes: 3 additions & 3 deletions credsweeper/rules/config.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -270,14 +270,14 @@
- src
- doc

- name: PEM Certificate
- name: PEM Private Key
severity: high
type: pem_key
values:
- (?P<value>-----BEGIN\s(?!ENCRYPTED|EC).*PRIVATE)
- (?P<value>-----BEGIN\s(?!ENCRYPTED|EC)[^-]*PRIVATE[^-]*KEY[^-]*-----)
filter_type:
- LineSpecificKeyCheck
min_line_len: 20
min_line_len: 27
usage_list:
- src
- doc
Expand Down
4 changes: 3 additions & 1 deletion credsweeper/scanner/scan_type/pem_key_pattern.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,8 @@
from credsweeper.scanner.scan_type import ScanType
from credsweeper.utils import Util

PEM_END_PATTERN = "-----END"


class PemKeyPattern(ScanType):
"""Check if line is a start of a PEM key.
Expand Down Expand Up @@ -61,7 +63,7 @@ def is_pem_key(cls, lines: List[str], config: Config) -> bool:
for line_num, line in enumerate(lines):
if line_num >= 190:
return False
if "-----END" in line:
if PEM_END_PATTERN in line:
# Check if entropy is high enough
removed_by_entropy = not Util.is_entropy_validate(key_data)
# Check if have no substring with 5 same consecutive characters (like 'AAAAA')
Expand Down
4 changes: 2 additions & 2 deletions credsweeper/scanner/scanner.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@

from credsweeper.app import APP_PATH
from credsweeper.common.constants import RuleType, MIN_VARIABLE_LENGTH, MIN_SEPARATOR_LENGTH, MIN_VALUE_LENGTH, \
MAX_LINE_LENGTH, Separator
MAX_LINE_LENGTH, Separator, PEM_BEGIN_PATTERN
from credsweeper.config import Config
from credsweeper.credentials import Candidate
from credsweeper.file_handler.analysis_target import AnalysisTarget
Expand Down Expand Up @@ -102,7 +102,7 @@ def _select_and_group_targets(self, targets: List[AnalysisTarget]) -> Tuple[Targ
if target_line_trimmed_len >= self.min_pattern_len:
pattern_targets.append((target, target_line_trimmed_lower, target_line_trimmed_len))
# Check if have "BEGIN" substring. Cannot otherwise ba matched as a PEM key
if target_line_trimmed_len >= self.min_pem_key_len and "BEGIN" in target_line_trimmed:
if target_line_trimmed_len >= self.min_pem_key_len and PEM_BEGIN_PATTERN in target_line_trimmed:
pem_targets.append((target, target_line_trimmed_lower, target_line_trimmed_len))

return keyword_targets, pattern_targets, pem_targets
Expand Down
24 changes: 12 additions & 12 deletions tests/data/depth_3.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1479,15 +1479,15 @@
"api_validation": "NOT_AVAILABLE",
"ml_validation": "NOT_AVAILABLE",
"ml_probability": null,
"rule": "PEM Certificate",
"rule": "PEM Private Key",
"severity": "high",
"line_data_list": [
{
"line": "-----BEGIN RSA PRIVATE KEY-----",
"line_num": 1,
"path": "tests/samples/pem_key",
"info": "tests/samples/pem_key|RAW",
"value": "-----BEGIN RSA PRIVATE",
"value": "-----BEGIN RSA PRIVATE KEY-----",
"variable": null,
"entropy_validation": false
}
Expand All @@ -1497,15 +1497,15 @@
"api_validation": "NOT_AVAILABLE",
"ml_validation": "NOT_AVAILABLE",
"ml_probability": null,
"rule": "PEM Certificate",
"rule": "PEM Private Key",
"severity": "high",
"line_data_list": [
{
"line": "-----BEGIN RSA PRIVATE KEY-----",
"line_num": 1,
"path": "tests/samples/pem_key.apk",
"info": "tests/samples/pem_key.apk|ZIP|3.zip|ZIP|2.zip|ZIP|2|RAW",
"value": "-----BEGIN RSA PRIVATE",
"value": "-----BEGIN RSA PRIVATE KEY-----",
"variable": null,
"entropy_validation": false
}
Expand All @@ -1515,15 +1515,15 @@
"api_validation": "NOT_AVAILABLE",
"ml_validation": "NOT_AVAILABLE",
"ml_probability": null,
"rule": "PEM Certificate",
"rule": "PEM Private Key",
"severity": "high",
"line_data_list": [
{
"line": "-----BEGIN RSA PRIVATE KEY-----",
"line_num": 1,
"path": "tests/samples/pem_key.apk",
"info": "tests/samples/pem_key.apk|ZIP|cred/pem_key.zip|ZIP|key.pem|RAW",
"value": "-----BEGIN RSA PRIVATE",
"value": "-----BEGIN RSA PRIVATE KEY-----",
"variable": null,
"entropy_validation": false
}
Expand Down Expand Up @@ -1551,15 +1551,15 @@
"api_validation": "NOT_AVAILABLE",
"ml_validation": "NOT_AVAILABLE",
"ml_probability": null,
"rule": "PEM Certificate",
"rule": "PEM Private Key",
"severity": "high",
"line_data_list": [
{
"line": "-----BEGIN RSA PRIVATE KEY-----",
"line_num": 1,
"path": "tests/samples/pem_key.bz2",
"info": "tests/samples/pem_key.bz2|BZIP2|tests/samples/pem_key|RAW",
"value": "-----BEGIN RSA PRIVATE",
"value": "-----BEGIN RSA PRIVATE KEY-----",
"variable": null,
"entropy_validation": false
}
Expand All @@ -1569,15 +1569,15 @@
"api_validation": "NOT_AVAILABLE",
"ml_validation": "NOT_AVAILABLE",
"ml_probability": null,
"rule": "PEM Certificate",
"rule": "PEM Private Key",
"severity": "high",
"line_data_list": [
{
"line": "-----BEGIN RSA PRIVATE KEY-----",
"line_num": 1,
"path": "tests/samples/pem_key.gz",
"info": "tests/samples/pem_key.gz|GZIP|tests/samples/pem_key|RAW",
"value": "-----BEGIN RSA PRIVATE",
"value": "-----BEGIN RSA PRIVATE KEY-----",
"variable": null,
"entropy_validation": false
}
Expand All @@ -1587,15 +1587,15 @@
"api_validation": "NOT_AVAILABLE",
"ml_validation": "NOT_AVAILABLE",
"ml_probability": null,
"rule": "PEM Certificate",
"rule": "PEM Private Key",
"severity": "high",
"line_data_list": [
{
"line": "-----BEGIN RSA PRIVATE KEY-----",
"line_num": 1,
"path": "tests/samples/pem_key.zip",
"info": "tests/samples/pem_key.zip|ZIP|pem_key|RAW",
"value": "-----BEGIN RSA PRIVATE",
"value": "-----BEGIN RSA PRIVATE KEY-----",
"variable": null,
"entropy_validation": false
}
Expand Down
4 changes: 2 additions & 2 deletions tests/data/doc.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -993,15 +993,15 @@
"api_validation": "NOT_AVAILABLE",
"ml_validation": "NOT_AVAILABLE",
"ml_probability": null,
"rule": "PEM Certificate",
"rule": "PEM Private Key",
"severity": "high",
"line_data_list": [
{
"line": "-----BEGIN RSA PRIVATE KEY-----",
"line_num": 1,
"path": "tests/samples/pem_key",
"info": "tests/samples/pem_key|RAW",
"value": "-----BEGIN RSA PRIVATE",
"value": "-----BEGIN RSA PRIVATE KEY-----",
"variable": null,
"entropy_validation": false
}
Expand Down
4 changes: 2 additions & 2 deletions tests/data/ml_threshold_0.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1389,15 +1389,15 @@
"api_validation": "NOT_AVAILABLE",
"ml_validation": "NOT_AVAILABLE",
"ml_probability": null,
"rule": "PEM Certificate",
"rule": "PEM Private Key",
"severity": "high",
"line_data_list": [
{
"line": "-----BEGIN RSA PRIVATE KEY-----",
"line_num": 1,
"path": "tests/samples/pem_key",
"info": "",
"value": "-----BEGIN RSA PRIVATE",
"value": "-----BEGIN RSA PRIVATE KEY-----",
"variable": null,
"entropy_validation": false
}
Expand Down
4 changes: 2 additions & 2 deletions tests/data/output.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1317,15 +1317,15 @@
"api_validation": "NOT_AVAILABLE",
"ml_validation": "NOT_AVAILABLE",
"ml_probability": null,
"rule": "PEM Certificate",
"rule": "PEM Private Key",
"severity": "high",
"line_data_list": [
{
"line": "-----BEGIN RSA PRIVATE KEY-----",
"line_num": 1,
"path": "tests/samples/pem_key",
"info": "",
"value": "-----BEGIN RSA PRIVATE",
"value": "-----BEGIN RSA PRIVATE KEY-----",
"variable": null,
"entropy_validation": false
}
Expand Down
4 changes: 2 additions & 2 deletions tests/rules/test_pem_key.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -65,7 +65,7 @@ def lines(self, request) -> List[str]:

@pytest.fixture
def rule_name(self) -> str:
return "PEM Certificate"
return "PEM Private Key"


class TestEmptyPemKey:
Expand All @@ -85,7 +85,7 @@ def lines(self, request) -> List[str]:

@pytest.fixture
def rule_name(self) -> str:
return "PEM Certificate"
return "PEM Private Key"

def test_scan_no_division_by_zero_exception_n(self, file_path: pytest.fixture, lines: pytest.fixture,
scanner: pytest.fixture) -> None:
Expand Down
4 changes: 2 additions & 2 deletions tests/test_main.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -556,8 +556,8 @@ def test_yaml_p(self) -> None:
cred_sweeper.run(content_provider=content_provider)
found_credentials = cred_sweeper.credential_manager.get_credentials()
self.assertEqual(2, len(found_credentials))
self.assertSetEqual({"Secret", "PEM Certificate"}, set(i.rule_name for i in found_credentials))
self.assertSetEqual({"we5345d0f3da48544z1t1e275y05i161x995q485\n", "-----BEGIN RSA PRIVATE"},
self.assertSetEqual({"Secret", "PEM Private Key"}, set(i.rule_name for i in found_credentials))
self.assertSetEqual({"we5345d0f3da48544z1t1e275y05i161x995q485\n", "-----BEGIN RSA PRIVATE KEY-----"},
set(i.line_data_list[0].value for i in found_credentials))

# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
Expand Down