PEM slight changes (#371)

* PEM slight changes * [no ci] PEM END PATTERN * Update credsweeper/common/constants.py Co-authored-by: ShinHyung Choi <[email protected]> --------- Co-authored-by: ShinHyung Choi <[email protected]>
Samsung · Jul 3, 2023 · d6b96ff · d6b96ff
1 parent 448fe56
commit d6b96ff
Show file tree

Hide file tree

Showing 10 changed files with 33 additions and 28 deletions.
diff --git a/credsweeper/common/constants.py b/credsweeper/common/constants.py
@@ -140,3 +140,6 @@ class DiffRowType(Enum):
 
 # default value for config and ValuePemPatternCheck
 DEFAULT_PEM_PATTERN_LEN = 5
+
+# PEM x509 patterns
+PEM_BEGIN_PATTERN = "-----BEGIN"
diff --git a/credsweeper/rules/config.yaml b/credsweeper/rules/config.yaml
@@ -270,14 +270,14 @@
     - src
     - doc
 
-- name: PEM Certificate
+- name: PEM Private Key
   severity: high
   type: pem_key
   values:
-    - (?P<value>-----BEGIN\s(?!ENCRYPTED|EC).*PRIVATE)
+    - (?P<value>-----BEGIN\s(?!ENCRYPTED|EC)[^-]*PRIVATE[^-]*KEY[^-]*-----)
   filter_type:
     - LineSpecificKeyCheck
-  min_line_len: 20
+  min_line_len: 27
   usage_list:
     - src
     - doc

diff --git a/credsweeper/scanner/scan_type/pem_key_pattern.py b/credsweeper/scanner/scan_type/pem_key_pattern.py
@@ -8,6 +8,8 @@
 from credsweeper.scanner.scan_type import ScanType
 from credsweeper.utils import Util
 
+PEM_END_PATTERN = "-----END"
+
 
 class PemKeyPattern(ScanType):
     """Check if line is a start of a PEM key.
@@ -61,7 +63,7 @@ def is_pem_key(cls, lines: List[str], config: Config) -> bool:
         for line_num, line in enumerate(lines):
             if line_num >= 190:
                 return False
-            if "-----END" in line:
+            if PEM_END_PATTERN in line:
                 # Check if entropy is high enough
                 removed_by_entropy = not Util.is_entropy_validate(key_data)
                 # Check if have no substring with 5 same consecutive characters (like 'AAAAA')

diff --git a/credsweeper/scanner/scanner.py b/credsweeper/scanner/scanner.py
@@ -4,7 +4,7 @@
 
 from credsweeper.app import APP_PATH
 from credsweeper.common.constants import RuleType, MIN_VARIABLE_LENGTH, MIN_SEPARATOR_LENGTH, MIN_VALUE_LENGTH, \
-    MAX_LINE_LENGTH, Separator
+    MAX_LINE_LENGTH, Separator, PEM_BEGIN_PATTERN
 from credsweeper.config import Config
 from credsweeper.credentials import Candidate
 from credsweeper.file_handler.analysis_target import AnalysisTarget
@@ -102,7 +102,7 @@ def _select_and_group_targets(self, targets: List[AnalysisTarget]) -> Tuple[Targ
             if target_line_trimmed_len >= self.min_pattern_len:
                 pattern_targets.append((target, target_line_trimmed_lower, target_line_trimmed_len))
             # Check if have "BEGIN" substring. Cannot otherwise ba matched as a PEM key
-            if target_line_trimmed_len >= self.min_pem_key_len and "BEGIN" in target_line_trimmed:
+            if target_line_trimmed_len >= self.min_pem_key_len and PEM_BEGIN_PATTERN in target_line_trimmed:
                 pem_targets.append((target, target_line_trimmed_lower, target_line_trimmed_len))
 
         return keyword_targets, pattern_targets, pem_targets

diff --git a/tests/data/depth_3.json b/tests/data/depth_3.json
@@ -1479,15 +1479,15 @@
         "api_validation": "NOT_AVAILABLE",
         "ml_validation": "NOT_AVAILABLE",
         "ml_probability": null,
-        "rule": "PEM Certificate",
+        "rule": "PEM Private Key",
         "severity": "high",
         "line_data_list": [
             {
                 "line": "-----BEGIN RSA PRIVATE KEY-----",
                 "line_num": 1,
                 "path": "tests/samples/pem_key",
                 "info": "tests/samples/pem_key|RAW",
-                "value": "-----BEGIN RSA PRIVATE",
+                "value": "-----BEGIN RSA PRIVATE KEY-----",
                 "variable": null,
                 "entropy_validation": false
             }
@@ -1497,15 +1497,15 @@
         "api_validation": "NOT_AVAILABLE",
         "ml_validation": "NOT_AVAILABLE",
         "ml_probability": null,
-        "rule": "PEM Certificate",
+        "rule": "PEM Private Key",
         "severity": "high",
         "line_data_list": [
             {
                 "line": "-----BEGIN RSA PRIVATE KEY-----",
                 "line_num": 1,
                 "path": "tests/samples/pem_key.apk",
                 "info": "tests/samples/pem_key.apk|ZIP|3.zip|ZIP|2.zip|ZIP|2|RAW",
-                "value": "-----BEGIN RSA PRIVATE",
+                "value": "-----BEGIN RSA PRIVATE KEY-----",
                 "variable": null,
                 "entropy_validation": false
             }
@@ -1515,15 +1515,15 @@
         "api_validation": "NOT_AVAILABLE",
         "ml_validation": "NOT_AVAILABLE",
         "ml_probability": null,
-        "rule": "PEM Certificate",
+        "rule": "PEM Private Key",
         "severity": "high",
         "line_data_list": [
             {
                 "line": "-----BEGIN RSA PRIVATE KEY-----",
                 "line_num": 1,
                 "path": "tests/samples/pem_key.apk",
                 "info": "tests/samples/pem_key.apk|ZIP|cred/pem_key.zip|ZIP|key.pem|RAW",
-                "value": "-----BEGIN RSA PRIVATE",
+                "value": "-----BEGIN RSA PRIVATE KEY-----",
                 "variable": null,
                 "entropy_validation": false
             }
@@ -1551,15 +1551,15 @@
         "api_validation": "NOT_AVAILABLE",
         "ml_validation": "NOT_AVAILABLE",
         "ml_probability": null,
-        "rule": "PEM Certificate",
+        "rule": "PEM Private Key",
         "severity": "high",
         "line_data_list": [
             {
                 "line": "-----BEGIN RSA PRIVATE KEY-----",
                 "line_num": 1,
                 "path": "tests/samples/pem_key.bz2",
                 "info": "tests/samples/pem_key.bz2|BZIP2|tests/samples/pem_key|RAW",
-                "value": "-----BEGIN RSA PRIVATE",
+                "value": "-----BEGIN RSA PRIVATE KEY-----",
                 "variable": null,
                 "entropy_validation": false
             }
@@ -1569,15 +1569,15 @@
         "api_validation": "NOT_AVAILABLE",
         "ml_validation": "NOT_AVAILABLE",
         "ml_probability": null,
-        "rule": "PEM Certificate",
+        "rule": "PEM Private Key",
         "severity": "high",
         "line_data_list": [
             {
                 "line": "-----BEGIN RSA PRIVATE KEY-----",
                 "line_num": 1,
                 "path": "tests/samples/pem_key.gz",
                 "info": "tests/samples/pem_key.gz|GZIP|tests/samples/pem_key|RAW",
-                "value": "-----BEGIN RSA PRIVATE",
+                "value": "-----BEGIN RSA PRIVATE KEY-----",
                 "variable": null,
                 "entropy_validation": false
             }
@@ -1587,15 +1587,15 @@
         "api_validation": "NOT_AVAILABLE",
         "ml_validation": "NOT_AVAILABLE",
         "ml_probability": null,
-        "rule": "PEM Certificate",
+        "rule": "PEM Private Key",
         "severity": "high",
         "line_data_list": [
             {
                 "line": "-----BEGIN RSA PRIVATE KEY-----",
                 "line_num": 1,
                 "path": "tests/samples/pem_key.zip",
                 "info": "tests/samples/pem_key.zip|ZIP|pem_key|RAW",
-                "value": "-----BEGIN RSA PRIVATE",
+                "value": "-----BEGIN RSA PRIVATE KEY-----",
                 "variable": null,
                 "entropy_validation": false
             }

diff --git a/tests/data/doc.json b/tests/data/doc.json
@@ -993,15 +993,15 @@
         "api_validation": "NOT_AVAILABLE",
         "ml_validation": "NOT_AVAILABLE",
         "ml_probability": null,
-        "rule": "PEM Certificate",
+        "rule": "PEM Private Key",
         "severity": "high",
         "line_data_list": [
             {
                 "line": "-----BEGIN RSA PRIVATE KEY-----",
                 "line_num": 1,
                 "path": "tests/samples/pem_key",
                 "info": "tests/samples/pem_key|RAW",
-                "value": "-----BEGIN RSA PRIVATE",
+                "value": "-----BEGIN RSA PRIVATE KEY-----",
                 "variable": null,
                 "entropy_validation": false
             }

diff --git a/tests/data/ml_threshold_0.json b/tests/data/ml_threshold_0.json
@@ -1389,15 +1389,15 @@
         "api_validation": "NOT_AVAILABLE",
         "ml_validation": "NOT_AVAILABLE",
         "ml_probability": null,
-        "rule": "PEM Certificate",
+        "rule": "PEM Private Key",
         "severity": "high",
         "line_data_list": [
             {
                 "line": "-----BEGIN RSA PRIVATE KEY-----",
                 "line_num": 1,
                 "path": "tests/samples/pem_key",
                 "info": "",
-                "value": "-----BEGIN RSA PRIVATE",
+                "value": "-----BEGIN RSA PRIVATE KEY-----",
                 "variable": null,
                 "entropy_validation": false
             }

diff --git a/tests/data/output.json b/tests/data/output.json
@@ -1317,15 +1317,15 @@
         "api_validation": "NOT_AVAILABLE",
         "ml_validation": "NOT_AVAILABLE",
         "ml_probability": null,
-        "rule": "PEM Certificate",
+        "rule": "PEM Private Key",
         "severity": "high",
         "line_data_list": [
             {
                 "line": "-----BEGIN RSA PRIVATE KEY-----",
                 "line_num": 1,
                 "path": "tests/samples/pem_key",
                 "info": "",
-                "value": "-----BEGIN RSA PRIVATE",
+                "value": "-----BEGIN RSA PRIVATE KEY-----",
                 "variable": null,
                 "entropy_validation": false
             }

diff --git a/tests/rules/test_pem_key.py b/tests/rules/test_pem_key.py
@@ -65,7 +65,7 @@ def lines(self, request) -> List[str]:
 
     @pytest.fixture
     def rule_name(self) -> str:
-        return "PEM Certificate"
+        return "PEM Private Key"
 
 
 class TestEmptyPemKey:
@@ -85,7 +85,7 @@ def lines(self, request) -> List[str]:
 
     @pytest.fixture
     def rule_name(self) -> str:
-        return "PEM Certificate"
+        return "PEM Private Key"
 
     def test_scan_no_division_by_zero_exception_n(self, file_path: pytest.fixture, lines: pytest.fixture,
                                                   scanner: pytest.fixture) -> None:

diff --git a/tests/test_main.py b/tests/test_main.py
@@ -556,8 +556,8 @@ def test_yaml_p(self) -> None:
         cred_sweeper.run(content_provider=content_provider)
         found_credentials = cred_sweeper.credential_manager.get_credentials()
         self.assertEqual(2, len(found_credentials))
-        self.assertSetEqual({"Secret", "PEM Certificate"}, set(i.rule_name for i in found_credentials))
-        self.assertSetEqual({"we5345d0f3da48544z1t1e275y05i161x995q485\n", "-----BEGIN RSA PRIVATE"},
+        self.assertSetEqual({"Secret", "PEM Private Key"}, set(i.rule_name for i in found_credentials))
+        self.assertSetEqual({"we5345d0f3da48544z1t1e275y05i161x995q485\n", "-----BEGIN RSA PRIVATE KEY-----"},
                             set(i.line_data_list[0].value for i in found_credentials))
 
     # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #