From 72df576028792be177f973ae4d8ad9d744e91aa1 Mon Sep 17 00:00:00 2001 From: Roman Babenko Date: Mon, 5 Aug 2024 14:40:37 +0300 Subject: [PATCH 1/7] DOC BenchMark workflow (#593) * DOC BM push event * test * fix * Update benchmark.yml --- .github/workflows/benchmark.yml | 34 +++++++++++++++++++++------------ 1 file changed, 22 insertions(+), 12 deletions(-) diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index 5797f1acb..1caaa133f 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -287,7 +287,7 @@ jobs: exit_code=0 LOW_DELTA=10 THRESHOLD=250 - + # RELEASE if [ ${RELEASE_TIME} -le ${HEAD_TIME} ]; then d=$(( 1000 * ( ${HEAD_TIME} - ${RELEASE_TIME} ) / ${RELEASE_TIME} )) @@ -311,7 +311,7 @@ jobs: echo "Speed-up." fi fi - + # BASE if [ ${BASE_TIME} -le ${HEAD_TIME} ]; then d=$(( 1000 * ( ${HEAD_TIME} - ${BASE_TIME} ) / ${BASE_TIME} )) @@ -335,7 +335,7 @@ jobs: echo "Speed-up." fi fi - + exit ${exit_code} # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # @@ -428,24 +428,34 @@ jobs: exit 1 fi -# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # + # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # run_doc_benchmark: runs-on: ubuntu-latest - if: ${{ 'Samsung/CredSweeper' == github.event.pull_request.head.repo.full_name }} + if: ${{ 'push' == github.event_name }} or ${{ 'Samsung/CredSweeper' == github.event.pull_request.head.repo.full_name }} steps: - - name: Checkout CredSweeper + - name: Checkout CredSweeper PR if: ${{ 'pull_request' == github.event_name }} uses: actions/checkout@v4 with: ref: ${{ github.event.pull_request.head.sha }} + - name: Checkout CredSweeper HEAD + if: ${{ 'push' == github.event_name }} + uses: actions/checkout@v4 + with: + ref: ${{ github.event.head }} + - name: Send cURL request with the commit SHA - if: ${{ 'pull_request' == github.event_name }} run: | - COMMIT_SHA=$(git rev-parse HEAD) - curl -X POST ${{ secrets.SLACK_URL }} \ - --data-urlencode \ - "payload={'text':'[BMT Request] ${{ github.event.repository.html_url }}/commit/${COMMIT_SHA}'}" + if [[ "${{ secrets.SLACK_URL }}" =~ http.*/.*/.* ]]; then + COMMIT_SHA=$(git rev-parse HEAD) + echo ${COMMIT_SHA} + curl -X POST ${{ secrets.SLACK_URL }} \ + --data-urlencode \ + "payload={'text':'[BMT Request] ${{ github.event.repository.html_url }}/commit/${COMMIT_SHA}'}" + else + echo "secrets.SLACK_URL is not available" + fi -# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # + # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # From 4d89a190286fc46950388ffb4be62df69996c61f Mon Sep 17 00:00:00 2001 From: Roman Babenko Date: Tue, 6 Aug 2024 11:58:20 +0300 Subject: [PATCH 2/7] softreset (#592) --- credsweeper/filters/__init__.py | 1 + credsweeper/filters/group/group.py | 2 +- .../filters/value_base64_part_check.py | 56 +++++++++++++++++++ credsweeper/rules/config.yaml | 4 +- credsweeper/scanner/scan_type/scan_type.py | 33 ++++++----- tests/filters/test_value_base64_part_check.py | 33 +++++++++++ tests/samples/test.html | 2 +- 7 files changed, 111 insertions(+), 20 deletions(-) create mode 100644 credsweeper/filters/value_base64_part_check.py create mode 100644 tests/filters/test_value_base64_part_check.py diff --git a/credsweeper/filters/__init__.py b/credsweeper/filters/__init__.py index 66de37b97..986728d22 100644 --- a/credsweeper/filters/__init__.py +++ b/credsweeper/filters/__init__.py @@ -9,6 +9,7 @@ from credsweeper.filters.value_base64_data_check import ValueBase64DataCheck from credsweeper.filters.value_base64_encoded_pem_check import ValueBase64EncodedPem from credsweeper.filters.value_base64_key_check import ValueBase64KeyCheck +from credsweeper.filters.value_base64_part_check import ValueBase64PartCheck from credsweeper.filters.value_blocklist_check import ValueBlocklistCheck from credsweeper.filters.value_camel_case_check import ValueCamelCaseCheck from credsweeper.filters.value_couple_keyword_check import ValueCoupleKeywordCheck diff --git a/credsweeper/filters/group/group.py b/credsweeper/filters/group/group.py index 6ee25387d..7f1bc3c6a 100644 --- a/credsweeper/filters/group/group.py +++ b/credsweeper/filters/group/group.py @@ -60,5 +60,5 @@ def get_pattern_base_filters(config: Config) -> List[Filter]: return [ # LineSpecificKeyCheck(), # ValuePatternCheck(config), # - ValuePatternLengthCheck(config) + ValuePatternLengthCheck(config), # ] diff --git a/credsweeper/filters/value_base64_part_check.py b/credsweeper/filters/value_base64_part_check.py new file mode 100644 index 000000000..d41a7ed8c --- /dev/null +++ b/credsweeper/filters/value_base64_part_check.py @@ -0,0 +1,56 @@ +import contextlib +import statistics + +from credsweeper.common.constants import Chars +from credsweeper.config import Config +from credsweeper.credentials import LineData +from credsweeper.file_handler.analysis_target import AnalysisTarget +from credsweeper.filters import Filter +from credsweeper.utils import Util + + +class ValueBase64PartCheck(Filter): + """ + Check that candidate is NOT a part of base64 long line + """ + + def __init__(self, config: Config = None) -> None: + pass + + def run(self, line_data: LineData, target: AnalysisTarget) -> bool: + """Run filter checks on received weird base64 token which must be a random string + + Args: + line_data: credential candidate data + target: multiline target from which line data was obtained + + Return: + True, when need to filter candidate and False if left + + """ + + with contextlib.suppress(Exception): + if line_data.value_start and '/' == line_data.line[line_data.value_start - 1]: + if '-' in line_data.value or '_' in line_data.value: + # the value contains url-safe chars, so '/' is a delimiter + return False + value_entropy = Util.get_shannon_entropy(line_data.value, Chars.BASE64STD_CHARS.value) + left_start = line_data.value_start - len(line_data.value) + if 0 > left_start: + left_start = 0 + left_entropy = Util.get_shannon_entropy(line_data.line[left_start:line_data.value_start], + Chars.BASE64STD_CHARS.value) + right_end = line_data.value_end + len(line_data.value) + if len(line_data.line) < right_end: + right_end = len(line_data.line) + right_entropy = Util.get_shannon_entropy(line_data.line[line_data.value_end:right_end], + Chars.BASE64STD_CHARS.value) + data = [value_entropy, left_entropy, right_entropy] + avg = statistics.mean(data) + stdev = statistics.stdev(data, avg) + avg_min = avg - stdev + if avg_min < left_entropy and avg_min < right_entropy: + # high entropy of bound parts looks like a part of base64 long line + return True + + return False diff --git a/credsweeper/rules/config.yaml b/credsweeper/rules/config.yaml index fda7112eb..3b51f6507 100644 --- a/credsweeper/rules/config.yaml +++ b/credsweeper/rules/config.yaml @@ -233,7 +233,9 @@ type: pattern values: - (?EAA[0-9A-Za-z]{80,800}) - filter_type: GeneralPattern + filter_type: + - ValuePatternCheck + - ValueBase64PartCheck required_substrings: - EAA min_line_len: 80 diff --git a/credsweeper/scanner/scan_type/scan_type.py b/credsweeper/scanner/scan_type/scan_type.py index 8a95df687..ffcec526c 100644 --- a/credsweeper/scanner/scan_type/scan_type.py +++ b/credsweeper/scanner/scan_type/scan_type.py @@ -164,24 +164,23 @@ def _get_candidates(cls, config: Config, rule: Rule, target: AnalysisTarget) -> if config.exclude_lines and target.line_strip in config.exclude_lines: return candidates - line_data_list = cls.get_line_data_list(config=config, - target=target, - pattern=rule.patterns[0], - filters=rule.filters) - - for line_data in line_data_list: - if config.exclude_values and line_data.value.strip() in config.exclude_values: - continue - - candidate = Candidate([line_data], rule.patterns, rule.rule_name, rule.severity, config, rule.validations, - rule.use_ml, rule.confidence) - # single pattern with multiple values means all the patterns must matched in target - if 1 < len(rule.patterns) and rule.rule_type in (RuleType.PATTERN, RuleType.KEYWORD): - # additional check whether all patterns match - if not cls._aux_scan(config, rule, target, candidate): - # cannot find secondary values for the candidate + if line_data_list := cls.get_line_data_list(config=config, + target=target, + pattern=rule.patterns[0], + filters=rule.filters): + for line_data in line_data_list: + if config.exclude_values and line_data.value.strip() in config.exclude_values: continue - candidates.append(candidate) + + candidate = Candidate([line_data], rule.patterns, rule.rule_name, rule.severity, config, + rule.validations, rule.use_ml, rule.confidence) + # single pattern with multiple values means all the patterns must matched in target + if 1 < len(rule.patterns) and rule.rule_type in (RuleType.PATTERN, RuleType.KEYWORD): + # additional check whether all patterns match + if not cls._aux_scan(config, rule, target, candidate): + # cannot find secondary values for the candidate + continue + candidates.append(candidate) return candidates @classmethod diff --git a/tests/filters/test_value_base64_part_check.py b/tests/filters/test_value_base64_part_check.py new file mode 100644 index 000000000..790ca9008 --- /dev/null +++ b/tests/filters/test_value_base64_part_check.py @@ -0,0 +1,33 @@ +import re +import unittest + +from credsweeper.credentials import LineData +from credsweeper.filters import ValueBase64PartCheck +from tests.filters.conftest import DUMMY_ANALYSIS_TARGET + + +class TestValueBase64PartCheck(unittest.TestCase): + EAA_PATTERN = re.compile(r"(?P\bEAA[0-9A-Za-z]{32})") + + def test_value_check_n(self) -> None: + line_data = LineData(config=None, + path="dummy", + file_type="", + line="qcE81rS+FJHuvg39lz4T/EAACEb00Kse0BAlGy7KeQ5YnaCEd09Eo" + "se0cBAlGy7KeQ5Yna9CoDsup39tiYdoQ4jH9Coup39tiYdWoQ4jHFZD", + info="", + line_num=1, + line_pos=0, + pattern=TestValueBase64PartCheck.EAA_PATTERN) + self.assertTrue(ValueBase64PartCheck().run(line_data, DUMMY_ANALYSIS_TARGET)) + + def test_value_check_p(self) -> None: + line_data = LineData(config=None, + path="dummy", + file_type="", + line="http://meta.test/api/EAACRvAWiwzR8rcXFsLiUH13ybj0tdEa?x=login", + info="", + line_num=1, + line_pos=0, + pattern=TestValueBase64PartCheck.EAA_PATTERN) + self.assertFalse(ValueBase64PartCheck().run(line_data, DUMMY_ANALYSIS_TARGET)) diff --git a/tests/samples/test.html b/tests/samples/test.html index d07d0420f..65964bccb 100644 --- a/tests/samples/test.html +++ b/tests/samples/test.html @@ -114,7 +114,7 @@ nested table - + xml capability From 9cc0d580185714e796be8a83fff4c2b6d07bc7fd Mon Sep 17 00:00:00 2001 From: Roman Babenko Date: Tue, 6 Aug 2024 13:50:26 +0300 Subject: [PATCH 3/7] IP rules removing (#591) --- cicd/benchmark.txt | 102 +- credsweeper/filters/__init__.py | 1 - credsweeper/filters/value_ip_check.py | 51 - credsweeper/rules/config.yaml | 28 - docs/source/credsweeper.filters.rst | 8 - tests/__init__.py | 12 +- tests/data/depth_3.json | 3080 +++++-------------------- tests/data/ml_threshold.json | 2348 ++----------------- tests/data/output.json | 1952 +--------------- tests/samples/ipv4 | 42 - tests/samples/ipv6 | 15 - 11 files changed, 965 insertions(+), 6674 deletions(-) delete mode 100644 credsweeper/filters/value_ip_check.py delete mode 100644 tests/samples/ipv4 delete mode 100644 tests/samples/ipv6 diff --git a/cicd/benchmark.txt b/cicd/benchmark.txt index c8d8fa7f7..d75bff0b3 100644 --- a/cicd/benchmark.txt +++ b/cicd/benchmark.txt @@ -1,13 +1,13 @@ -DATA: 16988573 interested lines. MARKUP: 62869 items +DATA: 16979136 interested lines. MARKUP: 61880 items FileType FileNumber ValidLines Positives Negatives Templates --------------- ------------ ------------ ----------- ----------- ----------- - 194 28318 65 430 89 + 194 28318 64 427 89 .1 2 641 2 5 .admx 1 26 1 .adoc 1 158 11 6 1 .api 2 118 4 .asciidoc 96 14471 53 348 27 -.axaml 5 286 6 +.axaml 5 286 5 .backup 1 62 1 1 .bash 2 2158 2 1 .bat 4 233 1 13 2 @@ -16,8 +16,8 @@ FileType FileNumber ValidLines Positives Negatives Templat .build 2 40 3 .bundle 4 1512 570 .bzl 3 2503 11 -.c 179 284009 16 942 5 -.cc 30 30656 624 1 +.c 179 284009 9 942 5 +.cc 29 30562 622 1 .cf 3 126 2 1 .cfg 1 385 1 1 .cjs 1 725 3 6 @@ -25,20 +25,20 @@ FileType FileNumber ValidLines Positives Negatives Templat .cljc 5 2421 12 .cls 1 657 1 .cmd 4 401 2 3 -.cnf 8 858 18 46 18 +.cnf 8 858 18 45 18 .coffee 1 585 2 -.conf 61 4954 54 74 54 +.conf 61 4954 51 74 54 .config 20 492 16 33 1 -.cpp 15 5688 2 61 +.cpp 15 5688 1 61 .creds 1 10 1 1 .crlf 1 27 1 .crt 2 4979 253 -.cs 269 82442 121 910 99 +.cs 268 82410 120 908 99 .cshtml 5 180 12 .csp 3 379 11 .csproj 1 14 1 .css 6 13564 10 -.csv 1 109 81 +.csv 1 109 78 .dart 2 22 2 .deprecated 1 126 1 .development 1 5 1 @@ -61,10 +61,10 @@ FileType FileNumber ValidLines Positives Negatives Templat .gd 1 37 1 .gml 3 3075 26 .gni 3 5017 18 -.go 1084 569469 666 4347 742 +.go 1079 566327 621 4334 742 .golden 5 1168 1 14 29 .gradle 45 3265 4 91 100 -.graphql 8 445 1 13 +.graphql 7 420 13 .graphqls 1 30 1 .groovy 23 5011 25 211 1 .h 11 2038 38 @@ -73,30 +73,30 @@ FileType FileNumber ValidLines Positives Negatives Templat .hs 17 4509 37 71 5 .html 53 15327 14 115 18 .idl 2 777 4 -.iml 6 699 38 -.in 6 2130 4 81 12 +.iml 6 699 36 +.in 6 2130 1 80 12 .inc 2 56 2 1 .ini 11 1437 24 12 18 .ipynb 1 134 5 .j 1 241 4 -.j2 31 5601 8 214 10 -.java 621 134132 328 1356 170 +.j2 30 5530 6 213 10 +.java 621 134132 322 1354 170 .jenkinsfile 1 58 1 7 .jinja2 1 64 2 -.js 659 536413 541 2645 336 -.json 861 13670751 917 11012 143 +.js 659 536413 521 2642 336 +.json 860 13670669 623 10947 143 .jsp 13 3202 1 42 .jsx 7 857 19 .jwt 6 8 7 .key 83 2737 70 14 -.kt 123 20774 53 383 3 -.l 1 982 2 -.las 1 6656 48 -.lasso 1 230 7 +.kt 123 20774 51 383 3 +.l 1 982 1 +.las 1 6656 46 +.lasso 1 230 6 .lasso9 1 164 5 .ldif 2 286 20 .ldiff 1 20 1 -.ldml 1 6656 48 +.ldml 1 6656 46 .leex 1 9 2 .less 4 3023 12 .libsonnet 2 210 1 11 @@ -104,13 +104,13 @@ FileType FileNumber ValidLines Positives Negatives Templat .lkml 1 43 1 .lock 24 160912 144 .log 2 199 38 52 -.lua 10 1924 3 37 3 +.lua 10 1924 37 3 .m 16 13358 11 151 3 .manifest 3 102 3 .markdown 3 139 3 1 .markerb 3 12 3 .marko 1 21 2 -.md 679 149755 784 2584 671 +.md 675 149422 661 2365 671 .mdx 3 549 7 .mjml 1 18 1 .mjs 22 4424 108 310 @@ -120,9 +120,9 @@ FileType FileNumber ValidLines Positives Negatives Templat .mod 2 96 4 .moo 1 1404 26 .mqh 1 1023 2 -.msg 1 26644 1 1 +.msg 1 26644 1 .mysql 1 36 2 -.ndjson 2 5006 73 350 2 +.ndjson 2 5006 34 268 2 .nix 4 211 12 .nolint 1 2 1 .odd 1 1281 57 @@ -132,31 +132,31 @@ FileType FileNumber ValidLines Positives Negatives Templat .patch 4 109405 27 .pbxproj 1 941 1 .pem 48 1169 47 8 -.php 371 75710 135 1833 80 +.php 371 75710 130 1769 80 .pl 16 14727 6 47 .pm 3 744 8 .po 3 2994 15 -.pod 9 1859 8 26 +.pod 9 1859 2 24 .pony 1 83 4 .postinst 2 354 4 16 .pp 10 563 23 .ppk 1 45 37 .private 1 15 1 .proj 1 85 3 -.properties 48 1621 56 28 34 +.properties 48 1621 51 28 34 .proto 5 5768 58 -.ps1 17 8618 69 2 +.ps1 16 8509 68 2 .ps1xml 1 5022 1 .pug 2 193 2 .purs 1 69 4 .pxd 1 150 5 2 -.py 896 292902 676 3468 748 +.py 890 291553 618 3466 748 .pyi 4 1361 9 .pyp 1 167 1 .pyx 2 1094 21 .r 4 62 6 3 1 .rake 2 51 2 -.rb 862 131917 253 3465 615 +.rb 861 131867 239 3455 615 .re 1 31 1 .red 1 159 1 .release 1 13 4 @@ -168,22 +168,22 @@ FileType FileNumber ValidLines Positives Negatives Templat .rrc 39 1404 514 .rs 31 9855 2 238 11 .rsc 1 691 1 -.rsp 16 7101 23 11 28 -.rst 87 34151 64 360 68 +.rsp 16 7101 19 10 28 +.rst 86 33980 61 358 68 .rules 1 6 2 -.sample 2 25 1 7 4 -.sbt 3 570 7 2 +.sample 2 25 1 5 4 +.sbt 3 570 6 2 .scala 40 5071 13 102 .scss 16 8553 32 1 .secrets 1 11 1 -.sh 144 21573 63 476 30 +.sh 143 21525 50 474 30 .slim 1 153 2 2 .sln 1 306 2 .smali 1 775 12 .snap 3 1708 1 34 2 .spec 2 332 2 .spin 1 565 1 -.sql 27 6606 29 69 4 +.sql 27 6606 29 62 4 .storyboard 20 1802 401 .strings 20 1240 184 .stub 3 84 6 @@ -193,7 +193,7 @@ FileType FileNumber ValidLines Positives Negatives Templat .swift 6 278 16 .t 9 1767 28 56 14 .td 2 14002 6 -.template 19 1633 5 42 11 +.template 19 1633 4 42 11 .test 2 24 25 4 .testsettings 1 21 5 .tf 21 1377 3 32 2 @@ -202,13 +202,13 @@ FileType FileNumber ValidLines Positives Negatives Templat .tl 2 2161 165 2 .tmpl 5 336 3 9 .token 1 1 3 -.toml 83 2379 55 72 172 +.toml 83 2379 54 72 172 .tpl 1 43 1 .travis 1 34 4 3 1 -.ts 585 106846 172 1930 203 -.tsx 55 9846 1 128 5 -.ttar 2 6050 8 3 -.txt 444 78553 1830 14283 50 +.ts 584 106807 166 1930 203 +.tsx 54 7914 1 124 5 +.ttar 2 6050 3 +.txt 443 78152 1775 14282 50 .utf8 1 77 2 .vsixmanifest 1 36 1 .vsmdi 1 6 1 @@ -218,12 +218,12 @@ FileType FileNumber ValidLines Positives Negatives Templat .xib 11 503 174 .xml 9 689 9 .xsl 1 311 1 -.yaml 149 20563 140 383 44 -.yml 418 36162 467 920 384 +.yaml 137 19004 116 356 44 +.yml 418 36162 460 916 384 .zsh 6 872 12 .zsh-theme 1 97 1 -TOTAL: 10333 16988573 8377 60439 5233 -credsweeper result_cnt : 7800, lost_cnt : 0, true_cnt : 7231, false_cnt : 569 +TOTAL: 10294 16979136 7615 59903 5233 +credsweeper result_cnt : 6697, lost_cnt : 0, true_cnt : 6470, false_cnt : 227 Rules Positives Negatives Templates Reported TP FP TN FN FPR FNR ACC PRC RCL F1 ------------------------------ ----------- ----------- ----------- ---------- ---- ---- ----- ---- -------- -------- -------- -------- -------- -------- API 123 3163 185 112 109 3 3345 14 0.000896 0.113821 0.995102 0.973214 0.886179 0.927660 @@ -250,8 +250,6 @@ Google API Key 12 0 0 1 Google Multi 10 2 0 11 10 1 1 0 0.500000 0.000000 0.916667 0.909091 1.000000 0.952381 Google OAuth Access Token 3 0 0 3 3 0 0 0 0.000000 1.000000 1.000000 1.000000 1.000000 Grafana Provisioned API Key 22 1 0 1 1 0 1 21 0.000000 0.954545 0.086957 1.000000 0.045455 0.086957 -IPv4 729 405 0 1205 728 342 63 1 0.844444 0.001372 0.697531 0.680374 0.998628 0.809339 -IPv6 33 131 0 33 33 0 131 0 0.000000 0.000000 1.000000 1.000000 1.000000 1.000000 JSON Web Token 284 11 2 274 271 3 10 13 0.230769 0.045775 0.946128 0.989051 0.954225 0.971326 Jira / Confluence PAT token 0 4 0 0 0 4 0 0.000000 1.000000 Jira 2FA 14 6 0 10 10 0 6 4 0.000000 0.285714 0.800000 1.000000 0.714286 0.833333 @@ -267,4 +265,4 @@ Slack Token 4 1 0 Token 585 3972 439 519 511 8 4403 74 0.001814 0.126496 0.983587 0.984586 0.873504 0.925725 Twilio API Key 0 5 2 0 0 7 0 0.000000 1.000000 URL Credentials 194 125 251 184 184 0 376 10 0.000000 0.051546 0.982456 1.000000 0.948454 0.973545 - 8377 60439 5233 7942 7231 569 59870 1146 0.009414 0.136803 0.975078 0.927051 0.863197 0.893985 + 7615 59903 5233 6704 6470 227 59676 1145 0.003789 0.150361 0.979679 0.966104 0.849639 0.904136 diff --git a/credsweeper/filters/__init__.py b/credsweeper/filters/__init__.py index 986728d22..f88bcac57 100644 --- a/credsweeper/filters/__init__.py +++ b/credsweeper/filters/__init__.py @@ -25,7 +25,6 @@ from credsweeper.filters.value_grafana_check import ValueGrafanaCheck from credsweeper.filters.value_grafana_service_check import ValueGrafanaServiceCheck from credsweeper.filters.value_hex_number_check import ValueHexNumberCheck -from credsweeper.filters.value_ip_check import ValueIPCheck from credsweeper.filters.value_jfrog_token_check import ValueJfrogTokenCheck from credsweeper.filters.value_json_web_token_check import ValueJsonWebTokenCheck from credsweeper.filters.value_last_word_check import ValueLastWordCheck diff --git a/credsweeper/filters/value_ip_check.py b/credsweeper/filters/value_ip_check.py deleted file mode 100644 index eae2d6dec..000000000 --- a/credsweeper/filters/value_ip_check.py +++ /dev/null @@ -1,51 +0,0 @@ -import contextlib -import ipaddress -import re - -from credsweeper.common.constants import ML_HUNK -from credsweeper.config import Config -from credsweeper.credentials import LineData -from credsweeper.file_handler.analysis_target import AnalysisTarget -from credsweeper.filters import Filter -from credsweeper.utils import Util - - -class ValueIPCheck(Filter): - """Filter out some of insensible IP""" - - TRUE_POSITIVE_MARKERS = [r"\bip\b", "server", "addr", "login"] - TRUE_POSITIVE_PATTERN = re.compile(Util.get_regex_combine_or(TRUE_POSITIVE_MARKERS), flags=re.IGNORECASE) - - FALSE_POSITIVE_MARKERS = ["version", "oid", "section", "rfc"] - FALSE_POSITIVE_PATTERN = re.compile(Util.get_regex_combine_or(FALSE_POSITIVE_MARKERS), flags=re.IGNORECASE) - - def __init__(self, config: Config = None) -> None: - pass - - def run(self, line_data: LineData, target: AnalysisTarget) -> bool: - """Run filter checks on received credential candidate data 'line_data'. - - Args: - line_data: credential candidate data - target: multiline target from which line data was obtained - - Return: - True, if need to filter candidate and False if left - - """ - - with contextlib.suppress(Exception): - ip = ipaddress.ip_address(line_data.value) - if 4 == ip.version: - byte_sum = sum(x for x in ip.packed) - if 100 > (byte_sum >> 2): - # versions usually have low average of sum the bytes - search_text = Util.subtext(line_data.line, line_data.value_start, ML_HUNK) - if self.FALSE_POSITIVE_PATTERN.search(search_text) \ - and not self.TRUE_POSITIVE_PATTERN.search(search_text): - return True - if ip.is_loopback or ip.is_private or ip.is_reserved or ip.is_link_local or ip.is_multicast: - return True - return False - - return True diff --git a/credsweeper/rules/config.yaml b/credsweeper/rules/config.yaml index 3b51f6507..ce05d3b4b 100644 --- a/credsweeper/rules/config.yaml +++ b/credsweeper/rules/config.yaml @@ -126,34 +126,6 @@ target: - code -- name: IPv4 - severity: info - confidence: weak - type: pattern - values: - - (?[0-2]?[0-9]{1,2}\.[0-2]?[0-9]{1,2}\.[0-2]?[0-9]{1,2}\.[0-2]?[0-9]{1,2})(?![.0-9a-zA-Z]) - filter_type: - - ValueIPCheck - min_line_len: 10 - required_substrings: - - "." - target: - - code - -- name: IPv6 - severity: info - confidence: strong - type: pattern - values: - - (?[0-9A-Fa-f]{0,4}:(:?[0-9A-Fa-f]{1,4}:?){0,6}:[0-9A-Fa-f]{1,4})(?![:0-9A-Za-z]) - filter_type: - - ValueIPCheck - min_line_len: 10 - required_substrings: - - ":" - target: - - code - - name: AWS Client ID severity: high confidence: moderate diff --git a/docs/source/credsweeper.filters.rst b/docs/source/credsweeper.filters.rst index f2da332f5..7599043ae 100644 --- a/docs/source/credsweeper.filters.rst +++ b/docs/source/credsweeper.filters.rst @@ -188,14 +188,6 @@ credsweeper.filters.value\_grafana\_check module :undoc-members: :show-inheritance: -credsweeper.filters.value\_ip\_check module -------------------------------------------- - -.. automodule:: credsweeper.filters.value_ip_check - :members: - :undoc-members: - :show-inheritance: - credsweeper.filters.value\_jfrog\_token\_check module ----------------------------------------------------- diff --git a/tests/__init__.py b/tests/__init__.py index bff4d614a..4f6c5b878 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -1,24 +1,24 @@ from pathlib import Path # total number of files in test samples -SAMPLES_FILES_COUNT: int = 131 +SAMPLES_FILES_COUNT: int = 129 # the lowest value of ML threshold is used to display possible lowest values NEGLIGIBLE_ML_THRESHOLD = 0.0001 # credentials count after scan -SAMPLES_CRED_COUNT: int = 430 -SAMPLES_CRED_LINE_COUNT: int = 447 +SAMPLES_CRED_COUNT: int = 362 +SAMPLES_CRED_LINE_COUNT: int = 379 # credentials count after post-processing -SAMPLES_POST_CRED_COUNT: int = 388 +SAMPLES_POST_CRED_COUNT: int = 320 # with option --doc SAMPLES_IN_DOC = 411 # archived credentials that are not found without --depth -SAMPLES_IN_DEEP_1 = SAMPLES_POST_CRED_COUNT + 25 -SAMPLES_IN_DEEP_2 = SAMPLES_IN_DEEP_1 + 18 +SAMPLES_IN_DEEP_1 = SAMPLES_POST_CRED_COUNT + 24 +SAMPLES_IN_DEEP_2 = SAMPLES_IN_DEEP_1 + 17 SAMPLES_IN_DEEP_3 = SAMPLES_IN_DEEP_2 + 1 # well known string with all latin letters diff --git a/tests/data/depth_3.json b/tests/data/depth_3.json index 369eeee14..bf0894ec2 100644 --- a/tests/data/depth_3.json +++ b/tests/data/depth_3.json @@ -2369,33 +2369,6 @@ } ] }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "IPv4", - "severity": "info", - "confidence": "weak", - "line_data_list": [ - { - "line": "(98.76.54.32)ID:master PW:dipPr149Gg!", - "line_num": 50, - "path": "tests/samples/doc_id_pair_passwd_pair", - "info": "tests/samples/doc_id_pair_passwd_pair|RAW", - "value": "98.76.54.32", - "value_start": 1, - "value_end": 12, - "variable": null, - "variable_start": -2, - "variable_end": -2, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 2.515950268099853, - "valid": false - } - } - ] - }, { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", @@ -3179,33 +3152,6 @@ } ] }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "IPv4", - "severity": "info", - "confidence": "weak", - "line_data_list": [ - { - "line": "ANYid:master pw:dipPr194Gg! ip:98.76.54.32", - "line_num": 95, - "path": "tests/samples/doc_id_pair_passwd_pair", - "info": "tests/samples/doc_id_pair_passwd_pair|RAW", - "value": "98.76.54.32", - "value_start": 31, - "value_end": 42, - "variable": null, - "variable_start": -2, - "variable_end": -2, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 2.515950268099853, - "valid": false - } - } - ] - }, { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", @@ -3233,33 +3179,6 @@ } ] }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "IPv4", - "severity": "info", - "confidence": "weak", - "line_data_list": [ - { - "line": "master@98.76.54.32 mailto:{1} (password-dipPr196Gg!) # skip", - "line_num": 97, - "path": "tests/samples/doc_id_pair_passwd_pair", - "info": "tests/samples/doc_id_pair_passwd_pair|RAW", - "value": "98.76.54.32", - "value_start": 7, - "value_end": 18, - "variable": null, - "variable_start": -2, - "variable_end": -2, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 2.515950268099853, - "valid": false - } - } - ] - }, { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", @@ -3341,60 +3260,6 @@ } ] }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "IPv4", - "severity": "info", - "confidence": "weak", - "line_data_list": [ - { - "line": "98.76.54.32(ID:master/PW:iPp10@GRq) # todo: move into other sample ?", - "line_num": 11, - "path": "tests/samples/doc_id_passwd_pair", - "info": "tests/samples/doc_id_passwd_pair|RAW", - "value": "98.76.54.32", - "value_start": 0, - "value_end": 11, - "variable": null, - "variable_start": -2, - "variable_end": -2, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 2.515950268099853, - "valid": false - } - } - ] - }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "IPv4", - "severity": "info", - "confidence": "weak", - "line_data_list": [ - { - "line": "98.76.54.32 id/pw:master/iPp19@GRq", - "line_num": 20, - "path": "tests/samples/doc_id_passwd_pair", - "info": "tests/samples/doc_id_passwd_pair|RAW", - "value": "98.76.54.32", - "value_start": 0, - "value_end": 11, - "variable": null, - "variable_start": -2, - "variable_end": -2, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 2.515950268099853, - "valid": false - } - } - ] - }, { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", @@ -3638,33 +3503,6 @@ } ] }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "IPv4", - "severity": "info", - "confidence": "weak", - "line_data_list": [ - { - "line": "master@98.76.54.32 password:Prl23Db#@", - "line_num": 14, - "path": "tests/samples/doc_passwd_pair", - "info": "tests/samples/doc_passwd_pair|RAW", - "value": "98.76.54.32", - "value_start": 7, - "value_end": 18, - "variable": null, - "variable_start": -2, - "variable_end": -2, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 2.515950268099853, - "valid": false - } - } - ] - }, { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", @@ -4853,87 +4691,6 @@ } ] }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "IPv4", - "severity": "info", - "confidence": "weak", - "line_data_list": [ - { - "line": "98.76.54.32 (master/IhqSb1Gg)", - "line_num": 1, - "path": "tests/samples/doc_various", - "info": "tests/samples/doc_various|RAW", - "value": "98.76.54.32", - "value_start": 0, - "value_end": 11, - "variable": null, - "variable_start": -2, - "variable_end": -2, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 2.515950268099853, - "valid": false - } - } - ] - }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "IPv4", - "severity": "info", - "confidence": "weak", - "line_data_list": [ - { - "line": "98.76.54.32(master/IhqSb1Gg)", - "line_num": 2, - "path": "tests/samples/doc_various", - "info": "tests/samples/doc_various|RAW", - "value": "98.76.54.32", - "value_start": 0, - "value_end": 11, - "variable": null, - "variable_start": -2, - "variable_end": -2, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 2.515950268099853, - "valid": false - } - } - ] - }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "IPv4", - "severity": "info", - "confidence": "weak", - "line_data_list": [ - { - "line": "master@98.76.54.32(pw:IhqSb1Gg)", - "line_num": 3, - "path": "tests/samples/doc_various", - "info": "tests/samples/doc_various|RAW", - "value": "98.76.54.32", - "value_start": 7, - "value_end": 18, - "variable": null, - "variable_start": -2, - "variable_end": -2, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 2.515950268099853, - "valid": false - } - } - ] - }, { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", @@ -5015,33 +4772,6 @@ } ] }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "IPv4", - "severity": "info", - "confidence": "weak", - "line_data_list": [ - { - "line": "ssh gildong.hong@98.76.54.32 mailto:{1} (PW:IhqSb1Gg)", - "line_num": 6, - "path": "tests/samples/doc_various", - "info": "tests/samples/doc_various|RAW", - "value": "98.76.54.32", - "value_start": 17, - "value_end": 28, - "variable": null, - "variable_start": -2, - "variable_end": -2, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 2.515950268099853, - "valid": false - } - } - ] - }, { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", @@ -5069,33 +4799,6 @@ } ] }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "IPv4", - "severity": "info", - "confidence": "weak", - "line_data_list": [ - { - "line": "ssh gildong.hong@98.76.54.32 mailto:{1} password:IhqSb1Gg", - "line_num": 7, - "path": "tests/samples/doc_various", - "info": "tests/samples/doc_various|RAW", - "value": "98.76.54.32", - "value_start": 17, - "value_end": 28, - "variable": null, - "variable_start": -2, - "variable_end": -2, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 2.515950268099853, - "valid": false - } - } - ] - }, { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", @@ -5123,33 +4826,6 @@ } ] }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "IPv4", - "severity": "info", - "confidence": "weak", - "line_data_list": [ - { - "line": "ssh -P IhqSb1Gg gildong.hong@98.76.54.32 mailto:{1} (password:IhqSb1Gg)", - "line_num": 11, - "path": "tests/samples/doc_various", - "info": "tests/samples/doc_various|RAW", - "value": "98.76.54.32", - "value_start": 29, - "value_end": 40, - "variable": null, - "variable_start": -2, - "variable_end": -2, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 2.515950268099853, - "valid": false - } - } - ] - }, { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", @@ -5177,60 +4853,6 @@ } ] }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "IPv4", - "severity": "info", - "confidence": "weak", - "line_data_list": [ - { - "line": "98.76.54.32 xxxx (master/IhqSb1Gg)", - "line_num": 14, - "path": "tests/samples/doc_various", - "info": "tests/samples/doc_various|RAW", - "value": "98.76.54.32", - "value_start": 0, - "value_end": 11, - "variable": null, - "variable_start": -2, - "variable_end": -2, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 2.515950268099853, - "valid": false - } - } - ] - }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "IPv4", - "severity": "info", - "confidence": "weak", - "line_data_list": [ - { - "line": "gildong.hong@98.76.54.32 pwd:IhqSb1Gg", - "line_num": 15, - "path": "tests/samples/doc_various", - "info": "tests/samples/doc_various|RAW", - "value": "98.76.54.32", - "value_start": 13, - "value_end": 24, - "variable": null, - "variable_start": -2, - "variable_end": -2, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 2.515950268099853, - "valid": false - } - } - ] - }, { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", @@ -5285,33 +4907,6 @@ } ] }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "IPv4", - "severity": "info", - "confidence": "weak", - "line_data_list": [ - { - "line": "98.76.54.32(pw:IhqSb1Gg)", - "line_num": 17, - "path": "tests/samples/doc_various", - "info": "tests/samples/doc_various|RAW", - "value": "98.76.54.32", - "value_start": 0, - "value_end": 11, - "variable": null, - "variable_start": -2, - "variable_end": -2, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 2.515950268099853, - "valid": false - } - } - ] - }, { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", @@ -5339,33 +4934,6 @@ } ] }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "IPv4", - "severity": "info", - "confidence": "weak", - "line_data_list": [ - { - "line": "98.76.54.32/pw:IhqSb1Gg", - "line_num": 19, - "path": "tests/samples/doc_various", - "info": "tests/samples/doc_various|RAW", - "value": "98.76.54.32", - "value_start": 0, - "value_end": 11, - "variable": null, - "variable_start": -2, - "variable_end": -2, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 2.515950268099853, - "valid": false - } - } - ] - }, { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", @@ -5447,33 +5015,6 @@ } ] }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "IPv4", - "severity": "info", - "confidence": "weak", - "line_data_list": [ - { - "line": "sftp gildong.hong@98.76.54.32 mailto:{1} (pw:IhqSb1Gg)", - "line_num": 22, - "path": "tests/samples/doc_various", - "info": "tests/samples/doc_various|RAW", - "value": "98.76.54.32", - "value_start": 18, - "value_end": 29, - "variable": null, - "variable_start": -2, - "variable_end": -2, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 2.515950268099853, - "valid": false - } - } - ] - }, { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", @@ -5501,33 +5042,6 @@ } ] }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "IPv4", - "severity": "info", - "confidence": "weak", - "line_data_list": [ - { - "line": "gildong.hong@98.76.54.32 mailto:{1} (pw:IhqSb1Gg)", - "line_num": 23, - "path": "tests/samples/doc_various", - "info": "tests/samples/doc_various|RAW", - "value": "98.76.54.32", - "value_start": 13, - "value_end": 24, - "variable": null, - "variable_start": -2, - "variable_end": -2, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 2.515950268099853, - "valid": false - } - } - ] - }, { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", @@ -5582,33 +5096,6 @@ } ] }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "IPv4", - "severity": "info", - "confidence": "weak", - "line_data_list": [ - { - "line": "ssh gildong.hong@98.76.54.32 mailto:{1} (password:IhqSb1Gg)", - "line_num": 25, - "path": "tests/samples/doc_various", - "info": "tests/samples/doc_various|RAW", - "value": "98.76.54.32", - "value_start": 17, - "value_end": 28, - "variable": null, - "variable_start": -2, - "variable_end": -2, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 2.515950268099853, - "valid": false - } - } - ] - }, { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", @@ -5636,33 +5123,6 @@ } ] }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "IPv4", - "severity": "info", - "confidence": "weak", - "line_data_list": [ - { - "line": "ssh gildong.hong@98.76.54.32 mailto:{1} master/IhqSb1Gg", - "line_num": 26, - "path": "tests/samples/doc_various", - "info": "tests/samples/doc_various|RAW", - "value": "98.76.54.32", - "value_start": 17, - "value_end": 28, - "variable": null, - "variable_start": -2, - "variable_end": -2, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 2.515950268099853, - "valid": false - } - } - ] - }, { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", @@ -5771,33 +5231,6 @@ } ] }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "IPv4", - "severity": "info", - "confidence": "weak", - "line_data_list": [ - { - "line": "98.76.54.32 ANY_PW:IhqSb1Gg", - "line_num": 34, - "path": "tests/samples/doc_various", - "info": "tests/samples/doc_various|RAW", - "value": "98.76.54.32", - "value_start": 0, - "value_end": 11, - "variable": null, - "variable_start": -2, - "variable_end": -2, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 2.515950268099853, - "valid": false - } - } - ] - }, { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", @@ -5825,33 +5258,6 @@ } ] }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "IPv4", - "severity": "info", - "confidence": "weak", - "line_data_list": [ - { - "line": "98.76.54.32(ID/PW:IhqSb1Gg)", - "line_num": 36, - "path": "tests/samples/doc_various", - "info": "tests/samples/doc_various|RAW", - "value": "98.76.54.32", - "value_start": 0, - "value_end": 11, - "variable": null, - "variable_start": -2, - "variable_end": -2, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 2.515950268099853, - "valid": false - } - } - ] - }, { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", @@ -5879,33 +5285,6 @@ } ] }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "IPv4", - "severity": "info", - "confidence": "weak", - "line_data_list": [ - { - "line": "gildong.hong@98.76.54.32 (pwd:IhqSb1Gg)", - "line_num": 38, - "path": "tests/samples/doc_various", - "info": "tests/samples/doc_various|RAW", - "value": "98.76.54.32", - "value_start": 13, - "value_end": 24, - "variable": null, - "variable_start": -2, - "variable_end": -2, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 2.515950268099853, - "valid": false - } - } - ] - }, { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", @@ -5960,33 +5339,6 @@ } ] }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "IPv4", - "severity": "info", - "confidence": "weak", - "line_data_list": [ - { - "line": "gildong.hong@98.76.54.32(master/IhqSb1Gg)", - "line_num": 44, - "path": "tests/samples/doc_various", - "info": "tests/samples/doc_various|RAW", - "value": "98.76.54.32", - "value_start": 13, - "value_end": 24, - "variable": null, - "variable_start": -2, - "variable_end": -2, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 2.515950268099853, - "valid": false - } - } - ] - }, { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", @@ -6014,141 +5366,6 @@ } ] }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "IPv4", - "severity": "info", - "confidence": "weak", - "line_data_list": [ - { - "line": "gildong.hong@98.76.54.32,pw:IhqSb1Gg", - "line_num": 47, - "path": "tests/samples/doc_various", - "info": "tests/samples/doc_various|RAW", - "value": "98.76.54.32", - "value_start": 13, - "value_end": 24, - "variable": null, - "variable_start": -2, - "variable_end": -2, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 2.515950268099853, - "valid": false - } - } - ] - }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "IPv4", - "severity": "info", - "confidence": "weak", - "line_data_list": [ - { - "line": "98.76.54.32(master/IhqSb1Gg,master/IhqSb1Gg)", - "line_num": 48, - "path": "tests/samples/doc_various", - "info": "tests/samples/doc_various|RAW", - "value": "98.76.54.32", - "value_start": 0, - "value_end": 11, - "variable": null, - "variable_start": -2, - "variable_end": -2, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 2.515950268099853, - "valid": false - } - } - ] - }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "IPv4", - "severity": "info", - "confidence": "weak", - "line_data_list": [ - { - "line": "98.76.54.32(master/IhqSb1Gg master/IhqSb1Gg)", - "line_num": 49, - "path": "tests/samples/doc_various", - "info": "tests/samples/doc_various|RAW", - "value": "98.76.54.32", - "value_start": 0, - "value_end": 11, - "variable": null, - "variable_start": -2, - "variable_end": -2, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 2.515950268099853, - "valid": false - } - } - ] - }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "IPv4", - "severity": "info", - "confidence": "weak", - "line_data_list": [ - { - "line": "xxxx:98.76.54.32(master/IhqSb1Gg)", - "line_num": 50, - "path": "tests/samples/doc_various", - "info": "tests/samples/doc_various|RAW", - "value": "98.76.54.32", - "value_start": 5, - "value_end": 16, - "variable": null, - "variable_start": -2, - "variable_end": -2, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 2.515950268099853, - "valid": false - } - } - ] - }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "IPv4", - "severity": "info", - "confidence": "weak", - "line_data_list": [ - { - "line": "98.76.54.32:xxxx(PW:IhqSb1Gg)", - "line_num": 51, - "path": "tests/samples/doc_various", - "info": "tests/samples/doc_various|RAW", - "value": "98.76.54.32", - "value_start": 0, - "value_end": 11, - "variable": null, - "variable_start": -2, - "variable_end": -2, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 2.515950268099853, - "valid": false - } - } - ] - }, { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", @@ -6176,33 +5393,6 @@ } ] }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "IPv4", - "severity": "info", - "confidence": "weak", - "line_data_list": [ - { - "line": "gildong.hong@98.76.54.32 PW:IhqSb1Gg", - "line_num": 56, - "path": "tests/samples/doc_various", - "info": "tests/samples/doc_various|RAW", - "value": "98.76.54.32", - "value_start": 13, - "value_end": 24, - "variable": null, - "variable_start": -2, - "variable_end": -2, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 2.515950268099853, - "valid": false - } - } - ] - }, { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", @@ -6230,33 +5420,6 @@ } ] }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "IPv4", - "severity": "info", - "confidence": "weak", - "line_data_list": [ - { - "line": "http|https://98.76.54.32/xxxx(master/IhqSb1Gg)", - "line_num": 59, - "path": "tests/samples/doc_various", - "info": "tests/samples/doc_various|RAW", - "value": "98.76.54.32", - "value_start": 13, - "value_end": 24, - "variable": null, - "variable_start": -2, - "variable_end": -2, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 2.515950268099853, - "valid": false - } - } - ] - }, { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", @@ -6394,26 +5557,26 @@ }, { "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "IPv4", - "severity": "info", - "confidence": "weak", + "ml_validation": "VALIDATED_KEY", + "ml_probability": 0.956, + "rule": "Password", + "severity": "medium", + "confidence": "moderate", "line_data_list": [ { - "line": "98.76.54.32(ID/PW:master/IhqSb1Gg)", - "line_num": 70, + "line": "master@98.76.54.32(pw:IhqSb1Gg)", + "line_num": 73, "path": "tests/samples/doc_various", "info": "tests/samples/doc_various|RAW", - "value": "98.76.54.32", - "value_start": 0, - "value_end": 11, - "variable": null, - "variable_start": -2, - "variable_end": -2, + "value": "IhqSb1Gg)", + "value_start": 22, + "value_end": 31, + "variable": "pw", + "variable_start": 19, + "variable_end": 21, "entropy_validation": { "iterator": "BASE64_CHARS", - "entropy": 2.515950268099853, + "entropy": 2.8177111123931664, "valid": false } } @@ -6421,26 +5584,26 @@ }, { "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "IPv4", - "severity": "info", - "confidence": "weak", + "ml_validation": "VALIDATED_KEY", + "ml_probability": 0.99, + "rule": "Password", + "severity": "medium", + "confidence": "moderate", "line_data_list": [ { - "line": "master@98.76.54.32(pw:IhqSb1Gg)", - "line_num": 73, + "line": "98.76.54.32 pw:IhqSb1Gg", + "line_num": 75, "path": "tests/samples/doc_various", "info": "tests/samples/doc_various|RAW", - "value": "98.76.54.32", - "value_start": 7, - "value_end": 18, - "variable": null, - "variable_start": -2, - "variable_end": -2, + "value": "IhqSb1Gg", + "value_start": 15, + "value_end": 23, + "variable": "pw", + "variable_start": 12, + "variable_end": 14, "entropy_validation": { "iterator": "BASE64_CHARS", - "entropy": 2.515950268099853, + "entropy": 3.0, "valid": false } } @@ -6449,52 +5612,25 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.956, + "ml_probability": 0.976, "rule": "Password", "severity": "medium", "confidence": "moderate", "line_data_list": [ { - "line": "master@98.76.54.32(pw:IhqSb1Gg)", - "line_num": 73, - "path": "tests/samples/doc_various", - "info": "tests/samples/doc_various|RAW", - "value": "IhqSb1Gg)", - "value_start": 22, - "value_end": 31, - "variable": "pw", - "variable_start": 19, - "variable_end": 21, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 2.8177111123931664, - "valid": false - } - } - ] - }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "IPv4", - "severity": "info", - "confidence": "weak", - "line_data_list": [ - { - "line": "master@98.76.54.32,PW:IhqSb1Gg", - "line_num": 74, + "line": "config:xxxx,PW:IhqSb1Gg", + "line_num": 78, "path": "tests/samples/doc_various", "info": "tests/samples/doc_various|RAW", - "value": "98.76.54.32", - "value_start": 7, - "value_end": 18, - "variable": null, - "variable_start": -2, - "variable_end": -2, + "value": "IhqSb1Gg", + "value_start": 15, + "value_end": 23, + "variable": "xxxx,PW", + "variable_start": 7, + "variable_end": 14, "entropy_validation": { "iterator": "BASE64_CHARS", - "entropy": 2.515950268099853, + "entropy": 3.0, "valid": false } } @@ -6502,26 +5638,26 @@ }, { "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "IPv4", - "severity": "info", - "confidence": "weak", + "ml_validation": "VALIDATED_KEY", + "ml_probability": 0.988, + "rule": "Password", + "severity": "medium", + "confidence": "moderate", "line_data_list": [ { - "line": "98.76.54.32 pw:IhqSb1Gg", - "line_num": 75, + "line": "scp gildong.hong@98.76.54.32 mailto:{1} pw:IhqSb1Gg", + "line_num": 82, "path": "tests/samples/doc_various", "info": "tests/samples/doc_various|RAW", - "value": "98.76.54.32", - "value_start": 0, - "value_end": 11, - "variable": null, - "variable_start": -2, - "variable_end": -2, + "value": "IhqSb1Gg", + "value_start": 43, + "value_end": 51, + "variable": "pw", + "variable_start": 40, + "variable_end": 42, "entropy_validation": { "iterator": "BASE64_CHARS", - "entropy": 2.515950268099853, + "entropy": 3.0, "valid": false } } @@ -6530,22 +5666,22 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.99, + "ml_probability": 0.952, "rule": "Password", "severity": "medium", "confidence": "moderate", "line_data_list": [ { - "line": "98.76.54.32 pw:IhqSb1Gg", - "line_num": 75, + "line": "id:gildong.hong@xxx.com mailto:{1} pw:IhqSb1Gg", + "line_num": 83, "path": "tests/samples/doc_various", "info": "tests/samples/doc_various|RAW", "value": "IhqSb1Gg", - "value_start": 15, - "value_end": 23, + "value_start": 38, + "value_end": 46, "variable": "pw", - "variable_start": 12, - "variable_end": 14, + "variable_start": 35, + "variable_end": 37, "entropy_validation": { "iterator": "BASE64_CHARS", "entropy": 3.0, @@ -6557,22 +5693,22 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.976, + "ml_probability": 0.943, "rule": "Password", "severity": "medium", "confidence": "moderate", "line_data_list": [ { - "line": "config:xxxx,PW:IhqSb1Gg", - "line_num": 78, + "line": "gildong.hong@98.76.54.32 mailto:{1} pw:IhqSb1Gg", + "line_num": 84, "path": "tests/samples/doc_various", "info": "tests/samples/doc_various|RAW", "value": "IhqSb1Gg", - "value_start": 15, - "value_end": 23, - "variable": "xxxx,PW", - "variable_start": 7, - "variable_end": 14, + "value_start": 39, + "value_end": 47, + "variable": "pw", + "variable_start": 36, + "variable_end": 38, "entropy_validation": { "iterator": "BASE64_CHARS", "entropy": 3.0, @@ -6583,26 +5719,26 @@ }, { "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "IPv4", - "severity": "info", - "confidence": "weak", + "ml_validation": "VALIDATED_KEY", + "ml_probability": 0.996, + "rule": "Password", + "severity": "medium", + "confidence": "moderate", "line_data_list": [ { - "line": "98.76.54.32-->master/IhqSb1Gg", - "line_num": 81, + "line": "ssh gildong.hong@98.76.54.32 mailto:{1},pw:IhqSb1Gg", + "line_num": 85, "path": "tests/samples/doc_various", "info": "tests/samples/doc_various|RAW", - "value": "98.76.54.32", - "value_start": 0, - "value_end": 11, - "variable": null, - "variable_start": -2, - "variable_end": -2, + "value": "IhqSb1Gg", + "value_start": 43, + "value_end": 51, + "variable": "pw", + "variable_start": 40, + "variable_end": 42, "entropy_validation": { "iterator": "BASE64_CHARS", - "entropy": 2.515950268099853, + "entropy": 3.0, "valid": false } } @@ -6610,26 +5746,26 @@ }, { "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "IPv4", - "severity": "info", - "confidence": "weak", + "ml_validation": "VALIDATED_KEY", + "ml_probability": 0.995, + "rule": "Password", + "severity": "medium", + "confidence": "moderate", "line_data_list": [ { - "line": "scp gildong.hong@98.76.54.32 mailto:{1} pw:IhqSb1Gg", - "line_num": 82, + "line": "(ssh gildong.hong@98.76.54.32 mailto{1}) pwd:IhqSb1Gg", + "line_num": 87, "path": "tests/samples/doc_various", "info": "tests/samples/doc_various|RAW", - "value": "98.76.54.32", - "value_start": 17, - "value_end": 28, - "variable": null, - "variable_start": -2, - "variable_end": -2, + "value": "IhqSb1Gg", + "value_start": 45, + "value_end": 53, + "variable": "pwd", + "variable_start": 41, + "variable_end": 44, "entropy_validation": { "iterator": "BASE64_CHARS", - "entropy": 2.515950268099853, + "entropy": 3.0, "valid": false } } @@ -6638,22 +5774,22 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.988, + "ml_probability": 0.996, "rule": "Password", "severity": "medium", "confidence": "moderate", "line_data_list": [ { - "line": "scp gildong.hong@98.76.54.32 mailto:{1} pw:IhqSb1Gg", - "line_num": 82, + "line": "ssh gildong.hong@98.76.54.32 mailto:{1}, pw:IhqSb1Gg", + "line_num": 90, "path": "tests/samples/doc_various", "info": "tests/samples/doc_various|RAW", "value": "IhqSb1Gg", - "value_start": 43, - "value_end": 51, + "value_start": 44, + "value_end": 52, "variable": "pw", - "variable_start": 40, - "variable_end": 42, + "variable_start": 41, + "variable_end": 43, "entropy_validation": { "iterator": "BASE64_CHARS", "entropy": 3.0, @@ -6665,25 +5801,25 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.952, + "ml_probability": 0.998, "rule": "Password", "severity": "medium", "confidence": "moderate", "line_data_list": [ { - "line": "id:gildong.hong@xxx.com mailto:{1} pw:IhqSb1Gg", - "line_num": 83, + "line": "ssh gildong.hong@98.76.54.32 mailto:{1} (pwd:IhqSb1Gg)", + "line_num": 93, "path": "tests/samples/doc_various", "info": "tests/samples/doc_various|RAW", - "value": "IhqSb1Gg", - "value_start": 38, - "value_end": 46, - "variable": "pw", - "variable_start": 35, - "variable_end": 37, + "value": "IhqSb1Gg)", + "value_start": 45, + "value_end": 54, + "variable": "pwd", + "variable_start": 41, + "variable_end": 44, "entropy_validation": { "iterator": "BASE64_CHARS", - "entropy": 3.0, + "entropy": 2.8177111123931664, "valid": false } } @@ -6691,26 +5827,26 @@ }, { "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "IPv4", - "severity": "info", - "confidence": "weak", + "ml_validation": "VALIDATED_KEY", + "ml_probability": 0.997, + "rule": "Password", + "severity": "medium", + "confidence": "moderate", "line_data_list": [ { - "line": "gildong.hong@98.76.54.32 mailto:{1} pw:IhqSb1Gg", - "line_num": 84, + "line": "gildong.hong@98.76.54.32 mailto:{1} (password:IhqSb1Gg)", + "line_num": 94, "path": "tests/samples/doc_various", "info": "tests/samples/doc_various|RAW", - "value": "98.76.54.32", - "value_start": 13, - "value_end": 24, - "variable": null, - "variable_start": -2, - "variable_end": -2, + "value": "IhqSb1Gg)", + "value_start": 46, + "value_end": 55, + "variable": "password", + "variable_start": 37, + "variable_end": 45, "entropy_validation": { "iterator": "BASE64_CHARS", - "entropy": 2.515950268099853, + "entropy": 2.8177111123931664, "valid": false } } @@ -6719,22 +5855,22 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.943, + "ml_probability": 0.995, "rule": "Password", "severity": "medium", "confidence": "moderate", "line_data_list": [ { - "line": "gildong.hong@98.76.54.32 mailto:{1} pw:IhqSb1Gg", - "line_num": 84, + "line": "gildong.hong@98.76.54.32 mailto:{1} Password:IhqSb1Gg", + "line_num": 96, "path": "tests/samples/doc_various", "info": "tests/samples/doc_various|RAW", "value": "IhqSb1Gg", - "value_start": 39, - "value_end": 47, - "variable": "pw", + "value_start": 45, + "value_end": 53, + "variable": "Password", "variable_start": 36, - "variable_end": 38, + "variable_end": 44, "entropy_validation": { "iterator": "BASE64_CHARS", "entropy": 3.0, @@ -6745,26 +5881,26 @@ }, { "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "IPv4", - "severity": "info", - "confidence": "weak", + "ml_validation": "VALIDATED_KEY", + "ml_probability": 0.979, + "rule": "Password", + "severity": "medium", + "confidence": "moderate", "line_data_list": [ { - "line": "ssh gildong.hong@98.76.54.32 mailto:{1},pw:IhqSb1Gg", - "line_num": 85, + "line": "gildong.hong@98.76.54.32 mailto:{1} (pass:IhqSb1Gg)", + "line_num": 97, "path": "tests/samples/doc_various", "info": "tests/samples/doc_various|RAW", - "value": "98.76.54.32", - "value_start": 17, - "value_end": 28, - "variable": null, - "variable_start": -2, - "variable_end": -2, + "value": "IhqSb1Gg)", + "value_start": 42, + "value_end": 51, + "variable": "pass", + "variable_start": 37, + "variable_end": 41, "entropy_validation": { "iterator": "BASE64_CHARS", - "entropy": 2.515950268099853, + "entropy": 2.8177111123931664, "valid": false } } @@ -6773,14 +5909,14 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.996, + "ml_probability": 0.994, "rule": "Password", "severity": "medium", "confidence": "moderate", "line_data_list": [ { - "line": "ssh gildong.hong@98.76.54.32 mailto:{1},pw:IhqSb1Gg", - "line_num": 85, + "line": "ssh gildong.hong@98.76.54.32 mailto:{1} pw:IhqSb1Gg", + "line_num": 100, "path": "tests/samples/doc_various", "info": "tests/samples/doc_various|RAW", "value": "IhqSb1Gg", @@ -6799,76 +5935,22 @@ }, { "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "IPv4", - "severity": "info", - "confidence": "weak", + "ml_validation": "VALIDATED_KEY", + "ml_probability": 0.999, + "rule": "Password", + "severity": "medium", + "confidence": "moderate", "line_data_list": [ { - "line": "gildong.hong@98.76.54.32 mailto:{1} (master/IhqSb1Gg)", - "line_num": 86, - "path": "tests/samples/doc_various", - "info": "tests/samples/doc_various|RAW", - "value": "98.76.54.32", - "value_start": 13, - "value_end": 24, - "variable": null, - "variable_start": -2, - "variable_end": -2, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 2.515950268099853, - "valid": false - } - } - ] - }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "IPv4", - "severity": "info", - "confidence": "weak", - "line_data_list": [ - { - "line": "(ssh gildong.hong@98.76.54.32 mailto{1}) pwd:IhqSb1Gg", - "line_num": 87, - "path": "tests/samples/doc_various", - "info": "tests/samples/doc_various|RAW", - "value": "98.76.54.32", - "value_start": 18, - "value_end": 29, - "variable": null, - "variable_start": -2, - "variable_end": -2, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 2.515950268099853, - "valid": false - } - } - ] - }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.995, - "rule": "Password", - "severity": "medium", - "confidence": "moderate", - "line_data_list": [ - { - "line": "(ssh gildong.hong@98.76.54.32 mailto{1}) pwd:IhqSb1Gg", - "line_num": 87, + "line": "ssh gildong.hong@98.76.54.32 mailto:{1} pass:IhqSb1Gg", + "line_num": 101, "path": "tests/samples/doc_various", "info": "tests/samples/doc_various|RAW", "value": "IhqSb1Gg", "value_start": 45, "value_end": 53, - "variable": "pwd", - "variable_start": 41, + "variable": "pass", + "variable_start": 40, "variable_end": 44, "entropy_validation": { "iterator": "BASE64_CHARS", @@ -6878,105 +5960,24 @@ } ] }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "IPv4", - "severity": "info", - "confidence": "weak", - "line_data_list": [ - { - "line": "ssh gildong.hong@98.76.54.32 (master/IhqSb1Gg)", - "line_num": 88, - "path": "tests/samples/doc_various", - "info": "tests/samples/doc_various|RAW", - "value": "98.76.54.32", - "value_start": 17, - "value_end": 28, - "variable": null, - "variable_start": -2, - "variable_end": -2, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 2.515950268099853, - "valid": false - } - } - ] - }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "IPv4", - "severity": "info", - "confidence": "weak", - "line_data_list": [ - { - "line": "ssh gildong.hong@98.76.54.32 mailto:[1} \uacc4\uc815master/IhqSb1Gg", - "line_num": 89, - "path": "tests/samples/doc_various", - "info": "tests/samples/doc_various|RAW", - "value": "98.76.54.32", - "value_start": 17, - "value_end": 28, - "variable": null, - "variable_start": -2, - "variable_end": -2, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 2.515950268099853, - "valid": false - } - } - ] - }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "IPv4", - "severity": "info", - "confidence": "weak", - "line_data_list": [ - { - "line": "ssh gildong.hong@98.76.54.32 mailto:{1}, pw:IhqSb1Gg", - "line_num": 90, - "path": "tests/samples/doc_various", - "info": "tests/samples/doc_various|RAW", - "value": "98.76.54.32", - "value_start": 17, - "value_end": 28, - "variable": null, - "variable_start": -2, - "variable_end": -2, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 2.515950268099853, - "valid": false - } - } - ] - }, { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.996, + "ml_probability": 0.999, "rule": "Password", "severity": "medium", "confidence": "moderate", "line_data_list": [ { - "line": "ssh gildong.hong@98.76.54.32 mailto:{1}, pw:IhqSb1Gg", - "line_num": 90, + "line": "id:gildong.hong@xxx.com mailto:{1}/password:IhqSb1Gg", + "line_num": 102, "path": "tests/samples/doc_various", "info": "tests/samples/doc_various|RAW", "value": "IhqSb1Gg", "value_start": 44, "value_end": 52, - "variable": "pw", - "variable_start": 41, + "variable": "password", + "variable_start": 35, "variable_end": 43, "entropy_validation": { "iterator": "BASE64_CHARS", @@ -6988,26 +5989,26 @@ }, { "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "IPv4", - "severity": "info", - "confidence": "weak", + "ml_validation": "VALIDATED_KEY", + "ml_probability": 1.0, + "rule": "Password", + "severity": "medium", + "confidence": "moderate", "line_data_list": [ { - "line": "gildong.hong@98.76.54.32 mailto:{1} (\ube44\ubc88 IhqSb1Gg)", - "line_num": 91, + "line": "ssh gildong.hong@98.76.54.32 mailto:{1}/password:IhqSb1Gg", + "line_num": 104, "path": "tests/samples/doc_various", "info": "tests/samples/doc_various|RAW", - "value": "98.76.54.32", - "value_start": 13, - "value_end": 24, - "variable": null, - "variable_start": -2, - "variable_end": -2, + "value": "IhqSb1Gg", + "value_start": 49, + "value_end": 57, + "variable": "password", + "variable_start": 40, + "variable_end": 48, "entropy_validation": { "iterator": "BASE64_CHARS", - "entropy": 2.515950268099853, + "entropy": 3.0, "valid": false } } @@ -7015,26 +6016,26 @@ }, { "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "IPv4", - "severity": "info", - "confidence": "weak", + "ml_validation": "VALIDATED_KEY", + "ml_probability": 0.934, + "rule": "Password", + "severity": "medium", + "confidence": "moderate", "line_data_list": [ { - "line": "ssh gildong.hong@98.76.54.32 mailto:{1} password IhqSb1Gg", - "line_num": 92, + "line": "-ANYID:gildong.hong@example.com mailto:{1} -pw:IhqSb1Gg", + "line_num": 105, "path": "tests/samples/doc_various", "info": "tests/samples/doc_various|RAW", - "value": "98.76.54.32", - "value_start": 17, - "value_end": 28, - "variable": null, - "variable_start": -2, - "variable_end": -2, + "value": "IhqSb1Gg", + "value_start": 47, + "value_end": 55, + "variable": "pw", + "variable_start": 44, + "variable_end": 46, "entropy_validation": { "iterator": "BASE64_CHARS", - "entropy": 2.515950268099853, + "entropy": 3.0, "valid": false } } @@ -7042,26 +6043,26 @@ }, { "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "IPv4", - "severity": "info", - "confidence": "weak", + "ml_validation": "VALIDATED_KEY", + "ml_probability": 0.97, + "rule": "Password", + "severity": "medium", + "confidence": "moderate", "line_data_list": [ { - "line": "ssh gildong.hong@98.76.54.32 mailto:{1} (pwd:IhqSb1Gg)", - "line_num": 93, + "line": "ID:gildong.hong@xxxx.net mailto:{1} pw:IhqSb1Gg", + "line_num": 106, "path": "tests/samples/doc_various", "info": "tests/samples/doc_various|RAW", - "value": "98.76.54.32", - "value_start": 17, - "value_end": 28, - "variable": null, - "variable_start": -2, - "variable_end": -2, + "value": "IhqSb1Gg", + "value_start": 39, + "value_end": 47, + "variable": "pw", + "variable_start": 36, + "variable_end": 38, "entropy_validation": { "iterator": "BASE64_CHARS", - "entropy": 2.515950268099853, + "entropy": 3.0, "valid": false } } @@ -7070,22 +6071,22 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.998, + "ml_probability": 0.978, "rule": "Password", "severity": "medium", "confidence": "moderate", "line_data_list": [ { - "line": "ssh gildong.hong@98.76.54.32 mailto:{1} (pwd:IhqSb1Gg)", - "line_num": 93, + "line": "http://98.76.54.32:xxx(pw:IhqSb1Gg)", + "line_num": 108, "path": "tests/samples/doc_various", "info": "tests/samples/doc_various|RAW", "value": "IhqSb1Gg)", - "value_start": 45, - "value_end": 54, - "variable": "pwd", - "variable_start": 41, - "variable_end": 44, + "value_start": 26, + "value_end": 35, + "variable": "pw", + "variable_start": 23, + "variable_end": 25, "entropy_validation": { "iterator": "BASE64_CHARS", "entropy": 2.8177111123931664, @@ -7098,781 +6099,25 @@ "api_validation": "NOT_AVAILABLE", "ml_validation": "NOT_AVAILABLE", "ml_probability": null, - "rule": "IPv4", - "severity": "info", + "rule": "Dropbox API secret (long term)", + "severity": "high", "confidence": "weak", "line_data_list": [ { - "line": "gildong.hong@98.76.54.32 mailto:{1} (password:IhqSb1Gg)", - "line_num": 94, - "path": "tests/samples/doc_various", - "info": "tests/samples/doc_various|RAW", - "value": "98.76.54.32", - "value_start": 13, - "value_end": 24, + "line": "var g = '7rBynGo0b1cAAAAAAAAAAc72L3T6rQK51mB5a06ijnwRG91deTxvSqdZNAlxq8pZ'", + "line_num": 1, + "path": "tests/samples/dropbox_api_secret_long_term", + "info": "tests/samples/dropbox_api_secret_long_term|RAW", + "value": "7rBynGo0b1cAAAAAAAAAAc72L3T6rQK51mB5a06ijnwRG91deTxvSqdZNAlxq8pZ", + "value_start": 9, + "value_end": 73, "variable": null, "variable_start": -2, "variable_end": -2, "entropy_validation": { "iterator": "BASE64_CHARS", - "entropy": 2.515950268099853, - "valid": false - } - } - ] - }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.997, - "rule": "Password", - "severity": "medium", - "confidence": "moderate", - "line_data_list": [ - { - "line": "gildong.hong@98.76.54.32 mailto:{1} (password:IhqSb1Gg)", - "line_num": 94, - "path": "tests/samples/doc_various", - "info": "tests/samples/doc_various|RAW", - "value": "IhqSb1Gg)", - "value_start": 46, - "value_end": 55, - "variable": "password", - "variable_start": 37, - "variable_end": 45, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 2.8177111123931664, - "valid": false - } - } - ] - }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "IPv4", - "severity": "info", - "confidence": "weak", - "line_data_list": [ - { - "line": "gildong.hong@98.76.54.32 mailto:{1} Password:IhqSb1Gg", - "line_num": 96, - "path": "tests/samples/doc_various", - "info": "tests/samples/doc_various|RAW", - "value": "98.76.54.32", - "value_start": 13, - "value_end": 24, - "variable": null, - "variable_start": -2, - "variable_end": -2, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 2.515950268099853, - "valid": false - } - } - ] - }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.995, - "rule": "Password", - "severity": "medium", - "confidence": "moderate", - "line_data_list": [ - { - "line": "gildong.hong@98.76.54.32 mailto:{1} Password:IhqSb1Gg", - "line_num": 96, - "path": "tests/samples/doc_various", - "info": "tests/samples/doc_various|RAW", - "value": "IhqSb1Gg", - "value_start": 45, - "value_end": 53, - "variable": "Password", - "variable_start": 36, - "variable_end": 44, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 3.0, - "valid": false - } - } - ] - }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "IPv4", - "severity": "info", - "confidence": "weak", - "line_data_list": [ - { - "line": "gildong.hong@98.76.54.32 mailto:{1} (pass:IhqSb1Gg)", - "line_num": 97, - "path": "tests/samples/doc_various", - "info": "tests/samples/doc_various|RAW", - "value": "98.76.54.32", - "value_start": 13, - "value_end": 24, - "variable": null, - "variable_start": -2, - "variable_end": -2, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 2.515950268099853, - "valid": false - } - } - ] - }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.979, - "rule": "Password", - "severity": "medium", - "confidence": "moderate", - "line_data_list": [ - { - "line": "gildong.hong@98.76.54.32 mailto:{1} (pass:IhqSb1Gg)", - "line_num": 97, - "path": "tests/samples/doc_various", - "info": "tests/samples/doc_various|RAW", - "value": "IhqSb1Gg)", - "value_start": 42, - "value_end": 51, - "variable": "pass", - "variable_start": 37, - "variable_end": 41, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 2.8177111123931664, - "valid": false - } - } - ] - }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "IPv4", - "severity": "info", - "confidence": "weak", - "line_data_list": [ - { - "line": "ssh gildong.hong@98.76.54.32 mailto:{1} (xxxx//IhqSb1Gg)", - "line_num": 99, - "path": "tests/samples/doc_various", - "info": "tests/samples/doc_various|RAW", - "value": "98.76.54.32", - "value_start": 17, - "value_end": 28, - "variable": null, - "variable_start": -2, - "variable_end": -2, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 2.515950268099853, - "valid": false - } - } - ] - }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "IPv4", - "severity": "info", - "confidence": "weak", - "line_data_list": [ - { - "line": "ssh gildong.hong@98.76.54.32 mailto:{1} pw:IhqSb1Gg", - "line_num": 100, - "path": "tests/samples/doc_various", - "info": "tests/samples/doc_various|RAW", - "value": "98.76.54.32", - "value_start": 17, - "value_end": 28, - "variable": null, - "variable_start": -2, - "variable_end": -2, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 2.515950268099853, - "valid": false - } - } - ] - }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.994, - "rule": "Password", - "severity": "medium", - "confidence": "moderate", - "line_data_list": [ - { - "line": "ssh gildong.hong@98.76.54.32 mailto:{1} pw:IhqSb1Gg", - "line_num": 100, - "path": "tests/samples/doc_various", - "info": "tests/samples/doc_various|RAW", - "value": "IhqSb1Gg", - "value_start": 43, - "value_end": 51, - "variable": "pw", - "variable_start": 40, - "variable_end": 42, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 3.0, - "valid": false - } - } - ] - }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "IPv4", - "severity": "info", - "confidence": "weak", - "line_data_list": [ - { - "line": "ssh gildong.hong@98.76.54.32 mailto:{1} pass:IhqSb1Gg", - "line_num": 101, - "path": "tests/samples/doc_various", - "info": "tests/samples/doc_various|RAW", - "value": "98.76.54.32", - "value_start": 17, - "value_end": 28, - "variable": null, - "variable_start": -2, - "variable_end": -2, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 2.515950268099853, - "valid": false - } - } - ] - }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.999, - "rule": "Password", - "severity": "medium", - "confidence": "moderate", - "line_data_list": [ - { - "line": "ssh gildong.hong@98.76.54.32 mailto:{1} pass:IhqSb1Gg", - "line_num": 101, - "path": "tests/samples/doc_various", - "info": "tests/samples/doc_various|RAW", - "value": "IhqSb1Gg", - "value_start": 45, - "value_end": 53, - "variable": "pass", - "variable_start": 40, - "variable_end": 44, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 3.0, - "valid": false - } - } - ] - }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.999, - "rule": "Password", - "severity": "medium", - "confidence": "moderate", - "line_data_list": [ - { - "line": "id:gildong.hong@xxx.com mailto:{1}/password:IhqSb1Gg", - "line_num": 102, - "path": "tests/samples/doc_various", - "info": "tests/samples/doc_various|RAW", - "value": "IhqSb1Gg", - "value_start": 44, - "value_end": 52, - "variable": "password", - "variable_start": 35, - "variable_end": 43, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 3.0, - "valid": false - } - } - ] - }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "IPv4", - "severity": "info", - "confidence": "weak", - "line_data_list": [ - { - "line": "ssh gildong.hong@98.76.54.32 mailto:{1}/password:IhqSb1Gg", - "line_num": 104, - "path": "tests/samples/doc_various", - "info": "tests/samples/doc_various|RAW", - "value": "98.76.54.32", - "value_start": 17, - "value_end": 28, - "variable": null, - "variable_start": -2, - "variable_end": -2, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 2.515950268099853, - "valid": false - } - } - ] - }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "VALIDATED_KEY", - "ml_probability": 1.0, - "rule": "Password", - "severity": "medium", - "confidence": "moderate", - "line_data_list": [ - { - "line": "ssh gildong.hong@98.76.54.32 mailto:{1}/password:IhqSb1Gg", - "line_num": 104, - "path": "tests/samples/doc_various", - "info": "tests/samples/doc_various|RAW", - "value": "IhqSb1Gg", - "value_start": 49, - "value_end": 57, - "variable": "password", - "variable_start": 40, - "variable_end": 48, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 3.0, - "valid": false - } - } - ] - }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.934, - "rule": "Password", - "severity": "medium", - "confidence": "moderate", - "line_data_list": [ - { - "line": "-ANYID:gildong.hong@example.com mailto:{1} -pw:IhqSb1Gg", - "line_num": 105, - "path": "tests/samples/doc_various", - "info": "tests/samples/doc_various|RAW", - "value": "IhqSb1Gg", - "value_start": 47, - "value_end": 55, - "variable": "pw", - "variable_start": 44, - "variable_end": 46, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 3.0, - "valid": false - } - } - ] - }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.97, - "rule": "Password", - "severity": "medium", - "confidence": "moderate", - "line_data_list": [ - { - "line": "ID:gildong.hong@xxxx.net mailto:{1} pw:IhqSb1Gg", - "line_num": 106, - "path": "tests/samples/doc_various", - "info": "tests/samples/doc_various|RAW", - "value": "IhqSb1Gg", - "value_start": 39, - "value_end": 47, - "variable": "pw", - "variable_start": 36, - "variable_end": 38, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 3.0, - "valid": false - } - } - ] - }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "IPv4", - "severity": "info", - "confidence": "weak", - "line_data_list": [ - { - "line": "ID/PW 98.76.54.32:xxx master/IhqSb1Gg", - "line_num": 107, - "path": "tests/samples/doc_various", - "info": "tests/samples/doc_various|RAW", - "value": "98.76.54.32", - "value_start": 6, - "value_end": 17, - "variable": null, - "variable_start": -2, - "variable_end": -2, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 2.515950268099853, - "valid": false - } - } - ] - }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "IPv4", - "severity": "info", - "confidence": "weak", - "line_data_list": [ - { - "line": "http://98.76.54.32:xxx(pw:IhqSb1Gg)", - "line_num": 108, - "path": "tests/samples/doc_various", - "info": "tests/samples/doc_various|RAW", - "value": "98.76.54.32", - "value_start": 7, - "value_end": 18, - "variable": null, - "variable_start": -2, - "variable_end": -2, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 2.515950268099853, - "valid": false - } - } - ] - }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.978, - "rule": "Password", - "severity": "medium", - "confidence": "moderate", - "line_data_list": [ - { - "line": "http://98.76.54.32:xxx(pw:IhqSb1Gg)", - "line_num": 108, - "path": "tests/samples/doc_various", - "info": "tests/samples/doc_various|RAW", - "value": "IhqSb1Gg)", - "value_start": 26, - "value_end": 35, - "variable": "pw", - "variable_start": 23, - "variable_end": 25, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 2.8177111123931664, - "valid": false - } - } - ] - }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "Dropbox API secret (long term)", - "severity": "high", - "confidence": "weak", - "line_data_list": [ - { - "line": "var g = '7rBynGo0b1cAAAAAAAAAAc72L3T6rQK51mB5a06ijnwRG91deTxvSqdZNAlxq8pZ'", - "line_num": 1, - "path": "tests/samples/dropbox_api_secret_long_term", - "info": "tests/samples/dropbox_api_secret_long_term|RAW", - "value": "7rBynGo0b1cAAAAAAAAAAc72L3T6rQK51mB5a06ijnwRG91deTxvSqdZNAlxq8pZ", - "value_start": 9, - "value_end": 73, - "variable": null, - "variable_start": -2, - "variable_end": -2, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 4.89361507332541, - "valid": true - } - } - ] - }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "Dropbox App secret", - "severity": "info", - "confidence": "weak", - "line_data_list": [ - { - "line": "var app_unique_val_s = 'wpv1jq9xwanbn3n';", - "line_num": 1, - "path": "tests/samples/dropbox_app_secret", - "info": "tests/samples/dropbox_app_secret|RAW", - "value": "wpv1jq9xwanbn3n", - "value_start": 24, - "value_end": 39, - "variable": null, - "variable_start": -2, - "variable_end": -2, - "entropy_validation": { - "iterator": "BASE36_CHARS", - "entropy": 3.4565647621309536, - "valid": true - } - } - ] - }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "Dropbox OAuth2 API Access Token", - "severity": "high", - "confidence": "moderate", - "line_data_list": [ - { - "line": "var dropbox = 'sl.BdmpmC82mhhySscKk2oQGyE5l--8LdAQftLTXVGQhP39Z8FtAK1BhePhyevurA-Elt7ToIr6OpwzKAYE7RBqpu6VVyQU5WlCTL_Q7N4gElXahaWou6aPpOIwgGCIOq9aeC3YFoc';", - "line_num": 1, - "path": "tests/samples/dropbox_oauth_token", - "info": "tests/samples/dropbox_oauth_token|RAW", - "value": "sl.BdmpmC82mhhySscKk2oQGyE5l--8LdAQftLTXVGQhP39Z8FtAK1BhePhyevurA-Elt7ToIr6OpwzKAYE7RBqpu6VVyQU5WlCTL_Q7N4gElXahaWou6aPpOIwgGCIOq9aeC3YFoc", - "value_start": 15, - "value_end": 153, - "variable": null, - "variable_start": -2, - "variable_end": -2, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 5.395844179446957, - "valid": true - } - } - ] - }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "Dynatrace API Token", - "severity": "high", - "confidence": "moderate", - "line_data_list": [ - { - "line": "dt0c01.ST2EY72KQINMH574WMNVI7YN.G3DFPBEJYMODIDAEX454M7YWBUVEFOWKPRVMWFASS64NFH52PX6BNDVFFM572RZM", - "line_num": 1, - "path": "tests/samples/dynatrace_api.hs", - "info": "tests/samples/dynatrace_api.hs|RAW", - "value": "dt0c01.ST2EY72KQINMH574WMNVI7YN.G3DFPBEJYMODIDAEX454M7YWBUVEFOWKPRVMWFASS64NFH52PX6BNDVFFM572RZM", - "value_start": 0, - "value_end": 96, - "variable": null, - "variable_start": -2, - "variable_end": -2, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 4.808191506786782, - "valid": true - } - } - ] - }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.756, - "rule": "Github Old Token", - "severity": "high", - "confidence": "moderate", - "line_data_list": [ - { - "line": "\\ngit_token = \"gireogicracklecrackle1231567190113413981\"\\n\\n", - "line_num": 1, - "path": "tests/samples/encoded_data", - "info": "tests/samples/encoded_data|ENCODED|RAW", - "value": "gireogicracklecrackle1231567190113413981", - "value_start": 15, - "value_end": 55, - "variable": null, - "variable_start": -2, - "variable_end": -2, - "entropy_validation": { - "iterator": "BASE36_CHARS", - "entropy": 3.97402442086502, - "valid": true - } - } - ] - }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.756, - "rule": "Token", - "severity": "medium", - "confidence": "moderate", - "line_data_list": [ - { - "line": "\\ngit_token = \"gireogicracklecrackle1231567190113413981\"\\n\\n", - "line_num": 1, - "path": "tests/samples/encoded_data", - "info": "tests/samples/encoded_data|ENCODED|RAW", - "value": "gireogicracklecrackle1231567190113413981", - "value_start": 15, - "value_end": 55, - "variable": "git_token", - "variable_start": 2, - "variable_end": 11, - "entropy_validation": { - "iterator": "BASE36_CHARS", - "entropy": 3.97402442086502, - "valid": true - } - } - ] - }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "Facebook Access Token", - "severity": "high", - "confidence": "moderate", - "line_data_list": [ - { - "line": "GI_REO_GI_FACEBOOK_TOKEN = \"EAACEb00Kse0BAlGy7KeQ5YnaCEd09Eose0cBAlGy7KeQ5Yna9CoDsup39tiYdoQ4jH9Coup39tiYdWoQ4jHFZD\"", - "line_num": 1, - "path": "tests/samples/facebook_key", - "info": "tests/samples/facebook_key|RAW", - "value": "EAACEb00Kse0BAlGy7KeQ5YnaCEd09Eose0cBAlGy7KeQ5Yna9CoDsup39tiYdoQ4jH9Coup39tiYdWoQ4jHFZD", - "value_start": 28, - "value_end": 115, - "variable": null, - "variable_start": -2, - "variable_end": -2, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 4.936120692057916, - "valid": true - } - } - ] - }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.999, - "rule": "Token", - "severity": "medium", - "confidence": "moderate", - "line_data_list": [ - { - "line": "GI_REO_GI_FACEBOOK_TOKEN = \"EAACEb00Kse0BAlGy7KeQ5YnaCEd09Eose0cBAlGy7KeQ5Yna9CoDsup39tiYdoQ4jH9Coup39tiYdWoQ4jHFZD\"", - "line_num": 1, - "path": "tests/samples/facebook_key", - "info": "tests/samples/facebook_key|RAW", - "value": "EAACEb00Kse0BAlGy7KeQ5YnaCEd09Eose0cBAlGy7KeQ5Yna9CoDsup39tiYdoQ4jH9Coup39tiYdWoQ4jHFZD", - "value_start": 28, - "value_end": 115, - "variable": "GI_REO_GI_FACEBOOK_TOKEN", - "variable_start": 0, - "variable_end": 24, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 4.936120692057916, - "valid": true - } - } - ] - }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "Facebook App Token", - "severity": "high", - "confidence": "moderate", - "line_data_list": [ - { - "line": "1527194624358273|qbBf2-fdB9zZpqLA0_2nNzZDw2M", - "line_num": 2, - "path": "tests/samples/facebook_key", - "info": "tests/samples/facebook_key|RAW", - "value": "1527194624358273|qbBf2-fdB9zZpqLA0_2nNzZDw2M", - "value_start": 0, - "value_end": 44, - "variable": null, - "variable_start": -2, - "variable_end": -2, - "entropy_validation": { - "iterator": "BASE36_CHARS", - "entropy": 3.2089099270924217, - "valid": true - } - } - ] - }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "Firebase Domain", - "severity": "info", - "confidence": "moderate", - "line_data_list": [ - { - "line": "test-app-domain-42.firebaseapp.com", - "line_num": 1, - "path": "tests/samples/firebase_domain", - "info": "tests/samples/firebase_domain|RAW", - "value": "test-app-domain-42.firebaseapp.com", - "value_start": 0, - "value_end": 34, - "variable": null, - "variable_start": -2, - "variable_end": -2, - "entropy_validation": { - "iterator": "BASE36_CHARS", - "entropy": 3.4347510262969525, - "valid": true + "entropy": 4.89361507332541, + "valid": true } } ] @@ -7881,24 +6126,24 @@ "api_validation": "NOT_AVAILABLE", "ml_validation": "NOT_AVAILABLE", "ml_probability": null, - "rule": "Firebase Domain", - "severity": "info", - "confidence": "moderate", - "line_data_list": [ - { - "line": "test2.io.firebaseio.com", - "line_num": 2, - "path": "tests/samples/firebase_domain", - "info": "tests/samples/firebase_domain|RAW", - "value": "test2.io.firebaseio.com", - "value_start": 0, - "value_end": 23, + "rule": "Dropbox App secret", + "severity": "info", + "confidence": "weak", + "line_data_list": [ + { + "line": "var app_unique_val_s = 'wpv1jq9xwanbn3n';", + "line_num": 1, + "path": "tests/samples/dropbox_app_secret", + "info": "tests/samples/dropbox_app_secret|RAW", + "value": "wpv1jq9xwanbn3n", + "value_start": 24, + "value_end": 39, "variable": null, "variable_start": -2, "variable_end": -2, "entropy_validation": { "iterator": "BASE36_CHARS", - "entropy": 3.1394163745499943, + "entropy": 3.4565647621309536, "valid": true } } @@ -7908,25 +6153,25 @@ "api_validation": "NOT_AVAILABLE", "ml_validation": "NOT_AVAILABLE", "ml_probability": null, - "rule": "Github Classic Token", + "rule": "Dropbox OAuth2 API Access Token", "severity": "high", - "confidence": "strong", + "confidence": "moderate", "line_data_list": [ { - "line": "ghp_00000000000000000000000000000004WZ4EQ # classic", + "line": "var dropbox = 'sl.BdmpmC82mhhySscKk2oQGyE5l--8LdAQftLTXVGQhP39Z8FtAK1BhePhyevurA-Elt7ToIr6OpwzKAYE7RBqpu6VVyQU5WlCTL_Q7N4gElXahaWou6aPpOIwgGCIOq9aeC3YFoc';", "line_num": 1, - "path": "tests/samples/github_classic_token", - "info": "tests/samples/github_classic_token|RAW", - "value": "ghp_00000000000000000000000000000004WZ4EQ", - "value_start": 0, - "value_end": 41, + "path": "tests/samples/dropbox_oauth_token", + "info": "tests/samples/dropbox_oauth_token|RAW", + "value": "sl.BdmpmC82mhhySscKk2oQGyE5l--8LdAQftLTXVGQhP39Z8FtAK1BhePhyevurA-Elt7ToIr6OpwzKAYE7RBqpu6VVyQU5WlCTL_Q7N4gElXahaWou6aPpOIwgGCIOq9aeC3YFoc", + "value_start": 15, + "value_end": 153, "variable": null, "variable_start": -2, "variable_end": -2, "entropy_validation": { "iterator": "BASE64_CHARS", - "entropy": 1.4322437698226884, - "valid": false + "entropy": 5.395844179446957, + "valid": true } } ] @@ -7935,24 +6180,24 @@ "api_validation": "NOT_AVAILABLE", "ml_validation": "NOT_AVAILABLE", "ml_probability": null, - "rule": "Github Fine-granted Token", + "rule": "Dynatrace API Token", "severity": "high", - "confidence": "strong", + "confidence": "moderate", "line_data_list": [ { - "line": "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", - "line_num": 2, - "path": "tests/samples/github_fine_granted_token", - "info": "tests/samples/github_fine_granted_token|RAW", - "value": "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", + "line": "dt0c01.ST2EY72KQINMH574WMNVI7YN.G3DFPBEJYMODIDAEX454M7YWBUVEFOWKPRVMWFASS64NFH52PX6BNDVFFM572RZM", + "line_num": 1, + "path": "tests/samples/dynatrace_api.hs", + "info": "tests/samples/dynatrace_api.hs|RAW", + "value": "dt0c01.ST2EY72KQINMH574WMNVI7YN.G3DFPBEJYMODIDAEX454M7YWBUVEFOWKPRVMWFASS64NFH52PX6BNDVFFM572RZM", "value_start": 0, - "value_end": 93, + "value_end": 96, "variable": null, "variable_start": -2, "variable_end": -2, "entropy_validation": { "iterator": "BASE64_CHARS", - "entropy": 5.255374790203285, + "entropy": 4.808191506786782, "valid": true } } @@ -7961,25 +6206,25 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 1.0, + "ml_probability": 0.756, "rule": "Github Old Token", "severity": "high", "confidence": "moderate", "line_data_list": [ { - "line": "GITHUB_ACCESS_TOKEN = \"lbyxnhqjfnzhlpnvcvaxjumgvdlnktgugdofmzyy\"", + "line": "\\ngit_token = \"gireogicracklecrackle1231567190113413981\"\\n\\n", "line_num": 1, - "path": "tests/samples/github_key.groovy", - "info": "tests/samples/github_key.groovy|RAW", - "value": "lbyxnhqjfnzhlpnvcvaxjumgvdlnktgugdofmzyy", - "value_start": 23, - "value_end": 63, + "path": "tests/samples/encoded_data", + "info": "tests/samples/encoded_data|ENCODED|RAW", + "value": "gireogicracklecrackle1231567190113413981", + "value_start": 15, + "value_end": 55, "variable": null, "variable_start": -2, "variable_end": -2, "entropy_validation": { "iterator": "BASE36_CHARS", - "entropy": 4.246439344671015, + "entropy": 3.97402442086502, "valid": true } } @@ -7988,52 +6233,25 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 1.0, + "ml_probability": 0.756, "rule": "Token", "severity": "medium", "confidence": "moderate", "line_data_list": [ { - "line": "GITHUB_ACCESS_TOKEN = \"lbyxnhqjfnzhlpnvcvaxjumgvdlnktgugdofmzyy\"", - "line_num": 1, - "path": "tests/samples/github_key.groovy", - "info": "tests/samples/github_key.groovy|RAW", - "value": "lbyxnhqjfnzhlpnvcvaxjumgvdlnktgugdofmzyy", - "value_start": 23, - "value_end": 63, - "variable": "GITHUB_ACCESS_TOKEN", - "variable_start": 0, - "variable_end": 19, - "entropy_validation": { - "iterator": "BASE36_CHARS", - "entropy": 4.246439344671015, - "valid": true - } - } - ] - }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "Gitlab Incoming Email Token", - "severity": "info", - "confidence": "weak", - "line_data_list": [ - { - "line": "var email_t = '7e4v6v5j2nepcc8f5zvatgl9g';", + "line": "\\ngit_token = \"gireogicracklecrackle1231567190113413981\"\\n\\n", "line_num": 1, - "path": "tests/samples/gitlab_email_token", - "info": "tests/samples/gitlab_email_token|RAW", - "value": "7e4v6v5j2nepcc8f5zvatgl9g", + "path": "tests/samples/encoded_data", + "info": "tests/samples/encoded_data|ENCODED|RAW", + "value": "gireogicracklecrackle1231567190113413981", "value_start": 15, - "value_end": 40, - "variable": null, - "variable_start": -2, - "variable_end": -2, + "value_end": 55, + "variable": "git_token", + "variable_start": 2, + "variable_end": 11, "entropy_validation": { "iterator": "BASE36_CHARS", - "entropy": 4.133660689688185, + "entropy": 3.97402442086502, "valid": true } } @@ -8043,52 +6261,52 @@ "api_validation": "NOT_AVAILABLE", "ml_validation": "NOT_AVAILABLE", "ml_probability": null, - "rule": "Gitlab Feed Token", - "severity": "info", - "confidence": "weak", + "rule": "Facebook Access Token", + "severity": "high", + "confidence": "moderate", "line_data_list": [ { - "line": "feed_n = 'o9aEaH32LN618KhF7e_L'", + "line": "GI_REO_GI_FACEBOOK_TOKEN = \"EAACEb00Kse0BAlGy7KeQ5YnaCEd09Eose0cBAlGy7KeQ5Yna9CoDsup39tiYdoQ4jH9Coup39tiYdWoQ4jHFZD\"", "line_num": 1, - "path": "tests/samples/gitlab_feed_token", - "info": "tests/samples/gitlab_feed_token|RAW", - "value": "o9aEaH32LN618KhF7e_L", - "value_start": 10, - "value_end": 30, + "path": "tests/samples/facebook_key", + "info": "tests/samples/facebook_key|RAW", + "value": "EAACEb00Kse0BAlGy7KeQ5YnaCEd09Eose0cBAlGy7KeQ5Yna9CoDsup39tiYdoQ4jH9Coup39tiYdWoQ4jHFZD", + "value_start": 28, + "value_end": 115, "variable": null, "variable_start": -2, "variable_end": -2, "entropy_validation": { "iterator": "BASE64_CHARS", - "entropy": 3.9058316901429944, - "valid": false + "entropy": 4.936120692057916, + "valid": true } } ] }, { "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "Gitlab PAT", - "severity": "high", - "confidence": "strong", + "ml_validation": "VALIDATED_KEY", + "ml_probability": 0.999, + "rule": "Token", + "severity": "medium", + "confidence": "moderate", "line_data_list": [ { - "line": "var pat = 'glpat-a6N2pFAr2L2A6iRsA_mw';", + "line": "GI_REO_GI_FACEBOOK_TOKEN = \"EAACEb00Kse0BAlGy7KeQ5YnaCEd09Eose0cBAlGy7KeQ5Yna9CoDsup39tiYdoQ4jH9Coup39tiYdWoQ4jHFZD\"", "line_num": 1, - "path": "tests/samples/gitlab_pat_api", - "info": "tests/samples/gitlab_pat_api|RAW", - "value": "glpat-a6N2pFAr2L2A6iRsA_mw", - "value_start": 11, - "value_end": 37, - "variable": null, - "variable_start": -2, - "variable_end": -2, + "path": "tests/samples/facebook_key", + "info": "tests/samples/facebook_key|RAW", + "value": "EAACEb00Kse0BAlGy7KeQ5YnaCEd09Eose0cBAlGy7KeQ5Yna9CoDsup39tiYdoQ4jH9Coup39tiYdWoQ4jHFZD", + "value_start": 28, + "value_end": 115, + "variable": "GI_REO_GI_FACEBOOK_TOKEN", + "variable_start": 0, + "variable_end": 24, "entropy_validation": { "iterator": "BASE64_CHARS", - "entropy": 3.7423376242715105, - "valid": false + "entropy": 4.936120692057916, + "valid": true } } ] @@ -8097,24 +6315,24 @@ "api_validation": "NOT_AVAILABLE", "ml_validation": "NOT_AVAILABLE", "ml_probability": null, - "rule": "Gitlab Pipeline Trigger Token", + "rule": "Facebook App Token", "severity": "high", - "confidence": "strong", + "confidence": "moderate", "line_data_list": [ { - "line": "trigger = 'glptt-33276248c9748113e978392e5c074b7f974f8683';", - "line_num": 1, - "path": "tests/samples/gitlab_pipeline_trigger_token", - "info": "tests/samples/gitlab_pipeline_trigger_token|RAW", - "value": "glptt-33276248c9748113e978392e5c074b7f974f8683", - "value_start": 11, - "value_end": 57, + "line": "1527194624358273|qbBf2-fdB9zZpqLA0_2nNzZDw2M", + "line_num": 2, + "path": "tests/samples/facebook_key", + "info": "tests/samples/facebook_key|RAW", + "value": "1527194624358273|qbBf2-fdB9zZpqLA0_2nNzZDw2M", + "value_start": 0, + "value_end": 44, "variable": null, "variable_start": -2, "variable_end": -2, "entropy_validation": { "iterator": "BASE36_CHARS", - "entropy": 3.8494857514609038, + "entropy": 3.2089099270924217, "valid": true } } @@ -8124,24 +6342,24 @@ "api_validation": "NOT_AVAILABLE", "ml_validation": "NOT_AVAILABLE", "ml_probability": null, - "rule": "Gitlab Registration Runner Token", - "severity": "high", - "confidence": "strong", + "rule": "Firebase Domain", + "severity": "info", + "confidence": "moderate", "line_data_list": [ { - "line": "gitlab_runner = 'GR1348941jG6xeSsmN8DFVKoyBYu2';", + "line": "test-app-domain-42.firebaseapp.com", "line_num": 1, - "path": "tests/samples/gitlab_registration_runner", - "info": "tests/samples/gitlab_registration_runner|RAW", - "value": "GR1348941jG6xeSsmN8DFVKoyBYu2", - "value_start": 17, - "value_end": 46, + "path": "tests/samples/firebase_domain", + "info": "tests/samples/firebase_domain|RAW", + "value": "test-app-domain-42.firebaseapp.com", + "value_start": 0, + "value_end": 34, "variable": null, "variable_start": -2, "variable_end": -2, "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 4.582118926162056, + "iterator": "BASE36_CHARS", + "entropy": 3.4347510262969525, "valid": true } } @@ -8151,25 +6369,25 @@ "api_validation": "NOT_AVAILABLE", "ml_validation": "NOT_AVAILABLE", "ml_probability": null, - "rule": "Gitlab Registration Runner Token 2023", - "severity": "high", - "confidence": "strong", + "rule": "Firebase Domain", + "severity": "info", + "confidence": "moderate", "line_data_list": [ { - "line": "return \"glrt-2CR8_eVxiio-1QmzPZwa\"", - "line_num": 1, - "path": "tests/samples/gitlab_registration_runner_2023", - "info": "tests/samples/gitlab_registration_runner_2023|RAW", - "value": "glrt-2CR8_eVxiio-1QmzPZwa", - "value_start": 8, - "value_end": 33, + "line": "test2.io.firebaseio.com", + "line_num": 2, + "path": "tests/samples/firebase_domain", + "info": "tests/samples/firebase_domain|RAW", + "value": "test2.io.firebaseio.com", + "value_start": 0, + "value_end": 23, "variable": null, "variable_start": -2, "variable_end": -2, "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 4.006593447001756, - "valid": false + "iterator": "BASE36_CHARS", + "entropy": 3.1394163745499943, + "valid": true } } ] @@ -8178,25 +6396,25 @@ "api_validation": "NOT_AVAILABLE", "ml_validation": "NOT_AVAILABLE", "ml_probability": null, - "rule": "Google API Key", + "rule": "Github Classic Token", "severity": "high", - "confidence": "moderate", + "confidence": "strong", "line_data_list": [ { - "line": "AIzaGiReoG-CrackleCrackle12315618_12315", + "line": "ghp_00000000000000000000000000000004WZ4EQ # classic", "line_num": 1, - "path": "tests/samples/google_api_key.toml", - "info": "tests/samples/google_api_key.toml|RAW", - "value": "AIzaGiReoG-CrackleCrackle12315618_12315", + "path": "tests/samples/github_classic_token", + "info": "tests/samples/github_classic_token|RAW", + "value": "ghp_00000000000000000000000000000004WZ4EQ", "value_start": 0, - "value_end": 39, + "value_end": 41, "variable": null, "variable_start": -2, "variable_end": -2, "entropy_validation": { - "iterator": "BASE36_CHARS", - "entropy": 3.165196181720608, - "valid": true + "iterator": "BASE64_CHARS", + "entropy": 1.4322437698226884, + "valid": false } } ] @@ -8205,113 +6423,79 @@ "api_validation": "NOT_AVAILABLE", "ml_validation": "NOT_AVAILABLE", "ml_probability": null, - "rule": "Google Multi", + "rule": "Github Fine-granted Token", "severity": "high", - "confidence": "moderate", + "confidence": "strong", "line_data_list": [ { - "line": "194206074328-qp89pdv6fi35vsi71258g1eh31q6h7c3.apps.googleusercontent.com", + "line": "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", "line_num": 2, - "path": "tests/samples/google_multi", - "info": "tests/samples/google_multi|RAW", - "value": "194206074328-qp89pdv6fi35vsi71258g1eh31q6h7c3.apps.googleusercontent.com", + "path": "tests/samples/github_fine_granted_token", + "info": "tests/samples/github_fine_granted_token|RAW", + "value": "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", "value_start": 0, - "value_end": 72, + "value_end": 93, "variable": null, "variable_start": -2, "variable_end": -2, "entropy_validation": { "iterator": "BASE64_CHARS", - "entropy": 4.531537327540733, + "entropy": 5.255374790203285, "valid": true } - }, - { - "line": "4L2QMyTm6Rr0o46ytGiReoG1", - "line_num": 4, - "path": "tests/samples/google_multi", - "info": "tests/samples/google_multi|RAW", - "value": "4L2QMyTm6Rr0o46ytGiReoG1", - "value_start": 0, - "value_end": 24, - "variable": null, - "variable_start": -2, - "variable_end": -2, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 4.084962500721157, - "valid": false - } } ] }, { "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "Google Multi", + "ml_validation": "VALIDATED_KEY", + "ml_probability": 1.0, + "rule": "Github Old Token", "severity": "high", "confidence": "moderate", "line_data_list": [ { - "line": "\"id\":\"194206074328-qdv6fi3eh31q6h7c35vsi7p89p1258g1.apps.googleusercontent.com\",\"CEKPET\":\"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\",", - "line_num": 18, - "path": "tests/samples/google_multi", - "info": "tests/samples/google_multi|RAW", - "value": "194206074328-qdv6fi3eh31q6h7c35vsi7p89p1258g1.apps.googleusercontent.com", - "value_start": 6, - "value_end": 78, + "line": "GITHUB_ACCESS_TOKEN = \"lbyxnhqjfnzhlpnvcvaxjumgvdlnktgugdofmzyy\"", + "line_num": 1, + "path": "tests/samples/github_key.groovy", + "info": "tests/samples/github_key.groovy|RAW", + "value": "lbyxnhqjfnzhlpnvcvaxjumgvdlnktgugdofmzyy", + "value_start": 23, + "value_end": 63, "variable": null, "variable_start": -2, "variable_end": -2, "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 4.531537327540733, + "iterator": "BASE36_CHARS", + "entropy": 4.246439344671015, "valid": true } - }, - { - "line": "\"id\":\"194206074328-qdv6fi3eh31q6h7c35vsi7p89p1258g1.apps.googleusercontent.com\",\"CEKPET\":\"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\",", - "line_num": 18, - "path": "tests/samples/google_multi", - "info": "tests/samples/google_multi|RAW", - "value": "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", - "value_start": 90, - "value_end": 125, - "variable": null, - "variable_start": -2, - "variable_end": -2, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 4.436181130262395, - "valid": false - } } ] }, { "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "Google OAuth Secret", - "severity": "high", - "confidence": "strong", + "ml_validation": "VALIDATED_KEY", + "ml_probability": 1.0, + "rule": "Token", + "severity": "medium", + "confidence": "moderate", "line_data_list": [ { - "line": "\"id\":\"194206074328-qdv6fi3eh31q6h7c35vsi7p89p1258g1.apps.googleusercontent.com\",\"CEKPET\":\"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\",", - "line_num": 18, - "path": "tests/samples/google_multi", - "info": "tests/samples/google_multi|RAW", - "value": "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", - "value_start": 90, - "value_end": 125, - "variable": null, - "variable_start": -2, - "variable_end": -2, + "line": "GITHUB_ACCESS_TOKEN = \"lbyxnhqjfnzhlpnvcvaxjumgvdlnktgugdofmzyy\"", + "line_num": 1, + "path": "tests/samples/github_key.groovy", + "info": "tests/samples/github_key.groovy|RAW", + "value": "lbyxnhqjfnzhlpnvcvaxjumgvdlnktgugdofmzyy", + "value_start": 23, + "value_end": 63, + "variable": "GITHUB_ACCESS_TOKEN", + "variable_start": 0, + "variable_end": 19, "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 4.436181130262395, - "valid": false + "iterator": "BASE36_CHARS", + "entropy": 4.246439344671015, + "valid": true } } ] @@ -8320,24 +6504,24 @@ "api_validation": "NOT_AVAILABLE", "ml_validation": "NOT_AVAILABLE", "ml_probability": null, - "rule": "Google OAuth Access Token", - "severity": "high", - "confidence": "moderate", + "rule": "Gitlab Incoming Email Token", + "severity": "info", + "confidence": "weak", "line_data_list": [ { - "line": "google_oauth_key = \"ya29.gi_reo_gi_crackle_ln22\"", + "line": "var email_t = '7e4v6v5j2nepcc8f5zvatgl9g';", "line_num": 1, - "path": "tests/samples/google_oauth_key", - "info": "tests/samples/google_oauth_key|RAW", - "value": "ya29.gi_reo_gi_crackle_ln22", - "value_start": 20, - "value_end": 47, + "path": "tests/samples/gitlab_email_token", + "info": "tests/samples/gitlab_email_token|RAW", + "value": "7e4v6v5j2nepcc8f5zvatgl9g", + "value_start": 15, + "value_end": 40, "variable": null, "variable_start": -2, "variable_end": -2, "entropy_validation": { "iterator": "BASE36_CHARS", - "entropy": 3.1797273164975133, + "entropy": 4.133660689688185, "valid": true } } @@ -8347,25 +6531,25 @@ "api_validation": "NOT_AVAILABLE", "ml_validation": "NOT_AVAILABLE", "ml_probability": null, - "rule": "Grafana Access Policy Token", - "severity": "high", - "confidence": "strong", + "rule": "Gitlab Feed Token", + "severity": "info", + "confidence": "weak", "line_data_list": [ { - "line": "grafana_policy = 'glc_eyJvIjoiMjA0NjMwIiwibiI6InRlc3QtdG9rZW4iLCJrIjoidklnbjJ2WHc5MTVXOWtNOWxsNHcyZHEyIiwibSI6eyJyIjoicHJvZC0wIn19'", + "line": "feed_n = 'o9aEaH32LN618KhF7e_L'", "line_num": 1, - "path": "tests/samples/grafana_access_policy_token", - "info": "tests/samples/grafana_access_policy_token|RAW", - "value": "glc_eyJvIjoiMjA0NjMwIiwibiI6InRlc3QtdG9rZW4iLCJrIjoidklnbjJ2WHc5MTVXOWtNOWxsNHcyZHEyIiwibSI6eyJyIjoicHJvZC0wIn19", - "value_start": 18, - "value_end": 130, + "path": "tests/samples/gitlab_feed_token", + "info": "tests/samples/gitlab_feed_token|RAW", + "value": "o9aEaH32LN618KhF7e_L", + "value_start": 10, + "value_end": 30, "variable": null, "variable_start": -2, "variable_end": -2, "entropy_validation": { "iterator": "BASE64_CHARS", - "entropy": 5.097632476604024, - "valid": true + "entropy": 3.9058316901429944, + "valid": false } } ] @@ -8374,24 +6558,24 @@ "api_validation": "NOT_AVAILABLE", "ml_validation": "NOT_AVAILABLE", "ml_probability": null, - "rule": "Grafana Provisioned API Key", + "rule": "Gitlab PAT", "severity": "high", "confidence": "strong", "line_data_list": [ { - "line": "grafana = 'eyJrIjoiMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMCIsIm4iOiJ4eHh4IiwiaWQiOjIwNDM2MH0='", + "line": "var pat = 'glpat-a6N2pFAr2L2A6iRsA_mw';", "line_num": 1, - "path": "tests/samples/grafana_provisioned_api_key", - "info": "tests/samples/grafana_provisioned_api_key|RAW", - "value": "eyJrIjoiMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMCIsIm4iOiJ4eHh4IiwiaWQiOjIwNDM2MH0=", + "path": "tests/samples/gitlab_pat_api", + "info": "tests/samples/gitlab_pat_api|RAW", + "value": "glpat-a6N2pFAr2L2A6iRsA_mw", "value_start": 11, - "value_end": 107, + "value_end": 37, "variable": null, "variable_start": -2, "variable_end": -2, "entropy_validation": { "iterator": "BASE64_CHARS", - "entropy": 3.8153130511409934, + "entropy": 3.7423376242715105, "valid": false } } @@ -8399,27 +6583,27 @@ }, { "api_validation": "NOT_AVAILABLE", - "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.985, - "rule": "JSON Web Token", - "severity": "medium", - "confidence": "moderate", + "ml_validation": "NOT_AVAILABLE", + "ml_probability": null, + "rule": "Gitlab Pipeline Trigger Token", + "severity": "high", + "confidence": "strong", "line_data_list": [ { - "line": "grafana = 'eyJrIjoiMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMCIsIm4iOiJ4eHh4IiwiaWQiOjIwNDM2MH0='", + "line": "trigger = 'glptt-33276248c9748113e978392e5c074b7f974f8683';", "line_num": 1, - "path": "tests/samples/grafana_provisioned_api_key", - "info": "tests/samples/grafana_provisioned_api_key|RAW", - "value": "eyJrIjoiMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMCIsIm4iOiJ4eHh4IiwiaWQiOjIwNDM2MH0=", + "path": "tests/samples/gitlab_pipeline_trigger_token", + "info": "tests/samples/gitlab_pipeline_trigger_token|RAW", + "value": "glptt-33276248c9748113e978392e5c074b7f974f8683", "value_start": 11, - "value_end": 107, + "value_end": 57, "variable": null, "variable_start": -2, "variable_end": -2, "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 3.8153130511409934, - "valid": false + "iterator": "BASE36_CHARS", + "entropy": 3.8494857514609038, + "valid": true } } ] @@ -8428,24 +6612,24 @@ "api_validation": "NOT_AVAILABLE", "ml_validation": "NOT_AVAILABLE", "ml_probability": null, - "rule": "Grafana Service Account Token", + "rule": "Gitlab Registration Runner Token", "severity": "high", "confidence": "strong", "line_data_list": [ { - "line": "glsa_ThisI5NtTheTok3nYou8reLo0k1ngF0r_0a2a3df7", + "line": "gitlab_runner = 'GR1348941jG6xeSsmN8DFVKoyBYu2';", "line_num": 1, - "path": "tests/samples/grafana_service_accounts", - "info": "tests/samples/grafana_service_accounts|RAW", - "value": "glsa_ThisI5NtTheTok3nYou8reLo0k1ngF0r_0a2a3df7", - "value_start": 0, + "path": "tests/samples/gitlab_registration_runner", + "info": "tests/samples/gitlab_registration_runner|RAW", + "value": "GR1348941jG6xeSsmN8DFVKoyBYu2", + "value_start": 17, "value_end": 46, "variable": null, "variable_start": -2, "variable_end": -2, "entropy_validation": { "iterator": "BASE64_CHARS", - "entropy": 4.52211252299684, + "entropy": 4.582118926162056, "valid": true } } @@ -8455,25 +6639,25 @@ "api_validation": "NOT_AVAILABLE", "ml_validation": "NOT_AVAILABLE", "ml_probability": null, - "rule": "Hashicorp Terraform Token", + "rule": "Gitlab Registration Runner Token 2023", "severity": "high", "confidence": "strong", "line_data_list": [ { - "line": "Z28P3STmkBQi1Y.atlasv1.YE7RBqu6VVyQIOq9a1eC3YFU5Elt7ToIr6OwzKAWlCTQ7N4gElXaWou6aPpOIwGCoc0", + "line": "return \"glrt-2CR8_eVxiio-1QmzPZwa\"", "line_num": 1, - "path": "tests/samples/hashicorp_terraform", - "info": "tests/samples/hashicorp_terraform|RAW", - "value": "Z28P3STmkBQi1Y.atlasv1.YE7RBqu6VVyQIOq9a1eC3YFU5Elt7ToIr6OwzKAWlCTQ7N4gElXaWou6aPpOIwGCoc0", - "value_start": 0, - "value_end": 90, + "path": "tests/samples/gitlab_registration_runner_2023", + "info": "tests/samples/gitlab_registration_runner_2023|RAW", + "value": "glrt-2CR8_eVxiio-1QmzPZwa", + "value_start": 8, + "value_end": 33, "variable": null, "variable_start": -2, "variable_end": -2, "entropy_validation": { "iterator": "BASE64_CHARS", - "entropy": 5.348551883097512, - "valid": true + "entropy": 4.006593447001756, + "valid": false } } ] @@ -8482,24 +6666,24 @@ "api_validation": "NOT_AVAILABLE", "ml_validation": "NOT_AVAILABLE", "ml_probability": null, - "rule": "Heroku API Key", + "rule": "Google API Key", "severity": "high", "confidence": "moderate", "line_data_list": [ { - "line": "HerOkUa04b8c1d-A147-b252-3b6a8f9c2b16", + "line": "AIzaGiReoG-CrackleCrackle12315618_12315", "line_num": 1, - "path": "tests/samples/heroku_api.toml", - "info": "tests/samples/heroku_api.toml|RAW", - "value": "HerOkUa04b8c1d-A147-b252-3b6a8f9c2b16", + "path": "tests/samples/google_api_key.toml", + "info": "tests/samples/google_api_key.toml|RAW", + "value": "AIzaGiReoG-CrackleCrackle12315618_12315", "value_start": 0, - "value_end": 37, + "value_end": 39, "variable": null, "variable_start": -2, "variable_end": -2, "entropy_validation": { "iterator": "BASE36_CHARS", - "entropy": 3.48037367471734, + "entropy": 3.165196181720608, "valid": true } } @@ -8509,51 +6693,41 @@ "api_validation": "NOT_AVAILABLE", "ml_validation": "NOT_AVAILABLE", "ml_probability": null, - "rule": "Instagram Access Token", + "rule": "Google Multi", "severity": "high", - "confidence": "strong", + "confidence": "moderate", "line_data_list": [ { - "line": "IGQVJWS3pUNmZA2MUJDVlRwLW9ac1lrU05nZAmpzWHQtWHFJSEFRMF9tWVpRdEd70HQ5Wk8wSnY0R0VEQnVQdUU0MnpxNWxocUYyNmZAXSTUtVVNrMmh1ZAHZAQQno2ZA0VHR0lLWkk1N1R5RDFvM0dmVEpIYQZGZX", - "line_num": 1, - "path": "tests/samples/instagram_access_token", - "info": "tests/samples/instagram_access_token|RAW", - "value": "IGQVJWS3pUNmZA2MUJDVlRwLW9ac1lrU05nZAmpzWHQtWHFJSEFRMF9tWVpRdEd70HQ5Wk8wSnY0R0VEQnVQdUU0MnpxNWxocUYyNmZAXSTUtVVNrMmh1ZAHZAQQno2ZA0VHR0lLWkk1N1R5RDFvM0dmVEpIYQZGZX", + "line": "194206074328-qp89pdv6fi35vsi71258g1eh31q6h7c3.apps.googleusercontent.com", + "line_num": 2, + "path": "tests/samples/google_multi", + "info": "tests/samples/google_multi|RAW", + "value": "194206074328-qp89pdv6fi35vsi71258g1eh31q6h7c3.apps.googleusercontent.com", "value_start": 0, - "value_end": 162, + "value_end": 72, "variable": null, "variable_start": -2, "variable_end": -2, "entropy_validation": { "iterator": "BASE64_CHARS", - "entropy": 5.231644837540696, + "entropy": 4.531537327540733, "valid": true } - } - ] - }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "IPv4", - "severity": "info", - "confidence": "weak", - "line_data_list": [ + }, { - "line": "100.64.0.0/10", - "line_num": 13, - "path": "tests/samples/ipv4", - "info": "tests/samples/ipv4|RAW", - "value": "100.64.0.0", + "line": "4L2QMyTm6Rr0o46ytGiReoG1", + "line_num": 4, + "path": "tests/samples/google_multi", + "info": "tests/samples/google_multi|RAW", + "value": "4L2QMyTm6Rr0o46ytGiReoG1", "value_start": 0, - "value_end": 10, + "value_end": 24, "variable": null, "variable_start": -2, "variable_end": -2, "entropy_validation": { "iterator": "BASE64_CHARS", - "entropy": 1.5253496664211537, + "entropy": 4.084962500721157, "valid": false } } @@ -8563,51 +6737,41 @@ "api_validation": "NOT_AVAILABLE", "ml_validation": "NOT_AVAILABLE", "ml_probability": null, - "rule": "IPv4", - "severity": "info", - "confidence": "weak", + "rule": "Google Multi", + "severity": "high", + "confidence": "moderate", "line_data_list": [ { - "line": "100.64.0.0\u2013100.127.255.255", - "line_num": 14, - "path": "tests/samples/ipv4", - "info": "tests/samples/ipv4|RAW", - "value": "100.127.255.255", - "value_start": 11, - "value_end": 26, + "line": "\"id\":\"194206074328-qdv6fi3eh31q6h7c35vsi7p89p1258g1.apps.googleusercontent.com\",\"CEKPET\":\"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\",", + "line_num": 18, + "path": "tests/samples/google_multi", + "info": "tests/samples/google_multi|RAW", + "value": "194206074328-qdv6fi3eh31q6h7c35vsi7p89p1258g1.apps.googleusercontent.com", + "value_start": 6, + "value_end": 78, "variable": null, "variable_start": -2, "variable_end": -2, "entropy_validation": { - "iterator": "BASE36_CHARS", - "entropy": 2.008519976342584, - "valid": false + "iterator": "BASE64_CHARS", + "entropy": 4.531537327540733, + "valid": true } - } - ] - }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "IPv4", - "severity": "info", - "confidence": "weak", - "line_data_list": [ + }, { - "line": "100.64.0.0\u2013100.127.255.255", - "line_num": 14, - "path": "tests/samples/ipv4", - "info": "tests/samples/ipv4|RAW", - "value": "100.64.0.0", - "value_start": 0, - "value_end": 10, + "line": "\"id\":\"194206074328-qdv6fi3eh31q6h7c35vsi7p89p1258g1.apps.googleusercontent.com\",\"CEKPET\":\"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\",", + "line_num": 18, + "path": "tests/samples/google_multi", + "info": "tests/samples/google_multi|RAW", + "value": "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", + "value_start": 90, + "value_end": 125, "variable": null, "variable_start": -2, "variable_end": -2, "entropy_validation": { "iterator": "BASE64_CHARS", - "entropy": 1.5253496664211537, + "entropy": 4.436181130262395, "valid": false } } @@ -8617,24 +6781,24 @@ "api_validation": "NOT_AVAILABLE", "ml_validation": "NOT_AVAILABLE", "ml_probability": null, - "rule": "IPv4", - "severity": "info", - "confidence": "weak", + "rule": "Google OAuth Secret", + "severity": "high", + "confidence": "strong", "line_data_list": [ { - "line": "192.0.0.0\u2013192.0.0.255", - "line_num": 22, - "path": "tests/samples/ipv4", - "info": "tests/samples/ipv4|RAW", - "value": "192.0.0.255", - "value_start": 10, - "value_end": 21, + "line": "\"id\":\"194206074328-qdv6fi3eh31q6h7c35vsi7p89p1258g1.apps.googleusercontent.com\",\"CEKPET\":\"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\",", + "line_num": 18, + "path": "tests/samples/google_multi", + "info": "tests/samples/google_multi|RAW", + "value": "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", + "value_start": 90, + "value_end": 125, "variable": null, "variable_start": -2, "variable_end": -2, "entropy_validation": { "iterator": "BASE64_CHARS", - "entropy": 1.9704957226453073, + "entropy": 4.436181130262395, "valid": false } } @@ -8644,25 +6808,25 @@ "api_validation": "NOT_AVAILABLE", "ml_validation": "NOT_AVAILABLE", "ml_probability": null, - "rule": "IPv4", - "severity": "info", - "confidence": "weak", + "rule": "Google OAuth Access Token", + "severity": "high", + "confidence": "moderate", "line_data_list": [ { - "line": "192.88.99.0/24", - "line_num": 25, - "path": "tests/samples/ipv4", - "info": "tests/samples/ipv4|RAW", - "value": "192.88.99.0", - "value_start": 0, - "value_end": 11, + "line": "google_oauth_key = \"ya29.gi_reo_gi_crackle_ln22\"", + "line_num": 1, + "path": "tests/samples/google_oauth_key", + "info": "tests/samples/google_oauth_key|RAW", + "value": "ya29.gi_reo_gi_crackle_ln22", + "value_start": 20, + "value_end": 47, "variable": null, "variable_start": -2, "variable_end": -2, "entropy_validation": { "iterator": "BASE36_CHARS", - "entropy": 1.9018695860849921, - "valid": false + "entropy": 3.1797273164975133, + "valid": true } } ] @@ -8671,25 +6835,25 @@ "api_validation": "NOT_AVAILABLE", "ml_validation": "NOT_AVAILABLE", "ml_probability": null, - "rule": "IPv4", - "severity": "info", - "confidence": "weak", + "rule": "Grafana Access Policy Token", + "severity": "high", + "confidence": "strong", "line_data_list": [ { - "line": "192.88.99.0\u2013192.88.99.255", - "line_num": 26, - "path": "tests/samples/ipv4", - "info": "tests/samples/ipv4|RAW", - "value": "192.88.99.0", - "value_start": 0, - "value_end": 11, + "line": "grafana_policy = 'glc_eyJvIjoiMjA0NjMwIiwibiI6InRlc3QtdG9rZW4iLCJrIjoidklnbjJ2WHc5MTVXOWtNOWxsNHcyZHEyIiwibSI6eyJyIjoicHJvZC0wIn19'", + "line_num": 1, + "path": "tests/samples/grafana_access_policy_token", + "info": "tests/samples/grafana_access_policy_token|RAW", + "value": "glc_eyJvIjoiMjA0NjMwIiwibiI6InRlc3QtdG9rZW4iLCJrIjoidklnbjJ2WHc5MTVXOWtNOWxsNHcyZHEyIiwibSI6eyJyIjoicHJvZC0wIn19", + "value_start": 18, + "value_end": 130, "variable": null, "variable_start": -2, "variable_end": -2, "entropy_validation": { - "iterator": "BASE36_CHARS", - "entropy": 1.9018695860849921, - "valid": false + "iterator": "BASE64_CHARS", + "entropy": 5.097632476604024, + "valid": true } } ] @@ -8698,24 +6862,24 @@ "api_validation": "NOT_AVAILABLE", "ml_validation": "NOT_AVAILABLE", "ml_probability": null, - "rule": "IPv4", - "severity": "info", - "confidence": "weak", + "rule": "Grafana Provisioned API Key", + "severity": "high", + "confidence": "strong", "line_data_list": [ { - "line": "192.88.99.0\u2013192.88.99.255", - "line_num": 26, - "path": "tests/samples/ipv4", - "info": "tests/samples/ipv4|RAW", - "value": "192.88.99.255", - "value_start": 12, - "value_end": 25, + "line": "grafana = 'eyJrIjoiMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMCIsIm4iOiJ4eHh4IiwiaWQiOjIwNDM2MH0='", + "line_num": 1, + "path": "tests/samples/grafana_provisioned_api_key", + "info": "tests/samples/grafana_provisioned_api_key|RAW", + "value": "eyJrIjoiMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMCIsIm4iOiJ4eHh4IiwiaWQiOjIwNDM2MH0=", + "value_start": 11, + "value_end": 107, "variable": null, "variable_start": -2, "variable_end": -2, "entropy_validation": { "iterator": "BASE64_CHARS", - "entropy": 2.019193052249804, + "entropy": 3.8153130511409934, "valid": false } } @@ -8723,26 +6887,26 @@ }, { "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "IPv6", - "severity": "info", - "confidence": "strong", + "ml_validation": "VALIDATED_KEY", + "ml_probability": 0.985, + "rule": "JSON Web Token", + "severity": "medium", + "confidence": "moderate", "line_data_list": [ { - "line": "2004:5678::9324", - "line_num": 12, - "path": "tests/samples/ipv6", - "info": "tests/samples/ipv6|RAW", - "value": "2004:5678::9324", - "value_start": 0, - "value_end": 15, + "line": "grafana = 'eyJrIjoiMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMCIsIm4iOiJ4eHh4IiwiaWQiOjIwNDM2MH0='", + "line_num": 1, + "path": "tests/samples/grafana_provisioned_api_key", + "info": "tests/samples/grafana_provisioned_api_key|RAW", + "value": "eyJrIjoiMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMCIsIm4iOiJ4eHh4IiwiaWQiOjIwNDM2MH0=", + "value_start": 11, + "value_end": 107, "variable": null, "variable_start": -2, "variable_end": -2, "entropy_validation": { - "iterator": "BASE36_CHARS", - "entropy": 2.725512476486815, + "iterator": "BASE64_CHARS", + "entropy": 3.8153130511409934, "valid": false } } @@ -8752,25 +6916,25 @@ "api_validation": "NOT_AVAILABLE", "ml_validation": "NOT_AVAILABLE", "ml_probability": null, - "rule": "IPv6", - "severity": "info", + "rule": "Grafana Service Account Token", + "severity": "high", "confidence": "strong", "line_data_list": [ { - "line": "2004::5678:9", - "line_num": 13, - "path": "tests/samples/ipv6", - "info": "tests/samples/ipv6|RAW", - "value": "2004::5678:9", + "line": "glsa_ThisI5NtTheTok3nYou8reLo0k1ngF0r_0a2a3df7", + "line_num": 1, + "path": "tests/samples/grafana_service_accounts", + "info": "tests/samples/grafana_service_accounts|RAW", + "value": "glsa_ThisI5NtTheTok3nYou8reLo0k1ngF0r_0a2a3df7", "value_start": 0, - "value_end": 12, + "value_end": 46, "variable": null, "variable_start": -2, "variable_end": -2, "entropy_validation": { - "iterator": "BASE36_CHARS", - "entropy": 2.5220552088742005, - "valid": false + "iterator": "BASE64_CHARS", + "entropy": 4.52211252299684, + "valid": true } } ] @@ -8779,25 +6943,25 @@ "api_validation": "NOT_AVAILABLE", "ml_validation": "NOT_AVAILABLE", "ml_probability": null, - "rule": "IPv6", - "severity": "info", + "rule": "Hashicorp Terraform Token", + "severity": "high", "confidence": "strong", "line_data_list": [ { - "line": "2041:0000:140F::875B:131B", - "line_num": 14, - "path": "tests/samples/ipv6", - "info": "tests/samples/ipv6|RAW", - "value": "2041:0000:140F::875B:131B", + "line": "Z28P3STmkBQi1Y.atlasv1.YE7RBqu6VVyQIOq9a1eC3YFU5Elt7ToIr6OwzKAWlCTQ7N4gElXaWou6aPpOIwGCoc0", + "line_num": 1, + "path": "tests/samples/hashicorp_terraform", + "info": "tests/samples/hashicorp_terraform|RAW", + "value": "Z28P3STmkBQi1Y.atlasv1.YE7RBqu6VVyQIOq9a1eC3YFU5Elt7ToIr6OwzKAWlCTQ7N4gElXaWou6aPpOIwGCoc0", "value_start": 0, - "value_end": 25, + "value_end": 90, "variable": null, "variable_start": -2, "variable_end": -2, "entropy_validation": { - "iterator": "HEX_CHARS", - "entropy": 2.6146939516467023, - "valid": false + "iterator": "BASE64_CHARS", + "entropy": 5.348551883097512, + "valid": true } } ] @@ -8806,25 +6970,25 @@ "api_validation": "NOT_AVAILABLE", "ml_validation": "NOT_AVAILABLE", "ml_probability": null, - "rule": "IPv6", - "severity": "info", - "confidence": "strong", + "rule": "Heroku API Key", + "severity": "high", + "confidence": "moderate", "line_data_list": [ { - "line": "LONG LINE TEST: 2041:0:140F::875B:131B 2041:0:140F::875B:131B ;", - "line_num": 15, - "path": "tests/samples/ipv6", - "info": "tests/samples/ipv6|RAW", - "value": "2041:0:140F::875B:131B", - "value_start": 7010, - "value_end": 7032, + "line": "HerOkUa04b8c1d-A147-b252-3b6a8f9c2b16", + "line_num": 1, + "path": "tests/samples/heroku_api.toml", + "info": "tests/samples/heroku_api.toml|RAW", + "value": "HerOkUa04b8c1d-A147-b252-3b6a8f9c2b16", + "value_start": 0, + "value_end": 37, "variable": null, "variable_start": -2, "variable_end": -2, "entropy_validation": { - "iterator": "HEX_CHARS", - "entropy": 2.684338637030481, - "valid": false + "iterator": "BASE36_CHARS", + "entropy": 3.48037367471734, + "valid": true } } ] @@ -8833,25 +6997,25 @@ "api_validation": "NOT_AVAILABLE", "ml_validation": "NOT_AVAILABLE", "ml_probability": null, - "rule": "IPv6", - "severity": "info", + "rule": "Instagram Access Token", + "severity": "high", "confidence": "strong", "line_data_list": [ { - "line": "LONG LINE TEST: 2041:0:140F::875B:131B 2041:0:140F::875B:131B ;", - "line_num": 15, - "path": "tests/samples/ipv6", - "info": "tests/samples/ipv6|RAW", - "value": "2041:0:140F::875B:131B", - "value_start": 7989, - "value_end": 8011, + "line": "IGQVJWS3pUNmZA2MUJDVlRwLW9ac1lrU05nZAmpzWHQtWHFJSEFRMF9tWVpRdEd70HQ5Wk8wSnY0R0VEQnVQdUU0MnpxNWxocUYyNmZAXSTUtVVNrMmh1ZAHZAQQno2ZA0VHR0lLWkk1N1R5RDFvM0dmVEpIYQZGZX", + "line_num": 1, + "path": "tests/samples/instagram_access_token", + "info": "tests/samples/instagram_access_token|RAW", + "value": "IGQVJWS3pUNmZA2MUJDVlRwLW9ac1lrU05nZAmpzWHQtWHFJSEFRMF9tWVpRdEd70HQ5Wk8wSnY0R0VEQnVQdUU0MnpxNWxocUYyNmZAXSTUtVVNrMmh1ZAHZAQQno2ZA0VHR0lLWkk1N1R5RDFvM0dmVEpIYQZGZX", + "value_start": 0, + "value_end": 162, "variable": null, "variable_start": -2, "variable_end": -2, "entropy_validation": { - "iterator": "HEX_CHARS", - "entropy": 2.684338637030481, - "valid": false + "iterator": "BASE64_CHARS", + "entropy": 5.231644837540696, + "valid": true } } ] @@ -11324,60 +9488,6 @@ } ] }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "IPv4", - "severity": "info", - "confidence": "weak", - "line_data_list": [ - { - "line": "18LibreOffice/7.6.2.1$Linux_X86_64 LibreOffice_project/9d0b4c0791fc17bc4181a67fd90c5aaed576d1c015.00001459612", - "line_num": 2, - "path": "tests/samples/sample.docx", - "info": "tests/samples/sample.docx.gz|GZIP|tests/samples/sample.docx|ZIP|docProps/app.xml|RAW", - "value": "7.6.2.1", - "value_start": 245, - "value_end": 252, - "variable": null, - "variable_start": -2, - "variable_end": -2, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 1.6042028126043455, - "valid": false - } - } - ] - }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "IPv4", - "severity": "info", - "confidence": "weak", - "line_data_list": [ - { - "line": "18LibreOffice/7.6.2.1$Linux_X86_64 LibreOffice_project/9d0b4c0791fc17bc4181a67fd90c5aaed576d1c015.00001460622", - "line_num": 2, - "path": "tests/samples/sample.docx", - "info": "tests/samples/sample.docx|ZIP|docProps/app.xml|RAW", - "value": "7.6.2.1", - "value_start": 245, - "value_end": 252, - "variable": null, - "variable_start": -2, - "variable_end": -2, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 1.6042028126043455, - "valid": false - } - } - ] - }, { "api_validation": "NOT_AVAILABLE", "ml_validation": "NOT_AVAILABLE", diff --git a/tests/data/ml_threshold.json b/tests/data/ml_threshold.json index d6e5801d2..71958bd1a 100644 --- a/tests/data/ml_threshold.json +++ b/tests/data/ml_threshold.json @@ -2328,33 +2328,6 @@ } ] }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "IPv4", - "severity": "info", - "confidence": "weak", - "line_data_list": [ - { - "line": "(98.76.54.32)ID:master PW:dipPr149Gg!", - "line_num": 50, - "path": "tests/samples/doc_id_pair_passwd_pair", - "info": "", - "value": "98.76.54.32", - "value_start": 1, - "value_end": 12, - "variable": null, - "variable_start": -2, - "variable_end": -2, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 2.515950268099853, - "valid": false - } - } - ] - }, { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", @@ -3246,33 +3219,6 @@ } ] }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "IPv4", - "severity": "info", - "confidence": "weak", - "line_data_list": [ - { - "line": "ANYid:master pw:dipPr194Gg! ip:98.76.54.32", - "line_num": 95, - "path": "tests/samples/doc_id_pair_passwd_pair", - "info": "", - "value": "98.76.54.32", - "value_start": 31, - "value_end": 42, - "variable": null, - "variable_start": -2, - "variable_end": -2, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 2.515950268099853, - "valid": false - } - } - ] - }, { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", @@ -3300,33 +3246,6 @@ } ] }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "IPv4", - "severity": "info", - "confidence": "weak", - "line_data_list": [ - { - "line": "master@98.76.54.32 mailto:{1} (password-dipPr196Gg!) # skip", - "line_num": 97, - "path": "tests/samples/doc_id_pair_passwd_pair", - "info": "", - "value": "98.76.54.32", - "value_start": 7, - "value_end": 18, - "variable": null, - "variable_start": -2, - "variable_end": -2, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 2.515950268099853, - "valid": false - } - } - ] - }, { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", @@ -3597,33 +3516,6 @@ } ] }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "IPv4", - "severity": "info", - "confidence": "weak", - "line_data_list": [ - { - "line": "98.76.54.32(ID:master/PW:iPp10@GRq) # todo: move into other sample ?", - "line_num": 11, - "path": "tests/samples/doc_id_passwd_pair", - "info": "", - "value": "98.76.54.32", - "value_start": 0, - "value_end": 11, - "variable": null, - "variable_start": -2, - "variable_end": -2, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 2.515950268099853, - "valid": false - } - } - ] - }, { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", @@ -3705,33 +3597,6 @@ } ] }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "IPv4", - "severity": "info", - "confidence": "weak", - "line_data_list": [ - { - "line": "98.76.54.32 id/pw:master/iPp19@GRq", - "line_num": 20, - "path": "tests/samples/doc_id_passwd_pair", - "info": "", - "value": "98.76.54.32", - "value_start": 0, - "value_end": 11, - "variable": null, - "variable_start": -2, - "variable_end": -2, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 2.515950268099853, - "valid": false - } - } - ] - }, { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", @@ -4083,33 +3948,6 @@ } ] }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "IPv4", - "severity": "info", - "confidence": "weak", - "line_data_list": [ - { - "line": "master@98.76.54.32 password:Prl23Db#@", - "line_num": 14, - "path": "tests/samples/doc_passwd_pair", - "info": "", - "value": "98.76.54.32", - "value_start": 7, - "value_end": 18, - "variable": null, - "variable_start": -2, - "variable_end": -2, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 2.515950268099853, - "valid": false - } - } - ] - }, { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", @@ -5406,87 +5244,6 @@ } ] }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "IPv4", - "severity": "info", - "confidence": "weak", - "line_data_list": [ - { - "line": "98.76.54.32 (master/IhqSb1Gg)", - "line_num": 1, - "path": "tests/samples/doc_various", - "info": "", - "value": "98.76.54.32", - "value_start": 0, - "value_end": 11, - "variable": null, - "variable_start": -2, - "variable_end": -2, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 2.515950268099853, - "valid": false - } - } - ] - }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "IPv4", - "severity": "info", - "confidence": "weak", - "line_data_list": [ - { - "line": "98.76.54.32(master/IhqSb1Gg)", - "line_num": 2, - "path": "tests/samples/doc_various", - "info": "", - "value": "98.76.54.32", - "value_start": 0, - "value_end": 11, - "variable": null, - "variable_start": -2, - "variable_end": -2, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 2.515950268099853, - "valid": false - } - } - ] - }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "IPv4", - "severity": "info", - "confidence": "weak", - "line_data_list": [ - { - "line": "master@98.76.54.32(pw:IhqSb1Gg)", - "line_num": 3, - "path": "tests/samples/doc_various", - "info": "", - "value": "98.76.54.32", - "value_start": 7, - "value_end": 18, - "variable": null, - "variable_start": -2, - "variable_end": -2, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 2.515950268099853, - "valid": false - } - } - ] - }, { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", @@ -5568,33 +5325,6 @@ } ] }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "IPv4", - "severity": "info", - "confidence": "weak", - "line_data_list": [ - { - "line": "ssh gildong.hong@98.76.54.32 mailto:{1} (PW:IhqSb1Gg)", - "line_num": 6, - "path": "tests/samples/doc_various", - "info": "", - "value": "98.76.54.32", - "value_start": 17, - "value_end": 28, - "variable": null, - "variable_start": -2, - "variable_end": -2, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 2.515950268099853, - "valid": false - } - } - ] - }, { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", @@ -5624,80 +5354,26 @@ }, { "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "IPv4", - "severity": "info", - "confidence": "weak", + "ml_validation": "VALIDATED_KEY", + "ml_probability": 1.0, + "rule": "Password", + "severity": "medium", + "confidence": "moderate", "line_data_list": [ { "line": "ssh gildong.hong@98.76.54.32 mailto:{1} password:IhqSb1Gg", "line_num": 7, "path": "tests/samples/doc_various", "info": "", - "value": "98.76.54.32", - "value_start": 17, - "value_end": 28, - "variable": null, - "variable_start": -2, - "variable_end": -2, + "value": "IhqSb1Gg", + "value_start": 49, + "value_end": 57, + "variable": "password", + "variable_start": 40, + "variable_end": 48, "entropy_validation": { "iterator": "BASE64_CHARS", - "entropy": 2.515950268099853, - "valid": false - } - } - ] - }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "VALIDATED_KEY", - "ml_probability": 1.0, - "rule": "Password", - "severity": "medium", - "confidence": "moderate", - "line_data_list": [ - { - "line": "ssh gildong.hong@98.76.54.32 mailto:{1} password:IhqSb1Gg", - "line_num": 7, - "path": "tests/samples/doc_various", - "info": "", - "value": "IhqSb1Gg", - "value_start": 49, - "value_end": 57, - "variable": "password", - "variable_start": 40, - "variable_end": 48, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 3.0, - "valid": false - } - } - ] - }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "IPv4", - "severity": "info", - "confidence": "weak", - "line_data_list": [ - { - "line": "ssh -P IhqSb1Gg gildong.hong@98.76.54.32 mailto:{1} (password:IhqSb1Gg)", - "line_num": 11, - "path": "tests/samples/doc_various", - "info": "", - "value": "98.76.54.32", - "value_start": 29, - "value_end": 40, - "variable": null, - "variable_start": -2, - "variable_end": -2, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 2.515950268099853, + "entropy": 3.0, "valid": false } } @@ -5730,60 +5406,6 @@ } ] }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "IPv4", - "severity": "info", - "confidence": "weak", - "line_data_list": [ - { - "line": "98.76.54.32 xxxx (master/IhqSb1Gg)", - "line_num": 14, - "path": "tests/samples/doc_various", - "info": "", - "value": "98.76.54.32", - "value_start": 0, - "value_end": 11, - "variable": null, - "variable_start": -2, - "variable_end": -2, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 2.515950268099853, - "valid": false - } - } - ] - }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "IPv4", - "severity": "info", - "confidence": "weak", - "line_data_list": [ - { - "line": "gildong.hong@98.76.54.32 pwd:IhqSb1Gg", - "line_num": 15, - "path": "tests/samples/doc_various", - "info": "", - "value": "98.76.54.32", - "value_start": 13, - "value_end": 24, - "variable": null, - "variable_start": -2, - "variable_end": -2, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 2.515950268099853, - "valid": false - } - } - ] - }, { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", @@ -5838,33 +5460,6 @@ } ] }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "IPv4", - "severity": "info", - "confidence": "weak", - "line_data_list": [ - { - "line": "98.76.54.32(pw:IhqSb1Gg)", - "line_num": 17, - "path": "tests/samples/doc_various", - "info": "", - "value": "98.76.54.32", - "value_start": 0, - "value_end": 11, - "variable": null, - "variable_start": -2, - "variable_end": -2, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 2.515950268099853, - "valid": false - } - } - ] - }, { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", @@ -5892,33 +5487,6 @@ } ] }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "IPv4", - "severity": "info", - "confidence": "weak", - "line_data_list": [ - { - "line": "98.76.54.32/pw:IhqSb1Gg", - "line_num": 19, - "path": "tests/samples/doc_various", - "info": "", - "value": "98.76.54.32", - "value_start": 0, - "value_end": 11, - "variable": null, - "variable_start": -2, - "variable_end": -2, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 2.515950268099853, - "valid": false - } - } - ] - }, { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", @@ -6000,33 +5568,6 @@ } ] }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "IPv4", - "severity": "info", - "confidence": "weak", - "line_data_list": [ - { - "line": "sftp gildong.hong@98.76.54.32 mailto:{1} (pw:IhqSb1Gg)", - "line_num": 22, - "path": "tests/samples/doc_various", - "info": "", - "value": "98.76.54.32", - "value_start": 18, - "value_end": 29, - "variable": null, - "variable_start": -2, - "variable_end": -2, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 2.515950268099853, - "valid": false - } - } - ] - }, { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", @@ -6054,33 +5595,6 @@ } ] }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "IPv4", - "severity": "info", - "confidence": "weak", - "line_data_list": [ - { - "line": "gildong.hong@98.76.54.32 mailto:{1} (pw:IhqSb1Gg)", - "line_num": 23, - "path": "tests/samples/doc_various", - "info": "", - "value": "98.76.54.32", - "value_start": 13, - "value_end": 24, - "variable": null, - "variable_start": -2, - "variable_end": -2, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 2.515950268099853, - "valid": false - } - } - ] - }, { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", @@ -6135,33 +5649,6 @@ } ] }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "IPv4", - "severity": "info", - "confidence": "weak", - "line_data_list": [ - { - "line": "ssh gildong.hong@98.76.54.32 mailto:{1} (password:IhqSb1Gg)", - "line_num": 25, - "path": "tests/samples/doc_various", - "info": "", - "value": "98.76.54.32", - "value_start": 17, - "value_end": 28, - "variable": null, - "variable_start": -2, - "variable_end": -2, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 2.515950268099853, - "valid": false - } - } - ] - }, { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", @@ -6189,33 +5676,6 @@ } ] }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "IPv4", - "severity": "info", - "confidence": "weak", - "line_data_list": [ - { - "line": "ssh gildong.hong@98.76.54.32 mailto:{1} master/IhqSb1Gg", - "line_num": 26, - "path": "tests/samples/doc_various", - "info": "", - "value": "98.76.54.32", - "value_start": 17, - "value_end": 28, - "variable": null, - "variable_start": -2, - "variable_end": -2, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 2.515950268099853, - "valid": false - } - } - ] - }, { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", @@ -6326,80 +5786,26 @@ }, { "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "IPv4", - "severity": "info", - "confidence": "weak", + "ml_validation": "VALIDATED_KEY", + "ml_probability": 0.997, + "rule": "Password", + "severity": "medium", + "confidence": "moderate", "line_data_list": [ { "line": "98.76.54.32 ANY_PW:IhqSb1Gg", "line_num": 34, "path": "tests/samples/doc_various", "info": "", - "value": "98.76.54.32", - "value_start": 0, - "value_end": 11, - "variable": null, - "variable_start": -2, - "variable_end": -2, + "value": "IhqSb1Gg", + "value_start": 19, + "value_end": 27, + "variable": "ANY_PW", + "variable_start": 12, + "variable_end": 18, "entropy_validation": { "iterator": "BASE64_CHARS", - "entropy": 2.515950268099853, - "valid": false - } - } - ] - }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.997, - "rule": "Password", - "severity": "medium", - "confidence": "moderate", - "line_data_list": [ - { - "line": "98.76.54.32 ANY_PW:IhqSb1Gg", - "line_num": 34, - "path": "tests/samples/doc_various", - "info": "", - "value": "IhqSb1Gg", - "value_start": 19, - "value_end": 27, - "variable": "ANY_PW", - "variable_start": 12, - "variable_end": 18, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 3.0, - "valid": false - } - } - ] - }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "IPv4", - "severity": "info", - "confidence": "weak", - "line_data_list": [ - { - "line": "98.76.54.32(ID/PW:IhqSb1Gg)", - "line_num": 36, - "path": "tests/samples/doc_various", - "info": "", - "value": "98.76.54.32", - "value_start": 0, - "value_end": 11, - "variable": null, - "variable_start": -2, - "variable_end": -2, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 2.515950268099853, + "entropy": 3.0, "valid": false } } @@ -6432,33 +5838,6 @@ } ] }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "IPv4", - "severity": "info", - "confidence": "weak", - "line_data_list": [ - { - "line": "gildong.hong@98.76.54.32 (pwd:IhqSb1Gg)", - "line_num": 38, - "path": "tests/samples/doc_various", - "info": "", - "value": "98.76.54.32", - "value_start": 13, - "value_end": 24, - "variable": null, - "variable_start": -2, - "variable_end": -2, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 2.515950268099853, - "valid": false - } - } - ] - }, { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", @@ -6513,33 +5892,6 @@ } ] }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "IPv4", - "severity": "info", - "confidence": "weak", - "line_data_list": [ - { - "line": "gildong.hong@98.76.54.32(master/IhqSb1Gg)", - "line_num": 44, - "path": "tests/samples/doc_various", - "info": "", - "value": "98.76.54.32", - "value_start": 13, - "value_end": 24, - "variable": null, - "variable_start": -2, - "variable_end": -2, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 2.515950268099853, - "valid": false - } - } - ] - }, { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", @@ -6569,26 +5921,26 @@ }, { "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "IPv4", - "severity": "info", - "confidence": "weak", + "ml_validation": "VALIDATED_KEY", + "ml_probability": 0.1, + "rule": "Password", + "severity": "medium", + "confidence": "moderate", "line_data_list": [ { "line": "gildong.hong@98.76.54.32,pw:IhqSb1Gg", "line_num": 47, "path": "tests/samples/doc_various", "info": "", - "value": "98.76.54.32", - "value_start": 13, - "value_end": 24, - "variable": null, - "variable_start": -2, - "variable_end": -2, + "value": "IhqSb1Gg", + "value_start": 28, + "value_end": 36, + "variable": "gildong.hong@98.76.54.32,pw", + "variable_start": 0, + "variable_end": 27, "entropy_validation": { "iterator": "BASE64_CHARS", - "entropy": 2.515950268099853, + "entropy": 3.0, "valid": false } } @@ -6597,25 +5949,25 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.1, + "ml_probability": 0.896, "rule": "Password", "severity": "medium", "confidence": "moderate", "line_data_list": [ { - "line": "gildong.hong@98.76.54.32,pw:IhqSb1Gg", - "line_num": 47, + "line": "98.76.54.32:xxxx(PW:IhqSb1Gg)", + "line_num": 51, "path": "tests/samples/doc_various", "info": "", - "value": "IhqSb1Gg", - "value_start": 28, - "value_end": 36, - "variable": "gildong.hong@98.76.54.32,pw", - "variable_start": 0, - "variable_end": 27, + "value": "IhqSb1Gg)", + "value_start": 20, + "value_end": 29, + "variable": "PW", + "variable_start": 17, + "variable_end": 19, "entropy_validation": { "iterator": "BASE64_CHARS", - "entropy": 3.0, + "entropy": 2.8177111123931664, "valid": false } } @@ -6623,26 +5975,26 @@ }, { "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "IPv4", - "severity": "info", - "confidence": "weak", + "ml_validation": "VALIDATED_KEY", + "ml_probability": 0.01, + "rule": "Password", + "severity": "medium", + "confidence": "moderate", "line_data_list": [ { - "line": "98.76.54.32(master/IhqSb1Gg,master/IhqSb1Gg)", - "line_num": 48, + "line": "\uacc4\uc815/Password-xxxx:master/IhqSb1Gg", + "line_num": 55, "path": "tests/samples/doc_various", "info": "", - "value": "98.76.54.32", - "value_start": 0, - "value_end": 11, - "variable": null, - "variable_start": -2, - "variable_end": -2, + "value": "master/IhqSb1Gg", + "value_start": 17, + "value_end": 32, + "variable": "Password-xxxx", + "variable_start": 3, + "variable_end": 16, "entropy_validation": { "iterator": "BASE64_CHARS", - "entropy": 2.515950268099853, + "entropy": 3.906890595608518, "valid": false } } @@ -6650,26 +6002,26 @@ }, { "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "IPv4", - "severity": "info", - "confidence": "weak", + "ml_validation": "VALIDATED_KEY", + "ml_probability": 0.949, + "rule": "Password", + "severity": "medium", + "confidence": "moderate", "line_data_list": [ { - "line": "98.76.54.32(master/IhqSb1Gg master/IhqSb1Gg)", - "line_num": 49, + "line": "gildong.hong@98.76.54.32 PW:IhqSb1Gg", + "line_num": 56, "path": "tests/samples/doc_various", "info": "", - "value": "98.76.54.32", - "value_start": 0, - "value_end": 11, - "variable": null, - "variable_start": -2, - "variable_end": -2, + "value": "IhqSb1Gg", + "value_start": 28, + "value_end": 36, + "variable": "PW", + "variable_start": 25, + "variable_end": 27, "entropy_validation": { "iterator": "BASE64_CHARS", - "entropy": 2.515950268099853, + "entropy": 3.0, "valid": false } } @@ -6677,26 +6029,26 @@ }, { "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "IPv4", - "severity": "info", - "confidence": "weak", + "ml_validation": "VALIDATED_KEY", + "ml_probability": 0.999, + "rule": "Password", + "severity": "medium", + "confidence": "moderate", "line_data_list": [ { - "line": "xxxx:98.76.54.32(master/IhqSb1Gg)", - "line_num": 50, + "line": "ANY_user:xxxx ANY_pwd:IhqSb1Gg", + "line_num": 61, "path": "tests/samples/doc_various", "info": "", - "value": "98.76.54.32", - "value_start": 5, - "value_end": 16, - "variable": null, - "variable_start": -2, - "variable_end": -2, + "value": "IhqSb1Gg", + "value_start": 22, + "value_end": 30, + "variable": "ANY_pwd", + "variable_start": 14, + "variable_end": 21, "entropy_validation": { "iterator": "BASE64_CHARS", - "entropy": 2.515950268099853, + "entropy": 3.0, "valid": false } } @@ -6704,26 +6056,26 @@ }, { "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "IPv4", - "severity": "info", - "confidence": "weak", + "ml_validation": "VALIDATED_KEY", + "ml_probability": 0.998, + "rule": "Password", + "severity": "medium", + "confidence": "moderate", "line_data_list": [ { - "line": "98.76.54.32:xxxx(PW:IhqSb1Gg)", - "line_num": 51, + "line": "Acount name:xxxx Initial Password:IhqSb1Gg", + "line_num": 62, "path": "tests/samples/doc_various", "info": "", - "value": "98.76.54.32", - "value_start": 0, - "value_end": 11, - "variable": null, - "variable_start": -2, - "variable_end": -2, + "value": "IhqSb1Gg", + "value_start": 34, + "value_end": 42, + "variable": "Password", + "variable_start": 25, + "variable_end": 33, "entropy_validation": { "iterator": "BASE64_CHARS", - "entropy": 2.515950268099853, + "entropy": 3.0, "valid": false } } @@ -6732,14 +6084,14 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.896, + "ml_probability": 0.953, "rule": "Password", "severity": "medium", "confidence": "moderate", "line_data_list": [ { - "line": "98.76.54.32:xxxx(PW:IhqSb1Gg)", - "line_num": 51, + "line": "Access wifi:xxxx(PW:IhqSb1Gg)", + "line_num": 63, "path": "tests/samples/doc_various", "info": "", "value": "IhqSb1Gg)", @@ -6759,25 +6111,25 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.01, + "ml_probability": 0.998, "rule": "Password", "severity": "medium", "confidence": "moderate", "line_data_list": [ { - "line": "\uacc4\uc815/Password-xxxx:master/IhqSb1Gg", - "line_num": 55, + "line": "-User:master -PasswordANY:IhqSb1Gg", + "line_num": 66, "path": "tests/samples/doc_various", "info": "", - "value": "master/IhqSb1Gg", - "value_start": 17, - "value_end": 32, - "variable": "Password-xxxx", - "variable_start": 3, - "variable_end": 16, + "value": "IhqSb1Gg", + "value_start": 26, + "value_end": 34, + "variable": "PasswordANY", + "variable_start": 14, + "variable_end": 25, "entropy_validation": { "iterator": "BASE64_CHARS", - "entropy": 3.906890595608518, + "entropy": 3.0, "valid": false } } @@ -6785,26 +6137,26 @@ }, { "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "IPv4", - "severity": "info", - "confidence": "weak", + "ml_validation": "VALIDATED_KEY", + "ml_probability": 0.998, + "rule": "Password", + "severity": "medium", + "confidence": "moderate", "line_data_list": [ { - "line": "gildong.hong@98.76.54.32 PW:IhqSb1Gg", - "line_num": 56, + "line": "password(default:IhqSb1Gg)", + "line_num": 68, "path": "tests/samples/doc_various", "info": "", - "value": "98.76.54.32", - "value_start": 13, - "value_end": 24, - "variable": null, - "variable_start": -2, - "variable_end": -2, + "value": "IhqSb1Gg)", + "value_start": 17, + "value_end": 26, + "variable": "password(default", + "variable_start": 0, + "variable_end": 16, "entropy_validation": { "iterator": "BASE64_CHARS", - "entropy": 2.515950268099853, + "entropy": 2.8177111123931664, "valid": false } } @@ -6813,25 +6165,25 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.949, + "ml_probability": 0.131, "rule": "Password", "severity": "medium", "confidence": "moderate", "line_data_list": [ { - "line": "gildong.hong@98.76.54.32 PW:IhqSb1Gg", - "line_num": 56, + "line": "98.76.54.32(ID/PW:master/IhqSb1Gg)", + "line_num": 70, "path": "tests/samples/doc_various", "info": "", - "value": "IhqSb1Gg", - "value_start": 28, - "value_end": 36, + "value": "master/IhqSb1Gg)", + "value_start": 18, + "value_end": 34, "variable": "PW", - "variable_start": 25, - "variable_end": 27, + "variable_start": 15, + "variable_end": 17, "entropy_validation": { "iterator": "BASE64_CHARS", - "entropy": 3.0, + "entropy": 3.75, "valid": false } } @@ -6839,26 +6191,26 @@ }, { "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "IPv4", - "severity": "info", - "confidence": "weak", + "ml_validation": "VALIDATED_KEY", + "ml_probability": 0.603, + "rule": "Password", + "severity": "medium", + "confidence": "moderate", "line_data_list": [ { - "line": "http|https://98.76.54.32/xxxx(master/IhqSb1Gg)", - "line_num": 59, + "line": "(ID&PWD):master/IhqSb1Gg", + "line_num": 71, "path": "tests/samples/doc_various", "info": "", - "value": "98.76.54.32", - "value_start": 13, + "value": "master/IhqSb1Gg", + "value_start": 9, "value_end": 24, - "variable": null, - "variable_start": -2, - "variable_end": -2, + "variable": "PWD)", + "variable_start": 4, + "variable_end": 8, "entropy_validation": { "iterator": "BASE64_CHARS", - "entropy": 2.515950268099853, + "entropy": 3.906890595608518, "valid": false } } @@ -6867,25 +6219,25 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.999, + "ml_probability": 0.956, "rule": "Password", "severity": "medium", "confidence": "moderate", "line_data_list": [ { - "line": "ANY_user:xxxx ANY_pwd:IhqSb1Gg", - "line_num": 61, + "line": "master@98.76.54.32(pw:IhqSb1Gg)", + "line_num": 73, "path": "tests/samples/doc_various", "info": "", - "value": "IhqSb1Gg", + "value": "IhqSb1Gg)", "value_start": 22, - "value_end": 30, - "variable": "ANY_pwd", - "variable_start": 14, + "value_end": 31, + "variable": "pw", + "variable_start": 19, "variable_end": 21, "entropy_validation": { "iterator": "BASE64_CHARS", - "entropy": 3.0, + "entropy": 2.8177111123931664, "valid": false } } @@ -6894,22 +6246,22 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.998, + "ml_probability": 0.029, "rule": "Password", "severity": "medium", "confidence": "moderate", "line_data_list": [ { - "line": "Acount name:xxxx Initial Password:IhqSb1Gg", - "line_num": 62, + "line": "master@98.76.54.32,PW:IhqSb1Gg", + "line_num": 74, "path": "tests/samples/doc_various", "info": "", "value": "IhqSb1Gg", - "value_start": 34, - "value_end": 42, - "variable": "Password", - "variable_start": 25, - "variable_end": 33, + "value_start": 22, + "value_end": 30, + "variable": "master@98.76.54.32,PW", + "variable_start": 0, + "variable_end": 21, "entropy_validation": { "iterator": "BASE64_CHARS", "entropy": 3.0, @@ -6921,25 +6273,25 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.953, + "ml_probability": 0.99, "rule": "Password", "severity": "medium", "confidence": "moderate", "line_data_list": [ { - "line": "Access wifi:xxxx(PW:IhqSb1Gg)", - "line_num": 63, + "line": "98.76.54.32 pw:IhqSb1Gg", + "line_num": 75, "path": "tests/samples/doc_various", "info": "", - "value": "IhqSb1Gg)", - "value_start": 20, - "value_end": 29, - "variable": "PW", - "variable_start": 17, - "variable_end": 19, + "value": "IhqSb1Gg", + "value_start": 15, + "value_end": 23, + "variable": "pw", + "variable_start": 12, + "variable_end": 14, "entropy_validation": { "iterator": "BASE64_CHARS", - "entropy": 2.8177111123931664, + "entropy": 3.0, "valid": false } } @@ -6948,22 +6300,22 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.998, + "ml_probability": 0.976, "rule": "Password", "severity": "medium", "confidence": "moderate", "line_data_list": [ { - "line": "-User:master -PasswordANY:IhqSb1Gg", - "line_num": 66, + "line": "config:xxxx,PW:IhqSb1Gg", + "line_num": 78, "path": "tests/samples/doc_various", "info": "", "value": "IhqSb1Gg", - "value_start": 26, - "value_end": 34, - "variable": "PasswordANY", - "variable_start": 14, - "variable_end": 25, + "value_start": 15, + "value_end": 23, + "variable": "xxxx,PW", + "variable_start": 7, + "variable_end": 14, "entropy_validation": { "iterator": "BASE64_CHARS", "entropy": 3.0, @@ -6975,52 +6327,25 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.998, + "ml_probability": 0.988, "rule": "Password", "severity": "medium", "confidence": "moderate", "line_data_list": [ { - "line": "password(default:IhqSb1Gg)", - "line_num": 68, - "path": "tests/samples/doc_various", - "info": "", - "value": "IhqSb1Gg)", - "value_start": 17, - "value_end": 26, - "variable": "password(default", - "variable_start": 0, - "variable_end": 16, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 2.8177111123931664, - "valid": false - } - } - ] - }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "IPv4", - "severity": "info", - "confidence": "weak", - "line_data_list": [ - { - "line": "98.76.54.32(ID/PW:master/IhqSb1Gg)", - "line_num": 70, + "line": "scp gildong.hong@98.76.54.32 mailto:{1} pw:IhqSb1Gg", + "line_num": 82, "path": "tests/samples/doc_various", "info": "", - "value": "98.76.54.32", - "value_start": 0, - "value_end": 11, - "variable": null, - "variable_start": -2, - "variable_end": -2, + "value": "IhqSb1Gg", + "value_start": 43, + "value_end": 51, + "variable": "pw", + "variable_start": 40, + "variable_end": 42, "entropy_validation": { "iterator": "BASE64_CHARS", - "entropy": 2.515950268099853, + "entropy": 3.0, "valid": false } } @@ -7029,25 +6354,25 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.131, + "ml_probability": 0.952, "rule": "Password", "severity": "medium", "confidence": "moderate", "line_data_list": [ { - "line": "98.76.54.32(ID/PW:master/IhqSb1Gg)", - "line_num": 70, + "line": "id:gildong.hong@xxx.com mailto:{1} pw:IhqSb1Gg", + "line_num": 83, "path": "tests/samples/doc_various", "info": "", - "value": "master/IhqSb1Gg)", - "value_start": 18, - "value_end": 34, - "variable": "PW", - "variable_start": 15, - "variable_end": 17, + "value": "IhqSb1Gg", + "value_start": 38, + "value_end": 46, + "variable": "pw", + "variable_start": 35, + "variable_end": 37, "entropy_validation": { "iterator": "BASE64_CHARS", - "entropy": 3.75, + "entropy": 3.0, "valid": false } } @@ -7056,25 +6381,25 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.603, + "ml_probability": 0.943, "rule": "Password", "severity": "medium", "confidence": "moderate", "line_data_list": [ { - "line": "(ID&PWD):master/IhqSb1Gg", - "line_num": 71, + "line": "gildong.hong@98.76.54.32 mailto:{1} pw:IhqSb1Gg", + "line_num": 84, "path": "tests/samples/doc_various", "info": "", - "value": "master/IhqSb1Gg", - "value_start": 9, - "value_end": 24, - "variable": "PWD)", - "variable_start": 4, - "variable_end": 8, + "value": "IhqSb1Gg", + "value_start": 39, + "value_end": 47, + "variable": "pw", + "variable_start": 36, + "variable_end": 38, "entropy_validation": { "iterator": "BASE64_CHARS", - "entropy": 3.906890595608518, + "entropy": 3.0, "valid": false } } @@ -7082,458 +6407,26 @@ }, { "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "IPv4", - "severity": "info", - "confidence": "weak", + "ml_validation": "VALIDATED_KEY", + "ml_probability": 0.996, + "rule": "Password", + "severity": "medium", + "confidence": "moderate", "line_data_list": [ { - "line": "master@98.76.54.32(pw:IhqSb1Gg)", - "line_num": 73, + "line": "ssh gildong.hong@98.76.54.32 mailto:{1},pw:IhqSb1Gg", + "line_num": 85, "path": "tests/samples/doc_various", "info": "", - "value": "98.76.54.32", - "value_start": 7, - "value_end": 18, - "variable": null, - "variable_start": -2, - "variable_end": -2, + "value": "IhqSb1Gg", + "value_start": 43, + "value_end": 51, + "variable": "pw", + "variable_start": 40, + "variable_end": 42, "entropy_validation": { "iterator": "BASE64_CHARS", - "entropy": 2.515950268099853, - "valid": false - } - } - ] - }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.956, - "rule": "Password", - "severity": "medium", - "confidence": "moderate", - "line_data_list": [ - { - "line": "master@98.76.54.32(pw:IhqSb1Gg)", - "line_num": 73, - "path": "tests/samples/doc_various", - "info": "", - "value": "IhqSb1Gg)", - "value_start": 22, - "value_end": 31, - "variable": "pw", - "variable_start": 19, - "variable_end": 21, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 2.8177111123931664, - "valid": false - } - } - ] - }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "IPv4", - "severity": "info", - "confidence": "weak", - "line_data_list": [ - { - "line": "master@98.76.54.32,PW:IhqSb1Gg", - "line_num": 74, - "path": "tests/samples/doc_various", - "info": "", - "value": "98.76.54.32", - "value_start": 7, - "value_end": 18, - "variable": null, - "variable_start": -2, - "variable_end": -2, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 2.515950268099853, - "valid": false - } - } - ] - }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.029, - "rule": "Password", - "severity": "medium", - "confidence": "moderate", - "line_data_list": [ - { - "line": "master@98.76.54.32,PW:IhqSb1Gg", - "line_num": 74, - "path": "tests/samples/doc_various", - "info": "", - "value": "IhqSb1Gg", - "value_start": 22, - "value_end": 30, - "variable": "master@98.76.54.32,PW", - "variable_start": 0, - "variable_end": 21, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 3.0, - "valid": false - } - } - ] - }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "IPv4", - "severity": "info", - "confidence": "weak", - "line_data_list": [ - { - "line": "98.76.54.32 pw:IhqSb1Gg", - "line_num": 75, - "path": "tests/samples/doc_various", - "info": "", - "value": "98.76.54.32", - "value_start": 0, - "value_end": 11, - "variable": null, - "variable_start": -2, - "variable_end": -2, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 2.515950268099853, - "valid": false - } - } - ] - }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.99, - "rule": "Password", - "severity": "medium", - "confidence": "moderate", - "line_data_list": [ - { - "line": "98.76.54.32 pw:IhqSb1Gg", - "line_num": 75, - "path": "tests/samples/doc_various", - "info": "", - "value": "IhqSb1Gg", - "value_start": 15, - "value_end": 23, - "variable": "pw", - "variable_start": 12, - "variable_end": 14, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 3.0, - "valid": false - } - } - ] - }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.976, - "rule": "Password", - "severity": "medium", - "confidence": "moderate", - "line_data_list": [ - { - "line": "config:xxxx,PW:IhqSb1Gg", - "line_num": 78, - "path": "tests/samples/doc_various", - "info": "", - "value": "IhqSb1Gg", - "value_start": 15, - "value_end": 23, - "variable": "xxxx,PW", - "variable_start": 7, - "variable_end": 14, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 3.0, - "valid": false - } - } - ] - }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "IPv4", - "severity": "info", - "confidence": "weak", - "line_data_list": [ - { - "line": "98.76.54.32-->master/IhqSb1Gg", - "line_num": 81, - "path": "tests/samples/doc_various", - "info": "", - "value": "98.76.54.32", - "value_start": 0, - "value_end": 11, - "variable": null, - "variable_start": -2, - "variable_end": -2, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 2.515950268099853, - "valid": false - } - } - ] - }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "IPv4", - "severity": "info", - "confidence": "weak", - "line_data_list": [ - { - "line": "scp gildong.hong@98.76.54.32 mailto:{1} pw:IhqSb1Gg", - "line_num": 82, - "path": "tests/samples/doc_various", - "info": "", - "value": "98.76.54.32", - "value_start": 17, - "value_end": 28, - "variable": null, - "variable_start": -2, - "variable_end": -2, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 2.515950268099853, - "valid": false - } - } - ] - }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.988, - "rule": "Password", - "severity": "medium", - "confidence": "moderate", - "line_data_list": [ - { - "line": "scp gildong.hong@98.76.54.32 mailto:{1} pw:IhqSb1Gg", - "line_num": 82, - "path": "tests/samples/doc_various", - "info": "", - "value": "IhqSb1Gg", - "value_start": 43, - "value_end": 51, - "variable": "pw", - "variable_start": 40, - "variable_end": 42, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 3.0, - "valid": false - } - } - ] - }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.952, - "rule": "Password", - "severity": "medium", - "confidence": "moderate", - "line_data_list": [ - { - "line": "id:gildong.hong@xxx.com mailto:{1} pw:IhqSb1Gg", - "line_num": 83, - "path": "tests/samples/doc_various", - "info": "", - "value": "IhqSb1Gg", - "value_start": 38, - "value_end": 46, - "variable": "pw", - "variable_start": 35, - "variable_end": 37, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 3.0, - "valid": false - } - } - ] - }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "IPv4", - "severity": "info", - "confidence": "weak", - "line_data_list": [ - { - "line": "gildong.hong@98.76.54.32 mailto:{1} pw:IhqSb1Gg", - "line_num": 84, - "path": "tests/samples/doc_various", - "info": "", - "value": "98.76.54.32", - "value_start": 13, - "value_end": 24, - "variable": null, - "variable_start": -2, - "variable_end": -2, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 2.515950268099853, - "valid": false - } - } - ] - }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.943, - "rule": "Password", - "severity": "medium", - "confidence": "moderate", - "line_data_list": [ - { - "line": "gildong.hong@98.76.54.32 mailto:{1} pw:IhqSb1Gg", - "line_num": 84, - "path": "tests/samples/doc_various", - "info": "", - "value": "IhqSb1Gg", - "value_start": 39, - "value_end": 47, - "variable": "pw", - "variable_start": 36, - "variable_end": 38, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 3.0, - "valid": false - } - } - ] - }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "IPv4", - "severity": "info", - "confidence": "weak", - "line_data_list": [ - { - "line": "ssh gildong.hong@98.76.54.32 mailto:{1},pw:IhqSb1Gg", - "line_num": 85, - "path": "tests/samples/doc_various", - "info": "", - "value": "98.76.54.32", - "value_start": 17, - "value_end": 28, - "variable": null, - "variable_start": -2, - "variable_end": -2, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 2.515950268099853, - "valid": false - } - } - ] - }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.996, - "rule": "Password", - "severity": "medium", - "confidence": "moderate", - "line_data_list": [ - { - "line": "ssh gildong.hong@98.76.54.32 mailto:{1},pw:IhqSb1Gg", - "line_num": 85, - "path": "tests/samples/doc_various", - "info": "", - "value": "IhqSb1Gg", - "value_start": 43, - "value_end": 51, - "variable": "pw", - "variable_start": 40, - "variable_end": 42, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 3.0, - "valid": false - } - } - ] - }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "IPv4", - "severity": "info", - "confidence": "weak", - "line_data_list": [ - { - "line": "gildong.hong@98.76.54.32 mailto:{1} (master/IhqSb1Gg)", - "line_num": 86, - "path": "tests/samples/doc_various", - "info": "", - "value": "98.76.54.32", - "value_start": 13, - "value_end": 24, - "variable": null, - "variable_start": -2, - "variable_end": -2, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 2.515950268099853, - "valid": false - } - } - ] - }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "IPv4", - "severity": "info", - "confidence": "weak", - "line_data_list": [ - { - "line": "(ssh gildong.hong@98.76.54.32 mailto{1}) pwd:IhqSb1Gg", - "line_num": 87, - "path": "tests/samples/doc_various", - "info": "", - "value": "98.76.54.32", - "value_start": 18, - "value_end": 29, - "variable": null, - "variable_start": -2, - "variable_end": -2, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 2.515950268099853, + "entropy": 3.0, "valid": false } } @@ -7543,186 +6436,24 @@ "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", "ml_probability": 0.995, - "rule": "Password", - "severity": "medium", - "confidence": "moderate", - "line_data_list": [ - { - "line": "(ssh gildong.hong@98.76.54.32 mailto{1}) pwd:IhqSb1Gg", - "line_num": 87, - "path": "tests/samples/doc_various", - "info": "", - "value": "IhqSb1Gg", - "value_start": 45, - "value_end": 53, - "variable": "pwd", - "variable_start": 41, - "variable_end": 44, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 3.0, - "valid": false - } - } - ] - }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "IPv4", - "severity": "info", - "confidence": "weak", - "line_data_list": [ - { - "line": "ssh gildong.hong@98.76.54.32 (master/IhqSb1Gg)", - "line_num": 88, - "path": "tests/samples/doc_various", - "info": "", - "value": "98.76.54.32", - "value_start": 17, - "value_end": 28, - "variable": null, - "variable_start": -2, - "variable_end": -2, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 2.515950268099853, - "valid": false - } - } - ] - }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "IPv4", - "severity": "info", - "confidence": "weak", - "line_data_list": [ - { - "line": "ssh gildong.hong@98.76.54.32 mailto:[1} \uacc4\uc815master/IhqSb1Gg", - "line_num": 89, - "path": "tests/samples/doc_various", - "info": "", - "value": "98.76.54.32", - "value_start": 17, - "value_end": 28, - "variable": null, - "variable_start": -2, - "variable_end": -2, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 2.515950268099853, - "valid": false - } - } - ] - }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "IPv4", - "severity": "info", - "confidence": "weak", - "line_data_list": [ - { - "line": "ssh gildong.hong@98.76.54.32 mailto:{1}, pw:IhqSb1Gg", - "line_num": 90, - "path": "tests/samples/doc_various", - "info": "", - "value": "98.76.54.32", - "value_start": 17, - "value_end": 28, - "variable": null, - "variable_start": -2, - "variable_end": -2, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 2.515950268099853, - "valid": false - } - } - ] - }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.996, - "rule": "Password", - "severity": "medium", - "confidence": "moderate", - "line_data_list": [ - { - "line": "ssh gildong.hong@98.76.54.32 mailto:{1}, pw:IhqSb1Gg", - "line_num": 90, - "path": "tests/samples/doc_various", - "info": "", - "value": "IhqSb1Gg", - "value_start": 44, - "value_end": 52, - "variable": "pw", - "variable_start": 41, - "variable_end": 43, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 3.0, - "valid": false - } - } - ] - }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "IPv4", - "severity": "info", - "confidence": "weak", - "line_data_list": [ - { - "line": "gildong.hong@98.76.54.32 mailto:{1} (\ube44\ubc88 IhqSb1Gg)", - "line_num": 91, - "path": "tests/samples/doc_various", - "info": "", - "value": "98.76.54.32", - "value_start": 13, - "value_end": 24, - "variable": null, - "variable_start": -2, - "variable_end": -2, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 2.515950268099853, - "valid": false - } - } - ] - }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "IPv4", - "severity": "info", - "confidence": "weak", + "rule": "Password", + "severity": "medium", + "confidence": "moderate", "line_data_list": [ { - "line": "ssh gildong.hong@98.76.54.32 mailto:{1} password IhqSb1Gg", - "line_num": 92, + "line": "(ssh gildong.hong@98.76.54.32 mailto{1}) pwd:IhqSb1Gg", + "line_num": 87, "path": "tests/samples/doc_various", "info": "", - "value": "98.76.54.32", - "value_start": 17, - "value_end": 28, - "variable": null, - "variable_start": -2, - "variable_end": -2, + "value": "IhqSb1Gg", + "value_start": 45, + "value_end": 53, + "variable": "pwd", + "variable_start": 41, + "variable_end": 44, "entropy_validation": { "iterator": "BASE64_CHARS", - "entropy": 2.515950268099853, + "entropy": 3.0, "valid": false } } @@ -7730,26 +6461,26 @@ }, { "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "IPv4", - "severity": "info", - "confidence": "weak", + "ml_validation": "VALIDATED_KEY", + "ml_probability": 0.996, + "rule": "Password", + "severity": "medium", + "confidence": "moderate", "line_data_list": [ { - "line": "ssh gildong.hong@98.76.54.32 mailto:{1} (pwd:IhqSb1Gg)", - "line_num": 93, + "line": "ssh gildong.hong@98.76.54.32 mailto:{1}, pw:IhqSb1Gg", + "line_num": 90, "path": "tests/samples/doc_various", "info": "", - "value": "98.76.54.32", - "value_start": 17, - "value_end": 28, - "variable": null, - "variable_start": -2, - "variable_end": -2, + "value": "IhqSb1Gg", + "value_start": 44, + "value_end": 52, + "variable": "pw", + "variable_start": 41, + "variable_end": 43, "entropy_validation": { "iterator": "BASE64_CHARS", - "entropy": 2.515950268099853, + "entropy": 3.0, "valid": false } } @@ -7782,33 +6513,6 @@ } ] }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "IPv4", - "severity": "info", - "confidence": "weak", - "line_data_list": [ - { - "line": "gildong.hong@98.76.54.32 mailto:{1} (password:IhqSb1Gg)", - "line_num": 94, - "path": "tests/samples/doc_various", - "info": "", - "value": "98.76.54.32", - "value_start": 13, - "value_end": 24, - "variable": null, - "variable_start": -2, - "variable_end": -2, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 2.515950268099853, - "valid": false - } - } - ] - }, { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", @@ -7836,33 +6540,6 @@ } ] }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "IPv4", - "severity": "info", - "confidence": "weak", - "line_data_list": [ - { - "line": "gildong.hong@98.76.54.32 mailto:{1} Password:IhqSb1Gg", - "line_num": 96, - "path": "tests/samples/doc_various", - "info": "", - "value": "98.76.54.32", - "value_start": 13, - "value_end": 24, - "variable": null, - "variable_start": -2, - "variable_end": -2, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 2.515950268099853, - "valid": false - } - } - ] - }, { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", @@ -7890,33 +6567,6 @@ } ] }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "IPv4", - "severity": "info", - "confidence": "weak", - "line_data_list": [ - { - "line": "gildong.hong@98.76.54.32 mailto:{1} (pass:IhqSb1Gg)", - "line_num": 97, - "path": "tests/samples/doc_various", - "info": "", - "value": "98.76.54.32", - "value_start": 13, - "value_end": 24, - "variable": null, - "variable_start": -2, - "variable_end": -2, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 2.515950268099853, - "valid": false - } - } - ] - }, { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", @@ -7944,60 +6594,6 @@ } ] }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "IPv4", - "severity": "info", - "confidence": "weak", - "line_data_list": [ - { - "line": "ssh gildong.hong@98.76.54.32 mailto:{1} (xxxx//IhqSb1Gg)", - "line_num": 99, - "path": "tests/samples/doc_various", - "info": "", - "value": "98.76.54.32", - "value_start": 17, - "value_end": 28, - "variable": null, - "variable_start": -2, - "variable_end": -2, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 2.515950268099853, - "valid": false - } - } - ] - }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "IPv4", - "severity": "info", - "confidence": "weak", - "line_data_list": [ - { - "line": "ssh gildong.hong@98.76.54.32 mailto:{1} pw:IhqSb1Gg", - "line_num": 100, - "path": "tests/samples/doc_various", - "info": "", - "value": "98.76.54.32", - "value_start": 17, - "value_end": 28, - "variable": null, - "variable_start": -2, - "variable_end": -2, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 2.515950268099853, - "valid": false - } - } - ] - }, { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", @@ -8025,33 +6621,6 @@ } ] }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "IPv4", - "severity": "info", - "confidence": "weak", - "line_data_list": [ - { - "line": "ssh gildong.hong@98.76.54.32 mailto:{1} pass:IhqSb1Gg", - "line_num": 101, - "path": "tests/samples/doc_various", - "info": "", - "value": "98.76.54.32", - "value_start": 17, - "value_end": 28, - "variable": null, - "variable_start": -2, - "variable_end": -2, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 2.515950268099853, - "valid": false - } - } - ] - }, { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", @@ -8106,33 +6675,6 @@ } ] }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "IPv4", - "severity": "info", - "confidence": "weak", - "line_data_list": [ - { - "line": "ssh gildong.hong@98.76.54.32 mailto:{1}/password:IhqSb1Gg", - "line_num": 104, - "path": "tests/samples/doc_various", - "info": "", - "value": "98.76.54.32", - "value_start": 17, - "value_end": 28, - "variable": null, - "variable_start": -2, - "variable_end": -2, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 2.515950268099853, - "valid": false - } - } - ] - }, { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", @@ -8208,61 +6750,7 @@ "variable_end": 38, "entropy_validation": { "iterator": "BASE64_CHARS", - "entropy": 3.0, - "valid": false - } - } - ] - }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "IPv4", - "severity": "info", - "confidence": "weak", - "line_data_list": [ - { - "line": "ID/PW 98.76.54.32:xxx master/IhqSb1Gg", - "line_num": 107, - "path": "tests/samples/doc_various", - "info": "", - "value": "98.76.54.32", - "value_start": 6, - "value_end": 17, - "variable": null, - "variable_start": -2, - "variable_end": -2, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 2.515950268099853, - "valid": false - } - } - ] - }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "IPv4", - "severity": "info", - "confidence": "weak", - "line_data_list": [ - { - "line": "http://98.76.54.32:xxx(pw:IhqSb1Gg)", - "line_num": 108, - "path": "tests/samples/doc_various", - "info": "", - "value": "98.76.54.32", - "value_start": 7, - "value_end": 18, - "variable": null, - "variable_start": -2, - "variable_end": -2, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 2.515950268099853, + "entropy": 3.0, "valid": false } } @@ -9166,330 +7654,6 @@ } ] }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "IPv4", - "severity": "info", - "confidence": "weak", - "line_data_list": [ - { - "line": "100.64.0.0/10", - "line_num": 13, - "path": "tests/samples/ipv4", - "info": "", - "value": "100.64.0.0", - "value_start": 0, - "value_end": 10, - "variable": null, - "variable_start": -2, - "variable_end": -2, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 1.5253496664211537, - "valid": false - } - } - ] - }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "IPv4", - "severity": "info", - "confidence": "weak", - "line_data_list": [ - { - "line": "100.64.0.0\u2013100.127.255.255", - "line_num": 14, - "path": "tests/samples/ipv4", - "info": "", - "value": "100.127.255.255", - "value_start": 11, - "value_end": 26, - "variable": null, - "variable_start": -2, - "variable_end": -2, - "entropy_validation": { - "iterator": "BASE36_CHARS", - "entropy": 2.008519976342584, - "valid": false - } - } - ] - }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "IPv4", - "severity": "info", - "confidence": "weak", - "line_data_list": [ - { - "line": "100.64.0.0\u2013100.127.255.255", - "line_num": 14, - "path": "tests/samples/ipv4", - "info": "", - "value": "100.64.0.0", - "value_start": 0, - "value_end": 10, - "variable": null, - "variable_start": -2, - "variable_end": -2, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 1.5253496664211537, - "valid": false - } - } - ] - }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "IPv4", - "severity": "info", - "confidence": "weak", - "line_data_list": [ - { - "line": "192.0.0.0\u2013192.0.0.255", - "line_num": 22, - "path": "tests/samples/ipv4", - "info": "", - "value": "192.0.0.255", - "value_start": 10, - "value_end": 21, - "variable": null, - "variable_start": -2, - "variable_end": -2, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 1.9704957226453073, - "valid": false - } - } - ] - }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "IPv4", - "severity": "info", - "confidence": "weak", - "line_data_list": [ - { - "line": "192.88.99.0/24", - "line_num": 25, - "path": "tests/samples/ipv4", - "info": "", - "value": "192.88.99.0", - "value_start": 0, - "value_end": 11, - "variable": null, - "variable_start": -2, - "variable_end": -2, - "entropy_validation": { - "iterator": "BASE36_CHARS", - "entropy": 1.9018695860849921, - "valid": false - } - } - ] - }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "IPv4", - "severity": "info", - "confidence": "weak", - "line_data_list": [ - { - "line": "192.88.99.0\u2013192.88.99.255", - "line_num": 26, - "path": "tests/samples/ipv4", - "info": "", - "value": "192.88.99.0", - "value_start": 0, - "value_end": 11, - "variable": null, - "variable_start": -2, - "variable_end": -2, - "entropy_validation": { - "iterator": "BASE36_CHARS", - "entropy": 1.9018695860849921, - "valid": false - } - } - ] - }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "IPv4", - "severity": "info", - "confidence": "weak", - "line_data_list": [ - { - "line": "192.88.99.0\u2013192.88.99.255", - "line_num": 26, - "path": "tests/samples/ipv4", - "info": "", - "value": "192.88.99.255", - "value_start": 12, - "value_end": 25, - "variable": null, - "variable_start": -2, - "variable_end": -2, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 2.019193052249804, - "valid": false - } - } - ] - }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "IPv6", - "severity": "info", - "confidence": "strong", - "line_data_list": [ - { - "line": "2004:5678::9324", - "line_num": 12, - "path": "tests/samples/ipv6", - "info": "", - "value": "2004:5678::9324", - "value_start": 0, - "value_end": 15, - "variable": null, - "variable_start": -2, - "variable_end": -2, - "entropy_validation": { - "iterator": "BASE36_CHARS", - "entropy": 2.725512476486815, - "valid": false - } - } - ] - }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "IPv6", - "severity": "info", - "confidence": "strong", - "line_data_list": [ - { - "line": "2004::5678:9", - "line_num": 13, - "path": "tests/samples/ipv6", - "info": "", - "value": "2004::5678:9", - "value_start": 0, - "value_end": 12, - "variable": null, - "variable_start": -2, - "variable_end": -2, - "entropy_validation": { - "iterator": "BASE36_CHARS", - "entropy": 2.5220552088742005, - "valid": false - } - } - ] - }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "IPv6", - "severity": "info", - "confidence": "strong", - "line_data_list": [ - { - "line": "2041:0000:140F::875B:131B", - "line_num": 14, - "path": "tests/samples/ipv6", - "info": "", - "value": "2041:0000:140F::875B:131B", - "value_start": 0, - "value_end": 25, - "variable": null, - "variable_start": -2, - "variable_end": -2, - "entropy_validation": { - "iterator": "HEX_CHARS", - "entropy": 2.6146939516467023, - "valid": false - } - } - ] - }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "IPv6", - "severity": "info", - "confidence": "strong", - "line_data_list": [ - { - "line": "LONG LINE TEST: 2041:0:140F::875B:131B 2041:0:140F::875B:131B ;", - "line_num": 15, - "path": "tests/samples/ipv6", - "info": "", - "value": "2041:0:140F::875B:131B", - "value_start": 7010, - "value_end": 7032, - "variable": null, - "variable_start": -2, - "variable_end": -2, - "entropy_validation": { - "iterator": "HEX_CHARS", - "entropy": 2.684338637030481, - "valid": false - } - } - ] - }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "IPv6", - "severity": "info", - "confidence": "strong", - "line_data_list": [ - { - "line": "LONG LINE TEST: 2041:0:140F::875B:131B 2041:0:140F::875B:131B ;", - "line_num": 15, - "path": "tests/samples/ipv6", - "info": "", - "value": "2041:0:140F::875B:131B", - "value_start": 7989, - "value_end": 8011, - "variable": null, - "variable_start": -2, - "variable_end": -2, - "entropy_validation": { - "iterator": "HEX_CHARS", - "entropy": 2.684338637030481, - "valid": false - } - } - ] - }, { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", diff --git a/tests/data/output.json b/tests/data/output.json index 2f3bee438..f5f87e544 100644 --- a/tests/data/output.json +++ b/tests/data/output.json @@ -2166,33 +2166,6 @@ } ] }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "IPv4", - "severity": "info", - "confidence": "weak", - "line_data_list": [ - { - "line": "(98.76.54.32)ID:master PW:dipPr149Gg!", - "line_num": 50, - "path": "tests/samples/doc_id_pair_passwd_pair", - "info": "", - "value": "98.76.54.32", - "value_start": 1, - "value_end": 12, - "variable": null, - "variable_start": -2, - "variable_end": -2, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 2.515950268099853, - "valid": false - } - } - ] - }, { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", @@ -2976,33 +2949,6 @@ } ] }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "IPv4", - "severity": "info", - "confidence": "weak", - "line_data_list": [ - { - "line": "ANYid:master pw:dipPr194Gg! ip:98.76.54.32", - "line_num": 95, - "path": "tests/samples/doc_id_pair_passwd_pair", - "info": "", - "value": "98.76.54.32", - "value_start": 31, - "value_end": 42, - "variable": null, - "variable_start": -2, - "variable_end": -2, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 2.515950268099853, - "valid": false - } - } - ] - }, { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", @@ -3030,33 +2976,6 @@ } ] }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "IPv4", - "severity": "info", - "confidence": "weak", - "line_data_list": [ - { - "line": "master@98.76.54.32 mailto:{1} (password-dipPr196Gg!) # skip", - "line_num": 97, - "path": "tests/samples/doc_id_pair_passwd_pair", - "info": "", - "value": "98.76.54.32", - "value_start": 7, - "value_end": 18, - "variable": null, - "variable_start": -2, - "variable_end": -2, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 2.515950268099853, - "valid": false - } - } - ] - }, { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", @@ -3138,60 +3057,6 @@ } ] }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "IPv4", - "severity": "info", - "confidence": "weak", - "line_data_list": [ - { - "line": "98.76.54.32(ID:master/PW:iPp10@GRq) # todo: move into other sample ?", - "line_num": 11, - "path": "tests/samples/doc_id_passwd_pair", - "info": "", - "value": "98.76.54.32", - "value_start": 0, - "value_end": 11, - "variable": null, - "variable_start": -2, - "variable_end": -2, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 2.515950268099853, - "valid": false - } - } - ] - }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "IPv4", - "severity": "info", - "confidence": "weak", - "line_data_list": [ - { - "line": "98.76.54.32 id/pw:master/iPp19@GRq", - "line_num": 20, - "path": "tests/samples/doc_id_passwd_pair", - "info": "", - "value": "98.76.54.32", - "value_start": 0, - "value_end": 11, - "variable": null, - "variable_start": -2, - "variable_end": -2, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 2.515950268099853, - "valid": false - } - } - ] - }, { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", @@ -3435,33 +3300,6 @@ } ] }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "IPv4", - "severity": "info", - "confidence": "weak", - "line_data_list": [ - { - "line": "master@98.76.54.32 password:Prl23Db#@", - "line_num": 14, - "path": "tests/samples/doc_passwd_pair", - "info": "", - "value": "98.76.54.32", - "value_start": 7, - "value_end": 18, - "variable": null, - "variable_start": -2, - "variable_end": -2, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 2.515950268099853, - "valid": false - } - } - ] - }, { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", @@ -4650,87 +4488,6 @@ } ] }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "IPv4", - "severity": "info", - "confidence": "weak", - "line_data_list": [ - { - "line": "98.76.54.32 (master/IhqSb1Gg)", - "line_num": 1, - "path": "tests/samples/doc_various", - "info": "", - "value": "98.76.54.32", - "value_start": 0, - "value_end": 11, - "variable": null, - "variable_start": -2, - "variable_end": -2, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 2.515950268099853, - "valid": false - } - } - ] - }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "IPv4", - "severity": "info", - "confidence": "weak", - "line_data_list": [ - { - "line": "98.76.54.32(master/IhqSb1Gg)", - "line_num": 2, - "path": "tests/samples/doc_various", - "info": "", - "value": "98.76.54.32", - "value_start": 0, - "value_end": 11, - "variable": null, - "variable_start": -2, - "variable_end": -2, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 2.515950268099853, - "valid": false - } - } - ] - }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "IPv4", - "severity": "info", - "confidence": "weak", - "line_data_list": [ - { - "line": "master@98.76.54.32(pw:IhqSb1Gg)", - "line_num": 3, - "path": "tests/samples/doc_various", - "info": "", - "value": "98.76.54.32", - "value_start": 7, - "value_end": 18, - "variable": null, - "variable_start": -2, - "variable_end": -2, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 2.515950268099853, - "valid": false - } - } - ] - }, { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", @@ -4812,33 +4569,6 @@ } ] }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "IPv4", - "severity": "info", - "confidence": "weak", - "line_data_list": [ - { - "line": "ssh gildong.hong@98.76.54.32 mailto:{1} (PW:IhqSb1Gg)", - "line_num": 6, - "path": "tests/samples/doc_various", - "info": "", - "value": "98.76.54.32", - "value_start": 17, - "value_end": 28, - "variable": null, - "variable_start": -2, - "variable_end": -2, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 2.515950268099853, - "valid": false - } - } - ] - }, { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", @@ -4866,33 +4596,6 @@ } ] }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "IPv4", - "severity": "info", - "confidence": "weak", - "line_data_list": [ - { - "line": "ssh gildong.hong@98.76.54.32 mailto:{1} password:IhqSb1Gg", - "line_num": 7, - "path": "tests/samples/doc_various", - "info": "", - "value": "98.76.54.32", - "value_start": 17, - "value_end": 28, - "variable": null, - "variable_start": -2, - "variable_end": -2, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 2.515950268099853, - "valid": false - } - } - ] - }, { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", @@ -4920,33 +4623,6 @@ } ] }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "IPv4", - "severity": "info", - "confidence": "weak", - "line_data_list": [ - { - "line": "ssh -P IhqSb1Gg gildong.hong@98.76.54.32 mailto:{1} (password:IhqSb1Gg)", - "line_num": 11, - "path": "tests/samples/doc_various", - "info": "", - "value": "98.76.54.32", - "value_start": 29, - "value_end": 40, - "variable": null, - "variable_start": -2, - "variable_end": -2, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 2.515950268099853, - "valid": false - } - } - ] - }, { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", @@ -4974,60 +4650,6 @@ } ] }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "IPv4", - "severity": "info", - "confidence": "weak", - "line_data_list": [ - { - "line": "98.76.54.32 xxxx (master/IhqSb1Gg)", - "line_num": 14, - "path": "tests/samples/doc_various", - "info": "", - "value": "98.76.54.32", - "value_start": 0, - "value_end": 11, - "variable": null, - "variable_start": -2, - "variable_end": -2, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 2.515950268099853, - "valid": false - } - } - ] - }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "IPv4", - "severity": "info", - "confidence": "weak", - "line_data_list": [ - { - "line": "gildong.hong@98.76.54.32 pwd:IhqSb1Gg", - "line_num": 15, - "path": "tests/samples/doc_various", - "info": "", - "value": "98.76.54.32", - "value_start": 13, - "value_end": 24, - "variable": null, - "variable_start": -2, - "variable_end": -2, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 2.515950268099853, - "valid": false - } - } - ] - }, { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", @@ -5082,33 +4704,6 @@ } ] }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "IPv4", - "severity": "info", - "confidence": "weak", - "line_data_list": [ - { - "line": "98.76.54.32(pw:IhqSb1Gg)", - "line_num": 17, - "path": "tests/samples/doc_various", - "info": "", - "value": "98.76.54.32", - "value_start": 0, - "value_end": 11, - "variable": null, - "variable_start": -2, - "variable_end": -2, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 2.515950268099853, - "valid": false - } - } - ] - }, { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", @@ -5136,33 +4731,6 @@ } ] }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "IPv4", - "severity": "info", - "confidence": "weak", - "line_data_list": [ - { - "line": "98.76.54.32/pw:IhqSb1Gg", - "line_num": 19, - "path": "tests/samples/doc_various", - "info": "", - "value": "98.76.54.32", - "value_start": 0, - "value_end": 11, - "variable": null, - "variable_start": -2, - "variable_end": -2, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 2.515950268099853, - "valid": false - } - } - ] - }, { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", @@ -5244,33 +4812,6 @@ } ] }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "IPv4", - "severity": "info", - "confidence": "weak", - "line_data_list": [ - { - "line": "sftp gildong.hong@98.76.54.32 mailto:{1} (pw:IhqSb1Gg)", - "line_num": 22, - "path": "tests/samples/doc_various", - "info": "", - "value": "98.76.54.32", - "value_start": 18, - "value_end": 29, - "variable": null, - "variable_start": -2, - "variable_end": -2, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 2.515950268099853, - "valid": false - } - } - ] - }, { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", @@ -5298,33 +4839,6 @@ } ] }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "IPv4", - "severity": "info", - "confidence": "weak", - "line_data_list": [ - { - "line": "gildong.hong@98.76.54.32 mailto:{1} (pw:IhqSb1Gg)", - "line_num": 23, - "path": "tests/samples/doc_various", - "info": "", - "value": "98.76.54.32", - "value_start": 13, - "value_end": 24, - "variable": null, - "variable_start": -2, - "variable_end": -2, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 2.515950268099853, - "valid": false - } - } - ] - }, { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", @@ -5379,33 +4893,6 @@ } ] }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "IPv4", - "severity": "info", - "confidence": "weak", - "line_data_list": [ - { - "line": "ssh gildong.hong@98.76.54.32 mailto:{1} (password:IhqSb1Gg)", - "line_num": 25, - "path": "tests/samples/doc_various", - "info": "", - "value": "98.76.54.32", - "value_start": 17, - "value_end": 28, - "variable": null, - "variable_start": -2, - "variable_end": -2, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 2.515950268099853, - "valid": false - } - } - ] - }, { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", @@ -5433,33 +4920,6 @@ } ] }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "IPv4", - "severity": "info", - "confidence": "weak", - "line_data_list": [ - { - "line": "ssh gildong.hong@98.76.54.32 mailto:{1} master/IhqSb1Gg", - "line_num": 26, - "path": "tests/samples/doc_various", - "info": "", - "value": "98.76.54.32", - "value_start": 17, - "value_end": 28, - "variable": null, - "variable_start": -2, - "variable_end": -2, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 2.515950268099853, - "valid": false - } - } - ] - }, { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", @@ -5568,33 +5028,6 @@ } ] }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "IPv4", - "severity": "info", - "confidence": "weak", - "line_data_list": [ - { - "line": "98.76.54.32 ANY_PW:IhqSb1Gg", - "line_num": 34, - "path": "tests/samples/doc_various", - "info": "", - "value": "98.76.54.32", - "value_start": 0, - "value_end": 11, - "variable": null, - "variable_start": -2, - "variable_end": -2, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 2.515950268099853, - "valid": false - } - } - ] - }, { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", @@ -5622,33 +5055,6 @@ } ] }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "IPv4", - "severity": "info", - "confidence": "weak", - "line_data_list": [ - { - "line": "98.76.54.32(ID/PW:IhqSb1Gg)", - "line_num": 36, - "path": "tests/samples/doc_various", - "info": "", - "value": "98.76.54.32", - "value_start": 0, - "value_end": 11, - "variable": null, - "variable_start": -2, - "variable_end": -2, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 2.515950268099853, - "valid": false - } - } - ] - }, { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", @@ -5676,33 +5082,6 @@ } ] }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "IPv4", - "severity": "info", - "confidence": "weak", - "line_data_list": [ - { - "line": "gildong.hong@98.76.54.32 (pwd:IhqSb1Gg)", - "line_num": 38, - "path": "tests/samples/doc_various", - "info": "", - "value": "98.76.54.32", - "value_start": 13, - "value_end": 24, - "variable": null, - "variable_start": -2, - "variable_end": -2, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 2.515950268099853, - "valid": false - } - } - ] - }, { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", @@ -5757,33 +5136,6 @@ } ] }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "IPv4", - "severity": "info", - "confidence": "weak", - "line_data_list": [ - { - "line": "gildong.hong@98.76.54.32(master/IhqSb1Gg)", - "line_num": 44, - "path": "tests/samples/doc_various", - "info": "", - "value": "98.76.54.32", - "value_start": 13, - "value_end": 24, - "variable": null, - "variable_start": -2, - "variable_end": -2, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 2.515950268099853, - "valid": false - } - } - ] - }, { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", @@ -5811,141 +5163,6 @@ } ] }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "IPv4", - "severity": "info", - "confidence": "weak", - "line_data_list": [ - { - "line": "gildong.hong@98.76.54.32,pw:IhqSb1Gg", - "line_num": 47, - "path": "tests/samples/doc_various", - "info": "", - "value": "98.76.54.32", - "value_start": 13, - "value_end": 24, - "variable": null, - "variable_start": -2, - "variable_end": -2, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 2.515950268099853, - "valid": false - } - } - ] - }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "IPv4", - "severity": "info", - "confidence": "weak", - "line_data_list": [ - { - "line": "98.76.54.32(master/IhqSb1Gg,master/IhqSb1Gg)", - "line_num": 48, - "path": "tests/samples/doc_various", - "info": "", - "value": "98.76.54.32", - "value_start": 0, - "value_end": 11, - "variable": null, - "variable_start": -2, - "variable_end": -2, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 2.515950268099853, - "valid": false - } - } - ] - }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "IPv4", - "severity": "info", - "confidence": "weak", - "line_data_list": [ - { - "line": "98.76.54.32(master/IhqSb1Gg master/IhqSb1Gg)", - "line_num": 49, - "path": "tests/samples/doc_various", - "info": "", - "value": "98.76.54.32", - "value_start": 0, - "value_end": 11, - "variable": null, - "variable_start": -2, - "variable_end": -2, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 2.515950268099853, - "valid": false - } - } - ] - }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "IPv4", - "severity": "info", - "confidence": "weak", - "line_data_list": [ - { - "line": "xxxx:98.76.54.32(master/IhqSb1Gg)", - "line_num": 50, - "path": "tests/samples/doc_various", - "info": "", - "value": "98.76.54.32", - "value_start": 5, - "value_end": 16, - "variable": null, - "variable_start": -2, - "variable_end": -2, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 2.515950268099853, - "valid": false - } - } - ] - }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "IPv4", - "severity": "info", - "confidence": "weak", - "line_data_list": [ - { - "line": "98.76.54.32:xxxx(PW:IhqSb1Gg)", - "line_num": 51, - "path": "tests/samples/doc_various", - "info": "", - "value": "98.76.54.32", - "value_start": 0, - "value_end": 11, - "variable": null, - "variable_start": -2, - "variable_end": -2, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 2.515950268099853, - "valid": false - } - } - ] - }, { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", @@ -5973,33 +5190,6 @@ } ] }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "IPv4", - "severity": "info", - "confidence": "weak", - "line_data_list": [ - { - "line": "gildong.hong@98.76.54.32 PW:IhqSb1Gg", - "line_num": 56, - "path": "tests/samples/doc_various", - "info": "", - "value": "98.76.54.32", - "value_start": 13, - "value_end": 24, - "variable": null, - "variable_start": -2, - "variable_end": -2, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 2.515950268099853, - "valid": false - } - } - ] - }, { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", @@ -6027,33 +5217,6 @@ } ] }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "IPv4", - "severity": "info", - "confidence": "weak", - "line_data_list": [ - { - "line": "http|https://98.76.54.32/xxxx(master/IhqSb1Gg)", - "line_num": 59, - "path": "tests/samples/doc_various", - "info": "", - "value": "98.76.54.32", - "value_start": 13, - "value_end": 24, - "variable": null, - "variable_start": -2, - "variable_end": -2, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 2.515950268099853, - "valid": false - } - } - ] - }, { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", @@ -6191,26 +5354,26 @@ }, { "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "IPv4", - "severity": "info", - "confidence": "weak", + "ml_validation": "VALIDATED_KEY", + "ml_probability": 0.956, + "rule": "Password", + "severity": "medium", + "confidence": "moderate", "line_data_list": [ { - "line": "98.76.54.32(ID/PW:master/IhqSb1Gg)", - "line_num": 70, + "line": "master@98.76.54.32(pw:IhqSb1Gg)", + "line_num": 73, "path": "tests/samples/doc_various", "info": "", - "value": "98.76.54.32", - "value_start": 0, - "value_end": 11, - "variable": null, - "variable_start": -2, - "variable_end": -2, + "value": "IhqSb1Gg)", + "value_start": 22, + "value_end": 31, + "variable": "pw", + "variable_start": 19, + "variable_end": 21, "entropy_validation": { "iterator": "BASE64_CHARS", - "entropy": 2.515950268099853, + "entropy": 2.8177111123931664, "valid": false } } @@ -6218,26 +5381,26 @@ }, { "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "IPv4", - "severity": "info", - "confidence": "weak", + "ml_validation": "VALIDATED_KEY", + "ml_probability": 0.99, + "rule": "Password", + "severity": "medium", + "confidence": "moderate", "line_data_list": [ { - "line": "master@98.76.54.32(pw:IhqSb1Gg)", - "line_num": 73, + "line": "98.76.54.32 pw:IhqSb1Gg", + "line_num": 75, "path": "tests/samples/doc_various", "info": "", - "value": "98.76.54.32", - "value_start": 7, - "value_end": 18, - "variable": null, - "variable_start": -2, - "variable_end": -2, + "value": "IhqSb1Gg", + "value_start": 15, + "value_end": 23, + "variable": "pw", + "variable_start": 12, + "variable_end": 14, "entropy_validation": { "iterator": "BASE64_CHARS", - "entropy": 2.515950268099853, + "entropy": 3.0, "valid": false } } @@ -6246,115 +5409,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.956, - "rule": "Password", - "severity": "medium", - "confidence": "moderate", - "line_data_list": [ - { - "line": "master@98.76.54.32(pw:IhqSb1Gg)", - "line_num": 73, - "path": "tests/samples/doc_various", - "info": "", - "value": "IhqSb1Gg)", - "value_start": 22, - "value_end": 31, - "variable": "pw", - "variable_start": 19, - "variable_end": 21, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 2.8177111123931664, - "valid": false - } - } - ] - }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "IPv4", - "severity": "info", - "confidence": "weak", - "line_data_list": [ - { - "line": "master@98.76.54.32,PW:IhqSb1Gg", - "line_num": 74, - "path": "tests/samples/doc_various", - "info": "", - "value": "98.76.54.32", - "value_start": 7, - "value_end": 18, - "variable": null, - "variable_start": -2, - "variable_end": -2, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 2.515950268099853, - "valid": false - } - } - ] - }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "IPv4", - "severity": "info", - "confidence": "weak", - "line_data_list": [ - { - "line": "98.76.54.32 pw:IhqSb1Gg", - "line_num": 75, - "path": "tests/samples/doc_various", - "info": "", - "value": "98.76.54.32", - "value_start": 0, - "value_end": 11, - "variable": null, - "variable_start": -2, - "variable_end": -2, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 2.515950268099853, - "valid": false - } - } - ] - }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.99, - "rule": "Password", - "severity": "medium", - "confidence": "moderate", - "line_data_list": [ - { - "line": "98.76.54.32 pw:IhqSb1Gg", - "line_num": 75, - "path": "tests/samples/doc_various", - "info": "", - "value": "IhqSb1Gg", - "value_start": 15, - "value_end": 23, - "variable": "pw", - "variable_start": 12, - "variable_end": 14, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 3.0, - "valid": false - } - } - ] - }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.976, + "ml_probability": 0.976, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -6378,60 +5433,6 @@ } ] }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "IPv4", - "severity": "info", - "confidence": "weak", - "line_data_list": [ - { - "line": "98.76.54.32-->master/IhqSb1Gg", - "line_num": 81, - "path": "tests/samples/doc_various", - "info": "", - "value": "98.76.54.32", - "value_start": 0, - "value_end": 11, - "variable": null, - "variable_start": -2, - "variable_end": -2, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 2.515950268099853, - "valid": false - } - } - ] - }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "IPv4", - "severity": "info", - "confidence": "weak", - "line_data_list": [ - { - "line": "scp gildong.hong@98.76.54.32 mailto:{1} pw:IhqSb1Gg", - "line_num": 82, - "path": "tests/samples/doc_various", - "info": "", - "value": "98.76.54.32", - "value_start": 17, - "value_end": 28, - "variable": null, - "variable_start": -2, - "variable_end": -2, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 2.515950268099853, - "valid": false - } - } - ] - }, { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", @@ -6486,33 +5487,6 @@ } ] }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "IPv4", - "severity": "info", - "confidence": "weak", - "line_data_list": [ - { - "line": "gildong.hong@98.76.54.32 mailto:{1} pw:IhqSb1Gg", - "line_num": 84, - "path": "tests/samples/doc_various", - "info": "", - "value": "98.76.54.32", - "value_start": 13, - "value_end": 24, - "variable": null, - "variable_start": -2, - "variable_end": -2, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 2.515950268099853, - "valid": false - } - } - ] - }, { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", @@ -6540,33 +5514,6 @@ } ] }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "IPv4", - "severity": "info", - "confidence": "weak", - "line_data_list": [ - { - "line": "ssh gildong.hong@98.76.54.32 mailto:{1},pw:IhqSb1Gg", - "line_num": 85, - "path": "tests/samples/doc_various", - "info": "", - "value": "98.76.54.32", - "value_start": 17, - "value_end": 28, - "variable": null, - "variable_start": -2, - "variable_end": -2, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 2.515950268099853, - "valid": false - } - } - ] - }, { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", @@ -6594,244 +5541,28 @@ } ] }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "IPv4", - "severity": "info", - "confidence": "weak", - "line_data_list": [ - { - "line": "gildong.hong@98.76.54.32 mailto:{1} (master/IhqSb1Gg)", - "line_num": 86, - "path": "tests/samples/doc_various", - "info": "", - "value": "98.76.54.32", - "value_start": 13, - "value_end": 24, - "variable": null, - "variable_start": -2, - "variable_end": -2, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 2.515950268099853, - "valid": false - } - } - ] - }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "IPv4", - "severity": "info", - "confidence": "weak", - "line_data_list": [ - { - "line": "(ssh gildong.hong@98.76.54.32 mailto{1}) pwd:IhqSb1Gg", - "line_num": 87, - "path": "tests/samples/doc_various", - "info": "", - "value": "98.76.54.32", - "value_start": 18, - "value_end": 29, - "variable": null, - "variable_start": -2, - "variable_end": -2, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 2.515950268099853, - "valid": false - } - } - ] - }, { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", "ml_probability": 0.995, - "rule": "Password", - "severity": "medium", - "confidence": "moderate", - "line_data_list": [ - { - "line": "(ssh gildong.hong@98.76.54.32 mailto{1}) pwd:IhqSb1Gg", - "line_num": 87, - "path": "tests/samples/doc_various", - "info": "", - "value": "IhqSb1Gg", - "value_start": 45, - "value_end": 53, - "variable": "pwd", - "variable_start": 41, - "variable_end": 44, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 3.0, - "valid": false - } - } - ] - }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "IPv4", - "severity": "info", - "confidence": "weak", - "line_data_list": [ - { - "line": "ssh gildong.hong@98.76.54.32 (master/IhqSb1Gg)", - "line_num": 88, - "path": "tests/samples/doc_various", - "info": "", - "value": "98.76.54.32", - "value_start": 17, - "value_end": 28, - "variable": null, - "variable_start": -2, - "variable_end": -2, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 2.515950268099853, - "valid": false - } - } - ] - }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "IPv4", - "severity": "info", - "confidence": "weak", - "line_data_list": [ - { - "line": "ssh gildong.hong@98.76.54.32 mailto:[1} \uacc4\uc815master/IhqSb1Gg", - "line_num": 89, - "path": "tests/samples/doc_various", - "info": "", - "value": "98.76.54.32", - "value_start": 17, - "value_end": 28, - "variable": null, - "variable_start": -2, - "variable_end": -2, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 2.515950268099853, - "valid": false - } - } - ] - }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "IPv4", - "severity": "info", - "confidence": "weak", - "line_data_list": [ - { - "line": "ssh gildong.hong@98.76.54.32 mailto:{1}, pw:IhqSb1Gg", - "line_num": 90, - "path": "tests/samples/doc_various", - "info": "", - "value": "98.76.54.32", - "value_start": 17, - "value_end": 28, - "variable": null, - "variable_start": -2, - "variable_end": -2, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 2.515950268099853, - "valid": false - } - } - ] - }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.996, - "rule": "Password", - "severity": "medium", - "confidence": "moderate", - "line_data_list": [ - { - "line": "ssh gildong.hong@98.76.54.32 mailto:{1}, pw:IhqSb1Gg", - "line_num": 90, - "path": "tests/samples/doc_various", - "info": "", - "value": "IhqSb1Gg", - "value_start": 44, - "value_end": 52, - "variable": "pw", - "variable_start": 41, - "variable_end": 43, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 3.0, - "valid": false - } - } - ] - }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "IPv4", - "severity": "info", - "confidence": "weak", - "line_data_list": [ - { - "line": "gildong.hong@98.76.54.32 mailto:{1} (\ube44\ubc88 IhqSb1Gg)", - "line_num": 91, - "path": "tests/samples/doc_various", - "info": "", - "value": "98.76.54.32", - "value_start": 13, - "value_end": 24, - "variable": null, - "variable_start": -2, - "variable_end": -2, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 2.515950268099853, - "valid": false - } - } - ] - }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "IPv4", - "severity": "info", - "confidence": "weak", + "rule": "Password", + "severity": "medium", + "confidence": "moderate", "line_data_list": [ { - "line": "ssh gildong.hong@98.76.54.32 mailto:{1} password IhqSb1Gg", - "line_num": 92, + "line": "(ssh gildong.hong@98.76.54.32 mailto{1}) pwd:IhqSb1Gg", + "line_num": 87, "path": "tests/samples/doc_various", "info": "", - "value": "98.76.54.32", - "value_start": 17, - "value_end": 28, - "variable": null, - "variable_start": -2, - "variable_end": -2, + "value": "IhqSb1Gg", + "value_start": 45, + "value_end": 53, + "variable": "pwd", + "variable_start": 41, + "variable_end": 44, "entropy_validation": { "iterator": "BASE64_CHARS", - "entropy": 2.515950268099853, + "entropy": 3.0, "valid": false } } @@ -6839,26 +5570,26 @@ }, { "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "IPv4", - "severity": "info", - "confidence": "weak", + "ml_validation": "VALIDATED_KEY", + "ml_probability": 0.996, + "rule": "Password", + "severity": "medium", + "confidence": "moderate", "line_data_list": [ { - "line": "ssh gildong.hong@98.76.54.32 mailto:{1} (pwd:IhqSb1Gg)", - "line_num": 93, + "line": "ssh gildong.hong@98.76.54.32 mailto:{1}, pw:IhqSb1Gg", + "line_num": 90, "path": "tests/samples/doc_various", "info": "", - "value": "98.76.54.32", - "value_start": 17, - "value_end": 28, - "variable": null, - "variable_start": -2, - "variable_end": -2, + "value": "IhqSb1Gg", + "value_start": 44, + "value_end": 52, + "variable": "pw", + "variable_start": 41, + "variable_end": 43, "entropy_validation": { "iterator": "BASE64_CHARS", - "entropy": 2.515950268099853, + "entropy": 3.0, "valid": false } } @@ -6891,33 +5622,6 @@ } ] }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "IPv4", - "severity": "info", - "confidence": "weak", - "line_data_list": [ - { - "line": "gildong.hong@98.76.54.32 mailto:{1} (password:IhqSb1Gg)", - "line_num": 94, - "path": "tests/samples/doc_various", - "info": "", - "value": "98.76.54.32", - "value_start": 13, - "value_end": 24, - "variable": null, - "variable_start": -2, - "variable_end": -2, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 2.515950268099853, - "valid": false - } - } - ] - }, { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", @@ -6945,33 +5649,6 @@ } ] }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "IPv4", - "severity": "info", - "confidence": "weak", - "line_data_list": [ - { - "line": "gildong.hong@98.76.54.32 mailto:{1} Password:IhqSb1Gg", - "line_num": 96, - "path": "tests/samples/doc_various", - "info": "", - "value": "98.76.54.32", - "value_start": 13, - "value_end": 24, - "variable": null, - "variable_start": -2, - "variable_end": -2, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 2.515950268099853, - "valid": false - } - } - ] - }, { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", @@ -6999,33 +5676,6 @@ } ] }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "IPv4", - "severity": "info", - "confidence": "weak", - "line_data_list": [ - { - "line": "gildong.hong@98.76.54.32 mailto:{1} (pass:IhqSb1Gg)", - "line_num": 97, - "path": "tests/samples/doc_various", - "info": "", - "value": "98.76.54.32", - "value_start": 13, - "value_end": 24, - "variable": null, - "variable_start": -2, - "variable_end": -2, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 2.515950268099853, - "valid": false - } - } - ] - }, { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", @@ -7053,60 +5703,6 @@ } ] }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "IPv4", - "severity": "info", - "confidence": "weak", - "line_data_list": [ - { - "line": "ssh gildong.hong@98.76.54.32 mailto:{1} (xxxx//IhqSb1Gg)", - "line_num": 99, - "path": "tests/samples/doc_various", - "info": "", - "value": "98.76.54.32", - "value_start": 17, - "value_end": 28, - "variable": null, - "variable_start": -2, - "variable_end": -2, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 2.515950268099853, - "valid": false - } - } - ] - }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "IPv4", - "severity": "info", - "confidence": "weak", - "line_data_list": [ - { - "line": "ssh gildong.hong@98.76.54.32 mailto:{1} pw:IhqSb1Gg", - "line_num": 100, - "path": "tests/samples/doc_various", - "info": "", - "value": "98.76.54.32", - "value_start": 17, - "value_end": 28, - "variable": null, - "variable_start": -2, - "variable_end": -2, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 2.515950268099853, - "valid": false - } - } - ] - }, { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", @@ -7134,33 +5730,6 @@ } ] }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "IPv4", - "severity": "info", - "confidence": "weak", - "line_data_list": [ - { - "line": "ssh gildong.hong@98.76.54.32 mailto:{1} pass:IhqSb1Gg", - "line_num": 101, - "path": "tests/samples/doc_various", - "info": "", - "value": "98.76.54.32", - "value_start": 17, - "value_end": 28, - "variable": null, - "variable_start": -2, - "variable_end": -2, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 2.515950268099853, - "valid": false - } - } - ] - }, { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", @@ -7215,33 +5784,6 @@ } ] }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "IPv4", - "severity": "info", - "confidence": "weak", - "line_data_list": [ - { - "line": "ssh gildong.hong@98.76.54.32 mailto:{1}/password:IhqSb1Gg", - "line_num": 104, - "path": "tests/samples/doc_various", - "info": "", - "value": "98.76.54.32", - "value_start": 17, - "value_end": 28, - "variable": null, - "variable_start": -2, - "variable_end": -2, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 2.515950268099853, - "valid": false - } - } - ] - }, { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", @@ -7313,65 +5855,11 @@ "value_start": 39, "value_end": 47, "variable": "pw", - "variable_start": 36, - "variable_end": 38, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 3.0, - "valid": false - } - } - ] - }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "IPv4", - "severity": "info", - "confidence": "weak", - "line_data_list": [ - { - "line": "ID/PW 98.76.54.32:xxx master/IhqSb1Gg", - "line_num": 107, - "path": "tests/samples/doc_various", - "info": "", - "value": "98.76.54.32", - "value_start": 6, - "value_end": 17, - "variable": null, - "variable_start": -2, - "variable_end": -2, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 2.515950268099853, - "valid": false - } - } - ] - }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "IPv4", - "severity": "info", - "confidence": "weak", - "line_data_list": [ - { - "line": "http://98.76.54.32:xxx(pw:IhqSb1Gg)", - "line_num": 108, - "path": "tests/samples/doc_various", - "info": "", - "value": "98.76.54.32", - "value_start": 7, - "value_end": 18, - "variable": null, - "variable_start": -2, - "variable_end": -2, + "variable_start": 36, + "variable_end": 38, "entropy_validation": { "iterator": "BASE64_CHARS", - "entropy": 2.515950268099853, + "entropy": 3.0, "valid": false } } @@ -8275,330 +6763,6 @@ } ] }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "IPv4", - "severity": "info", - "confidence": "weak", - "line_data_list": [ - { - "line": "100.64.0.0/10", - "line_num": 13, - "path": "tests/samples/ipv4", - "info": "", - "value": "100.64.0.0", - "value_start": 0, - "value_end": 10, - "variable": null, - "variable_start": -2, - "variable_end": -2, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 1.5253496664211537, - "valid": false - } - } - ] - }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "IPv4", - "severity": "info", - "confidence": "weak", - "line_data_list": [ - { - "line": "100.64.0.0\u2013100.127.255.255", - "line_num": 14, - "path": "tests/samples/ipv4", - "info": "", - "value": "100.127.255.255", - "value_start": 11, - "value_end": 26, - "variable": null, - "variable_start": -2, - "variable_end": -2, - "entropy_validation": { - "iterator": "BASE36_CHARS", - "entropy": 2.008519976342584, - "valid": false - } - } - ] - }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "IPv4", - "severity": "info", - "confidence": "weak", - "line_data_list": [ - { - "line": "100.64.0.0\u2013100.127.255.255", - "line_num": 14, - "path": "tests/samples/ipv4", - "info": "", - "value": "100.64.0.0", - "value_start": 0, - "value_end": 10, - "variable": null, - "variable_start": -2, - "variable_end": -2, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 1.5253496664211537, - "valid": false - } - } - ] - }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "IPv4", - "severity": "info", - "confidence": "weak", - "line_data_list": [ - { - "line": "192.0.0.0\u2013192.0.0.255", - "line_num": 22, - "path": "tests/samples/ipv4", - "info": "", - "value": "192.0.0.255", - "value_start": 10, - "value_end": 21, - "variable": null, - "variable_start": -2, - "variable_end": -2, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 1.9704957226453073, - "valid": false - } - } - ] - }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "IPv4", - "severity": "info", - "confidence": "weak", - "line_data_list": [ - { - "line": "192.88.99.0/24", - "line_num": 25, - "path": "tests/samples/ipv4", - "info": "", - "value": "192.88.99.0", - "value_start": 0, - "value_end": 11, - "variable": null, - "variable_start": -2, - "variable_end": -2, - "entropy_validation": { - "iterator": "BASE36_CHARS", - "entropy": 1.9018695860849921, - "valid": false - } - } - ] - }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "IPv4", - "severity": "info", - "confidence": "weak", - "line_data_list": [ - { - "line": "192.88.99.0\u2013192.88.99.255", - "line_num": 26, - "path": "tests/samples/ipv4", - "info": "", - "value": "192.88.99.0", - "value_start": 0, - "value_end": 11, - "variable": null, - "variable_start": -2, - "variable_end": -2, - "entropy_validation": { - "iterator": "BASE36_CHARS", - "entropy": 1.9018695860849921, - "valid": false - } - } - ] - }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "IPv4", - "severity": "info", - "confidence": "weak", - "line_data_list": [ - { - "line": "192.88.99.0\u2013192.88.99.255", - "line_num": 26, - "path": "tests/samples/ipv4", - "info": "", - "value": "192.88.99.255", - "value_start": 12, - "value_end": 25, - "variable": null, - "variable_start": -2, - "variable_end": -2, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 2.019193052249804, - "valid": false - } - } - ] - }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "IPv6", - "severity": "info", - "confidence": "strong", - "line_data_list": [ - { - "line": "2004:5678::9324", - "line_num": 12, - "path": "tests/samples/ipv6", - "info": "", - "value": "2004:5678::9324", - "value_start": 0, - "value_end": 15, - "variable": null, - "variable_start": -2, - "variable_end": -2, - "entropy_validation": { - "iterator": "BASE36_CHARS", - "entropy": 2.725512476486815, - "valid": false - } - } - ] - }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "IPv6", - "severity": "info", - "confidence": "strong", - "line_data_list": [ - { - "line": "2004::5678:9", - "line_num": 13, - "path": "tests/samples/ipv6", - "info": "", - "value": "2004::5678:9", - "value_start": 0, - "value_end": 12, - "variable": null, - "variable_start": -2, - "variable_end": -2, - "entropy_validation": { - "iterator": "BASE36_CHARS", - "entropy": 2.5220552088742005, - "valid": false - } - } - ] - }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "IPv6", - "severity": "info", - "confidence": "strong", - "line_data_list": [ - { - "line": "2041:0000:140F::875B:131B", - "line_num": 14, - "path": "tests/samples/ipv6", - "info": "", - "value": "2041:0000:140F::875B:131B", - "value_start": 0, - "value_end": 25, - "variable": null, - "variable_start": -2, - "variable_end": -2, - "entropy_validation": { - "iterator": "HEX_CHARS", - "entropy": 2.6146939516467023, - "valid": false - } - } - ] - }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "IPv6", - "severity": "info", - "confidence": "strong", - "line_data_list": [ - { - "line": "LONG LINE TEST: 2041:0:140F::875B:131B 2041:0:140F::875B:131B ;", - "line_num": 15, - "path": "tests/samples/ipv6", - "info": "", - "value": "2041:0:140F::875B:131B", - "value_start": 7010, - "value_end": 7032, - "variable": null, - "variable_start": -2, - "variable_end": -2, - "entropy_validation": { - "iterator": "HEX_CHARS", - "entropy": 2.684338637030481, - "valid": false - } - } - ] - }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "IPv6", - "severity": "info", - "confidence": "strong", - "line_data_list": [ - { - "line": "LONG LINE TEST: 2041:0:140F::875B:131B 2041:0:140F::875B:131B ;", - "line_num": 15, - "path": "tests/samples/ipv6", - "info": "", - "value": "2041:0:140F::875B:131B", - "value_start": 7989, - "value_end": 8011, - "variable": null, - "variable_start": -2, - "variable_end": -2, - "entropy_validation": { - "iterator": "HEX_CHARS", - "entropy": 2.684338637030481, - "valid": false - } - } - ] - }, { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", diff --git a/tests/samples/ipv4 b/tests/samples/ipv4 deleted file mode 100644 index d1da7b509..000000000 --- a/tests/samples/ipv4 +++ /dev/null @@ -1,42 +0,0 @@ -# wrong values: -299.199.99.0 -321.500.312.32 -99.199.299.0 -version 8.8.8.8 -long line check rfc 1.2.3.4 OVERSIZE; - -# list from https://en.wikipedia.org/wiki/Reserved_IP_addresses -0.0.0.0/8 -0.0.0.0–0.255.255.255 -10.0.0.0/8 -10.0.0.0–10.255.255.255 -100.64.0.0/10 -100.64.0.0–100.127.255.255 -127.0.0.0/8 -127.0.0.0–127.255.255.255 -169.254.0.0/16 -169.254.0.0–169.254.255.255 -172.16.0.0/12 -172.16.0.0–172.31.255.255 -192.0.0.0/24 -192.0.0.0–192.0.0.255 -192.0.2.0/24 -192.0.2.0–192.0.2.255 -192.88.99.0/24 -192.88.99.0–192.88.99.255 -192.168.0.0/16 -192.168.0.0–192.168.255.255 -198.18.0.0/15 -198.18.0.0–198.19.255.255 -198.51.100.0/24 -198.51.100.0–198.51.100.255 -203.0.113.0/24 -203.0.113.0–203.0.113.255 -224.0.0.0/4 -224.0.0.0–239.255.255.255 -233.252.0.0/24 -233.252.0.0-233.252.0.255 -240.0.0.0/4 -240.0.0.0–255.255.255.254 -255.255.255.255/32 -255.255.255.255 diff --git a/tests/samples/ipv6 b/tests/samples/ipv6 deleted file mode 100644 index e807faa9e..000000000 --- a/tests/samples/ipv6 +++ /dev/null @@ -1,15 +0,0 @@ -# not an ipv6 -abba:03911 -1234::5678::9 - -# loopback -0000:0000:0000:0000:0000:0000:0000:0001 -::1 # fill -2001:db8:85a3:8d3:1319:8a2e:370:7348 private -fe80::1ff:fe23:4567:890a # link_local - -# dummy but valid -2004:5678::9324 -2004::5678:9 -2041:0000:140F::875B:131B -LONG LINE TEST: 2041:0:140F::875B:131B 2041:0:140F::875B:131B ; From af4e9b0ef99fe3119f423fad276ba7098e5a60be Mon Sep 17 00:00:00 2001 From: Roman Babenko Date: Wed, 7 Aug 2024 13:17:37 +0300 Subject: [PATCH 4/7] JWT rule improvement (#549) * jwt * rollback some * style * [skip actions] [jwt] 2024-08-06T18:27:29+03:00 * [skip actions] [jwt] 2024-08-07T00:14:23+03:00 * custom BM ref * ref: jwt * testfix * more reserved words * BM scores upd * rollback embarrassing changes --- .github/workflows/benchmark.yml | 12 +- cicd/benchmark.txt | 48 ++--- credsweeper/filters/__init__.py | 1 + .../filters/value_azure_token_check.py | 52 +++++ .../filters/value_json_web_token_check.py | 47 ++++- credsweeper/rules/config.yaml | 13 +- credsweeper/utils/util.py | 9 +- docs/source/credsweeper.filters.rst | 48 +++++ tests/__init__.py | 10 +- tests/data/depth_3.json | 190 +++++++----------- tests/data/doc.json | 108 ++++++++++ tests/data/ml_threshold.json | 188 ++++++----------- tests/data/output.json | 161 ++++++--------- tests/filters/test_value_azure_token_check.py | 26 +++ .../test_value_json_web_token_check.py | 47 +++-- tests/rules/test_jwt.py | 10 +- tests/samples/azure_access_token | 4 +- tests/samples/json_web_token | 2 + tests/samples/json_web_token.hs | 1 - 19 files changed, 562 insertions(+), 415 deletions(-) create mode 100644 credsweeper/filters/value_azure_token_check.py create mode 100644 tests/filters/test_value_azure_token_check.py create mode 100644 tests/samples/json_web_token delete mode 100644 tests/samples/json_web_token.hs diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index 1caaa133f..d6017bb39 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -22,7 +22,8 @@ jobs: - name: Checkout CredData uses: actions/checkout@v4 with: - repository: Samsung/CredData + repository: babenek/CredData + ref: jwt - name: Markup hashing run: | @@ -72,7 +73,8 @@ jobs: - name: Checkout CredData uses: actions/checkout@v4 with: - repository: Samsung/CredData + repository: babenek/CredData + ref: jwt - name: Markup hashing run: | @@ -169,7 +171,8 @@ jobs: - name: Checkout CredData uses: actions/checkout@v4 with: - repository: Samsung/CredData + repository: babenek/CredData + ref: jwt - name: Markup hashing run: | @@ -350,7 +353,8 @@ jobs: - name: Checkout CredData uses: actions/checkout@v4 with: - repository: Samsung/CredData + repository: babenek/CredData + ref: jwt - name: Markup hashing run: | diff --git a/cicd/benchmark.txt b/cicd/benchmark.txt index d75bff0b3..c2e02262c 100644 --- a/cicd/benchmark.txt +++ b/cicd/benchmark.txt @@ -1,4 +1,4 @@ -DATA: 16979136 interested lines. MARKUP: 61880 items +DATA: 16978521 interested lines. MARKUP: 61845 items FileType FileNumber ValidLines Positives Negatives Templates --------------- ------------ ------------ ----------- ----------- ----------- 194 28318 64 427 89 @@ -27,7 +27,7 @@ FileType FileNumber ValidLines Positives Negatives Templat .cmd 4 401 2 3 .cnf 8 858 18 45 18 .coffee 1 585 2 -.conf 61 4954 51 74 54 +.conf 60 4945 50 74 54 .config 20 492 16 33 1 .cpp 15 5688 1 61 .creds 1 10 1 1 @@ -53,7 +53,7 @@ FileType FileNumber ValidLines Positives Negatives Templat .erb 13 323 27 .erl 4 96 8 .ex 25 4968 3 105 5 -.example 17 1838 74 37 55 +.example 17 1838 73 37 55 .exs 24 4842 3 188 4 .ext 5 211 1 4 2 .fsproj 1 75 1 @@ -61,16 +61,16 @@ FileType FileNumber ValidLines Positives Negatives Templat .gd 1 37 1 .gml 3 3075 26 .gni 3 5017 18 -.go 1079 566327 621 4334 742 +.go 1079 566327 619 4333 742 .golden 5 1168 1 14 29 .gradle 45 3265 4 91 100 .graphql 7 420 13 .graphqls 1 30 1 -.groovy 23 5011 25 211 1 +.groovy 22 4986 20 215 1 .h 11 2038 38 .haml 9 191 16 .hbs 2 54 3 -.hs 17 4509 37 71 5 +.hs 14 4140 31 72 5 .html 53 15327 14 115 18 .idl 2 777 4 .iml 6 699 36 @@ -80,16 +80,16 @@ FileType FileNumber ValidLines Positives Negatives Templat .ipynb 1 134 5 .j 1 241 4 .j2 30 5530 6 213 10 -.java 621 134132 322 1354 170 +.java 621 134132 314 1357 170 .jenkinsfile 1 58 1 7 .jinja2 1 64 2 .js 659 536413 521 2642 336 -.json 860 13670669 623 10947 143 +.json 860 13670669 623 10948 140 .jsp 13 3202 1 42 .jsx 7 857 19 -.jwt 6 8 7 +.jwt 1 1 2 .key 83 2737 70 14 -.kt 123 20774 51 383 3 +.kt 123 20774 50 384 3 .l 1 982 1 .las 1 6656 46 .lasso 1 230 6 @@ -110,10 +110,10 @@ FileType FileNumber ValidLines Positives Negatives Templat .markdown 3 139 3 1 .markerb 3 12 3 .marko 1 21 2 -.md 675 149422 661 2365 671 +.md 673 149294 646 2366 671 .mdx 3 549 7 .mjml 1 18 1 -.mjs 22 4424 108 310 +.mjs 22 4424 50 343 .mk 1 5878 16 .ml 1 1856 24 .mlir 2 1596 19 @@ -132,7 +132,7 @@ FileType FileNumber ValidLines Positives Negatives Templat .patch 4 109405 27 .pbxproj 1 941 1 .pem 48 1169 47 8 -.php 371 75710 130 1769 80 +.php 371 75710 129 1770 80 .pl 16 14727 6 47 .pm 3 744 8 .po 3 2994 15 @@ -150,13 +150,13 @@ FileType FileNumber ValidLines Positives Negatives Templat .pug 2 193 2 .purs 1 69 4 .pxd 1 150 5 2 -.py 890 291553 618 3466 748 +.py 890 291553 618 3465 748 .pyi 4 1361 9 .pyp 1 167 1 .pyx 2 1094 21 .r 4 62 6 3 1 .rake 2 51 2 -.rb 861 131867 239 3455 615 +.rb 861 131867 237 3457 615 .re 1 31 1 .red 1 159 1 .release 1 13 4 @@ -197,7 +197,7 @@ FileType FileNumber ValidLines Positives Negatives Templat .test 2 24 25 4 .testsettings 1 21 5 .tf 21 1377 3 32 2 -.tfstate 4 307 21 10 4 +.tfstate 4 307 18 11 4 .tfvars 1 31 3 3 .tl 2 2161 165 2 .tmpl 5 336 3 9 @@ -205,7 +205,7 @@ FileType FileNumber ValidLines Positives Negatives Templat .toml 83 2379 54 72 172 .tpl 1 43 1 .travis 1 34 4 3 1 -.ts 584 106807 166 1930 203 +.ts 583 106730 158 1935 203 .tsx 54 7914 1 124 5 .ttar 2 6050 3 .txt 443 78152 1775 14282 50 @@ -222,8 +222,8 @@ FileType FileNumber ValidLines Positives Negatives Templat .yml 418 36162 460 916 384 .zsh 6 872 12 .zsh-theme 1 97 1 -TOTAL: 10294 16979136 7615 59903 5233 -credsweeper result_cnt : 6697, lost_cnt : 0, true_cnt : 6470, false_cnt : 227 +TOTAL: 10281 16978521 7499 59954 5230 +credsweeper result_cnt : 6597, lost_cnt : 0, true_cnt : 6352, false_cnt : 245 Rules Positives Negatives Templates Reported TP FP TN FN FPR FNR ACC PRC RCL F1 ------------------------------ ----------- ----------- ----------- ---------- ---- ---- ----- ---- -------- -------- -------- -------- -------- -------- API 123 3163 185 112 109 3 3345 14 0.000896 0.113821 0.995102 0.973214 0.886179 0.927660 @@ -232,7 +232,7 @@ AWS Multi 75 12 0 8 AWS S3 Bucket 61 25 0 87 61 24 1 0 0.960000 0.000000 0.720930 0.717647 1.000000 0.835616 Atlassian Old PAT token 27 212 3 12 3 8 207 24 0.037209 0.888889 0.867769 0.272727 0.111111 0.157895 Auth 407 2725 77 372 351 21 2781 56 0.007495 0.137592 0.976005 0.943548 0.862408 0.901155 -Azure Access Token 19 0 0 0 0 0 19 1.000000 0.000000 0.000000 +Azure Access Token 19 0 0 12 12 0 0 7 0.368421 0.631579 1.000000 0.631579 0.774194 BASE64 Private Key 7 2 0 7 7 0 2 0 0.000000 0.000000 1.000000 1.000000 1.000000 1.000000 BASE64 encoded PEM Private Key 7 0 0 5 5 0 0 2 0.285714 0.714286 1.000000 0.714286 0.833333 Bitbucket Client ID 142 1813 9 46 27 18 1804 115 0.009879 0.809859 0.932281 0.600000 0.190141 0.288770 @@ -249,8 +249,8 @@ Gitlab Incoming Email Token 37 3 0 2 Google API Key 12 0 0 12 12 0 0 0 0.000000 1.000000 1.000000 1.000000 1.000000 Google Multi 10 2 0 11 10 1 1 0 0.500000 0.000000 0.916667 0.909091 1.000000 0.952381 Google OAuth Access Token 3 0 0 3 3 0 0 0 0.000000 1.000000 1.000000 1.000000 1.000000 -Grafana Provisioned API Key 22 1 0 1 1 0 1 21 0.000000 0.954545 0.086957 1.000000 0.045455 0.086957 -JSON Web Token 284 11 2 274 271 3 10 13 0.230769 0.045775 0.946128 0.989051 0.954225 0.971326 +Grafana Provisioned API Key 22 1 0 5 5 0 1 17 0.000000 0.772727 0.260870 1.000000 0.227273 0.370370 +JSON Web Token 169 61 0 158 137 21 40 32 0.344262 0.189349 0.769565 0.867089 0.810651 0.837920 Jira / Confluence PAT token 0 4 0 0 0 4 0 0.000000 1.000000 Jira 2FA 14 6 0 10 10 0 6 4 0.000000 0.285714 0.800000 1.000000 0.714286 0.833333 Key 483 8494 464 445 436 9 8949 47 0.001005 0.097308 0.994068 0.979775 0.902692 0.939655 @@ -262,7 +262,7 @@ Salt 42 76 2 3 Secret 1358 28497 869 1234 1229 5 29361 129 0.000170 0.094993 0.995639 0.995948 0.905007 0.948302 Seed 1 6 0 0 0 6 1 0.000000 1.000000 0.857143 0.000000 Slack Token 4 1 0 4 4 0 1 0 0.000000 0.000000 1.000000 1.000000 1.000000 1.000000 -Token 585 3972 439 519 511 8 4403 74 0.001814 0.126496 0.983587 0.984586 0.873504 0.925725 +Token 584 3973 438 519 511 8 4403 73 0.001814 0.125000 0.983784 0.984586 0.875000 0.926564 Twilio API Key 0 5 2 0 0 7 0 0.000000 1.000000 URL Credentials 194 125 251 184 184 0 376 10 0.000000 0.051546 0.982456 1.000000 0.948454 0.973545 - 7615 59903 5233 6704 6470 227 59676 1145 0.003789 0.150361 0.979679 0.966104 0.849639 0.904136 + 7499 59954 5230 6604 6352 245 59709 1147 0.004086 0.152954 0.979363 0.962862 0.847046 0.901249 diff --git a/credsweeper/filters/__init__.py b/credsweeper/filters/__init__.py index f88bcac57..f050671a5 100644 --- a/credsweeper/filters/__init__.py +++ b/credsweeper/filters/__init__.py @@ -5,6 +5,7 @@ from credsweeper.filters.value_allowlist_check import ValueAllowlistCheck from credsweeper.filters.value_array_dictionary_check import ValueArrayDictionaryCheck from credsweeper.filters.value_atlassian_token_check import ValueAtlassianTokenCheck +from credsweeper.filters.value_azure_token_check import ValueAzureTokenCheck from credsweeper.filters.value_base32_data_check import ValueBase32DataCheck from credsweeper.filters.value_base64_data_check import ValueBase64DataCheck from credsweeper.filters.value_base64_encoded_pem_check import ValueBase64EncodedPem diff --git a/credsweeper/filters/value_azure_token_check.py b/credsweeper/filters/value_azure_token_check.py new file mode 100644 index 000000000..d62293a54 --- /dev/null +++ b/credsweeper/filters/value_azure_token_check.py @@ -0,0 +1,52 @@ +import contextlib +import json + +from credsweeper.common.constants import Chars +from credsweeper.config import Config +from credsweeper.credentials import LineData +from credsweeper.file_handler.analysis_target import AnalysisTarget +from credsweeper.filters import Filter +from credsweeper.filters.value_entropy_base64_check import ValueEntropyBase64Check +from credsweeper.utils import Util + + +class ValueAzureTokenCheck(Filter): + """ + Azure tokens contains header, payload and signature + https://learn.microsoft.com/en-us/azure/active-directory-b2c/access-tokens + """ + + def __init__(self, config: Config = None) -> None: + pass + + def run(self, line_data: LineData, target: AnalysisTarget) -> bool: + """Run filter checks on received token which might be structured. + + Args: + line_data: credential candidate data + target: multiline target from which line data was obtained + + Return: + True, when need to filter candidate and False if left + + """ + with contextlib.suppress(Exception): + parts = line_data.value.split('.') + if 3 != len(parts): + return True + hdr = Util.decode_base64(parts[0], padding_safe=True, urlsafe_detect=True) + header = json.loads(hdr) + if not ("alg" in header and "typ" in header and "kid" in header): + # must be all parts in header + return True + pld = Util.decode_base64(parts[1], padding_safe=True, urlsafe_detect=True) + payload = json.loads(pld) + if not ("iss" in payload and "exp" in payload and "iat" in payload): + # must be all parts in payload + return True + min_entropy = ValueEntropyBase64Check.get_min_data_entropy(len(parts[2])) + entropy = Util.get_shannon_entropy(parts[2], Chars.BASE64URL_CHARS.value) + # good signature has to be like random bytes + return entropy < min_entropy + + return True diff --git a/credsweeper/filters/value_json_web_token_check.py b/credsweeper/filters/value_json_web_token_check.py index 7f6048a80..d7265dbce 100644 --- a/credsweeper/filters/value_json_web_token_check.py +++ b/credsweeper/filters/value_json_web_token_check.py @@ -11,9 +11,19 @@ class ValueJsonWebTokenCheck(Filter): """ Check that candidate is JWT which starts usually from 'eyJ' - only header is parsed with "typ" or "alg" member from example of RFC7519 - https://datatracker.ietf.org/doc/html/rfc7519 + registered keys are checked to be in the JWT parts + https://www.iana.org/assignments/jose/jose.xhtml """ + header_keys = { + "alg", "jku", "jwk", "kid", "x5u", "x5c", "x5t", "x5t#S256", "typ", "cty", "crit", "alg", "enc", "zip", "jku", + "jwk", "kid", "x5u", "x5c", "x5t", "x5t#S256", "typ", "cty", "crit", "epk", "apu", "apv", "iv", "tag", "p2s", + "p2c", "iss", "sub", "aud", "b64", "ppt", "url", "nonce", "svt" + } + payload_keys = { + "iss", "sub", "aud", "exp", "nbf", "iat", "jti", "kty", "use", "key_ops", "alg", "enc", "zip", "jku", "jwk", + "kid", "x5u", "x5c", "x5t", "x5t#S256", "crv", "x", "y", "d", "n", "e", "d", "p", "q", "dp", "dq", "qi", "oth", + "k", "crv", "d", "x", "ext", "crit", "keys", "id", "role", "token", "secret", "password", "nonce" + } def __init__(self, config: Config = None) -> None: pass @@ -29,12 +39,29 @@ def run(self, line_data: LineData, target: AnalysisTarget) -> bool: True, when need to filter candidate and False if left """ + header_check = False + payload_check = False + signature_check = False with contextlib.suppress(Exception): - delimiter_pos = line_data.value.find(".") - # jwt token. '.' must be always in given data, according regex in rule - value = line_data.value[:delimiter_pos] - decoded = Util.decode_base64(value, padding_safe=True, urlsafe_detect=True) - if header := json.loads(decoded): - if "alg" in header or "typ" in header: - return False - return True + jwt_parts = line_data.value.split('.') + for part in jwt_parts: + data = Util.decode_base64(part, padding_safe=True, urlsafe_detect=True) + if part.startswith("eyJ"): + # open part - just base64 encoded + json_keys = json.loads(data).keys() + # header will be checked first + if not header_check: + header_check = bool(ValueJsonWebTokenCheck.header_keys.intersection(json_keys)) + # payload follows the header + if not payload_check: + payload_check = bool(ValueJsonWebTokenCheck.payload_keys.intersection(json_keys)) + # any other payloads are allowed + elif header_check and payload_check and not signature_check: + # signature check or skip encrypted part + signature_check = not Util.is_ascii_entropy_validate(data) + else: + break + if header_check and payload_check and signature_check: + return False + else: + return True diff --git a/credsweeper/rules/config.yaml b/credsweeper/rules/config.yaml index ce05d3b4b..26e7e5198 100644 --- a/credsweeper/rules/config.yaml +++ b/credsweeper/rules/config.yaml @@ -338,17 +338,18 @@ - name: JSON Web Token severity: medium - confidence: moderate + confidence: strong type: pattern values: - - (?eyJ[0-9A-Za-z_=-]{15,8000}([.0-9A-Za-z_=-]{1,8000})?) - filter_type: GeneralPattern - use_ml: true + - (?eyJ[0-9A-Za-z_+/=-]{15,8000}(\.[0-9A-Za-z_+/=-]{0,8000}){2,16}) + filter_type: + - ValueJsonWebTokenCheck required_substrings: - eyJ min_line_len: 18 target: - code + - doc - name: MailChimp API Key severity: high @@ -841,9 +842,9 @@ confidence: strong type: pattern values: - - (?eyJ[A-Za-z0-9_=-]{50,500}\.eyJ[A-Za-z0-9_=-]{1,8000}\.[A-Za-z0-9_=-]{1,8000}) + - (?eyJ[A-Za-z0-9_=-]{50,500}\.eyJ[A-Za-z0-9_=-]{8,8000}\.[A-Za-z0-9_=-]{18,800}) filter_type: - - ValueJsonWebTokenCheck + - ValueAzureTokenCheck required_substrings: - eyJ min_line_len: 148 diff --git a/credsweeper/utils/util.py b/credsweeper/utils/util.py index 3f51d18d5..400d9c0a5 100644 --- a/credsweeper/utils/util.py +++ b/credsweeper/utils/util.py @@ -84,6 +84,8 @@ def get_shannon_entropy(data: str, iterator: str) -> float: 32: 3.25392803184602, 40: 3.64853567064867, 64: 4.57756933688035, + 384: 7.39, + 512: 7.55, } @staticmethod @@ -95,10 +97,13 @@ def get_min_data_entropy(x: int) -> float: # approximated for range 12 - 64 _x = x - 8 y = ((0.000016617804 * _x - 0.002695077) * _x + 0.170393) * _x + 0.4 - elif 64 < x: + elif 64 < x < 384: # logarithm base 2 - slow, but precise _x = x - 8 - y = 1.581026279659 * math.log2(_x) - 1.90156 + y = 1.095884 * math.log2(_x) - 1.90156 + elif 384 < x < 512: + # solved for 384 - 512 + y = -0.11215851 * math.log2(x)**2 + 2.34303484 * math.log2(x) - 4.4466237 else: # less or equal to 8 bytes might have 0 entropy y = 0 diff --git a/docs/source/credsweeper.filters.rst b/docs/source/credsweeper.filters.rst index 7599043ae..e91a76a69 100644 --- a/docs/source/credsweeper.filters.rst +++ b/docs/source/credsweeper.filters.rst @@ -20,6 +20,14 @@ credsweeper.filters.filter module :undoc-members: :show-inheritance: +credsweeper.filters.line\_git\_binary\_check module +--------------------------------------------------- + +.. automodule:: credsweeper.filters.line_git_binary_check + :members: + :undoc-members: + :show-inheritance: + credsweeper.filters.line\_specific\_key\_check module ----------------------------------------------------- @@ -60,6 +68,14 @@ credsweeper.filters.value\_atlassian\_token\_check module :undoc-members: :show-inheritance: +credsweeper.filters.value\_azure\_token\_check module +----------------------------------------------------- + +.. automodule:: credsweeper.filters.value_azure_token_check + :members: + :undoc-members: + :show-inheritance: + credsweeper.filters.value\_base32\_data\_check module ----------------------------------------------------- @@ -92,6 +108,14 @@ credsweeper.filters.value\_base64\_key\_check module :undoc-members: :show-inheritance: +credsweeper.filters.value\_base64\_part\_check module +----------------------------------------------------- + +.. automodule:: credsweeper.filters.value_base64_part_check + :members: + :undoc-members: + :show-inheritance: + credsweeper.filters.value\_blocklist\_check module -------------------------------------------------- @@ -132,6 +156,14 @@ credsweeper.filters.value\_dictionary\_value\_length\_check module :undoc-members: :show-inheritance: +credsweeper.filters.value\_discord\_bot\_check module +----------------------------------------------------- + +.. automodule:: credsweeper.filters.value_discord_bot_check + :members: + :undoc-members: + :show-inheritance: + credsweeper.filters.value\_entropy\_base32\_check module -------------------------------------------------------- @@ -188,6 +220,22 @@ credsweeper.filters.value\_grafana\_check module :undoc-members: :show-inheritance: +credsweeper.filters.value\_grafana\_service\_check module +--------------------------------------------------------- + +.. automodule:: credsweeper.filters.value_grafana_service_check + :members: + :undoc-members: + :show-inheritance: + +credsweeper.filters.value\_hex\_number\_check module +---------------------------------------------------- + +.. automodule:: credsweeper.filters.value_hex_number_check + :members: + :undoc-members: + :show-inheritance: + credsweeper.filters.value\_jfrog\_token\_check module ----------------------------------------------------- diff --git a/tests/__init__.py b/tests/__init__.py index 4f6c5b878..5bda59e59 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -7,18 +7,18 @@ NEGLIGIBLE_ML_THRESHOLD = 0.0001 # credentials count after scan -SAMPLES_CRED_COUNT: int = 362 -SAMPLES_CRED_LINE_COUNT: int = 379 +SAMPLES_CRED_COUNT: int = 360 +SAMPLES_CRED_LINE_COUNT: int = 377 # credentials count after post-processing -SAMPLES_POST_CRED_COUNT: int = 320 +SAMPLES_POST_CRED_COUNT: int = 319 # with option --doc -SAMPLES_IN_DOC = 411 +SAMPLES_IN_DOC = 415 # archived credentials that are not found without --depth SAMPLES_IN_DEEP_1 = SAMPLES_POST_CRED_COUNT + 24 -SAMPLES_IN_DEEP_2 = SAMPLES_IN_DEEP_1 + 17 +SAMPLES_IN_DEEP_2 = SAMPLES_IN_DEEP_1 + 16 SAMPLES_IN_DEEP_3 = SAMPLES_IN_DEEP_2 + 1 # well known string with all latin letters diff --git a/tests/data/depth_3.json b/tests/data/depth_3.json index bf0894ec2..4c7822ef9 100644 --- a/tests/data/depth_3.json +++ b/tests/data/depth_3.json @@ -394,7 +394,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.941, + "ml_probability": 0.963, "rule": "Auth", "severity": "medium", "confidence": "moderate", @@ -418,33 +418,6 @@ } ] }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.941, - "rule": "JSON Web Token", - "severity": "medium", - "confidence": "moderate", - "line_data_list": [ - { - "line": "curl -H \"Authorization: Bearer eyJGRpVu1c2VzY2-823r_db32hbf4W1lbj\" http://localhost:8080/.", - "line_num": 9, - "path": "tests/samples/auth_n.template", - "info": "tests/samples/auth_n.template|RAW", - "value": "eyJGRpVu1c2VzY2-823r_db32hbf4W1lbj", - "value_start": 31, - "value_end": 65, - "variable": null, - "variable_start": -2, - "variable_end": -2, - "entropy_validation": { - "iterator": "BASE36_CHARS", - "entropy": 3.2479906920322064, - "valid": true - } - } - ] - }, { "api_validation": "NOT_AVAILABLE", "ml_validation": "NOT_AVAILABLE", @@ -805,11 +778,11 @@ }, { "api_validation": "NOT_AVAILABLE", - "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.93, + "ml_validation": "NOT_AVAILABLE", + "ml_probability": null, "rule": "JSON Web Token", "severity": "medium", - "confidence": "moderate", + "confidence": "strong", "line_data_list": [ { "line": "eyJ0eXAiOiJKV1QiLCJhbGciOiJSUzI1NiIsIng1dCI6Imk2bEdrM0ZaenhSY1ViMkMzbkVRN3N5SEpsWSIsImtpZCI6Imk2bEdrM0ZaenhSY1ViMkMzbkVRN3N5SEpsWSJ9.eyJhdWQiOiJlZjFkYTlkNC1mZjc3LTRjM2UtYTAwNS04NDBjM2Y4MzA3NDUiLCJpc3MiOiJodHRwczovL3N0cy53aW5kb3dzLm5ldC9mYTE1ZDY5Mi1lOWM3LTQ0NjAtYTc0My0yOWYyOTUyMjIyOS8iLCJpYXQiOjE1MzcyMzMxMDYsIm5iZiI6MTUzNzIzMzEwNiwiZXhwIjoxNTM3MjM3MDA2LCJhY3IiOiIxIiwiYWlvIjoiQVhRQWkvOElBQUFBRm0rRS9RVEcrZ0ZuVnhMaldkdzhLKzYxQUdyU091TU1GNmViYU1qN1hPM0libUQzZkdtck95RCtOdlp5R24yVmFUL2tES1h3NE1JaHJnR1ZxNkJuOHdMWG9UMUxrSVorRnpRVmtKUFBMUU9WNEtjWHFTbENWUERTL0RpQ0RnRTIyMlRJbU12V05hRU1hVU9Uc0lHdlRRPT0iLCJhbXIiOlsid2lhIl0sImFwcGlkIjoiNzVkYmU3N2YtMTBhMy00ZTU5LTg1ZmQtOGMxMjc1NDRmMTdjIiwiYXBwaWRhY3IiOiIwIiwiZW1haWwiOiJBYmVMaUBtaWNyb3NvZnQuY29tIiwiZmFtaWx5X25hbWUiOiJMaW5jb2xuIiwiZ2l2ZW5fbmFtZSI6IkFiZSAoTVNGVCkiLCJpZHAiOiJodHRwczovL3N0cy53aW5kb3dzLm5ldC83MmY5ODhiZi04NmYxLTQxYWYtOTFhYi0yZDdjZDAxMjIyNDcvIiwiaXBhZGRyIjoiMjIyLjIyMi4yMjIuMjIiLCJuYW1lIjoiYWJlbGkiLCJvaWQiOiIwMjIyM2I2Yi1hYTFkLTQyZDQtOWVjMC0xYjJiYjkxOTQ0MzgiLCJyaCI6IkkiLCJzY3AiOiJ1c2VyX2ltcGVyc29uYXRpb24iLCJzdWIiOiJsM19yb0lTUVUyMjJiVUxTOXlpMmswWHBxcE9pTXo1SDNaQUNvMUdlWEEiLCJ0aWQiOiJmYTE1ZDY5Mi1lOWM3LTQ0NjAtYTc0My0yOWYyOTU2ZmQ0MjkiLCJ1bmlxdWVfbmFtZSI6ImFiZWxpQG1pY3Jvc29mdC5jb20iLCJ1dGkiOiJGVnNHeFlYSTMwLVR1aWt1dVVvRkFBIiwidmVyIjoiMS4wIn0.D3H6pMUtQnoJAGq6AHd", @@ -830,6 +803,60 @@ } ] }, + { + "api_validation": "NOT_AVAILABLE", + "ml_validation": "NOT_AVAILABLE", + "ml_probability": null, + "rule": "Azure Access Token", + "severity": "high", + "confidence": "strong", + "line_data_list": [ + { + "line": "eyJ0eXAiOiJKV1QiLCJhbGciOiJSUzI1NiIsImtpZCI6Imk2bEdrM0ZaenhSY1ViMkMzbkVRN3N5SEpsWSJ9.eyJhdWQiOiI2ZTc0MTcyYi1iZTU2LTQ4NDMtOWZmNC1lNjZhMzliYjEyZTMiLCJpc3MiOiJodHRwczovL2xvZ2luLm1pY3Jvc29mdG9ubGluZS5jb20vNzJmOTg4YmYtODZmMS00MWFmLTkxYWItMmQ3Y2QwMTFkYjQ3L3YyLjAiLCJpYXQiOjE1MzcyMzEwNDgsIm5iZiI6MTUzNzIzMTA0OCwiZXhwIjoxNTM3MjM0OTQ4LCJhaW8iOiJBWFFBaS84SUFBQUF0QWFaTG8zQ2hNaWY2S09udHRSQjdlQnE0L0RjY1F6amNKR3hQWXkvQzNqRGFOR3hYZDZ3TklJVkdSZ2hOUm53SjFsT2NBbk5aY2p2a295ckZ4Q3R0djMzMTQwUmlvT0ZKNGJDQ0dWdW9DYWcxdU9UVDIyMjIyZ0h3TFBZUS91Zjc5UVgrMEtJaWpkcm1wNjlSY3R6bVE9PSIsImF6cCI6IjZlNzQxNzJiLWJlNTYtNDg0My05ZmY0LWU2NmEzOWJiMTJlMyIsImF6cGFjciI6IjAiLCJuYW1lIjoiQWJlIExpbmNvbG4iLCJvaWQiOiI2OTAyMjJiZS1mZjFhLTRkNTYtYWJkMS03ZTRmN2QzOGU0NzQiLCJwcmVmZXJyZWRfdXNlcm5hbWUiOiJhYmVsaUBtaWNyb3NvZnQuY29tIiwicmgiOiJJIiwic2NwIjoiYWNjZXNzX2FzX3VzZXIiLCJzdWIiOiJIS1pwZmFIeVdhZGVPb3VZbGl0anJJLUtmZlRtMjIyWDVyclYzeERxZktRIiwidGlkIjoiNzJmOTg4YmYtODZmMS00MWFmLTkxYWItMmQ3Y2QwMTFkYjQ3IiwidXRpIjoiZnFpQnFYTFBqMGVRYTgyUy1JWUZBQSIsInZlciI6IjIuMCJ9.pj4N-w_3Us9DrBLfpCt", + "line_num": 2, + "path": "tests/samples/azure_access_token", + "info": "tests/samples/azure_access_token|RAW", + "value": "eyJ0eXAiOiJKV1QiLCJhbGciOiJSUzI1NiIsImtpZCI6Imk2bEdrM0ZaenhSY1ViMkMzbkVRN3N5SEpsWSJ9.eyJhdWQiOiI2ZTc0MTcyYi1iZTU2LTQ4NDMtOWZmNC1lNjZhMzliYjEyZTMiLCJpc3MiOiJodHRwczovL2xvZ2luLm1pY3Jvc29mdG9ubGluZS5jb20vNzJmOTg4YmYtODZmMS00MWFmLTkxYWItMmQ3Y2QwMTFkYjQ3L3YyLjAiLCJpYXQiOjE1MzcyMzEwNDgsIm5iZiI6MTUzNzIzMTA0OCwiZXhwIjoxNTM3MjM0OTQ4LCJhaW8iOiJBWFFBaS84SUFBQUF0QWFaTG8zQ2hNaWY2S09udHRSQjdlQnE0L0RjY1F6amNKR3hQWXkvQzNqRGFOR3hYZDZ3TklJVkdSZ2hOUm53SjFsT2NBbk5aY2p2a295ckZ4Q3R0djMzMTQwUmlvT0ZKNGJDQ0dWdW9DYWcxdU9UVDIyMjIyZ0h3TFBZUS91Zjc5UVgrMEtJaWpkcm1wNjlSY3R6bVE9PSIsImF6cCI6IjZlNzQxNzJiLWJlNTYtNDg0My05ZmY0LWU2NmEzOWJiMTJlMyIsImF6cGFjciI6IjAiLCJuYW1lIjoiQWJlIExpbmNvbG4iLCJvaWQiOiI2OTAyMjJiZS1mZjFhLTRkNTYtYWJkMS03ZTRmN2QzOGU0NzQiLCJwcmVmZXJyZWRfdXNlcm5hbWUiOiJhYmVsaUBtaWNyb3NvZnQuY29tIiwicmgiOiJJIiwic2NwIjoiYWNjZXNzX2FzX3VzZXIiLCJzdWIiOiJIS1pwZmFIeVdhZGVPb3VZbGl0anJJLUtmZlRtMjIyWDVyclYzeERxZktRIiwidGlkIjoiNzJmOTg4YmYtODZmMS00MWFmLTkxYWItMmQ3Y2QwMTFkYjQ3IiwidXRpIjoiZnFpQnFYTFBqMGVRYTgyUy1JWUZBQSIsInZlciI6IjIuMCJ9.pj4N-w_3Us9DrBLfpCt", + "value_start": 0, + "value_end": 1029, + "variable": null, + "variable_start": -2, + "variable_end": -2, + "entropy_validation": { + "iterator": "BASE64_CHARS", + "entropy": 5.6044494049575055, + "valid": true + } + } + ] + }, + { + "api_validation": "NOT_AVAILABLE", + "ml_validation": "NOT_AVAILABLE", + "ml_probability": null, + "rule": "JSON Web Token", + "severity": "medium", + "confidence": "strong", + "line_data_list": [ + { + "line": "eyJ0eXAiOiJKV1QiLCJhbGciOiJSUzI1NiIsImtpZCI6Imk2bEdrM0ZaenhSY1ViMkMzbkVRN3N5SEpsWSJ9.eyJhdWQiOiI2ZTc0MTcyYi1iZTU2LTQ4NDMtOWZmNC1lNjZhMzliYjEyZTMiLCJpc3MiOiJodHRwczovL2xvZ2luLm1pY3Jvc29mdG9ubGluZS5jb20vNzJmOTg4YmYtODZmMS00MWFmLTkxYWItMmQ3Y2QwMTFkYjQ3L3YyLjAiLCJpYXQiOjE1MzcyMzEwNDgsIm5iZiI6MTUzNzIzMTA0OCwiZXhwIjoxNTM3MjM0OTQ4LCJhaW8iOiJBWFFBaS84SUFBQUF0QWFaTG8zQ2hNaWY2S09udHRSQjdlQnE0L0RjY1F6amNKR3hQWXkvQzNqRGFOR3hYZDZ3TklJVkdSZ2hOUm53SjFsT2NBbk5aY2p2a295ckZ4Q3R0djMzMTQwUmlvT0ZKNGJDQ0dWdW9DYWcxdU9UVDIyMjIyZ0h3TFBZUS91Zjc5UVgrMEtJaWpkcm1wNjlSY3R6bVE9PSIsImF6cCI6IjZlNzQxNzJiLWJlNTYtNDg0My05ZmY0LWU2NmEzOWJiMTJlMyIsImF6cGFjciI6IjAiLCJuYW1lIjoiQWJlIExpbmNvbG4iLCJvaWQiOiI2OTAyMjJiZS1mZjFhLTRkNTYtYWJkMS03ZTRmN2QzOGU0NzQiLCJwcmVmZXJyZWRfdXNlcm5hbWUiOiJhYmVsaUBtaWNyb3NvZnQuY29tIiwicmgiOiJJIiwic2NwIjoiYWNjZXNzX2FzX3VzZXIiLCJzdWIiOiJIS1pwZmFIeVdhZGVPb3VZbGl0anJJLUtmZlRtMjIyWDVyclYzeERxZktRIiwidGlkIjoiNzJmOTg4YmYtODZmMS00MWFmLTkxYWItMmQ3Y2QwMTFkYjQ3IiwidXRpIjoiZnFpQnFYTFBqMGVRYTgyUy1JWUZBQSIsInZlciI6IjIuMCJ9.pj4N-w_3Us9DrBLfpCt", + "line_num": 2, + "path": "tests/samples/azure_access_token", + "info": "tests/samples/azure_access_token|RAW", + "value": "eyJ0eXAiOiJKV1QiLCJhbGciOiJSUzI1NiIsImtpZCI6Imk2bEdrM0ZaenhSY1ViMkMzbkVRN3N5SEpsWSJ9.eyJhdWQiOiI2ZTc0MTcyYi1iZTU2LTQ4NDMtOWZmNC1lNjZhMzliYjEyZTMiLCJpc3MiOiJodHRwczovL2xvZ2luLm1pY3Jvc29mdG9ubGluZS5jb20vNzJmOTg4YmYtODZmMS00MWFmLTkxYWItMmQ3Y2QwMTFkYjQ3L3YyLjAiLCJpYXQiOjE1MzcyMzEwNDgsIm5iZiI6MTUzNzIzMTA0OCwiZXhwIjoxNTM3MjM0OTQ4LCJhaW8iOiJBWFFBaS84SUFBQUF0QWFaTG8zQ2hNaWY2S09udHRSQjdlQnE0L0RjY1F6amNKR3hQWXkvQzNqRGFOR3hYZDZ3TklJVkdSZ2hOUm53SjFsT2NBbk5aY2p2a295ckZ4Q3R0djMzMTQwUmlvT0ZKNGJDQ0dWdW9DYWcxdU9UVDIyMjIyZ0h3TFBZUS91Zjc5UVgrMEtJaWpkcm1wNjlSY3R6bVE9PSIsImF6cCI6IjZlNzQxNzJiLWJlNTYtNDg0My05ZmY0LWU2NmEzOWJiMTJlMyIsImF6cGFjciI6IjAiLCJuYW1lIjoiQWJlIExpbmNvbG4iLCJvaWQiOiI2OTAyMjJiZS1mZjFhLTRkNTYtYWJkMS03ZTRmN2QzOGU0NzQiLCJwcmVmZXJyZWRfdXNlcm5hbWUiOiJhYmVsaUBtaWNyb3NvZnQuY29tIiwicmgiOiJJIiwic2NwIjoiYWNjZXNzX2FzX3VzZXIiLCJzdWIiOiJIS1pwZmFIeVdhZGVPb3VZbGl0anJJLUtmZlRtMjIyWDVyclYzeERxZktRIiwidGlkIjoiNzJmOTg4YmYtODZmMS00MWFmLTkxYWItMmQ3Y2QwMTFkYjQ3IiwidXRpIjoiZnFpQnFYTFBqMGVRYTgyUy1JWUZBQSIsInZlciI6IjIuMCJ9.pj4N-w_3Us9DrBLfpCt", + "value_start": 0, + "value_end": 1029, + "variable": null, + "variable_start": -2, + "variable_end": -2, + "entropy_validation": { + "iterator": "BASE64_CHARS", + "entropy": 5.6044494049575055, + "valid": true + } + } + ] + }, { "api_validation": "NOT_AVAILABLE", "ml_validation": "NOT_AVAILABLE", @@ -6885,33 +6912,6 @@ } ] }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.985, - "rule": "JSON Web Token", - "severity": "medium", - "confidence": "moderate", - "line_data_list": [ - { - "line": "grafana = 'eyJrIjoiMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMCIsIm4iOiJ4eHh4IiwiaWQiOjIwNDM2MH0='", - "line_num": 1, - "path": "tests/samples/grafana_provisioned_api_key", - "info": "tests/samples/grafana_provisioned_api_key|RAW", - "value": "eyJrIjoiMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMCIsIm4iOiJ4eHh4IiwiaWQiOjIwNDM2MH0=", - "value_start": 11, - "value_end": 107, - "variable": null, - "variable_start": -2, - "variable_end": -2, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 3.8153130511409934, - "valid": false - } - } - ] - }, { "api_validation": "NOT_AVAILABLE", "ml_validation": "NOT_AVAILABLE", @@ -7157,26 +7157,26 @@ }, { "api_validation": "NOT_AVAILABLE", - "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.931, + "ml_validation": "NOT_AVAILABLE", + "ml_probability": null, "rule": "JSON Web Token", "severity": "medium", - "confidence": "moderate", + "confidence": "strong", "line_data_list": [ { - "line": "$payload = 'eyJgsIZgeJhvNgFpSmlP.eyJcaaF9xCe7shE0ENPiBlEJOpS'", + "line": "detected: eyJhbGciOiJSUzI1NiJ9Cg.eyJleHAiOjY1NTM2fQo.Ce7sh0ENPiBlE_dose0cBA", "line_num": 1, - "path": "tests/samples/json_web_token.hs", - "info": "tests/samples/json_web_token.hs|RAW", - "value": "eyJgsIZgeJhvNgFpSmlP.eyJcaaF9xCe7shE0ENPiBlEJOpS", - "value_start": 12, - "value_end": 60, + "path": "tests/samples/json_web_token", + "info": "tests/samples/json_web_token|RAW", + "value": "eyJhbGciOiJSUzI1NiJ9Cg.eyJleHAiOjY1NTM2fQo.Ce7sh0ENPiBlE_dose0cBA", + "value_start": 10, + "value_end": 75, "variable": null, "variable_start": -2, "variable_end": -2, "entropy_validation": { "iterator": "BASE64_CHARS", - "entropy": 4.520488802699322, + "entropy": 4.790963630103494, "valid": true } } @@ -10487,33 +10487,6 @@ } ] }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "VALIDATED_KEY", - "ml_probability": 1.0, - "rule": "JSON Web Token", - "severity": "medium", - "confidence": "moderate", - "line_data_list": [ - { - "line": "token in zip: eyJrIjoiMDAwMDAwNDAwMDAwODAwMDAwMDAwMDAwNDAwMDAwMDAwMDAwMDAyMSIsIm4iOiJ4eHh4IiwiaWQiOjQzMDh9Cg", - "line_num": 1, - "path": "tests/samples/test2.eml", - "info": "tests/samples/test2.eml|EML-DATA|ZIP|token.txt|RAW", - "value": "eyJrIjoiMDAwMDAwNDAwMDAwODAwMDAwMDAwMDAwNDAwMDAwMDAwMDAwMDAyMSIsIm4iOiJ4eHh4IiwiaWQiOjQzMDh9Cg", - "value_start": 14, - "value_end": 108, - "variable": null, - "variable_start": -2, - "variable_end": -2, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 4.006147345318248, - "valid": false - } - } - ] - }, { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", @@ -10568,33 +10541,6 @@ } ] }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.999, - "rule": "JSON Web Token", - "severity": "medium", - "confidence": "moderate", - "line_data_list": [ - { - "line": "token in text: eyJrIjoiMDAwMDAwNDAwMDAwODAwNDAwMDAwMDAwNDAwMDAwMDAwMDAwMDAyMSIsIm4iOiJ4eHh4IiwiaWQiOjQzMDh9Cg", - "line_num": 8, - "path": "tests/samples/test2.eml", - "info": "tests/samples/test2.eml|EML-TEXT", - "value": "eyJrIjoiMDAwMDAwNDAwMDAwODAwNDAwMDAwMDAwNDAwMDAwMDAwMDAwMDAyMSIsIm4iOiJ4eHh4IiwiaWQiOjQzMDh9Cg", - "value_start": 15, - "value_end": 109, - "variable": null, - "variable_start": -2, - "variable_end": -2, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 4.0296677144829305, - "valid": false - } - } - ] - }, { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", diff --git a/tests/data/doc.json b/tests/data/doc.json index edef67d77..46c8cdb7d 100644 --- a/tests/data/doc.json +++ b/tests/data/doc.json @@ -438,6 +438,87 @@ } ] }, + { + "api_validation": "NOT_AVAILABLE", + "ml_validation": "NOT_AVAILABLE", + "ml_probability": null, + "rule": "JSON Web Token", + "severity": "medium", + "confidence": "strong", + "line_data_list": [ + { + "line": "eyJ0eXAiOiJKV1QiLCJhbGciOiJSUzI1NiIsIng1dCI6Imk2bEdrM0ZaenhSY1ViMkMzbkVRN3N5SEpsWSIsImtpZCI6Imk2bEdrM0ZaenhSY1ViMkMzbkVRN3N5SEpsWSJ9.eyJhdWQiOiJlZjFkYTlkNC1mZjc3LTRjM2UtYTAwNS04NDBjM2Y4MzA3NDUiLCJpc3MiOiJodHRwczovL3N0cy53aW5kb3dzLm5ldC9mYTE1ZDY5Mi1lOWM3LTQ0NjAtYTc0My0yOWYyOTUyMjIyOS8iLCJpYXQiOjE1MzcyMzMxMDYsIm5iZiI6MTUzNzIzMzEwNiwiZXhwIjoxNTM3MjM3MDA2LCJhY3IiOiIxIiwiYWlvIjoiQVhRQWkvOElBQUFBRm0rRS9RVEcrZ0ZuVnhMaldkdzhLKzYxQUdyU091TU1GNmViYU1qN1hPM0libUQzZkdtck95RCtOdlp5R24yVmFUL2tES1h3NE1JaHJnR1ZxNkJuOHdMWG9UMUxrSVorRnpRVmtKUFBMUU9WNEtjWHFTbENWUERTL0RpQ0RnRTIyMlRJbU12V05hRU1hVU9Uc0lHdlRRPT0iLCJhbXIiOlsid2lhIl0sImFwcGlkIjoiNzVkYmU3N2YtMTBhMy00ZTU5LTg1ZmQtOGMxMjc1NDRmMTdjIiwiYXBwaWRhY3IiOiIwIiwiZW1haWwiOiJBYmVMaUBtaWNyb3NvZnQuY29tIiwiZmFtaWx5X25hbWUiOiJMaW5jb2xuIiwiZ2l2ZW5fbmFtZSI6IkFiZSAoTVNGVCkiLCJpZHAiOiJodHRwczovL3N0cy53aW5kb3dzLm5ldC83MmY5ODhiZi04NmYxLTQxYWYtOTFhYi0yZDdjZDAxMjIyNDcvIiwiaXBhZGRyIjoiMjIyLjIyMi4yMjIuMjIiLCJuYW1lIjoiYWJlbGkiLCJvaWQiOiIwMjIyM2I2Yi1hYTFkLTQyZDQtOWVjMC0xYjJiYjkxOTQ0MzgiLCJyaCI6IkkiLCJzY3AiOiJ1c2VyX2ltcGVyc29uYXRpb24iLCJzdWIiOiJsM19yb0lTUVUyMjJiVUxTOXlpMmswWHBxcE9pTXo1SDNaQUNvMUdlWEEiLCJ0aWQiOiJmYTE1ZDY5Mi1lOWM3LTQ0NjAtYTc0My0yOWYyOTU2ZmQ0MjkiLCJ1bmlxdWVfbmFtZSI6ImFiZWxpQG1pY3Jvc29mdC5jb20iLCJ1dGkiOiJGVnNHeFlYSTMwLVR1aWt1dVVvRkFBIiwidmVyIjoiMS4wIn0.D3H6pMUtQnoJAGq6AHd", + "line_num": 1, + "path": "tests/samples/azure_access_token", + "info": "tests/samples/azure_access_token|RAW", + "value": "eyJ0eXAiOiJKV1QiLCJhbGciOiJSUzI1NiIsIng1dCI6Imk2bEdrM0ZaenhSY1ViMkMzbkVRN3N5SEpsWSIsImtpZCI6Imk2bEdrM0ZaenhSY1ViMkMzbkVRN3N5SEpsWSJ9.eyJhdWQiOiJlZjFkYTlkNC1mZjc3LTRjM2UtYTAwNS04NDBjM2Y4MzA3NDUiLCJpc3MiOiJodHRwczovL3N0cy53aW5kb3dzLm5ldC9mYTE1ZDY5Mi1lOWM3LTQ0NjAtYTc0My0yOWYyOTUyMjIyOS8iLCJpYXQiOjE1MzcyMzMxMDYsIm5iZiI6MTUzNzIzMzEwNiwiZXhwIjoxNTM3MjM3MDA2LCJhY3IiOiIxIiwiYWlvIjoiQVhRQWkvOElBQUFBRm0rRS9RVEcrZ0ZuVnhMaldkdzhLKzYxQUdyU091TU1GNmViYU1qN1hPM0libUQzZkdtck95RCtOdlp5R24yVmFUL2tES1h3NE1JaHJnR1ZxNkJuOHdMWG9UMUxrSVorRnpRVmtKUFBMUU9WNEtjWHFTbENWUERTL0RpQ0RnRTIyMlRJbU12V05hRU1hVU9Uc0lHdlRRPT0iLCJhbXIiOlsid2lhIl0sImFwcGlkIjoiNzVkYmU3N2YtMTBhMy00ZTU5LTg1ZmQtOGMxMjc1NDRmMTdjIiwiYXBwaWRhY3IiOiIwIiwiZW1haWwiOiJBYmVMaUBtaWNyb3NvZnQuY29tIiwiZmFtaWx5X25hbWUiOiJMaW5jb2xuIiwiZ2l2ZW5fbmFtZSI6IkFiZSAoTVNGVCkiLCJpZHAiOiJodHRwczovL3N0cy53aW5kb3dzLm5ldC83MmY5ODhiZi04NmYxLTQxYWYtOTFhYi0yZDdjZDAxMjIyNDcvIiwiaXBhZGRyIjoiMjIyLjIyMi4yMjIuMjIiLCJuYW1lIjoiYWJlbGkiLCJvaWQiOiIwMjIyM2I2Yi1hYTFkLTQyZDQtOWVjMC0xYjJiYjkxOTQ0MzgiLCJyaCI6IkkiLCJzY3AiOiJ1c2VyX2ltcGVyc29uYXRpb24iLCJzdWIiOiJsM19yb0lTUVUyMjJiVUxTOXlpMmswWHBxcE9pTXo1SDNaQUNvMUdlWEEiLCJ0aWQiOiJmYTE1ZDY5Mi1lOWM3LTQ0NjAtYTc0My0yOWYyOTU2ZmQ0MjkiLCJ1bmlxdWVfbmFtZSI6ImFiZWxpQG1pY3Jvc29mdC5jb20iLCJ1dGkiOiJGVnNHeFlYSTMwLVR1aWt1dVVvRkFBIiwidmVyIjoiMS4wIn0.D3H6pMUtQnoJAGq6AHd", + "value_start": 0, + "value_end": 1316, + "variable": null, + "variable_start": -2, + "variable_end": -2, + "entropy_validation": { + "iterator": "BASE64_CHARS", + "entropy": 5.615950458346115, + "valid": true + } + } + ] + }, + { + "api_validation": "NOT_AVAILABLE", + "ml_validation": "NOT_AVAILABLE", + "ml_probability": null, + "rule": "Azure Access Token", + "severity": "high", + "confidence": "strong", + "line_data_list": [ + { + "line": "eyJ0eXAiOiJKV1QiLCJhbGciOiJSUzI1NiIsImtpZCI6Imk2bEdrM0ZaenhSY1ViMkMzbkVRN3N5SEpsWSJ9.eyJhdWQiOiI2ZTc0MTcyYi1iZTU2LTQ4NDMtOWZmNC1lNjZhMzliYjEyZTMiLCJpc3MiOiJodHRwczovL2xvZ2luLm1pY3Jvc29mdG9ubGluZS5jb20vNzJmOTg4YmYtODZmMS00MWFmLTkxYWItMmQ3Y2QwMTFkYjQ3L3YyLjAiLCJpYXQiOjE1MzcyMzEwNDgsIm5iZiI6MTUzNzIzMTA0OCwiZXhwIjoxNTM3MjM0OTQ4LCJhaW8iOiJBWFFBaS84SUFBQUF0QWFaTG8zQ2hNaWY2S09udHRSQjdlQnE0L0RjY1F6amNKR3hQWXkvQzNqRGFOR3hYZDZ3TklJVkdSZ2hOUm53SjFsT2NBbk5aY2p2a295ckZ4Q3R0djMzMTQwUmlvT0ZKNGJDQ0dWdW9DYWcxdU9UVDIyMjIyZ0h3TFBZUS91Zjc5UVgrMEtJaWpkcm1wNjlSY3R6bVE9PSIsImF6cCI6IjZlNzQxNzJiLWJlNTYtNDg0My05ZmY0LWU2NmEzOWJiMTJlMyIsImF6cGFjciI6IjAiLCJuYW1lIjoiQWJlIExpbmNvbG4iLCJvaWQiOiI2OTAyMjJiZS1mZjFhLTRkNTYtYWJkMS03ZTRmN2QzOGU0NzQiLCJwcmVmZXJyZWRfdXNlcm5hbWUiOiJhYmVsaUBtaWNyb3NvZnQuY29tIiwicmgiOiJJIiwic2NwIjoiYWNjZXNzX2FzX3VzZXIiLCJzdWIiOiJIS1pwZmFIeVdhZGVPb3VZbGl0anJJLUtmZlRtMjIyWDVyclYzeERxZktRIiwidGlkIjoiNzJmOTg4YmYtODZmMS00MWFmLTkxYWItMmQ3Y2QwMTFkYjQ3IiwidXRpIjoiZnFpQnFYTFBqMGVRYTgyUy1JWUZBQSIsInZlciI6IjIuMCJ9.pj4N-w_3Us9DrBLfpCt", + "line_num": 2, + "path": "tests/samples/azure_access_token", + "info": "tests/samples/azure_access_token|RAW", + "value": "eyJ0eXAiOiJKV1QiLCJhbGciOiJSUzI1NiIsImtpZCI6Imk2bEdrM0ZaenhSY1ViMkMzbkVRN3N5SEpsWSJ9.eyJhdWQiOiI2ZTc0MTcyYi1iZTU2LTQ4NDMtOWZmNC1lNjZhMzliYjEyZTMiLCJpc3MiOiJodHRwczovL2xvZ2luLm1pY3Jvc29mdG9ubGluZS5jb20vNzJmOTg4YmYtODZmMS00MWFmLTkxYWItMmQ3Y2QwMTFkYjQ3L3YyLjAiLCJpYXQiOjE1MzcyMzEwNDgsIm5iZiI6MTUzNzIzMTA0OCwiZXhwIjoxNTM3MjM0OTQ4LCJhaW8iOiJBWFFBaS84SUFBQUF0QWFaTG8zQ2hNaWY2S09udHRSQjdlQnE0L0RjY1F6amNKR3hQWXkvQzNqRGFOR3hYZDZ3TklJVkdSZ2hOUm53SjFsT2NBbk5aY2p2a295ckZ4Q3R0djMzMTQwUmlvT0ZKNGJDQ0dWdW9DYWcxdU9UVDIyMjIyZ0h3TFBZUS91Zjc5UVgrMEtJaWpkcm1wNjlSY3R6bVE9PSIsImF6cCI6IjZlNzQxNzJiLWJlNTYtNDg0My05ZmY0LWU2NmEzOWJiMTJlMyIsImF6cGFjciI6IjAiLCJuYW1lIjoiQWJlIExpbmNvbG4iLCJvaWQiOiI2OTAyMjJiZS1mZjFhLTRkNTYtYWJkMS03ZTRmN2QzOGU0NzQiLCJwcmVmZXJyZWRfdXNlcm5hbWUiOiJhYmVsaUBtaWNyb3NvZnQuY29tIiwicmgiOiJJIiwic2NwIjoiYWNjZXNzX2FzX3VzZXIiLCJzdWIiOiJIS1pwZmFIeVdhZGVPb3VZbGl0anJJLUtmZlRtMjIyWDVyclYzeERxZktRIiwidGlkIjoiNzJmOTg4YmYtODZmMS00MWFmLTkxYWItMmQ3Y2QwMTFkYjQ3IiwidXRpIjoiZnFpQnFYTFBqMGVRYTgyUy1JWUZBQSIsInZlciI6IjIuMCJ9.pj4N-w_3Us9DrBLfpCt", + "value_start": 0, + "value_end": 1029, + "variable": null, + "variable_start": -2, + "variable_end": -2, + "entropy_validation": { + "iterator": "BASE64_CHARS", + "entropy": 5.6044494049575055, + "valid": true + } + } + ] + }, + { + "api_validation": "NOT_AVAILABLE", + "ml_validation": "NOT_AVAILABLE", + "ml_probability": null, + "rule": "JSON Web Token", + "severity": "medium", + "confidence": "strong", + "line_data_list": [ + { + "line": "eyJ0eXAiOiJKV1QiLCJhbGciOiJSUzI1NiIsImtpZCI6Imk2bEdrM0ZaenhSY1ViMkMzbkVRN3N5SEpsWSJ9.eyJhdWQiOiI2ZTc0MTcyYi1iZTU2LTQ4NDMtOWZmNC1lNjZhMzliYjEyZTMiLCJpc3MiOiJodHRwczovL2xvZ2luLm1pY3Jvc29mdG9ubGluZS5jb20vNzJmOTg4YmYtODZmMS00MWFmLTkxYWItMmQ3Y2QwMTFkYjQ3L3YyLjAiLCJpYXQiOjE1MzcyMzEwNDgsIm5iZiI6MTUzNzIzMTA0OCwiZXhwIjoxNTM3MjM0OTQ4LCJhaW8iOiJBWFFBaS84SUFBQUF0QWFaTG8zQ2hNaWY2S09udHRSQjdlQnE0L0RjY1F6amNKR3hQWXkvQzNqRGFOR3hYZDZ3TklJVkdSZ2hOUm53SjFsT2NBbk5aY2p2a295ckZ4Q3R0djMzMTQwUmlvT0ZKNGJDQ0dWdW9DYWcxdU9UVDIyMjIyZ0h3TFBZUS91Zjc5UVgrMEtJaWpkcm1wNjlSY3R6bVE9PSIsImF6cCI6IjZlNzQxNzJiLWJlNTYtNDg0My05ZmY0LWU2NmEzOWJiMTJlMyIsImF6cGFjciI6IjAiLCJuYW1lIjoiQWJlIExpbmNvbG4iLCJvaWQiOiI2OTAyMjJiZS1mZjFhLTRkNTYtYWJkMS03ZTRmN2QzOGU0NzQiLCJwcmVmZXJyZWRfdXNlcm5hbWUiOiJhYmVsaUBtaWNyb3NvZnQuY29tIiwicmgiOiJJIiwic2NwIjoiYWNjZXNzX2FzX3VzZXIiLCJzdWIiOiJIS1pwZmFIeVdhZGVPb3VZbGl0anJJLUtmZlRtMjIyWDVyclYzeERxZktRIiwidGlkIjoiNzJmOTg4YmYtODZmMS00MWFmLTkxYWItMmQ3Y2QwMTFkYjQ3IiwidXRpIjoiZnFpQnFYTFBqMGVRYTgyUy1JWUZBQSIsInZlciI6IjIuMCJ9.pj4N-w_3Us9DrBLfpCt", + "line_num": 2, + "path": "tests/samples/azure_access_token", + "info": "tests/samples/azure_access_token|RAW", + "value": "eyJ0eXAiOiJKV1QiLCJhbGciOiJSUzI1NiIsImtpZCI6Imk2bEdrM0ZaenhSY1ViMkMzbkVRN3N5SEpsWSJ9.eyJhdWQiOiI2ZTc0MTcyYi1iZTU2LTQ4NDMtOWZmNC1lNjZhMzliYjEyZTMiLCJpc3MiOiJodHRwczovL2xvZ2luLm1pY3Jvc29mdG9ubGluZS5jb20vNzJmOTg4YmYtODZmMS00MWFmLTkxYWItMmQ3Y2QwMTFkYjQ3L3YyLjAiLCJpYXQiOjE1MzcyMzEwNDgsIm5iZiI6MTUzNzIzMTA0OCwiZXhwIjoxNTM3MjM0OTQ4LCJhaW8iOiJBWFFBaS84SUFBQUF0QWFaTG8zQ2hNaWY2S09udHRSQjdlQnE0L0RjY1F6amNKR3hQWXkvQzNqRGFOR3hYZDZ3TklJVkdSZ2hOUm53SjFsT2NBbk5aY2p2a295ckZ4Q3R0djMzMTQwUmlvT0ZKNGJDQ0dWdW9DYWcxdU9UVDIyMjIyZ0h3TFBZUS91Zjc5UVgrMEtJaWpkcm1wNjlSY3R6bVE9PSIsImF6cCI6IjZlNzQxNzJiLWJlNTYtNDg0My05ZmY0LWU2NmEzOWJiMTJlMyIsImF6cGFjciI6IjAiLCJuYW1lIjoiQWJlIExpbmNvbG4iLCJvaWQiOiI2OTAyMjJiZS1mZjFhLTRkNTYtYWJkMS03ZTRmN2QzOGU0NzQiLCJwcmVmZXJyZWRfdXNlcm5hbWUiOiJhYmVsaUBtaWNyb3NvZnQuY29tIiwicmgiOiJJIiwic2NwIjoiYWNjZXNzX2FzX3VzZXIiLCJzdWIiOiJIS1pwZmFIeVdhZGVPb3VZbGl0anJJLUtmZlRtMjIyWDVyclYzeERxZktRIiwidGlkIjoiNzJmOTg4YmYtODZmMS00MWFmLTkxYWItMmQ3Y2QwMTFkYjQ3IiwidXRpIjoiZnFpQnFYTFBqMGVRYTgyUy1JWUZBQSIsInZlciI6IjIuMCJ9.pj4N-w_3Us9DrBLfpCt", + "value_start": 0, + "value_end": 1029, + "variable": null, + "variable_start": -2, + "variable_end": -2, + "entropy_validation": { + "iterator": "BASE64_CHARS", + "entropy": 5.6044494049575055, + "valid": true + } + } + ] + }, { "api_validation": "NOT_AVAILABLE", "ml_validation": "NOT_AVAILABLE", @@ -11439,6 +11520,33 @@ } ] }, + { + "api_validation": "NOT_AVAILABLE", + "ml_validation": "NOT_AVAILABLE", + "ml_probability": null, + "rule": "JSON Web Token", + "severity": "medium", + "confidence": "strong", + "line_data_list": [ + { + "line": "detected: eyJhbGciOiJSUzI1NiJ9Cg.eyJleHAiOjY1NTM2fQo.Ce7sh0ENPiBlE_dose0cBA", + "line_num": 1, + "path": "tests/samples/json_web_token", + "info": "tests/samples/json_web_token|RAW", + "value": "eyJhbGciOiJSUzI1NiJ9Cg.eyJleHAiOjY1NTM2fQo.Ce7sh0ENPiBlE_dose0cBA", + "value_start": 10, + "value_end": 75, + "variable": null, + "variable_start": -2, + "variable_end": -2, + "entropy_validation": { + "iterator": "BASE64_CHARS", + "entropy": 4.790963630103494, + "valid": true + } + } + ] + }, { "api_validation": "NOT_AVAILABLE", "ml_validation": "NOT_AVAILABLE", diff --git a/tests/data/ml_threshold.json b/tests/data/ml_threshold.json index 71958bd1a..c03e55fde 100644 --- a/tests/data/ml_threshold.json +++ b/tests/data/ml_threshold.json @@ -299,7 +299,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.941, + "ml_probability": 0.963, "rule": "Auth", "severity": "medium", "confidence": "moderate", @@ -323,33 +323,6 @@ } ] }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.941, - "rule": "JSON Web Token", - "severity": "medium", - "confidence": "moderate", - "line_data_list": [ - { - "line": "curl -H \"Authorization: Bearer eyJGRpVu1c2VzY2-823r_db32hbf4W1lbj\" http://localhost:8080/.", - "line_num": 9, - "path": "tests/samples/auth_n.template", - "info": "", - "value": "eyJGRpVu1c2VzY2-823r_db32hbf4W1lbj", - "value_start": 31, - "value_end": 65, - "variable": null, - "variable_start": -2, - "variable_end": -2, - "entropy_validation": { - "iterator": "BASE36_CHARS", - "entropy": 3.2479906920322064, - "valid": true - } - } - ] - }, { "api_validation": "NOT_AVAILABLE", "ml_validation": "NOT_AVAILABLE", @@ -764,11 +737,11 @@ }, { "api_validation": "NOT_AVAILABLE", - "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.93, + "ml_validation": "NOT_AVAILABLE", + "ml_probability": null, "rule": "JSON Web Token", "severity": "medium", - "confidence": "moderate", + "confidence": "strong", "line_data_list": [ { "line": "eyJ0eXAiOiJKV1QiLCJhbGciOiJSUzI1NiIsIng1dCI6Imk2bEdrM0ZaenhSY1ViMkMzbkVRN3N5SEpsWSIsImtpZCI6Imk2bEdrM0ZaenhSY1ViMkMzbkVRN3N5SEpsWSJ9.eyJhdWQiOiJlZjFkYTlkNC1mZjc3LTRjM2UtYTAwNS04NDBjM2Y4MzA3NDUiLCJpc3MiOiJodHRwczovL3N0cy53aW5kb3dzLm5ldC9mYTE1ZDY5Mi1lOWM3LTQ0NjAtYTc0My0yOWYyOTUyMjIyOS8iLCJpYXQiOjE1MzcyMzMxMDYsIm5iZiI6MTUzNzIzMzEwNiwiZXhwIjoxNTM3MjM3MDA2LCJhY3IiOiIxIiwiYWlvIjoiQVhRQWkvOElBQUFBRm0rRS9RVEcrZ0ZuVnhMaldkdzhLKzYxQUdyU091TU1GNmViYU1qN1hPM0libUQzZkdtck95RCtOdlp5R24yVmFUL2tES1h3NE1JaHJnR1ZxNkJuOHdMWG9UMUxrSVorRnpRVmtKUFBMUU9WNEtjWHFTbENWUERTL0RpQ0RnRTIyMlRJbU12V05hRU1hVU9Uc0lHdlRRPT0iLCJhbXIiOlsid2lhIl0sImFwcGlkIjoiNzVkYmU3N2YtMTBhMy00ZTU5LTg1ZmQtOGMxMjc1NDRmMTdjIiwiYXBwaWRhY3IiOiIwIiwiZW1haWwiOiJBYmVMaUBtaWNyb3NvZnQuY29tIiwiZmFtaWx5X25hbWUiOiJMaW5jb2xuIiwiZ2l2ZW5fbmFtZSI6IkFiZSAoTVNGVCkiLCJpZHAiOiJodHRwczovL3N0cy53aW5kb3dzLm5ldC83MmY5ODhiZi04NmYxLTQxYWYtOTFhYi0yZDdjZDAxMjIyNDcvIiwiaXBhZGRyIjoiMjIyLjIyMi4yMjIuMjIiLCJuYW1lIjoiYWJlbGkiLCJvaWQiOiIwMjIyM2I2Yi1hYTFkLTQyZDQtOWVjMC0xYjJiYjkxOTQ0MzgiLCJyaCI6IkkiLCJzY3AiOiJ1c2VyX2ltcGVyc29uYXRpb24iLCJzdWIiOiJsM19yb0lTUVUyMjJiVUxTOXlpMmswWHBxcE9pTXo1SDNaQUNvMUdlWEEiLCJ0aWQiOiJmYTE1ZDY5Mi1lOWM3LTQ0NjAtYTc0My0yOWYyOTU2ZmQ0MjkiLCJ1bmlxdWVfbmFtZSI6ImFiZWxpQG1pY3Jvc29mdC5jb20iLCJ1dGkiOiJGVnNHeFlYSTMwLVR1aWt1dVVvRkFBIiwidmVyIjoiMS4wIn0.D3H6pMUtQnoJAGq6AHd", @@ -789,6 +762,60 @@ } ] }, + { + "api_validation": "NOT_AVAILABLE", + "ml_validation": "NOT_AVAILABLE", + "ml_probability": null, + "rule": "Azure Access Token", + "severity": "high", + "confidence": "strong", + "line_data_list": [ + { + "line": "eyJ0eXAiOiJKV1QiLCJhbGciOiJSUzI1NiIsImtpZCI6Imk2bEdrM0ZaenhSY1ViMkMzbkVRN3N5SEpsWSJ9.eyJhdWQiOiI2ZTc0MTcyYi1iZTU2LTQ4NDMtOWZmNC1lNjZhMzliYjEyZTMiLCJpc3MiOiJodHRwczovL2xvZ2luLm1pY3Jvc29mdG9ubGluZS5jb20vNzJmOTg4YmYtODZmMS00MWFmLTkxYWItMmQ3Y2QwMTFkYjQ3L3YyLjAiLCJpYXQiOjE1MzcyMzEwNDgsIm5iZiI6MTUzNzIzMTA0OCwiZXhwIjoxNTM3MjM0OTQ4LCJhaW8iOiJBWFFBaS84SUFBQUF0QWFaTG8zQ2hNaWY2S09udHRSQjdlQnE0L0RjY1F6amNKR3hQWXkvQzNqRGFOR3hYZDZ3TklJVkdSZ2hOUm53SjFsT2NBbk5aY2p2a295ckZ4Q3R0djMzMTQwUmlvT0ZKNGJDQ0dWdW9DYWcxdU9UVDIyMjIyZ0h3TFBZUS91Zjc5UVgrMEtJaWpkcm1wNjlSY3R6bVE9PSIsImF6cCI6IjZlNzQxNzJiLWJlNTYtNDg0My05ZmY0LWU2NmEzOWJiMTJlMyIsImF6cGFjciI6IjAiLCJuYW1lIjoiQWJlIExpbmNvbG4iLCJvaWQiOiI2OTAyMjJiZS1mZjFhLTRkNTYtYWJkMS03ZTRmN2QzOGU0NzQiLCJwcmVmZXJyZWRfdXNlcm5hbWUiOiJhYmVsaUBtaWNyb3NvZnQuY29tIiwicmgiOiJJIiwic2NwIjoiYWNjZXNzX2FzX3VzZXIiLCJzdWIiOiJIS1pwZmFIeVdhZGVPb3VZbGl0anJJLUtmZlRtMjIyWDVyclYzeERxZktRIiwidGlkIjoiNzJmOTg4YmYtODZmMS00MWFmLTkxYWItMmQ3Y2QwMTFkYjQ3IiwidXRpIjoiZnFpQnFYTFBqMGVRYTgyUy1JWUZBQSIsInZlciI6IjIuMCJ9.pj4N-w_3Us9DrBLfpCt", + "line_num": 2, + "path": "tests/samples/azure_access_token", + "info": "", + "value": "eyJ0eXAiOiJKV1QiLCJhbGciOiJSUzI1NiIsImtpZCI6Imk2bEdrM0ZaenhSY1ViMkMzbkVRN3N5SEpsWSJ9.eyJhdWQiOiI2ZTc0MTcyYi1iZTU2LTQ4NDMtOWZmNC1lNjZhMzliYjEyZTMiLCJpc3MiOiJodHRwczovL2xvZ2luLm1pY3Jvc29mdG9ubGluZS5jb20vNzJmOTg4YmYtODZmMS00MWFmLTkxYWItMmQ3Y2QwMTFkYjQ3L3YyLjAiLCJpYXQiOjE1MzcyMzEwNDgsIm5iZiI6MTUzNzIzMTA0OCwiZXhwIjoxNTM3MjM0OTQ4LCJhaW8iOiJBWFFBaS84SUFBQUF0QWFaTG8zQ2hNaWY2S09udHRSQjdlQnE0L0RjY1F6amNKR3hQWXkvQzNqRGFOR3hYZDZ3TklJVkdSZ2hOUm53SjFsT2NBbk5aY2p2a295ckZ4Q3R0djMzMTQwUmlvT0ZKNGJDQ0dWdW9DYWcxdU9UVDIyMjIyZ0h3TFBZUS91Zjc5UVgrMEtJaWpkcm1wNjlSY3R6bVE9PSIsImF6cCI6IjZlNzQxNzJiLWJlNTYtNDg0My05ZmY0LWU2NmEzOWJiMTJlMyIsImF6cGFjciI6IjAiLCJuYW1lIjoiQWJlIExpbmNvbG4iLCJvaWQiOiI2OTAyMjJiZS1mZjFhLTRkNTYtYWJkMS03ZTRmN2QzOGU0NzQiLCJwcmVmZXJyZWRfdXNlcm5hbWUiOiJhYmVsaUBtaWNyb3NvZnQuY29tIiwicmgiOiJJIiwic2NwIjoiYWNjZXNzX2FzX3VzZXIiLCJzdWIiOiJIS1pwZmFIeVdhZGVPb3VZbGl0anJJLUtmZlRtMjIyWDVyclYzeERxZktRIiwidGlkIjoiNzJmOTg4YmYtODZmMS00MWFmLTkxYWItMmQ3Y2QwMTFkYjQ3IiwidXRpIjoiZnFpQnFYTFBqMGVRYTgyUy1JWUZBQSIsInZlciI6IjIuMCJ9.pj4N-w_3Us9DrBLfpCt", + "value_start": 0, + "value_end": 1029, + "variable": null, + "variable_start": -2, + "variable_end": -2, + "entropy_validation": { + "iterator": "BASE64_CHARS", + "entropy": 5.6044494049575055, + "valid": true + } + } + ] + }, + { + "api_validation": "NOT_AVAILABLE", + "ml_validation": "NOT_AVAILABLE", + "ml_probability": null, + "rule": "JSON Web Token", + "severity": "medium", + "confidence": "strong", + "line_data_list": [ + { + "line": "eyJ0eXAiOiJKV1QiLCJhbGciOiJSUzI1NiIsImtpZCI6Imk2bEdrM0ZaenhSY1ViMkMzbkVRN3N5SEpsWSJ9.eyJhdWQiOiI2ZTc0MTcyYi1iZTU2LTQ4NDMtOWZmNC1lNjZhMzliYjEyZTMiLCJpc3MiOiJodHRwczovL2xvZ2luLm1pY3Jvc29mdG9ubGluZS5jb20vNzJmOTg4YmYtODZmMS00MWFmLTkxYWItMmQ3Y2QwMTFkYjQ3L3YyLjAiLCJpYXQiOjE1MzcyMzEwNDgsIm5iZiI6MTUzNzIzMTA0OCwiZXhwIjoxNTM3MjM0OTQ4LCJhaW8iOiJBWFFBaS84SUFBQUF0QWFaTG8zQ2hNaWY2S09udHRSQjdlQnE0L0RjY1F6amNKR3hQWXkvQzNqRGFOR3hYZDZ3TklJVkdSZ2hOUm53SjFsT2NBbk5aY2p2a295ckZ4Q3R0djMzMTQwUmlvT0ZKNGJDQ0dWdW9DYWcxdU9UVDIyMjIyZ0h3TFBZUS91Zjc5UVgrMEtJaWpkcm1wNjlSY3R6bVE9PSIsImF6cCI6IjZlNzQxNzJiLWJlNTYtNDg0My05ZmY0LWU2NmEzOWJiMTJlMyIsImF6cGFjciI6IjAiLCJuYW1lIjoiQWJlIExpbmNvbG4iLCJvaWQiOiI2OTAyMjJiZS1mZjFhLTRkNTYtYWJkMS03ZTRmN2QzOGU0NzQiLCJwcmVmZXJyZWRfdXNlcm5hbWUiOiJhYmVsaUBtaWNyb3NvZnQuY29tIiwicmgiOiJJIiwic2NwIjoiYWNjZXNzX2FzX3VzZXIiLCJzdWIiOiJIS1pwZmFIeVdhZGVPb3VZbGl0anJJLUtmZlRtMjIyWDVyclYzeERxZktRIiwidGlkIjoiNzJmOTg4YmYtODZmMS00MWFmLTkxYWItMmQ3Y2QwMTFkYjQ3IiwidXRpIjoiZnFpQnFYTFBqMGVRYTgyUy1JWUZBQSIsInZlciI6IjIuMCJ9.pj4N-w_3Us9DrBLfpCt", + "line_num": 2, + "path": "tests/samples/azure_access_token", + "info": "", + "value": "eyJ0eXAiOiJKV1QiLCJhbGciOiJSUzI1NiIsImtpZCI6Imk2bEdrM0ZaenhSY1ViMkMzbkVRN3N5SEpsWSJ9.eyJhdWQiOiI2ZTc0MTcyYi1iZTU2LTQ4NDMtOWZmNC1lNjZhMzliYjEyZTMiLCJpc3MiOiJodHRwczovL2xvZ2luLm1pY3Jvc29mdG9ubGluZS5jb20vNzJmOTg4YmYtODZmMS00MWFmLTkxYWItMmQ3Y2QwMTFkYjQ3L3YyLjAiLCJpYXQiOjE1MzcyMzEwNDgsIm5iZiI6MTUzNzIzMTA0OCwiZXhwIjoxNTM3MjM0OTQ4LCJhaW8iOiJBWFFBaS84SUFBQUF0QWFaTG8zQ2hNaWY2S09udHRSQjdlQnE0L0RjY1F6amNKR3hQWXkvQzNqRGFOR3hYZDZ3TklJVkdSZ2hOUm53SjFsT2NBbk5aY2p2a295ckZ4Q3R0djMzMTQwUmlvT0ZKNGJDQ0dWdW9DYWcxdU9UVDIyMjIyZ0h3TFBZUS91Zjc5UVgrMEtJaWpkcm1wNjlSY3R6bVE9PSIsImF6cCI6IjZlNzQxNzJiLWJlNTYtNDg0My05ZmY0LWU2NmEzOWJiMTJlMyIsImF6cGFjciI6IjAiLCJuYW1lIjoiQWJlIExpbmNvbG4iLCJvaWQiOiI2OTAyMjJiZS1mZjFhLTRkNTYtYWJkMS03ZTRmN2QzOGU0NzQiLCJwcmVmZXJyZWRfdXNlcm5hbWUiOiJhYmVsaUBtaWNyb3NvZnQuY29tIiwicmgiOiJJIiwic2NwIjoiYWNjZXNzX2FzX3VzZXIiLCJzdWIiOiJIS1pwZmFIeVdhZGVPb3VZbGl0anJJLUtmZlRtMjIyWDVyclYzeERxZktRIiwidGlkIjoiNzJmOTg4YmYtODZmMS00MWFmLTkxYWItMmQ3Y2QwMTFkYjQ3IiwidXRpIjoiZnFpQnFYTFBqMGVRYTgyUy1JWUZBQSIsInZlciI6IjIuMCJ9.pj4N-w_3Us9DrBLfpCt", + "value_start": 0, + "value_end": 1029, + "variable": null, + "variable_start": -2, + "variable_end": -2, + "entropy_validation": { + "iterator": "BASE64_CHARS", + "entropy": 5.6044494049575055, + "valid": true + } + } + ] + }, { "api_validation": "NOT_AVAILABLE", "ml_validation": "NOT_AVAILABLE", @@ -7519,33 +7546,6 @@ } ] }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.985, - "rule": "JSON Web Token", - "severity": "medium", - "confidence": "moderate", - "line_data_list": [ - { - "line": "grafana = 'eyJrIjoiMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMCIsIm4iOiJ4eHh4IiwiaWQiOjIwNDM2MH0='", - "line_num": 1, - "path": "tests/samples/grafana_provisioned_api_key", - "info": "", - "value": "eyJrIjoiMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMCIsIm4iOiJ4eHh4IiwiaWQiOjIwNDM2MH0=", - "value_start": 11, - "value_end": 107, - "variable": null, - "variable_start": -2, - "variable_end": -2, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 3.8153130511409934, - "valid": false - } - } - ] - }, { "api_validation": "NOT_AVAILABLE", "ml_validation": "NOT_AVAILABLE", @@ -7791,26 +7791,26 @@ }, { "api_validation": "NOT_AVAILABLE", - "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.931, + "ml_validation": "NOT_AVAILABLE", + "ml_probability": null, "rule": "JSON Web Token", "severity": "medium", - "confidence": "moderate", + "confidence": "strong", "line_data_list": [ { - "line": "$payload = 'eyJgsIZgeJhvNgFpSmlP.eyJcaaF9xCe7shE0ENPiBlEJOpS'", + "line": "detected: eyJhbGciOiJSUzI1NiJ9Cg.eyJleHAiOjY1NTM2fQo.Ce7sh0ENPiBlE_dose0cBA", "line_num": 1, - "path": "tests/samples/json_web_token.hs", + "path": "tests/samples/json_web_token", "info": "", - "value": "eyJgsIZgeJhvNgFpSmlP.eyJcaaF9xCe7shE0ENPiBlEJOpS", - "value_start": 12, - "value_end": 60, + "value": "eyJhbGciOiJSUzI1NiJ9Cg.eyJleHAiOjY1NTM2fQo.Ce7sh0ENPiBlE_dose0cBA", + "value_start": 10, + "value_end": 75, "variable": null, "variable_start": -2, "variable_end": -2, "entropy_validation": { "iterator": "BASE64_CHARS", - "entropy": 4.520488802699322, + "entropy": 4.790963630103494, "valid": true } } @@ -9603,33 +9603,6 @@ } ] }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.217, - "rule": "JSON Web Token", - "severity": "medium", - "confidence": "moderate", - "line_data_list": [ - { - "line": "eyJUaGVyZSBpcyBub3QgdGhlIEpTT04geW91IGFyZSBsb29raW5nIGZvciJ9CjwvYm9keT4KPC9o", - "line_num": 17, - "path": "tests/samples/test.eml", - "info": "", - "value": "eyJUaGVyZSBpcyBub3QgdGhlIEpTT04geW91IGFyZSBsb29raW5nIGZvciJ9CjwvYm9keT4KPC9o", - "value_start": 0, - "value_end": 76, - "variable": null, - "variable_start": -2, - "variable_end": -2, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 5.282347539953402, - "valid": true - } - } - ] - }, { "api_validation": "NOT_AVAILABLE", "ml_validation": "NOT_AVAILABLE", @@ -9657,33 +9630,6 @@ } ] }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.999, - "rule": "JSON Web Token", - "severity": "medium", - "confidence": "moderate", - "line_data_list": [ - { - "line": "token in text: eyJrIjoiMDAwMDAwNDAwMDAwODAwNDAwMDAwMDAwNDAwMDAwMDAwMDAwMDAyMSIsIm4iOiJ4eHh4IiwiaWQiOjQzMDh9Cg", - "line_num": 18, - "path": "tests/samples/test2.eml", - "info": "", - "value": "eyJrIjoiMDAwMDAwNDAwMDAwODAwNDAwMDAwMDAwNDAwMDAwMDAwMDAwMDAyMSIsIm4iOiJ4eHh4IiwiaWQiOjQzMDh9Cg", - "value_start": 15, - "value_end": 109, - "variable": null, - "variable_start": -2, - "variable_end": -2, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 4.0296677144829305, - "valid": false - } - } - ] - }, { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", diff --git a/tests/data/output.json b/tests/data/output.json index f5f87e544..cfe4115d4 100644 --- a/tests/data/output.json +++ b/tests/data/output.json @@ -299,7 +299,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.941, + "ml_probability": 0.963, "rule": "Auth", "severity": "medium", "confidence": "moderate", @@ -323,33 +323,6 @@ } ] }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.941, - "rule": "JSON Web Token", - "severity": "medium", - "confidence": "moderate", - "line_data_list": [ - { - "line": "curl -H \"Authorization: Bearer eyJGRpVu1c2VzY2-823r_db32hbf4W1lbj\" http://localhost:8080/.", - "line_num": 9, - "path": "tests/samples/auth_n.template", - "info": "", - "value": "eyJGRpVu1c2VzY2-823r_db32hbf4W1lbj", - "value_start": 31, - "value_end": 65, - "variable": null, - "variable_start": -2, - "variable_end": -2, - "entropy_validation": { - "iterator": "BASE36_CHARS", - "entropy": 3.2479906920322064, - "valid": true - } - } - ] - }, { "api_validation": "NOT_AVAILABLE", "ml_validation": "NOT_AVAILABLE", @@ -710,11 +683,11 @@ }, { "api_validation": "NOT_AVAILABLE", - "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.93, + "ml_validation": "NOT_AVAILABLE", + "ml_probability": null, "rule": "JSON Web Token", "severity": "medium", - "confidence": "moderate", + "confidence": "strong", "line_data_list": [ { "line": "eyJ0eXAiOiJKV1QiLCJhbGciOiJSUzI1NiIsIng1dCI6Imk2bEdrM0ZaenhSY1ViMkMzbkVRN3N5SEpsWSIsImtpZCI6Imk2bEdrM0ZaenhSY1ViMkMzbkVRN3N5SEpsWSJ9.eyJhdWQiOiJlZjFkYTlkNC1mZjc3LTRjM2UtYTAwNS04NDBjM2Y4MzA3NDUiLCJpc3MiOiJodHRwczovL3N0cy53aW5kb3dzLm5ldC9mYTE1ZDY5Mi1lOWM3LTQ0NjAtYTc0My0yOWYyOTUyMjIyOS8iLCJpYXQiOjE1MzcyMzMxMDYsIm5iZiI6MTUzNzIzMzEwNiwiZXhwIjoxNTM3MjM3MDA2LCJhY3IiOiIxIiwiYWlvIjoiQVhRQWkvOElBQUFBRm0rRS9RVEcrZ0ZuVnhMaldkdzhLKzYxQUdyU091TU1GNmViYU1qN1hPM0libUQzZkdtck95RCtOdlp5R24yVmFUL2tES1h3NE1JaHJnR1ZxNkJuOHdMWG9UMUxrSVorRnpRVmtKUFBMUU9WNEtjWHFTbENWUERTL0RpQ0RnRTIyMlRJbU12V05hRU1hVU9Uc0lHdlRRPT0iLCJhbXIiOlsid2lhIl0sImFwcGlkIjoiNzVkYmU3N2YtMTBhMy00ZTU5LTg1ZmQtOGMxMjc1NDRmMTdjIiwiYXBwaWRhY3IiOiIwIiwiZW1haWwiOiJBYmVMaUBtaWNyb3NvZnQuY29tIiwiZmFtaWx5X25hbWUiOiJMaW5jb2xuIiwiZ2l2ZW5fbmFtZSI6IkFiZSAoTVNGVCkiLCJpZHAiOiJodHRwczovL3N0cy53aW5kb3dzLm5ldC83MmY5ODhiZi04NmYxLTQxYWYtOTFhYi0yZDdjZDAxMjIyNDcvIiwiaXBhZGRyIjoiMjIyLjIyMi4yMjIuMjIiLCJuYW1lIjoiYWJlbGkiLCJvaWQiOiIwMjIyM2I2Yi1hYTFkLTQyZDQtOWVjMC0xYjJiYjkxOTQ0MzgiLCJyaCI6IkkiLCJzY3AiOiJ1c2VyX2ltcGVyc29uYXRpb24iLCJzdWIiOiJsM19yb0lTUVUyMjJiVUxTOXlpMmswWHBxcE9pTXo1SDNaQUNvMUdlWEEiLCJ0aWQiOiJmYTE1ZDY5Mi1lOWM3LTQ0NjAtYTc0My0yOWYyOTU2ZmQ0MjkiLCJ1bmlxdWVfbmFtZSI6ImFiZWxpQG1pY3Jvc29mdC5jb20iLCJ1dGkiOiJGVnNHeFlYSTMwLVR1aWt1dVVvRkFBIiwidmVyIjoiMS4wIn0.D3H6pMUtQnoJAGq6AHd", @@ -735,6 +708,60 @@ } ] }, + { + "api_validation": "NOT_AVAILABLE", + "ml_validation": "NOT_AVAILABLE", + "ml_probability": null, + "rule": "Azure Access Token", + "severity": "high", + "confidence": "strong", + "line_data_list": [ + { + "line": "eyJ0eXAiOiJKV1QiLCJhbGciOiJSUzI1NiIsImtpZCI6Imk2bEdrM0ZaenhSY1ViMkMzbkVRN3N5SEpsWSJ9.eyJhdWQiOiI2ZTc0MTcyYi1iZTU2LTQ4NDMtOWZmNC1lNjZhMzliYjEyZTMiLCJpc3MiOiJodHRwczovL2xvZ2luLm1pY3Jvc29mdG9ubGluZS5jb20vNzJmOTg4YmYtODZmMS00MWFmLTkxYWItMmQ3Y2QwMTFkYjQ3L3YyLjAiLCJpYXQiOjE1MzcyMzEwNDgsIm5iZiI6MTUzNzIzMTA0OCwiZXhwIjoxNTM3MjM0OTQ4LCJhaW8iOiJBWFFBaS84SUFBQUF0QWFaTG8zQ2hNaWY2S09udHRSQjdlQnE0L0RjY1F6amNKR3hQWXkvQzNqRGFOR3hYZDZ3TklJVkdSZ2hOUm53SjFsT2NBbk5aY2p2a295ckZ4Q3R0djMzMTQwUmlvT0ZKNGJDQ0dWdW9DYWcxdU9UVDIyMjIyZ0h3TFBZUS91Zjc5UVgrMEtJaWpkcm1wNjlSY3R6bVE9PSIsImF6cCI6IjZlNzQxNzJiLWJlNTYtNDg0My05ZmY0LWU2NmEzOWJiMTJlMyIsImF6cGFjciI6IjAiLCJuYW1lIjoiQWJlIExpbmNvbG4iLCJvaWQiOiI2OTAyMjJiZS1mZjFhLTRkNTYtYWJkMS03ZTRmN2QzOGU0NzQiLCJwcmVmZXJyZWRfdXNlcm5hbWUiOiJhYmVsaUBtaWNyb3NvZnQuY29tIiwicmgiOiJJIiwic2NwIjoiYWNjZXNzX2FzX3VzZXIiLCJzdWIiOiJIS1pwZmFIeVdhZGVPb3VZbGl0anJJLUtmZlRtMjIyWDVyclYzeERxZktRIiwidGlkIjoiNzJmOTg4YmYtODZmMS00MWFmLTkxYWItMmQ3Y2QwMTFkYjQ3IiwidXRpIjoiZnFpQnFYTFBqMGVRYTgyUy1JWUZBQSIsInZlciI6IjIuMCJ9.pj4N-w_3Us9DrBLfpCt", + "line_num": 2, + "path": "tests/samples/azure_access_token", + "info": "", + "value": "eyJ0eXAiOiJKV1QiLCJhbGciOiJSUzI1NiIsImtpZCI6Imk2bEdrM0ZaenhSY1ViMkMzbkVRN3N5SEpsWSJ9.eyJhdWQiOiI2ZTc0MTcyYi1iZTU2LTQ4NDMtOWZmNC1lNjZhMzliYjEyZTMiLCJpc3MiOiJodHRwczovL2xvZ2luLm1pY3Jvc29mdG9ubGluZS5jb20vNzJmOTg4YmYtODZmMS00MWFmLTkxYWItMmQ3Y2QwMTFkYjQ3L3YyLjAiLCJpYXQiOjE1MzcyMzEwNDgsIm5iZiI6MTUzNzIzMTA0OCwiZXhwIjoxNTM3MjM0OTQ4LCJhaW8iOiJBWFFBaS84SUFBQUF0QWFaTG8zQ2hNaWY2S09udHRSQjdlQnE0L0RjY1F6amNKR3hQWXkvQzNqRGFOR3hYZDZ3TklJVkdSZ2hOUm53SjFsT2NBbk5aY2p2a295ckZ4Q3R0djMzMTQwUmlvT0ZKNGJDQ0dWdW9DYWcxdU9UVDIyMjIyZ0h3TFBZUS91Zjc5UVgrMEtJaWpkcm1wNjlSY3R6bVE9PSIsImF6cCI6IjZlNzQxNzJiLWJlNTYtNDg0My05ZmY0LWU2NmEzOWJiMTJlMyIsImF6cGFjciI6IjAiLCJuYW1lIjoiQWJlIExpbmNvbG4iLCJvaWQiOiI2OTAyMjJiZS1mZjFhLTRkNTYtYWJkMS03ZTRmN2QzOGU0NzQiLCJwcmVmZXJyZWRfdXNlcm5hbWUiOiJhYmVsaUBtaWNyb3NvZnQuY29tIiwicmgiOiJJIiwic2NwIjoiYWNjZXNzX2FzX3VzZXIiLCJzdWIiOiJIS1pwZmFIeVdhZGVPb3VZbGl0anJJLUtmZlRtMjIyWDVyclYzeERxZktRIiwidGlkIjoiNzJmOTg4YmYtODZmMS00MWFmLTkxYWItMmQ3Y2QwMTFkYjQ3IiwidXRpIjoiZnFpQnFYTFBqMGVRYTgyUy1JWUZBQSIsInZlciI6IjIuMCJ9.pj4N-w_3Us9DrBLfpCt", + "value_start": 0, + "value_end": 1029, + "variable": null, + "variable_start": -2, + "variable_end": -2, + "entropy_validation": { + "iterator": "BASE64_CHARS", + "entropy": 5.6044494049575055, + "valid": true + } + } + ] + }, + { + "api_validation": "NOT_AVAILABLE", + "ml_validation": "NOT_AVAILABLE", + "ml_probability": null, + "rule": "JSON Web Token", + "severity": "medium", + "confidence": "strong", + "line_data_list": [ + { + "line": "eyJ0eXAiOiJKV1QiLCJhbGciOiJSUzI1NiIsImtpZCI6Imk2bEdrM0ZaenhSY1ViMkMzbkVRN3N5SEpsWSJ9.eyJhdWQiOiI2ZTc0MTcyYi1iZTU2LTQ4NDMtOWZmNC1lNjZhMzliYjEyZTMiLCJpc3MiOiJodHRwczovL2xvZ2luLm1pY3Jvc29mdG9ubGluZS5jb20vNzJmOTg4YmYtODZmMS00MWFmLTkxYWItMmQ3Y2QwMTFkYjQ3L3YyLjAiLCJpYXQiOjE1MzcyMzEwNDgsIm5iZiI6MTUzNzIzMTA0OCwiZXhwIjoxNTM3MjM0OTQ4LCJhaW8iOiJBWFFBaS84SUFBQUF0QWFaTG8zQ2hNaWY2S09udHRSQjdlQnE0L0RjY1F6amNKR3hQWXkvQzNqRGFOR3hYZDZ3TklJVkdSZ2hOUm53SjFsT2NBbk5aY2p2a295ckZ4Q3R0djMzMTQwUmlvT0ZKNGJDQ0dWdW9DYWcxdU9UVDIyMjIyZ0h3TFBZUS91Zjc5UVgrMEtJaWpkcm1wNjlSY3R6bVE9PSIsImF6cCI6IjZlNzQxNzJiLWJlNTYtNDg0My05ZmY0LWU2NmEzOWJiMTJlMyIsImF6cGFjciI6IjAiLCJuYW1lIjoiQWJlIExpbmNvbG4iLCJvaWQiOiI2OTAyMjJiZS1mZjFhLTRkNTYtYWJkMS03ZTRmN2QzOGU0NzQiLCJwcmVmZXJyZWRfdXNlcm5hbWUiOiJhYmVsaUBtaWNyb3NvZnQuY29tIiwicmgiOiJJIiwic2NwIjoiYWNjZXNzX2FzX3VzZXIiLCJzdWIiOiJIS1pwZmFIeVdhZGVPb3VZbGl0anJJLUtmZlRtMjIyWDVyclYzeERxZktRIiwidGlkIjoiNzJmOTg4YmYtODZmMS00MWFmLTkxYWItMmQ3Y2QwMTFkYjQ3IiwidXRpIjoiZnFpQnFYTFBqMGVRYTgyUy1JWUZBQSIsInZlciI6IjIuMCJ9.pj4N-w_3Us9DrBLfpCt", + "line_num": 2, + "path": "tests/samples/azure_access_token", + "info": "", + "value": "eyJ0eXAiOiJKV1QiLCJhbGciOiJSUzI1NiIsImtpZCI6Imk2bEdrM0ZaenhSY1ViMkMzbkVRN3N5SEpsWSJ9.eyJhdWQiOiI2ZTc0MTcyYi1iZTU2LTQ4NDMtOWZmNC1lNjZhMzliYjEyZTMiLCJpc3MiOiJodHRwczovL2xvZ2luLm1pY3Jvc29mdG9ubGluZS5jb20vNzJmOTg4YmYtODZmMS00MWFmLTkxYWItMmQ3Y2QwMTFkYjQ3L3YyLjAiLCJpYXQiOjE1MzcyMzEwNDgsIm5iZiI6MTUzNzIzMTA0OCwiZXhwIjoxNTM3MjM0OTQ4LCJhaW8iOiJBWFFBaS84SUFBQUF0QWFaTG8zQ2hNaWY2S09udHRSQjdlQnE0L0RjY1F6amNKR3hQWXkvQzNqRGFOR3hYZDZ3TklJVkdSZ2hOUm53SjFsT2NBbk5aY2p2a295ckZ4Q3R0djMzMTQwUmlvT0ZKNGJDQ0dWdW9DYWcxdU9UVDIyMjIyZ0h3TFBZUS91Zjc5UVgrMEtJaWpkcm1wNjlSY3R6bVE9PSIsImF6cCI6IjZlNzQxNzJiLWJlNTYtNDg0My05ZmY0LWU2NmEzOWJiMTJlMyIsImF6cGFjciI6IjAiLCJuYW1lIjoiQWJlIExpbmNvbG4iLCJvaWQiOiI2OTAyMjJiZS1mZjFhLTRkNTYtYWJkMS03ZTRmN2QzOGU0NzQiLCJwcmVmZXJyZWRfdXNlcm5hbWUiOiJhYmVsaUBtaWNyb3NvZnQuY29tIiwicmgiOiJJIiwic2NwIjoiYWNjZXNzX2FzX3VzZXIiLCJzdWIiOiJIS1pwZmFIeVdhZGVPb3VZbGl0anJJLUtmZlRtMjIyWDVyclYzeERxZktRIiwidGlkIjoiNzJmOTg4YmYtODZmMS00MWFmLTkxYWItMmQ3Y2QwMTFkYjQ3IiwidXRpIjoiZnFpQnFYTFBqMGVRYTgyUy1JWUZBQSIsInZlciI6IjIuMCJ9.pj4N-w_3Us9DrBLfpCt", + "value_start": 0, + "value_end": 1029, + "variable": null, + "variable_start": -2, + "variable_end": -2, + "entropy_validation": { + "iterator": "BASE64_CHARS", + "entropy": 5.6044494049575055, + "valid": true + } + } + ] + }, { "api_validation": "NOT_AVAILABLE", "ml_validation": "NOT_AVAILABLE", @@ -6628,33 +6655,6 @@ } ] }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.985, - "rule": "JSON Web Token", - "severity": "medium", - "confidence": "moderate", - "line_data_list": [ - { - "line": "grafana = 'eyJrIjoiMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMCIsIm4iOiJ4eHh4IiwiaWQiOjIwNDM2MH0='", - "line_num": 1, - "path": "tests/samples/grafana_provisioned_api_key", - "info": "", - "value": "eyJrIjoiMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMCIsIm4iOiJ4eHh4IiwiaWQiOjIwNDM2MH0=", - "value_start": 11, - "value_end": 107, - "variable": null, - "variable_start": -2, - "variable_end": -2, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 3.8153130511409934, - "valid": false - } - } - ] - }, { "api_validation": "NOT_AVAILABLE", "ml_validation": "NOT_AVAILABLE", @@ -6900,26 +6900,26 @@ }, { "api_validation": "NOT_AVAILABLE", - "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.931, + "ml_validation": "NOT_AVAILABLE", + "ml_probability": null, "rule": "JSON Web Token", "severity": "medium", - "confidence": "moderate", + "confidence": "strong", "line_data_list": [ { - "line": "$payload = 'eyJgsIZgeJhvNgFpSmlP.eyJcaaF9xCe7shE0ENPiBlEJOpS'", + "line": "detected: eyJhbGciOiJSUzI1NiJ9Cg.eyJleHAiOjY1NTM2fQo.Ce7sh0ENPiBlE_dose0cBA", "line_num": 1, - "path": "tests/samples/json_web_token.hs", + "path": "tests/samples/json_web_token", "info": "", - "value": "eyJgsIZgeJhvNgFpSmlP.eyJcaaF9xCe7shE0ENPiBlEJOpS", - "value_start": 12, - "value_end": 60, + "value": "eyJhbGciOiJSUzI1NiJ9Cg.eyJleHAiOjY1NTM2fQo.Ce7sh0ENPiBlE_dose0cBA", + "value_start": 10, + "value_end": 75, "variable": null, "variable_start": -2, "variable_end": -2, "entropy_validation": { "iterator": "BASE64_CHARS", - "entropy": 4.520488802699322, + "entropy": 4.790963630103494, "valid": true } } @@ -8550,33 +8550,6 @@ } ] }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.999, - "rule": "JSON Web Token", - "severity": "medium", - "confidence": "moderate", - "line_data_list": [ - { - "line": "token in text: eyJrIjoiMDAwMDAwNDAwMDAwODAwNDAwMDAwMDAwNDAwMDAwMDAwMDAwMDAyMSIsIm4iOiJ4eHh4IiwiaWQiOjQzMDh9Cg", - "line_num": 18, - "path": "tests/samples/test2.eml", - "info": "", - "value": "eyJrIjoiMDAwMDAwNDAwMDAwODAwNDAwMDAwMDAwNDAwMDAwMDAwMDAwMDAyMSIsIm4iOiJ4eHh4IiwiaWQiOjQzMDh9Cg", - "value_start": 15, - "value_end": 109, - "variable": null, - "variable_start": -2, - "variable_end": -2, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 4.0296677144829305, - "valid": false - } - } - ] - }, { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", diff --git a/tests/filters/test_value_azure_token_check.py b/tests/filters/test_value_azure_token_check.py new file mode 100644 index 000000000..fab078169 --- /dev/null +++ b/tests/filters/test_value_azure_token_check.py @@ -0,0 +1,26 @@ +import unittest + +from credsweeper.filters import ValueAzureTokenCheck +from tests.filters.conftest import LINE_VALUE_PATTERN, DUMMY_ANALYSIS_TARGET +from tests.test_utils.dummy_line_data import get_line_data + + +class TestValueAzureTokenCheck(unittest.TestCase): + + def test_value_AzureToken_check_p(self): + self.assertTrue(ValueAzureTokenCheck().run(get_line_data(line=""), DUMMY_ANALYSIS_TARGET)) + self.assertTrue(ValueAzureTokenCheck().run(get_line_data(line="eyJungle", pattern=LINE_VALUE_PATTERN), + DUMMY_ANALYSIS_TARGET)) + self.assertTrue(ValueAzureTokenCheck().run( + get_line_data(line="eyJhbGciOjEsInR5cCI6Miwia2lkIjozfQo", pattern=LINE_VALUE_PATTERN), + DUMMY_ANALYSIS_TARGET)) + self.assertTrue(ValueAzureTokenCheck().run( + get_line_data(line="eyJhbGciOjEsInR5cCI6Miwia2lkIjozfQo.eyJhbGciOjEsInR5cCI6Miwia2lkIjozfQo" + ".eyJhbGciOjEsInR5cCI6Miwia2lkIjozfQo", + pattern=LINE_VALUE_PATTERN), DUMMY_ANALYSIS_TARGET)) + + def test_value_AzureToken_check_n(self): + self.assertFalse(ValueAzureTokenCheck().run( + get_line_data(line="eyJhbGciOjEsInR5cCI6Miwia2lkIjozfQo.eyJpc3MiOjEsImV4cCI6MiwiaWF0IjozfQo" + ".1234567890qwertyuiopasdfghjklzxc", + pattern=LINE_VALUE_PATTERN), DUMMY_ANALYSIS_TARGET)) diff --git a/tests/filters/test_value_json_web_token_check.py b/tests/filters/test_value_json_web_token_check.py index fc6f1d2d2..9aa85a752 100644 --- a/tests/filters/test_value_json_web_token_check.py +++ b/tests/filters/test_value_json_web_token_check.py @@ -1,28 +1,35 @@ -import base64 - -import pytest +import unittest from credsweeper.filters import ValueJsonWebTokenCheck from tests.filters.conftest import LINE_VALUE_PATTERN, DUMMY_ANALYSIS_TARGET from tests.test_utils.dummy_line_data import get_line_data -class TestValueJsonWebTokenCheck: +class TestValueJsonWebTokenCheck(unittest.TestCase): - @pytest.mark.parametrize("line", ["12345:asbdsa:28yd"]) - def test_value_jwt_check_p(self, file_path: pytest.fixture, line: str) -> None: - encoded_line = base64.b64encode(line.encode('ascii')).decode('ascii') - jwt_like_line = base64.b64encode('{"typ":"JWT", "dummy": false}'.encode('ascii')).decode('ascii') - jwt_line_data = get_line_data(file_path, line=f"{jwt_like_line}.{encoded_line}", pattern=LINE_VALUE_PATTERN) - assert ValueJsonWebTokenCheck().run(jwt_line_data, DUMMY_ANALYSIS_TARGET) is False - # partially line - jwt_line_data = get_line_data(file_path, line=f"{jwt_like_line}.AnyTailOfString", pattern=LINE_VALUE_PATTERN) - assert ValueJsonWebTokenCheck().run(jwt_line_data, DUMMY_ANALYSIS_TARGET) is False + def test_value_jwt_check_p(self): + self.assertTrue(ValueJsonWebTokenCheck().run(get_line_data(line="", pattern=LINE_VALUE_PATTERN), + DUMMY_ANALYSIS_TARGET)) + self.assertTrue(ValueJsonWebTokenCheck().run(get_line_data(line="eyJungle", pattern=LINE_VALUE_PATTERN), + DUMMY_ANALYSIS_TARGET)) + self.assertTrue(ValueJsonWebTokenCheck().run( + get_line_data(line="1234567890qwertyuiopasdfghjklzxc", pattern=LINE_VALUE_PATTERN), DUMMY_ANALYSIS_TARGET)) + self.assertTrue(ValueJsonWebTokenCheck().run( + get_line_data(line="eyJhbGciOiJSUzI1NiJ9Cg.eyJleHAiOjY1NTM2fQo.eyJleHAiOjY1NTM2fQo", + pattern=LINE_VALUE_PATTERN), DUMMY_ANALYSIS_TARGET)) + self.assertTrue(ValueJsonWebTokenCheck().run( + get_line_data(line="eyJhbGciOiJSUzI1NiJ9Cg.eyJleHAiOjY1NTM2fQo.AAAAAAAAAAAAAAAAAAAAAAA", + pattern=LINE_VALUE_PATTERN), DUMMY_ANALYSIS_TARGET)) - @pytest.mark.parametrize("line", ["1234f:asbdsa:28yd"]) - def test_value_jwt_check_n(self, file_path: pytest.fixture, line: str) -> None: - encoded_line = base64.b64encode(line.encode('ascii')).decode('ascii') - jwt_line_data = get_line_data(file_path, line=f"eyJungle.{encoded_line}", pattern=LINE_VALUE_PATTERN) - assert ValueJsonWebTokenCheck().run(jwt_line_data, DUMMY_ANALYSIS_TARGET) is True - jwt_line_data = get_line_data(file_path, line="eyJungle", pattern=LINE_VALUE_PATTERN) - assert ValueJsonWebTokenCheck().run(jwt_line_data, DUMMY_ANALYSIS_TARGET) is True + def test_value_jwt_check_n(self): + self.assertFalse(ValueJsonWebTokenCheck().run( + get_line_data(line="eyJhbGciOiJQUzM4NCJ9.eyJkdW1teSI6bnVsbH0.eyJpc3MiOiJqb2UifQ." \ + "_VP9ZxcPkOptWScOUMXriLH31bTcrg0YhlYL-A7TTHLX7LTDKjggtVq3Nmdl4GIS" \ + "gJdM7GHHZOJHckUjgD-T3X6oHQanKqjpWjU-GxcnOkM86e0joZgJUL7CpHUt7e3W" \ + "MhbUrOCyCFRrxOXtuTvGr2m_LdS7I5OyZ4xEP4JRcsOgOnGq-MEWmLqrRvc4zy5m" \ + "pM6tJwJXI8fr1tF4pcAZxXR17ITCrocVSRC6NuWOVzh_XyyEVRUfqlDbJnU2Z_I0" \ + "dfEQIcC6K5hAgQGSZQC_pQDA51RUoUHa9KfNskerI681fJ8mbjIlbf68CFdXZnjE" \ + "zobUhMn5Z544PF9DjW1BVtsQgXtHlSDFxl6MIMVdvM8oLRbrjlf6BYCRnCxuTA_y" \ + "Ui1o9ndy7ckISHQVhuYFKu78l7nqC4heghK_Gw4h7EB7s8eEuUC-D6JjVtX10IyS" \ + "vCRkRo7f8dWQTjFLs7mlPowjRz0cP5J-MmCoegKHYagOHZ_ArXOR91_u8jMdwmOf", + pattern=LINE_VALUE_PATTERN), DUMMY_ANALYSIS_TARGET)) diff --git a/tests/rules/test_jwt.py b/tests/rules/test_jwt.py index 10ec210ac..8ac1c4ec1 100644 --- a/tests/rules/test_jwt.py +++ b/tests/rules/test_jwt.py @@ -8,11 +8,11 @@ class TestJwt(BaseTestRule): @pytest.fixture(params=[[ - "jwt: 'eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiIxMjM0NTY3ODkwIiwibmFtZSI6IkpvaG4gRG9lIiwiaWF0IjoxN", - "TE2MjM5MDIyLCJ0ZXN0IjoiSSBuZWVkIHJlYWxseSByZWFsbHkgbG9uZyBsb25nIGxvbmcgbG9uZyBsb25nIGxvbmcgbG9uZyBsb25nI", - "GxvbmcgbG9uZyBsb25nIGxvbmcgbG9uZyBsb25nIGxvbmcgbG9uZyBsb25nIGxvbmcgbG9uZyBsb25nIGxvbmcgbG9uZyBsb25nIGxvb", - "mcgbG9uZyBsb25nIGxvbmcgbG9uZyBsb25nIGxvbmcgbG9uZyBsb25nIGxvbmcgbG9uZyBsb25nIGxvbmcgbG9uZyBsb25nIGxvbmcgb", - "G9uZyBsb25nIGxvbmcgbG9uZyBsb25nIGxvbmcgbG9uZyBsb25nIGxvbmcgbG9uZyBsb25nIGxvbmcgbG9uZyBsb25nIGxvbmcgbG9uZ", + "jwt: 'eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiIxMjM0NTY3ODkwIiwibmFtZSI6IkpvaG4gRG9lIiwiaWF0IjoxN" + "TE2MjM5MDIyLCJ0ZXN0IjoiSSBuZWVkIHJlYWxseSByZWFsbHkgbG9uZyBsb25nIGxvbmcgbG9uZyBsb25nIGxvbmcgbG9uZyBsb25nI" + "GxvbmcgbG9uZyBsb25nIGxvbmcgbG9uZyBsb25nIGxvbmcgbG9uZyBsb25nIGxvbmcgbG9uZyBsb25nIGxvbmcgbG9uZyBsb25nIGxvb" + "mcgbG9uZyBsb25nIGxvbmcgbG9uZyBsb25nIGxvbmcgbG9uZyBsb25nIGxvbmcgbG9uZyBsb25nIGxvbmcgbG9uZyBsb25nIGxvbmcgb" + "G9uZyBsb25nIGxvbmcgbG9uZyBsb25nIGxvbmcgbG9uZyBsb25nIGxvbmcgbG9uZyBsb25nIGxvbmcgbG9uZyBsb25nIGxvbmcgbG9uZ" "yBsb25nIGxvbmcgbG9uZyBsb25nIGxvbmcgbG9uZyBqd3QgdG9rZW4ifQ.4pWgA4mthx4FPPh1AZQY0luTKTQ7VOj6PGwwiANvtqg'" ]]) def lines(self, request) -> List[str]: diff --git a/tests/samples/azure_access_token b/tests/samples/azure_access_token index 4d5d0c454..d85be0700 100644 --- a/tests/samples/azure_access_token +++ b/tests/samples/azure_access_token @@ -1 +1,3 @@ -eyJ0eXAiOiJKV1QiLCJhbGciOiJSUzI1NiIsIng1dCI6Imk2bEdrM0ZaenhSY1ViMkMzbkVRN3N5SEpsWSIsImtpZCI6Imk2bEdrM0ZaenhSY1ViMkMzbkVRN3N5SEpsWSJ9.eyJhdWQiOiJlZjFkYTlkNC1mZjc3LTRjM2UtYTAwNS04NDBjM2Y4MzA3NDUiLCJpc3MiOiJodHRwczovL3N0cy53aW5kb3dzLm5ldC9mYTE1ZDY5Mi1lOWM3LTQ0NjAtYTc0My0yOWYyOTUyMjIyOS8iLCJpYXQiOjE1MzcyMzMxMDYsIm5iZiI6MTUzNzIzMzEwNiwiZXhwIjoxNTM3MjM3MDA2LCJhY3IiOiIxIiwiYWlvIjoiQVhRQWkvOElBQUFBRm0rRS9RVEcrZ0ZuVnhMaldkdzhLKzYxQUdyU091TU1GNmViYU1qN1hPM0libUQzZkdtck95RCtOdlp5R24yVmFUL2tES1h3NE1JaHJnR1ZxNkJuOHdMWG9UMUxrSVorRnpRVmtKUFBMUU9WNEtjWHFTbENWUERTL0RpQ0RnRTIyMlRJbU12V05hRU1hVU9Uc0lHdlRRPT0iLCJhbXIiOlsid2lhIl0sImFwcGlkIjoiNzVkYmU3N2YtMTBhMy00ZTU5LTg1ZmQtOGMxMjc1NDRmMTdjIiwiYXBwaWRhY3IiOiIwIiwiZW1haWwiOiJBYmVMaUBtaWNyb3NvZnQuY29tIiwiZmFtaWx5X25hbWUiOiJMaW5jb2xuIiwiZ2l2ZW5fbmFtZSI6IkFiZSAoTVNGVCkiLCJpZHAiOiJodHRwczovL3N0cy53aW5kb3dzLm5ldC83MmY5ODhiZi04NmYxLTQxYWYtOTFhYi0yZDdjZDAxMjIyNDcvIiwiaXBhZGRyIjoiMjIyLjIyMi4yMjIuMjIiLCJuYW1lIjoiYWJlbGkiLCJvaWQiOiIwMjIyM2I2Yi1hYTFkLTQyZDQtOWVjMC0xYjJiYjkxOTQ0MzgiLCJyaCI6IkkiLCJzY3AiOiJ1c2VyX2ltcGVyc29uYXRpb24iLCJzdWIiOiJsM19yb0lTUVUyMjJiVUxTOXlpMmswWHBxcE9pTXo1SDNaQUNvMUdlWEEiLCJ0aWQiOiJmYTE1ZDY5Mi1lOWM3LTQ0NjAtYTc0My0yOWYyOTU2ZmQ0MjkiLCJ1bmlxdWVfbmFtZSI6ImFiZWxpQG1pY3Jvc29mdC5jb20iLCJ1dGkiOiJGVnNHeFlYSTMwLVR1aWt1dVVvRkFBIiwidmVyIjoiMS4wIn0.D3H6pMUtQnoJAGq6AHd \ No newline at end of file +eyJ0eXAiOiJKV1QiLCJhbGciOiJSUzI1NiIsIng1dCI6Imk2bEdrM0ZaenhSY1ViMkMzbkVRN3N5SEpsWSIsImtpZCI6Imk2bEdrM0ZaenhSY1ViMkMzbkVRN3N5SEpsWSJ9.eyJhdWQiOiJlZjFkYTlkNC1mZjc3LTRjM2UtYTAwNS04NDBjM2Y4MzA3NDUiLCJpc3MiOiJodHRwczovL3N0cy53aW5kb3dzLm5ldC9mYTE1ZDY5Mi1lOWM3LTQ0NjAtYTc0My0yOWYyOTUyMjIyOS8iLCJpYXQiOjE1MzcyMzMxMDYsIm5iZiI6MTUzNzIzMzEwNiwiZXhwIjoxNTM3MjM3MDA2LCJhY3IiOiIxIiwiYWlvIjoiQVhRQWkvOElBQUFBRm0rRS9RVEcrZ0ZuVnhMaldkdzhLKzYxQUdyU091TU1GNmViYU1qN1hPM0libUQzZkdtck95RCtOdlp5R24yVmFUL2tES1h3NE1JaHJnR1ZxNkJuOHdMWG9UMUxrSVorRnpRVmtKUFBMUU9WNEtjWHFTbENWUERTL0RpQ0RnRTIyMlRJbU12V05hRU1hVU9Uc0lHdlRRPT0iLCJhbXIiOlsid2lhIl0sImFwcGlkIjoiNzVkYmU3N2YtMTBhMy00ZTU5LTg1ZmQtOGMxMjc1NDRmMTdjIiwiYXBwaWRhY3IiOiIwIiwiZW1haWwiOiJBYmVMaUBtaWNyb3NvZnQuY29tIiwiZmFtaWx5X25hbWUiOiJMaW5jb2xuIiwiZ2l2ZW5fbmFtZSI6IkFiZSAoTVNGVCkiLCJpZHAiOiJodHRwczovL3N0cy53aW5kb3dzLm5ldC83MmY5ODhiZi04NmYxLTQxYWYtOTFhYi0yZDdjZDAxMjIyNDcvIiwiaXBhZGRyIjoiMjIyLjIyMi4yMjIuMjIiLCJuYW1lIjoiYWJlbGkiLCJvaWQiOiIwMjIyM2I2Yi1hYTFkLTQyZDQtOWVjMC0xYjJiYjkxOTQ0MzgiLCJyaCI6IkkiLCJzY3AiOiJ1c2VyX2ltcGVyc29uYXRpb24iLCJzdWIiOiJsM19yb0lTUVUyMjJiVUxTOXlpMmswWHBxcE9pTXo1SDNaQUNvMUdlWEEiLCJ0aWQiOiJmYTE1ZDY5Mi1lOWM3LTQ0NjAtYTc0My0yOWYyOTU2ZmQ0MjkiLCJ1bmlxdWVfbmFtZSI6ImFiZWxpQG1pY3Jvc29mdC5jb20iLCJ1dGkiOiJGVnNHeFlYSTMwLVR1aWt1dVVvRkFBIiwidmVyIjoiMS4wIn0.D3H6pMUtQnoJAGq6AHd +eyJ0eXAiOiJKV1QiLCJhbGciOiJSUzI1NiIsImtpZCI6Imk2bEdrM0ZaenhSY1ViMkMzbkVRN3N5SEpsWSJ9.eyJhdWQiOiI2ZTc0MTcyYi1iZTU2LTQ4NDMtOWZmNC1lNjZhMzliYjEyZTMiLCJpc3MiOiJodHRwczovL2xvZ2luLm1pY3Jvc29mdG9ubGluZS5jb20vNzJmOTg4YmYtODZmMS00MWFmLTkxYWItMmQ3Y2QwMTFkYjQ3L3YyLjAiLCJpYXQiOjE1MzcyMzEwNDgsIm5iZiI6MTUzNzIzMTA0OCwiZXhwIjoxNTM3MjM0OTQ4LCJhaW8iOiJBWFFBaS84SUFBQUF0QWFaTG8zQ2hNaWY2S09udHRSQjdlQnE0L0RjY1F6amNKR3hQWXkvQzNqRGFOR3hYZDZ3TklJVkdSZ2hOUm53SjFsT2NBbk5aY2p2a295ckZ4Q3R0djMzMTQwUmlvT0ZKNGJDQ0dWdW9DYWcxdU9UVDIyMjIyZ0h3TFBZUS91Zjc5UVgrMEtJaWpkcm1wNjlSY3R6bVE9PSIsImF6cCI6IjZlNzQxNzJiLWJlNTYtNDg0My05ZmY0LWU2NmEzOWJiMTJlMyIsImF6cGFjciI6IjAiLCJuYW1lIjoiQWJlIExpbmNvbG4iLCJvaWQiOiI2OTAyMjJiZS1mZjFhLTRkNTYtYWJkMS03ZTRmN2QzOGU0NzQiLCJwcmVmZXJyZWRfdXNlcm5hbWUiOiJhYmVsaUBtaWNyb3NvZnQuY29tIiwicmgiOiJJIiwic2NwIjoiYWNjZXNzX2FzX3VzZXIiLCJzdWIiOiJIS1pwZmFIeVdhZGVPb3VZbGl0anJJLUtmZlRtMjIyWDVyclYzeERxZktRIiwidGlkIjoiNzJmOTg4YmYtODZmMS00MWFmLTkxYWItMmQ3Y2QwMTFkYjQ3IiwidXRpIjoiZnFpQnFYTFBqMGVRYTgyUy1JWUZBQSIsInZlciI6IjIuMCJ9.pj4N-w_3Us9DrBLfpCt +^^^ examples from https://learn.microsoft.com/en-us/entra/identity-platform/access-tokens \ No newline at end of file diff --git a/tests/samples/json_web_token b/tests/samples/json_web_token new file mode 100644 index 000000000..45e244772 --- /dev/null +++ b/tests/samples/json_web_token @@ -0,0 +1,2 @@ +detected: eyJhbGciOiJSUzI1NiJ9Cg.eyJleHAiOjY1NTM2fQo.Ce7sh0ENPiBlE_dose0cBA +not detected: eyJhbGciOiJSUzI1NiIsInR5cCI6IkpXVCJ9.NiIsInR5cCI6IkpXV.NiIsInR5cCI6IkpXV diff --git a/tests/samples/json_web_token.hs b/tests/samples/json_web_token.hs deleted file mode 100644 index 76f919b6c..000000000 --- a/tests/samples/json_web_token.hs +++ /dev/null @@ -1 +0,0 @@ -$payload = 'eyJgsIZgeJhvNgFpSmlP.eyJcaaF9xCe7shE0ENPiBlEJOpS' From e31ef71bbf532ebbdf075d7680682adc032e3f7c Mon Sep 17 00:00:00 2001 From: Roman Babenko Date: Fri, 9 Aug 2024 09:46:30 +0300 Subject: [PATCH 5/7] Filters optimisation (#588) * Filters optimisation * style * unicode cases in filter * tmp markup loan * BM scores fix * docs upd * removed unused filter * doc upd * [no ci] upd2 * test counters fix * reduce whitespaces during extracting subtext * aux BM ref * BM scores fix * Rollback BM * JWT fix * customBMref * JWT fix BC scor * BM scores fix --- .github/workflows/benchmark.yml | 8 +- cicd/benchmark.txt | 66 +++++++------- credsweeper/credentials/line_data.py | 2 + credsweeper/filters/__init__.py | 2 - credsweeper/filters/group/general_pattern.py | 2 + credsweeper/filters/group/group.py | 8 +- .../filters/group/url_credentials_group.py | 7 +- .../filters/line_specific_key_check.py | 12 ++- .../filters/separator_unusual_check.py | 49 ---------- .../value_dictionary_value_length_check.py | 7 +- .../filters/value_json_web_token_check.py | 2 +- credsweeper/filters/value_length_check.py | 26 ------ .../value_not_allowed_pattern_check.py | 2 +- .../filters/value_pattern_length_check.py | 10 --- .../filters/value_useless_word_check.py | 7 +- credsweeper/rules/config.yaml | 2 +- credsweeper/utils/util.py | 10 ++- docs/source/credsweeper.filters.rst | 24 ----- tests/__init__.py | 8 +- tests/data/depth_3.json | 89 ++++++++++++++++++- tests/data/ml_threshold.json | 62 ++++++++++++- tests/data/output.json | 62 ++++++++++++- tests/filters/test_line_specific_key_check.py | 4 +- .../test_value_json_web_token_check.py | 8 ++ tests/filters/test_value_length_check.py | 19 ---- .../filters/test_value_useless_word_check.py | 2 +- tests/ml_model/test_ml_validator.py | 16 ++-- tests/samples/aws_client_id | 2 + tests/samples/key.hs | 2 +- tests/samples/salt.py | 1 + tests/samples/url_cred.js | 2 + tests/test_main.py | 2 +- tests/utils/test_util.py | 16 ++-- 33 files changed, 318 insertions(+), 223 deletions(-) delete mode 100644 credsweeper/filters/separator_unusual_check.py delete mode 100644 credsweeper/filters/value_length_check.py delete mode 100644 credsweeper/filters/value_pattern_length_check.py delete mode 100644 tests/filters/test_value_length_check.py diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index d6017bb39..8b418ecf1 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -23,7 +23,7 @@ jobs: uses: actions/checkout@v4 with: repository: babenek/CredData - ref: jwt + ref: auxiliary - name: Markup hashing run: | @@ -74,7 +74,7 @@ jobs: uses: actions/checkout@v4 with: repository: babenek/CredData - ref: jwt + ref: auxiliary - name: Markup hashing run: | @@ -172,7 +172,7 @@ jobs: uses: actions/checkout@v4 with: repository: babenek/CredData - ref: jwt + ref: auxiliary - name: Markup hashing run: | @@ -354,7 +354,7 @@ jobs: uses: actions/checkout@v4 with: repository: babenek/CredData - ref: jwt + ref: auxiliary - name: Markup hashing run: | diff --git a/cicd/benchmark.txt b/cicd/benchmark.txt index c2e02262c..1626253d5 100644 --- a/cicd/benchmark.txt +++ b/cicd/benchmark.txt @@ -1,7 +1,7 @@ -DATA: 16978521 interested lines. MARKUP: 61845 items +DATA: 16978521 interested lines. MARKUP: 61855 items FileType FileNumber ValidLines Positives Negatives Templates --------------- ------------ ------------ ----------- ----------- ----------- - 194 28318 64 427 89 + 194 28318 64 429 87 .1 2 641 2 5 .admx 1 26 1 .adoc 1 158 11 6 1 @@ -11,7 +11,7 @@ FileType FileNumber ValidLines Positives Negatives Templat .backup 1 62 1 1 .bash 2 2158 2 1 .bat 4 233 1 13 2 -.bats 15 2804 12 52 9 +.bats 15 2804 14 50 9 .bazel 3 424 8 .build 2 40 3 .bundle 4 1512 570 @@ -27,7 +27,7 @@ FileType FileNumber ValidLines Positives Negatives Templat .cmd 4 401 2 3 .cnf 8 858 18 45 18 .coffee 1 585 2 -.conf 60 4945 50 74 54 +.conf 60 4945 54 69 54 .config 20 492 16 33 1 .cpp 15 5688 1 61 .creds 1 10 1 1 @@ -53,15 +53,15 @@ FileType FileNumber ValidLines Positives Negatives Templat .erb 13 323 27 .erl 4 96 8 .ex 25 4968 3 105 5 -.example 17 1838 73 37 55 -.exs 24 4842 3 188 4 +.example 17 1838 74 38 54 +.exs 24 4842 3 190 4 .ext 5 211 1 4 2 .fsproj 1 75 1 .g4 2 201 2 .gd 1 37 1 .gml 3 3075 26 .gni 3 5017 18 -.go 1079 566327 619 4333 742 +.go 1079 566327 623 4329 742 .golden 5 1168 1 14 29 .gradle 45 3265 4 91 100 .graphql 7 420 13 @@ -74,22 +74,22 @@ FileType FileNumber ValidLines Positives Negatives Templat .html 53 15327 14 115 18 .idl 2 777 4 .iml 6 699 36 -.in 6 2130 1 80 12 +.in 6 2130 3 78 12 .inc 2 56 2 1 .ini 11 1437 24 12 18 .ipynb 1 134 5 -.j 1 241 4 +.j 1 241 2 2 .j2 30 5530 6 213 10 -.java 621 134132 314 1357 170 +.java 621 134132 314 1361 170 .jenkinsfile 1 58 1 7 .jinja2 1 64 2 -.js 659 536413 521 2642 336 -.json 860 13670669 623 10948 140 +.js 659 536413 526 2638 336 +.json 860 13670669 624 10946 140 .jsp 13 3202 1 42 .jsx 7 857 19 .jwt 1 1 2 .key 83 2737 70 14 -.kt 123 20774 50 384 3 +.kt 123 20774 51 384 3 .l 1 982 1 .las 1 6656 46 .lasso 1 230 6 @@ -105,12 +105,12 @@ FileType FileNumber ValidLines Positives Negatives Templat .lock 24 160912 144 .log 2 199 38 52 .lua 10 1924 37 3 -.m 16 13358 11 151 3 +.m 16 13358 11 152 3 .manifest 3 102 3 .markdown 3 139 3 1 .markerb 3 12 3 .marko 1 21 2 -.md 673 149294 646 2366 671 +.md 673 149294 658 2362 664 .mdx 3 549 7 .mjml 1 18 1 .mjs 22 4424 50 343 @@ -122,7 +122,7 @@ FileType FileNumber ValidLines Positives Negatives Templat .mqh 1 1023 2 .msg 1 26644 1 .mysql 1 36 2 -.ndjson 2 5006 34 268 2 +.ndjson 2 5006 37 266 2 .nix 4 211 12 .nolint 1 2 1 .odd 1 1281 57 @@ -132,7 +132,7 @@ FileType FileNumber ValidLines Positives Negatives Templat .patch 4 109405 27 .pbxproj 1 941 1 .pem 48 1169 47 8 -.php 371 75710 129 1770 80 +.php 371 75710 130 1769 80 .pl 16 14727 6 47 .pm 3 744 8 .po 3 2994 15 @@ -150,13 +150,13 @@ FileType FileNumber ValidLines Positives Negatives Templat .pug 2 193 2 .purs 1 69 4 .pxd 1 150 5 2 -.py 890 291553 618 3465 748 +.py 890 291553 627 3460 744 .pyi 4 1361 9 .pyp 1 167 1 .pyx 2 1094 21 .r 4 62 6 3 1 .rake 2 51 2 -.rb 861 131867 237 3457 615 +.rb 861 131867 237 3458 615 .re 1 31 1 .red 1 159 1 .release 1 13 4 @@ -222,16 +222,16 @@ FileType FileNumber ValidLines Positives Negatives Templat .yml 418 36162 460 916 384 .zsh 6 872 12 .zsh-theme 1 97 1 -TOTAL: 10281 16978521 7499 59954 5230 -credsweeper result_cnt : 6597, lost_cnt : 0, true_cnt : 6352, false_cnt : 245 +TOTAL: 10281 16978521 7546 59932 5216 +credsweeper result_cnt : 6585, lost_cnt : 0, true_cnt : 6367, false_cnt : 218 Rules Positives Negatives Templates Reported TP FP TN FN FPR FNR ACC PRC RCL F1 ------------------------------ ----------- ----------- ----------- ---------- ---- ---- ----- ---- -------- -------- -------- -------- -------- -------- -API 123 3163 185 112 109 3 3345 14 0.000896 0.113821 0.995102 0.973214 0.886179 0.927660 +API 124 3162 185 112 109 3 3344 15 0.000896 0.120968 0.994814 0.973214 0.879032 0.923729 AWS Client ID 168 13 0 160 160 0 13 8 0.000000 0.047619 0.955801 1.000000 0.952381 0.975610 AWS Multi 75 12 0 87 75 11 1 0 0.916667 0.000000 0.873563 0.872093 1.000000 0.931677 -AWS S3 Bucket 61 25 0 87 61 24 1 0 0.960000 0.000000 0.720930 0.717647 1.000000 0.835616 +AWS S3 Bucket 61 25 0 92 61 25 0 0 1.000000 0.000000 0.709302 0.709302 1.000000 0.829932 Atlassian Old PAT token 27 212 3 12 3 8 207 24 0.037209 0.888889 0.867769 0.272727 0.111111 0.157895 -Auth 407 2725 77 372 351 21 2781 56 0.007495 0.137592 0.976005 0.943548 0.862408 0.901155 +Auth 408 2727 77 372 351 21 2783 57 0.007489 0.139706 0.975716 0.943548 0.860294 0.900000 Azure Access Token 19 0 0 12 12 0 0 7 0.368421 0.631579 1.000000 0.631579 0.774194 BASE64 Private Key 7 2 0 7 7 0 2 0 0.000000 0.000000 1.000000 1.000000 1.000000 1.000000 BASE64 encoded PEM Private Key 7 0 0 5 5 0 0 2 0.285714 0.714286 1.000000 0.714286 0.833333 @@ -250,19 +250,19 @@ Google API Key 12 0 0 1 Google Multi 10 2 0 11 10 1 1 0 0.500000 0.000000 0.916667 0.909091 1.000000 0.952381 Google OAuth Access Token 3 0 0 3 3 0 0 0 0.000000 1.000000 1.000000 1.000000 1.000000 Grafana Provisioned API Key 22 1 0 5 5 0 1 17 0.000000 0.772727 0.260870 1.000000 0.227273 0.370370 -JSON Web Token 169 61 0 158 137 21 40 32 0.344262 0.189349 0.769565 0.867089 0.810651 0.837920 +JSON Web Token 169 61 0 130 130 0 61 39 0.000000 0.230769 0.830435 1.000000 0.769231 0.869565 Jira / Confluence PAT token 0 4 0 0 0 4 0 0.000000 1.000000 Jira 2FA 14 6 0 10 10 0 6 4 0.000000 0.285714 0.800000 1.000000 0.714286 0.833333 -Key 483 8494 464 445 436 9 8949 47 0.001005 0.097308 0.994068 0.979775 0.902692 0.939655 -Nonce 83 53 0 85 79 6 47 4 0.113208 0.048193 0.926471 0.929412 0.951807 0.940476 +Key 497 8483 464 448 439 9 8938 58 0.001006 0.116700 0.992906 0.979911 0.883300 0.929101 +Nonce 90 47 0 84 83 1 46 7 0.021277 0.077778 0.941606 0.988095 0.922222 0.954023 Other 0 0 5 0 0 5 0 0.000000 1.000000 PEM Private Key 1019 1483 0 1023 1019 4 1479 0 0.002697 0.000000 0.998401 0.996090 1.000000 0.998041 -Password 1823 7474 2752 1681 1614 67 10159 209 0.006552 0.114646 0.977094 0.960143 0.885354 0.921233 -Salt 42 76 2 38 38 0 78 4 0.000000 0.095238 0.966667 1.000000 0.904762 0.950000 -Secret 1358 28497 869 1234 1229 5 29361 129 0.000170 0.094993 0.995639 0.995948 0.905007 0.948302 +Password 1834 7473 2741 1691 1627 64 10150 207 0.006266 0.112868 0.977507 0.962153 0.887132 0.923121 +Salt 45 73 2 39 39 0 75 6 0.000000 0.133333 0.950000 1.000000 0.866667 0.928571 +Secret 1362 28492 868 1236 1231 5 29355 131 0.000170 0.096182 0.995573 0.995955 0.903818 0.947652 Seed 1 6 0 0 0 6 1 0.000000 1.000000 0.857143 0.000000 Slack Token 4 1 0 4 4 0 1 0 0.000000 0.000000 1.000000 1.000000 1.000000 1.000000 -Token 584 3973 438 519 511 8 4403 73 0.001814 0.125000 0.983784 0.984586 0.875000 0.926564 +Token 586 3974 438 513 504 9 4403 82 0.002040 0.139932 0.981793 0.982456 0.860068 0.917197 Twilio API Key 0 5 2 0 0 7 0 0.000000 1.000000 -URL Credentials 194 125 251 184 184 0 376 10 0.000000 0.051546 0.982456 1.000000 0.948454 0.973545 - 7499 59954 5230 6604 6352 245 59709 1147 0.004086 0.152954 0.979363 0.962862 0.847046 0.901249 +URL Credentials 198 127 249 190 190 0 376 8 0.000000 0.040404 0.986063 1.000000 0.959596 0.979381 + 7546 59932 5216 6596 6367 218 59714 1179 0.003637 0.156242 0.979297 0.966894 0.843758 0.901139 diff --git a/credsweeper/credentials/line_data.py b/credsweeper/credentials/line_data.py index 06a68fb5b..d407b657a 100644 --- a/credsweeper/credentials/line_data.py +++ b/credsweeper/credentials/line_data.py @@ -193,6 +193,8 @@ def sanitize_variable(self) -> None: while self.variable and sanitized_var_len != len(self.variable): sanitized_var_len = len(self.variable) self.variable = self.variable.strip(self.variable_strip_pattern) + if self.variable.endswith('\\'): + self.variable = self.variable[:-1] if variable and len(self.variable) < len(variable) and 0 <= self.variable_start and 0 <= self.variable_end: start = variable.find(self.variable) self.variable_start += start diff --git a/credsweeper/filters/__init__.py b/credsweeper/filters/__init__.py index f050671a5..a91119e63 100644 --- a/credsweeper/filters/__init__.py +++ b/credsweeper/filters/__init__.py @@ -29,13 +29,11 @@ from credsweeper.filters.value_jfrog_token_check import ValueJfrogTokenCheck from credsweeper.filters.value_json_web_token_check import ValueJsonWebTokenCheck from credsweeper.filters.value_last_word_check import ValueLastWordCheck -from credsweeper.filters.value_length_check import ValueLengthCheck from credsweeper.filters.value_method_check import ValueMethodCheck from credsweeper.filters.value_not_allowed_pattern_check import ValueNotAllowedPatternCheck from credsweeper.filters.value_not_part_encoded_check import ValueNotPartEncodedCheck from credsweeper.filters.value_number_check import ValueNumberCheck from credsweeper.filters.value_pattern_check import ValuePatternCheck -from credsweeper.filters.value_pattern_length_check import ValuePatternLengthCheck from credsweeper.filters.value_similarity_check import ValueSimilarityCheck from credsweeper.filters.value_split_keyword_check import ValueSplitKeywordCheck from credsweeper.filters.value_string_type_check import ValueStringTypeCheck diff --git a/credsweeper/filters/group/general_pattern.py b/credsweeper/filters/group/general_pattern.py index b340055fd..96018a5f4 100644 --- a/credsweeper/filters/group/general_pattern.py +++ b/credsweeper/filters/group/general_pattern.py @@ -1,5 +1,6 @@ from credsweeper.common.constants import GroupType from credsweeper.config import Config +from credsweeper.filters import ValueUselessWordCheck from credsweeper.filters.group import Group @@ -8,3 +9,4 @@ class GeneralPattern(Group): def __init__(self, config: Config) -> None: super().__init__(config, GroupType.PATTERN) + self.filters.extend([ValueUselessWordCheck()]) diff --git a/credsweeper/filters/group/group.py b/credsweeper/filters/group/group.py index 7f1bc3c6a..37cea6948 100644 --- a/credsweeper/filters/group/group.py +++ b/credsweeper/filters/group/group.py @@ -5,9 +5,9 @@ from credsweeper.config import Config from credsweeper.filters import (Filter, LineSpecificKeyCheck, ValueAllowlistCheck, ValueArrayDictionaryCheck, ValueBlocklistCheck, ValueCamelCaseCheck, ValueFilePathCheck, ValueFirstWordCheck, - ValueLastWordCheck, ValueLengthCheck, ValueMethodCheck, ValueNotAllowedPatternCheck, - ValuePatternCheck, ValueSimilarityCheck, ValueStringTypeCheck, ValueTokenCheck, - VariableNotAllowedPatternCheck, ValuePatternLengthCheck, ValueHexNumberCheck) + ValueLastWordCheck, ValueMethodCheck, ValueNotAllowedPatternCheck, ValuePatternCheck, + ValueSimilarityCheck, ValueStringTypeCheck, ValueTokenCheck, + VariableNotAllowedPatternCheck, ValueHexNumberCheck) class Group(ABC): @@ -43,7 +43,6 @@ def get_keyword_base_filters(config: Config) -> List[Filter]: ValueFirstWordCheck(), ValueHexNumberCheck(), ValueLastWordCheck(), - ValueLengthCheck(config), ValueMethodCheck(), ValueSimilarityCheck(), ValueStringTypeCheck(config), @@ -60,5 +59,4 @@ def get_pattern_base_filters(config: Config) -> List[Filter]: return [ # LineSpecificKeyCheck(), # ValuePatternCheck(config), # - ValuePatternLengthCheck(config), # ] diff --git a/credsweeper/filters/group/url_credentials_group.py b/credsweeper/filters/group/url_credentials_group.py index 9a7477191..23aba1d3b 100644 --- a/credsweeper/filters/group/url_credentials_group.py +++ b/credsweeper/filters/group/url_credentials_group.py @@ -2,8 +2,8 @@ from credsweeper.config import Config from credsweeper.filters import (ValueAllowlistCheck, ValueArrayDictionaryCheck, ValueBlocklistCheck, ValueCamelCaseCheck, ValueDictionaryValueLengthCheck, ValueFilePathCheck, - ValueFirstWordCheck, ValueLastWordCheck, ValueLengthCheck, ValueMethodCheck, - ValueNotAllowedPatternCheck, ValuePatternCheck, ValueStringTypeCheck, ValueTokenCheck) + ValueFirstWordCheck, ValueLastWordCheck, ValueMethodCheck, ValueNotAllowedPatternCheck, + ValuePatternCheck, ValueStringTypeCheck, ValueTokenCheck) from credsweeper.filters.group import Group @@ -25,11 +25,10 @@ def __init__(self, config: Config) -> None: ValueFilePathCheck(), ValueFirstWordCheck(), ValueLastWordCheck(), - ValueLengthCheck(config), ValueMethodCheck(), ValueStringTypeCheck(config), ValueNotAllowedPatternCheck(), ValueTokenCheck(), - ValueDictionaryValueLengthCheck(), + ValueDictionaryValueLengthCheck(min_len=4, max_len=80), ValuePatternCheck(config) ] diff --git a/credsweeper/filters/line_specific_key_check.py b/credsweeper/filters/line_specific_key_check.py index 8bbfa15a1..71fec9dc0 100644 --- a/credsweeper/filters/line_specific_key_check.py +++ b/credsweeper/filters/line_specific_key_check.py @@ -1,5 +1,6 @@ import re +from credsweeper.common.constants import ML_HUNK from credsweeper.config import Config from credsweeper.credentials import LineData from credsweeper.file_handler.analysis_target import AnalysisTarget @@ -10,8 +11,8 @@ class LineSpecificKeyCheck(Filter): """Check that values from list below is not in candidate line.""" - NOT_ALLOWED = [r"example", r"enc\(", r"enc\[", r"true", r"false"] - NOT_ALLOWED_PATTERN = re.compile(Util.get_regex_combine_or(NOT_ALLOWED)) + NOT_ALLOWED = [r"example", r"\benc[\(\[]", r"\btrue\b", r"\bfalse\b"] + NOT_ALLOWED_PATTERN = re.compile(Util.get_regex_combine_or(NOT_ALLOWED), re.IGNORECASE) def __init__(self, config: Config = None) -> None: pass @@ -29,8 +30,13 @@ def run(self, line_data: LineData, target: AnalysisTarget) -> bool: """ if line_data.line is None: return True + if 0 <= line_data.variable_start: + # variable may be defined too + sub_line_start = 0 if ML_HUNK >= line_data.variable_start else line_data.variable_start - ML_HUNK + else: + sub_line_start = 0 if ML_HUNK >= line_data.value_start else line_data.value_start - ML_HUNK - if self.NOT_ALLOWED_PATTERN.search(target.line_lower): + if self.NOT_ALLOWED_PATTERN.search(line_data.line, sub_line_start, line_data.value_end + ML_HUNK): return True return False diff --git a/credsweeper/filters/separator_unusual_check.py b/credsweeper/filters/separator_unusual_check.py deleted file mode 100644 index b05da326b..000000000 --- a/credsweeper/filters/separator_unusual_check.py +++ /dev/null @@ -1,49 +0,0 @@ -import logging - -from credsweeper.config import Config -from credsweeper.credentials import LineData -from credsweeper.file_handler.analysis_target import AnalysisTarget -from credsweeper.filters import Filter - -logger = logging.getLogger(__name__) - - -class SeparatorUnusualCheck(Filter): - """Check that candidate have no double symbol ops (like ++, --, <<) or comparison ops (like != or ==) as separator. - - Example: - `pwd == 'value'` - `pwd != 'value'` - `pwd << value` - - """ - - def __init__(self, config: Config = None) -> None: - pass - - def run(self, line_data: LineData, target: AnalysisTarget) -> bool: - """Run filter checks on received credential candidate data 'line_data'. - - Args: - line_data: credential candidate data - target: multiline target from which line data was obtained - - Return: - True, if need to filter candidate and False if left - - """ - if line_data.separator is None: - return True - - if 1 > line_data.separator_start: - logger.warning(f"Wrong separator start position {line_data}") - return True - - try: - if line_data.separator == line_data.line[line_data.separator_start + 1] or \ - (line_data.separator == "=" and line_data.line[line_data.separator_start - 1] == "!"): - return True - except IndexError: - return True - - return False diff --git a/credsweeper/filters/value_dictionary_value_length_check.py b/credsweeper/filters/value_dictionary_value_length_check.py index c0b92a846..8186f8229 100644 --- a/credsweeper/filters/value_dictionary_value_length_check.py +++ b/credsweeper/filters/value_dictionary_value_length_check.py @@ -7,8 +7,9 @@ class ValueDictionaryValueLengthCheck(Filter): """Check that candidate length is between 5 and 30.""" - def __init__(self, config: Config = None) -> None: - pass + def __init__(self, config: Config = None, min_len: int = 4, max_len: int = 31) -> None: + self.min_len = min_len + self.max_len = max_len def run(self, line_data: LineData, target: AnalysisTarget) -> bool: """Run filter checks on received credential candidate data 'line_data'. @@ -21,7 +22,7 @@ def run(self, line_data: LineData, target: AnalysisTarget) -> bool: True, if need to filter candidate and False if left """ - if 4 <= len(line_data.value) <= 31: + if self.min_len <= len(line_data.value) <= self.max_len: return False else: return True diff --git a/credsweeper/filters/value_json_web_token_check.py b/credsweeper/filters/value_json_web_token_check.py index d7265dbce..ed6a2e2e1 100644 --- a/credsweeper/filters/value_json_web_token_check.py +++ b/credsweeper/filters/value_json_web_token_check.py @@ -53,7 +53,7 @@ def run(self, line_data: LineData, target: AnalysisTarget) -> bool: if not header_check: header_check = bool(ValueJsonWebTokenCheck.header_keys.intersection(json_keys)) # payload follows the header - if not payload_check: + elif not payload_check: payload_check = bool(ValueJsonWebTokenCheck.payload_keys.intersection(json_keys)) # any other payloads are allowed elif header_check and payload_check and not signature_check: diff --git a/credsweeper/filters/value_length_check.py b/credsweeper/filters/value_length_check.py deleted file mode 100644 index 57596f35e..000000000 --- a/credsweeper/filters/value_length_check.py +++ /dev/null @@ -1,26 +0,0 @@ -from credsweeper.config import Config -from credsweeper.credentials import LineData -from credsweeper.file_handler.analysis_target import AnalysisTarget -from credsweeper.filters import Filter - - -class ValueLengthCheck(Filter): - """Check if potential candidate value is not too short (longer or equal to `min_len`).""" - - def __init__(self, config: Config) -> None: - self.min_len = config.min_keyword_value_length - - def run(self, line_data: LineData, target: AnalysisTarget) -> bool: - """Run filter checks on received credential candidate data 'line_data'. - - Args: - line_data: credential candidate data - target: multiline target from which line data was obtained - - Return: - True, if need to filter candidate and False if left - - """ - if len(line_data.value) < self.min_len: - return True - return False diff --git a/credsweeper/filters/value_not_allowed_pattern_check.py b/credsweeper/filters/value_not_allowed_pattern_check.py index 944c9c34e..a0cc89aa1 100644 --- a/credsweeper/filters/value_not_allowed_pattern_check.py +++ b/credsweeper/filters/value_not_allowed_pattern_check.py @@ -10,7 +10,7 @@ class ValueNotAllowedPatternCheck(Filter): """Check that secret doesn't open or closes brackets or a new line.""" - NOT_ALLOWED = [r"[<>\[\]{}]\s+", r"^\s*\\", r"^\s*\\n\s*"] + NOT_ALLOWED = [r"[<>\[\]{}]\s+", r"\\u00(26|3c)gt;?(\s|\\+[nrt])?", r"^\s*\\", r"^\s*\\n\s*"] NOT_ALLOWED_PATTERN = re.compile( # f"{Util.get_regex_combine_or(NOT_ALLOWED)}$", # flags=re.IGNORECASE) diff --git a/credsweeper/filters/value_pattern_length_check.py b/credsweeper/filters/value_pattern_length_check.py deleted file mode 100644 index dd4531bf1..000000000 --- a/credsweeper/filters/value_pattern_length_check.py +++ /dev/null @@ -1,10 +0,0 @@ -from credsweeper.config import Config -from credsweeper.filters import ValueLengthCheck - - -class ValuePatternLengthCheck(ValueLengthCheck): - """Check if potential candidate value is not too short like ValueLengthCheck but with different min_len""" - - def __init__(self, config: Config) -> None: - super().__init__(config) - self.min_len = config.min_pattern_value_length diff --git a/credsweeper/filters/value_useless_word_check.py b/credsweeper/filters/value_useless_word_check.py index c921c937d..6182d1e3f 100644 --- a/credsweeper/filters/value_useless_word_check.py +++ b/credsweeper/filters/value_useless_word_check.py @@ -11,11 +11,10 @@ class ValueUselessWordCheck(Filter): """Check is candidate value contains sub-rows with operators (like ->).""" NOT_ALLOWED = [ - "((\\{)?(0x)+([0-9a-f]|\\%){1}.*)", # Check is contain \{0x or 0x - "(\\-\\>.*)", # Check if contain -> - "(xxxx.*)", # Check if contain xxxxx + "((\\{)?(0x)+([0-9a-f]|\\%){1})", # Check is contain \{0x or 0x + r"((\w+)?->)", # Check if contain -> + "(.*example)", # Check if contain `example` word "(\\$\\w+)", # Check whether it looks like a variable e.g. $word - "(\\s).*" # Check if contain \s ] NOT_ALLOWED_PATTERN = re.compile( # Util.get_regex_combine_or(NOT_ALLOWED), # diff --git a/credsweeper/rules/config.yaml b/credsweeper/rules/config.yaml index 26e7e5198..afe66d028 100644 --- a/credsweeper/rules/config.yaml +++ b/credsweeper/rules/config.yaml @@ -648,7 +648,7 @@ confidence: moderate type: pattern values: - - (?P[\"'])?(?P[+0-9A-Za-z-]{2,80}://)([^\s\'"<>\[\]^~`{|}@:/]{0,80}:){1,3}(?P[^\s\'"<>\[\]^~`{|}@:/]{3,80})@[^\s\'"<>\[\]^~`{|}@:/]{1,800}\\{0,8}(?P[\"'])? + - (?P[\"'])?(?P[+0-9A-Za-z-]{2,80}://)([^\s\'"<>\[\]^~`{|}:/]{0,80}:){1,3}(?P[^\s\'"<>\[\]^~`{|}@:/]{3,80})@[^\s\'"<>\[\]^~`{|}@:/]{1,800}\\{0,8}(?P[\"'])? filter_type: UrlCredentialsGroup use_ml: true required_substrings: diff --git a/credsweeper/utils/util.py b/credsweeper/utils/util.py index 400d9c0a5..80affdb29 100644 --- a/credsweeper/utils/util.py +++ b/credsweeper/utils/util.py @@ -4,6 +4,7 @@ import logging import math import os +import string import struct import tarfile from dataclasses import dataclass @@ -690,6 +691,13 @@ def subtext(text: str, pos: int, hunk_size: int) -> str: else: left_quota = hunk_size - pos left_pos = 0 + # skip leading whitespaces in result string + for i in range(left_pos, pos): + if text[i] in string.whitespace: + left_quota += 1 + left_pos += 1 + else: + break right_remain = len(text) - pos if hunk_size <= right_remain: right_quota = 0 @@ -703,4 +711,4 @@ def subtext(text: str, pos: int, hunk_size: int) -> str: left_pos -= right_quota if 0 > left_pos: left_pos = 0 - return text[left_pos:right_pos] + return text[left_pos:right_pos].rstrip() diff --git a/docs/source/credsweeper.filters.rst b/docs/source/credsweeper.filters.rst index e91a76a69..102580f39 100644 --- a/docs/source/credsweeper.filters.rst +++ b/docs/source/credsweeper.filters.rst @@ -36,14 +36,6 @@ credsweeper.filters.line\_specific\_key\_check module :undoc-members: :show-inheritance: -credsweeper.filters.separator\_unusual\_check module ----------------------------------------------------- - -.. automodule:: credsweeper.filters.separator_unusual_check - :members: - :undoc-members: - :show-inheritance: - credsweeper.filters.value\_allowlist\_check module -------------------------------------------------- @@ -260,14 +252,6 @@ credsweeper.filters.value\_last\_word\_check module :undoc-members: :show-inheritance: -credsweeper.filters.value\_length\_check module ------------------------------------------------ - -.. automodule:: credsweeper.filters.value_length_check - :members: - :undoc-members: - :show-inheritance: - credsweeper.filters.value\_method\_check module ----------------------------------------------- @@ -308,14 +292,6 @@ credsweeper.filters.value\_pattern\_check module :undoc-members: :show-inheritance: -credsweeper.filters.value\_pattern\_length\_check module --------------------------------------------------------- - -.. automodule:: credsweeper.filters.value_pattern_length_check - :members: - :undoc-members: - :show-inheritance: - credsweeper.filters.value\_similarity\_check module --------------------------------------------------- diff --git a/tests/__init__.py b/tests/__init__.py index 5bda59e59..a501eae08 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -7,18 +7,18 @@ NEGLIGIBLE_ML_THRESHOLD = 0.0001 # credentials count after scan -SAMPLES_CRED_COUNT: int = 360 -SAMPLES_CRED_LINE_COUNT: int = 377 +SAMPLES_CRED_COUNT: int = 362 +SAMPLES_CRED_LINE_COUNT: int = 379 # credentials count after post-processing -SAMPLES_POST_CRED_COUNT: int = 319 +SAMPLES_POST_CRED_COUNT: int = 321 # with option --doc SAMPLES_IN_DOC = 415 # archived credentials that are not found without --depth SAMPLES_IN_DEEP_1 = SAMPLES_POST_CRED_COUNT + 24 -SAMPLES_IN_DEEP_2 = SAMPLES_IN_DEEP_1 + 16 +SAMPLES_IN_DEEP_2 = SAMPLES_IN_DEEP_1 + 17 SAMPLES_IN_DEEP_3 = SAMPLES_IN_DEEP_2 + 1 # well known string with all latin letters diff --git a/tests/data/depth_3.json b/tests/data/depth_3.json index 4c7822ef9..01cdc64ff 100644 --- a/tests/data/depth_3.json +++ b/tests/data/depth_3.json @@ -7266,17 +7266,17 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.986, + "ml_probability": 0.999, "rule": "Secret", "severity": "medium", "confidence": "moderate", "line_data_list": [ { - "line": "secret_looks_like_linux_path__=\"VnpmUGWxhQW/9KQAwrL2ZYd/DJPNG1PQjEXAMPLE\"", + "line": "secret_looks_like_linux_path__=\"VnpmUGWxhQW/9KQAwrL2ZYd/DJPNG1PQjEXAMbLE\"", "line_num": 5, "path": "tests/samples/key.hs", "info": "tests/samples/key.hs|RAW", - "value": "VnpmUGWxhQW/9KQAwrL2ZYd/DJPNG1PQjEXAMPLE", + "value": "VnpmUGWxhQW/9KQAwrL2ZYd/DJPNG1PQjEXAMbLE", "value_start": 32, "value_end": 72, "variable": "secret_looks_like_linux_path__", @@ -7284,7 +7284,7 @@ "variable_end": 30, "entropy_validation": { "iterator": "BASE64_CHARS", - "entropy": 4.784183719779189, + "entropy": 4.8530559073332755, "valid": true } } @@ -9353,6 +9353,33 @@ } ] }, + { + "api_validation": "NOT_AVAILABLE", + "ml_validation": "VALIDATED_KEY", + "ml_probability": 0.911, + "rule": "Salt", + "severity": "medium", + "confidence": "moderate", + "line_data_list": [ + { + "line": "salt3 = \" 827634876\"; ", + "line_num": 1, + "path": "tests/samples/salt.py", + "info": "tests/samples/salt.py|STRUCT|STRUCT:2|KEYWORD:`salt3 = \" 827634876\"; `", + "value": " 827634876", + "value_start": 9, + "value_end": 19, + "variable": "salt3", + "variable_start": 0, + "variable_end": 5, + "entropy_validation": { + "iterator": "BASE64_CHARS", + "entropy": 2.389735285398626, + "valid": false + } + } + ] + }, { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", @@ -9434,6 +9461,33 @@ } ] }, + { + "api_validation": "NOT_AVAILABLE", + "ml_validation": "VALIDATED_KEY", + "ml_probability": 0.665, + "rule": "Salt", + "severity": "medium", + "confidence": "moderate", + "line_data_list": [ + { + "line": "json_escaped = \"{\\\\\\\"salt8\\\\\\\":\\\\\\\"4b9a6d8b638eb0c6\\\\\\\"}\"", + "line_num": 5, + "path": "tests/samples/salt.py", + "info": "tests/samples/salt.py|RAW", + "value": "4b9a6d8b638eb0c6", + "value_start": 35, + "value_end": 51, + "variable": "salt8", + "variable_start": 21, + "variable_end": 26, + "entropy_validation": { + "iterator": "BASE36_CHARS", + "entropy": 3.2806390622295662, + "valid": true + } + } + ] + }, { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", @@ -10838,6 +10892,33 @@ } ] }, + { + "api_validation": "NOT_AVAILABLE", + "ml_validation": "VALIDATED_KEY", + "ml_probability": 0.989, + "rule": "URL Credentials", + "severity": "high", + "confidence": "moderate", + "line_data_list": [ + { + "line": "email_as_login = \"smtps://example@gmail.com:FnD83JZs@smtp.gmail.com:465\";", + "line_num": 13, + "path": "tests/samples/url_cred.js", + "info": "tests/samples/url_cred.js|RAW", + "value": "FnD83JZs", + "value_start": 44, + "value_end": 52, + "variable": "smtps://", + "variable_start": 18, + "variable_end": 26, + "entropy_validation": { + "iterator": "BASE64_CHARS", + "entropy": 3.0, + "valid": false + } + } + ] + }, { "api_validation": "NOT_AVAILABLE", "ml_validation": "NOT_AVAILABLE", diff --git a/tests/data/ml_threshold.json b/tests/data/ml_threshold.json index c03e55fde..d905cd4b5 100644 --- a/tests/data/ml_threshold.json +++ b/tests/data/ml_threshold.json @@ -7927,17 +7927,17 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.986, + "ml_probability": 0.999, "rule": "Secret", "severity": "medium", "confidence": "moderate", "line_data_list": [ { - "line": "secret_looks_like_linux_path__=\"VnpmUGWxhQW/9KQAwrL2ZYd/DJPNG1PQjEXAMPLE\"", + "line": "secret_looks_like_linux_path__=\"VnpmUGWxhQW/9KQAwrL2ZYd/DJPNG1PQjEXAMbLE\"", "line_num": 5, "path": "tests/samples/key.hs", "info": "", - "value": "VnpmUGWxhQW/9KQAwrL2ZYd/DJPNG1PQjEXAMPLE", + "value": "VnpmUGWxhQW/9KQAwrL2ZYd/DJPNG1PQjEXAMbLE", "value_start": 32, "value_end": 72, "variable": "secret_looks_like_linux_path__", @@ -7945,7 +7945,7 @@ "variable_end": 30, "entropy_validation": { "iterator": "BASE64_CHARS", - "entropy": 4.784183719779189, + "entropy": 4.8530559073332755, "valid": true } } @@ -9198,6 +9198,33 @@ } ] }, + { + "api_validation": "NOT_AVAILABLE", + "ml_validation": "VALIDATED_KEY", + "ml_probability": 0.665, + "rule": "Salt", + "severity": "medium", + "confidence": "moderate", + "line_data_list": [ + { + "line": "json_escaped = \"{\\\\\\\"salt8\\\\\\\":\\\\\\\"4b9a6d8b638eb0c6\\\\\\\"}\"", + "line_num": 5, + "path": "tests/samples/salt.py", + "info": "", + "value": "4b9a6d8b638eb0c6", + "value_start": 35, + "value_end": 51, + "variable": "salt8", + "variable_start": 21, + "variable_end": 26, + "entropy_validation": { + "iterator": "BASE36_CHARS", + "entropy": 3.2806390622295662, + "valid": true + } + } + ] + }, { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", @@ -9927,6 +9954,33 @@ } ] }, + { + "api_validation": "NOT_AVAILABLE", + "ml_validation": "VALIDATED_KEY", + "ml_probability": 0.989, + "rule": "URL Credentials", + "severity": "high", + "confidence": "moderate", + "line_data_list": [ + { + "line": "email_as_login = \"smtps://example@gmail.com:FnD83JZs@smtp.gmail.com:465\";", + "line_num": 13, + "path": "tests/samples/url_cred.js", + "info": "", + "value": "FnD83JZs", + "value_start": 44, + "value_end": 52, + "variable": "smtps://", + "variable_start": 18, + "variable_end": 26, + "entropy_validation": { + "iterator": "BASE64_CHARS", + "entropy": 3.0, + "valid": false + } + } + ] + }, { "api_validation": "NOT_AVAILABLE", "ml_validation": "NOT_AVAILABLE", diff --git a/tests/data/output.json b/tests/data/output.json index cfe4115d4..1d75227d5 100644 --- a/tests/data/output.json +++ b/tests/data/output.json @@ -7009,17 +7009,17 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.986, + "ml_probability": 0.999, "rule": "Secret", "severity": "medium", "confidence": "moderate", "line_data_list": [ { - "line": "secret_looks_like_linux_path__=\"VnpmUGWxhQW/9KQAwrL2ZYd/DJPNG1PQjEXAMPLE\"", + "line": "secret_looks_like_linux_path__=\"VnpmUGWxhQW/9KQAwrL2ZYd/DJPNG1PQjEXAMbLE\"", "line_num": 5, "path": "tests/samples/key.hs", "info": "", - "value": "VnpmUGWxhQW/9KQAwrL2ZYd/DJPNG1PQjEXAMPLE", + "value": "VnpmUGWxhQW/9KQAwrL2ZYd/DJPNG1PQjEXAMbLE", "value_start": 32, "value_end": 72, "variable": "secret_looks_like_linux_path__", @@ -7027,7 +7027,7 @@ "variable_end": 30, "entropy_validation": { "iterator": "BASE64_CHARS", - "entropy": 4.784183719779189, + "entropy": 4.8530559073332755, "valid": true } } @@ -8145,6 +8145,33 @@ } ] }, + { + "api_validation": "NOT_AVAILABLE", + "ml_validation": "VALIDATED_KEY", + "ml_probability": 0.665, + "rule": "Salt", + "severity": "medium", + "confidence": "moderate", + "line_data_list": [ + { + "line": "json_escaped = \"{\\\\\\\"salt8\\\\\\\":\\\\\\\"4b9a6d8b638eb0c6\\\\\\\"}\"", + "line_num": 5, + "path": "tests/samples/salt.py", + "info": "", + "value": "4b9a6d8b638eb0c6", + "value_start": 35, + "value_end": 51, + "variable": "salt8", + "variable_start": 21, + "variable_end": 26, + "entropy_validation": { + "iterator": "BASE36_CHARS", + "entropy": 3.2806390622295662, + "valid": true + } + } + ] + }, { "api_validation": "NOT_AVAILABLE", "ml_validation": "NOT_AVAILABLE", @@ -8847,6 +8874,33 @@ } ] }, + { + "api_validation": "NOT_AVAILABLE", + "ml_validation": "VALIDATED_KEY", + "ml_probability": 0.989, + "rule": "URL Credentials", + "severity": "high", + "confidence": "moderate", + "line_data_list": [ + { + "line": "email_as_login = \"smtps://example@gmail.com:FnD83JZs@smtp.gmail.com:465\";", + "line_num": 13, + "path": "tests/samples/url_cred.js", + "info": "", + "value": "FnD83JZs", + "value_start": 44, + "value_end": 52, + "variable": "smtps://", + "variable_start": 18, + "variable_end": 26, + "entropy_validation": { + "iterator": "BASE64_CHARS", + "entropy": 3.0, + "valid": false + } + } + ] + }, { "api_validation": "NOT_AVAILABLE", "ml_validation": "NOT_AVAILABLE", diff --git a/tests/filters/test_line_specific_key_check.py b/tests/filters/test_line_specific_key_check.py index ca123d839..c55a3cb3f 100644 --- a/tests/filters/test_line_specific_key_check.py +++ b/tests/filters/test_line_specific_key_check.py @@ -18,9 +18,11 @@ def test_line_specific_key_check_p(self, file_path: pytest.fixture, line: str) - @pytest.mark.parametrize("line", [ '"AwsAccessKey": enc("AKIAGIREOGIAWSKEY123"),', - '"AwsAccessKey": "AKIAGIREXAMPLEKEY123"', + '"AwsAccessKey as example": "AKIAGIREXAMPLEKEY123"', ]) def test_line_specific_key_check_n(self, file_path: pytest.fixture, line: str) -> None: cred_candidate = get_line_data(file_path, line=line, pattern=LINE_VALUE_PATTERN) + # LINE_VALUE_PATTERN does not detect a value position + cred_candidate.value_start = line.find("AKIA") target = AnalysisTarget(line_pos=0, lines=[line], line_nums=[1], descriptor=DUMMY_DESCRIPTOR) assert LineSpecificKeyCheck().run(cred_candidate, target) is True diff --git a/tests/filters/test_value_json_web_token_check.py b/tests/filters/test_value_json_web_token_check.py index 9aa85a752..4cb701956 100644 --- a/tests/filters/test_value_json_web_token_check.py +++ b/tests/filters/test_value_json_web_token_check.py @@ -20,6 +20,9 @@ def test_value_jwt_check_p(self): self.assertTrue(ValueJsonWebTokenCheck().run( get_line_data(line="eyJhbGciOiJSUzI1NiJ9Cg.eyJleHAiOjY1NTM2fQo.AAAAAAAAAAAAAAAAAAAAAAA", pattern=LINE_VALUE_PATTERN), DUMMY_ANALYSIS_TARGET)) + self.assertTrue(ValueJsonWebTokenCheck().run( + get_line_data(line="eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.x3.GFsFyGiCUIP5VHI9CEJL9thWsGjSZf1fJfarNk-LGTM", + pattern=LINE_VALUE_PATTERN), DUMMY_ANALYSIS_TARGET)) def test_value_jwt_check_n(self): self.assertFalse(ValueJsonWebTokenCheck().run( @@ -33,3 +36,8 @@ def test_value_jwt_check_n(self): "Ui1o9ndy7ckISHQVhuYFKu78l7nqC4heghK_Gw4h7EB7s8eEuUC-D6JjVtX10IyS" \ "vCRkRo7f8dWQTjFLs7mlPowjRz0cP5J-MmCoegKHYagOHZ_ArXOR91_u8jMdwmOf", pattern=LINE_VALUE_PATTERN), DUMMY_ANALYSIS_TARGET)) + self.assertFalse(ValueJsonWebTokenCheck().run( + get_line_data(line="eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9." \ + "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9." \ + ".e30.GFsFyGiCUIP5VHI9CEJL9thWsGjSZf1fJfarNk-LGTM", + pattern=LINE_VALUE_PATTERN), DUMMY_ANALYSIS_TARGET)) diff --git a/tests/filters/test_value_length_check.py b/tests/filters/test_value_length_check.py deleted file mode 100644 index 69bd5f809..000000000 --- a/tests/filters/test_value_length_check.py +++ /dev/null @@ -1,19 +0,0 @@ -import pytest - -from credsweeper.config import Config -from credsweeper.filters import ValueLengthCheck -from tests.filters.conftest import LINE_VALUE_PATTERN, DUMMY_ANALYSIS_TARGET -from tests.test_utils.dummy_line_data import get_line_data - - -class TestValueLengthCheck: - - def test_value_length_check_p(self, file_path: pytest.fixture, config: Config, - success_line: pytest.fixture) -> None: - line_data = get_line_data(file_path, line=success_line, pattern=LINE_VALUE_PATTERN) - assert ValueLengthCheck(config).run(line_data, DUMMY_ANALYSIS_TARGET) is False - - @pytest.mark.parametrize("line", ["Cra"]) - def test_value_length_check_n(self, file_path: pytest.fixture, config: Config, line: str) -> None: - line_data = get_line_data(file_path, line=line, pattern=LINE_VALUE_PATTERN) - assert ValueLengthCheck(config).run(line_data, DUMMY_ANALYSIS_TARGET) is True diff --git a/tests/filters/test_value_useless_word_check.py b/tests/filters/test_value_useless_word_check.py index 2cbdcf192..911f3781f 100644 --- a/tests/filters/test_value_useless_word_check.py +++ b/tests/filters/test_value_useless_word_check.py @@ -11,7 +11,7 @@ def test_value_useless_word_check_p(self, file_path: pytest.fixture, success_lin line_data = get_line_data(file_path=file_path, line=success_line, pattern=LINE_VALUE_PATTERN) assert ValueUselessWordCheck().run(line_data, DUMMY_ANALYSIS_TARGET) is False - @pytest.mark.parametrize("line", ["{0x943058439}", "0x%", "->gi_reo_gi", "xxxxxGIREOGI", " GIREOGI"]) + @pytest.mark.parametrize("line", ["{0x943058439}", "0x%", "->gi_reo_gi", "GIREOGIEXAMPLE"]) def test_value_useless_word_check_n(self, file_path: pytest.fixture, line: str) -> None: line_data = get_line_data(file_path=file_path, line=line, pattern=LINE_VALUE_PATTERN) assert ValueUselessWordCheck().run(line_data, DUMMY_ANALYSIS_TARGET) is True diff --git a/tests/ml_model/test_ml_validator.py b/tests/ml_model/test_ml_validator.py index 9322eae63..ee7083ae9 100644 --- a/tests/ml_model/test_ml_validator.py +++ b/tests/ml_model/test_ml_validator.py @@ -10,7 +10,7 @@ from credsweeper.credentials import Candidate, CandidateKey from credsweeper.ml_model import MlValidator from credsweeper.utils import Util -from tests import AZ_STRING, NEGLIGIBLE_ML_THRESHOLD +from tests import NEGLIGIBLE_ML_THRESHOLD class TestMlValidator(unittest.TestCase): @@ -48,22 +48,22 @@ def validate(_candidate: Candidate) -> Tuple[bool, float]: candidate.line_data_list[0].value = "Ahga%$FiQ@Ei8" decision, probability = validate(candidate) - self.assertAlmostEqual(probability, 0.9997520446777344, delta=NEGLIGIBLE_ML_THRESHOLD) + self.assertAlmostEqual(0.9997520446777344, probability, delta=NEGLIGIBLE_ML_THRESHOLD) candidate.line_data_list[0].path = "sample.py" candidate.line_data_list[0].file_type = ".yaml" decision, probability = validate(candidate) - self.assertAlmostEqual(probability, 0.9994515776634216, delta=NEGLIGIBLE_ML_THRESHOLD) + self.assertAlmostEqual(0.9994515776634216, probability, delta=NEGLIGIBLE_ML_THRESHOLD) candidate.line_data_list[0].path = "test.zip" candidate.line_data_list[0].file_type = ".zip" decision, probability = validate(candidate) - self.assertAlmostEqual(probability, 0.9994281530380249, delta=NEGLIGIBLE_ML_THRESHOLD) + self.assertAlmostEqual(0.9994281530380249, probability, delta=NEGLIGIBLE_ML_THRESHOLD) candidate.line_data_list[0].path = "other.txt" candidate.line_data_list[0].file_type = ".txt" decision, probability = validate(candidate) - self.assertAlmostEqual(probability, 0.9980608820915222, delta=NEGLIGIBLE_ML_THRESHOLD) + self.assertAlmostEqual(0.9980608820915222, probability, delta=NEGLIGIBLE_ML_THRESHOLD) def test_extract_features_p(self): candidate1 = Candidate.get_dummy_candidate(self.config, "main.py", ".py", "info") @@ -74,10 +74,10 @@ def test_extract_features_p(self): candidate1.line_data_list[0].value = "123" candidate1.rule_name = "Password" features1 = self.ml_validator.extract_features([candidate1]) - self.assertEqual(18, np.count_nonzero(features1)) + self.assertAlmostEqual(18, np.count_nonzero(features1), delta=NEGLIGIBLE_ML_THRESHOLD) candidate2 = copy.deepcopy(candidate1) features2 = self.ml_validator.extract_features([candidate1, candidate2]) - self.assertEqual(18, np.count_nonzero(features2)) + self.assertAlmostEqual(18, np.count_nonzero(features2), delta=NEGLIGIBLE_ML_THRESHOLD) candidate2.rule_name = "Secret" features3 = self.ml_validator.extract_features([candidate1, candidate2]) - self.assertEqual(19, np.count_nonzero(features3)) + self.assertAlmostEqual(19, np.count_nonzero(features3), delta=NEGLIGIBLE_ML_THRESHOLD) diff --git a/tests/samples/aws_client_id b/tests/samples/aws_client_id index 3685378f5..b5a83275c 100644 --- a/tests/samples/aws_client_id +++ b/tests/samples/aws_client_id @@ -1,2 +1,4 @@ The items are AKIAGIREOGIAWSKEY123,AKIAGIREOGIAWSKEY45X the coma is necessary there ^ bariers thesting !!! +must be filtered: AKIAGIREOGIAEXAMPLE7 +filtered too: AKIALGSBKLIKEAREAL12 --access-key diff --git a/tests/samples/key.hs b/tests/samples/key.hs index 5a13fab50..3d197e623 100644 --- a/tests/samples/key.hs +++ b/tests/samples/key.hs @@ -2,6 +2,6 @@ prKeyValid=LS0tLS1CRUdJTiBQUklWQVRFIENDcUdTTTQ5QXdFSEJHMHdhd0lCQVFRZ0ViVnpmUGWxh secret_looks_like_linux_path_1="/VnpmUGWxhQW9KQAwrL2ZYdDJPNG1PQjYxMXNPaF" secret_looks_like_linux_path_2="VnpmUGWxhQW/9KQAwrL2ZYd/DJPNG1PQjYxMXNPF" secret_looks_like_linux_path_3="VnpmUGWxhQW/9KQAwrL2ZYdDJPNG1PQjYxMXNPF=" -secret_looks_like_linux_path__="VnpmUGWxhQW/9KQAwrL2ZYd/DJPNG1PQjEXAMPLE" +secret_looks_like_linux_path__="VnpmUGWxhQW/9KQAwrL2ZYd/DJPNG1PQjEXAMbLE" "https://example.com/api/js?key=dhd0lCQVFRZ0ViVnpmUGWxhQW9KQWwrLzZYdDJPNG1PQjYxMXNPaFJB&bug=true" diff --git a/tests/samples/salt.py b/tests/samples/salt.py index 4140c4e5a..60b2fcd07 100644 --- a/tests/samples/salt.py +++ b/tests/samples/salt.py @@ -2,3 +2,4 @@ salt2 = r"""\0x12\0x3s""" salt3 = u"\u0020827634876" salt4 = {"salt5": "my124%#$@s\x04clt\0"} +json_escaped = "{\\\"salt8\\\":\\\"4b9a6d8b638eb0c6\\\"}" diff --git a/tests/samples/url_cred.js b/tests/samples/url_cred.js index 4b21bc971..d555f7233 100644 --- a/tests/samples/url_cred.js +++ b/tests/samples/url_cred.js @@ -9,3 +9,5 @@ url = "https://secure.com/83675/39084?Credential=546DFS64N90P3AW7DX%2Fkeep%26cut /* partially line to sanitize url-like items 39084?Credential=546DFS64N90P3AW7DX&key=3487263-2384579834-234732875-345&hasToBefound=2 */ + +email_as_login = "smtps://example@gmail.com:FnD83JZs@smtp.gmail.com:465"; diff --git a/tests/test_main.py b/tests/test_main.py index d7066debf..6c774c4f9 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -749,7 +749,7 @@ def prepare(report: List[Dict[str, Any]]): tmp_file = Path(tmp_dir) / cfg["json_filename"] # apply the current path to keep equivalence in path os.chdir(TESTS_PATH.parent) - content_provider: AbstractProvider = FilesProvider(["tests/samples"]) + content_provider: AbstractProvider = FilesProvider([Path("tests") / "samples"]) # replace output report file to place in tmp_dir cfg["json_filename"] = str(tmp_file) cred_sweeper = CredSweeper(**cfg) diff --git a/tests/utils/test_util.py b/tests/utils/test_util.py index 2d67cccde..b2cf2cca0 100644 --- a/tests/utils/test_util.py +++ b/tests/utils/test_util.py @@ -599,13 +599,19 @@ def test_get_chunks_coverage_p(self): def test_subtext_n(self): self.assertEqual("", Util.subtext("", 0, 0)) + self.assertEqual("", Util.subtext(' ' * 42, 0, 0)) def test_subtext_p(self): - # self.assertEqual(AZ_STRING, Util.subtext(AZ_STRING, 37, 40)) - self.assertEqual("The quick ", Util.subtext(AZ_STRING, 0, 5)) - self.assertEqual("The quick ", Util.subtext(AZ_STRING, 3, 5)) - self.assertEqual(" fox jumps", Util.subtext(AZ_STRING, 20, 5)) + self.assertEqual("var=value0123456789;", Util.subtext(" var=value0123456789; ", 21, 10)) + self.assertEqual(AZ_STRING, Util.subtext(AZ_STRING, len(AZ_STRING) >> 1, 1 + len(AZ_STRING) >> 1)) + self.assertEqual("x jump", Util.subtext(AZ_STRING, len(AZ_STRING) >> 1, 3)) + self.assertEqual("ox jumps", Util.subtext(AZ_STRING, len(AZ_STRING) >> 1, 4)) + self.assertEqual("fox jumps", Util.subtext(AZ_STRING, len(AZ_STRING) >> 1, 5)) + self.assertEqual("fox jumps ov", Util.subtext(AZ_STRING, len(AZ_STRING) >> 1, 6)) + self.assertEqual("The quick", Util.subtext(AZ_STRING, 0, 5)) + self.assertEqual("The quick", Util.subtext(AZ_STRING, 3, 5)) + self.assertEqual("fox jumps", Util.subtext(AZ_STRING, AZ_STRING.find("jumps"), 5)) self.assertEqual("e lazy dog", Util.subtext(AZ_STRING, len(AZ_STRING) - 2, 5)) self.assertEqual("the lazy dog", Util.subtext(AZ_STRING, len(AZ_STRING) - 2, 6)) - self.assertEqual(AZ_STRING[:40], Util.subtext(AZ_STRING, 15, 20)) + self.assertEqual(AZ_STRING[:39], Util.subtext(AZ_STRING, 15, 20)) self.assertEqual(AZ_STRING[-40:], Util.subtext(AZ_STRING, 33, 20)) From 66c5bb9113cf6c3b7c98c8e9cf9e849ec855a607 Mon Sep 17 00:00:00 2001 From: Roman Babenko Date: Fri, 9 Aug 2024 11:26:02 +0300 Subject: [PATCH 6/7] UUID pattern (#577) * UUID pattern added * BM scores fix --- .github/workflows/benchmark.yml | 9 +- cicd/benchmark.txt | 150 ++++++++++++++-------------- credsweeper/rules/config.yaml | 17 ++++ tests/__init__.py | 10 +- tests/data/depth_3.json | 27 +++++ tests/data/doc.json | 27 +++++ tests/data/ml_threshold.json | 27 +++++ tests/data/output.json | 27 +++++ tests/ml_model/test_ml_validator.py | 26 +++++ tests/samples/uuid | 2 + tests/test_main.py | 5 +- 11 files changed, 240 insertions(+), 87 deletions(-) create mode 100644 tests/samples/uuid diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index 8b418ecf1..90dfa8f8d 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -23,7 +23,7 @@ jobs: uses: actions/checkout@v4 with: repository: babenek/CredData - ref: auxiliary + ref: uuid - name: Markup hashing run: | @@ -74,7 +74,7 @@ jobs: uses: actions/checkout@v4 with: repository: babenek/CredData - ref: auxiliary + ref: uuid - name: Markup hashing run: | @@ -172,7 +172,7 @@ jobs: uses: actions/checkout@v4 with: repository: babenek/CredData - ref: auxiliary + ref: uuid - name: Markup hashing run: | @@ -342,6 +342,7 @@ jobs: exit ${exit_code} # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # + experiment: # the ml train test is placed here to use cached data set needs: [ download_data ] @@ -354,7 +355,7 @@ jobs: uses: actions/checkout@v4 with: repository: babenek/CredData - ref: auxiliary + ref: uuid - name: Markup hashing run: | diff --git a/cicd/benchmark.txt b/cicd/benchmark.txt index 1626253d5..72246106f 100644 --- a/cicd/benchmark.txt +++ b/cicd/benchmark.txt @@ -1,22 +1,22 @@ -DATA: 16978521 interested lines. MARKUP: 61855 items +DATA: 16348035 interested lines. MARKUP: 62567 items FileType FileNumber ValidLines Positives Negatives Templates --------------- ------------ ------------ ----------- ----------- ----------- - 194 28318 64 429 87 + 194 28318 66 427 87 .1 2 641 2 5 .admx 1 26 1 -.adoc 1 158 11 6 1 +.adoc 1 158 13 6 1 .api 2 118 4 -.asciidoc 96 14471 53 348 27 +.asciidoc 96 14471 51 348 27 .axaml 5 286 5 -.backup 1 62 1 1 +.backup 1 62 2 1 .bash 2 2158 2 1 -.bat 4 233 1 13 2 +.bat 4 233 14 2 .bats 15 2804 14 50 9 .bazel 3 424 8 .build 2 40 3 .bundle 4 1512 570 .bzl 3 2503 11 -.c 179 284009 9 942 5 +.c 179 284009 9 943 5 .cc 29 30562 622 1 .cf 3 126 2 1 .cfg 1 385 1 1 @@ -27,13 +27,13 @@ FileType FileNumber ValidLines Positives Negatives Templat .cmd 4 401 2 3 .cnf 8 858 18 45 18 .coffee 1 585 2 -.conf 60 4945 54 69 54 +.conf 60 4945 54 71 53 .config 20 492 16 33 1 -.cpp 15 5688 1 61 +.cpp 15 5688 2 61 .creds 1 10 1 1 .crlf 1 27 1 .crt 2 4979 253 -.cs 268 82410 120 908 99 +.cs 268 82410 158 907 94 .cshtml 5 180 12 .csp 3 379 11 .csproj 1 14 1 @@ -52,44 +52,44 @@ FileType FileNumber ValidLines Positives Negatives Templat .env 10 136 11 3 17 .erb 13 323 27 .erl 4 96 8 -.ex 25 4968 3 105 5 -.example 17 1838 74 38 54 -.exs 24 4842 3 190 4 +.ex 25 4968 5 105 5 +.example 17 1838 75 38 54 +.exs 24 4842 8 190 4 .ext 5 211 1 4 2 -.fsproj 1 75 1 +.fsproj 1 75 1 1 .g4 2 201 2 .gd 1 37 1 .gml 3 3075 26 .gni 3 5017 18 -.go 1079 566327 623 4329 742 +.go 1080 566476 673 4319 741 .golden 5 1168 1 14 29 .gradle 45 3265 4 91 100 .graphql 7 420 13 .graphqls 1 30 1 -.groovy 22 4986 20 215 1 -.h 11 2038 38 +.groovy 22 4986 24 215 1 +.h 9 1958 36 .haml 9 191 16 .hbs 2 54 3 .hs 14 4140 31 72 5 -.html 53 15327 14 115 18 -.idl 2 777 4 +.html 53 15327 22 115 18 +.idl 2 777 1 4 .iml 6 699 36 .in 6 2130 3 78 12 .inc 2 56 2 1 -.ini 11 1437 24 12 18 +.ini 11 1437 25 12 18 .ipynb 1 134 5 .j 1 241 2 2 .j2 30 5530 6 213 10 -.java 621 134132 314 1361 170 -.jenkinsfile 1 58 1 7 +.java 621 134132 359 1360 170 +.jenkinsfile 1 58 2 7 .jinja2 1 64 2 -.js 659 536413 526 2638 336 -.json 860 13670669 624 10946 140 +.js 659 536413 536 2635 330 +.json 850 13046270 1074 10778 140 .jsp 13 3202 1 42 .jsx 7 857 19 .jwt 1 1 2 .key 83 2737 70 14 -.kt 123 20774 51 384 3 +.kt 123 20774 67 384 3 .l 1 982 1 .las 1 6656 46 .lasso 1 230 6 @@ -106,33 +106,33 @@ FileType FileNumber ValidLines Positives Negatives Templat .log 2 199 38 52 .lua 10 1924 37 3 .m 16 13358 11 152 3 -.manifest 3 102 3 +.manifest 3 102 9 3 .markdown 3 139 3 1 .markerb 3 12 3 .marko 1 21 2 -.md 673 149294 658 2362 664 +.md 674 149399 722 2365 662 .mdx 3 549 7 .mjml 1 18 1 -.mjs 22 4424 50 343 +.mjs 22 4424 78 343 .mk 1 5878 16 .ml 1 1856 24 .mlir 2 1596 19 .mod 2 96 4 .moo 1 1404 26 .mqh 1 1023 2 -.msg 1 26644 1 +.msg 1 26644 1 1 .mysql 1 36 2 -.ndjson 2 5006 37 266 2 +.ndjson 2 5006 70 266 2 .nix 4 211 12 .nolint 1 2 1 .odd 1 1281 57 .oracle 1 9 1 .p8 4 64 4 .pan 2 48 4 -.patch 4 109405 27 +.patch 4 109405 4 27 .pbxproj 1 941 1 .pem 48 1169 47 8 -.php 371 75710 130 1769 80 +.php 371 75710 131 1769 80 .pl 16 14727 6 47 .pm 3 744 8 .po 3 2994 15 @@ -143,20 +143,20 @@ FileType FileNumber ValidLines Positives Negatives Templat .ppk 1 45 37 .private 1 15 1 .proj 1 85 3 -.properties 48 1621 51 28 34 -.proto 5 5768 58 -.ps1 16 8509 68 2 +.properties 48 1621 53 28 33 +.proto 5 5768 2 58 +.ps1 16 8509 15 67 2 .ps1xml 1 5022 1 .pug 2 193 2 .purs 1 69 4 .pxd 1 150 5 2 -.py 890 291553 627 3460 744 +.py 890 291553 685 3456 729 .pyi 4 1361 9 .pyp 1 167 1 .pyx 2 1094 21 .r 4 62 6 3 1 .rake 2 51 2 -.rb 861 131867 237 3458 615 +.rb 860 131838 259 3451 612 .re 1 31 1 .red 1 159 1 .release 1 13 4 @@ -169,100 +169,98 @@ FileType FileNumber ValidLines Positives Negatives Templat .rs 31 9855 2 238 11 .rsc 1 691 1 .rsp 16 7101 19 10 28 -.rst 86 33980 61 358 68 +.rst 86 33980 69 358 68 .rules 1 6 2 -.sample 2 25 1 5 4 +.sample 2 25 3 4 4 .sbt 3 570 6 2 -.scala 40 5071 13 102 +.scala 40 5071 22 102 .scss 16 8553 32 1 .secrets 1 11 1 -.sh 143 21525 50 474 30 +.sh 143 21525 51 474 30 .slim 1 153 2 2 -.sln 1 306 2 .smali 1 775 12 -.snap 3 1708 1 34 2 +.snap 3 1708 9 30 2 .spec 2 332 2 .spin 1 565 1 -.sql 27 6606 29 62 4 +.sql 27 6606 126 62 4 .storyboard 20 1802 401 .strings 20 1240 184 .stub 3 84 6 .sublime-keymap 1 3 1 .sum 37 22854 283 .svg 1 638 12 -.swift 6 278 16 .t 9 1767 28 56 14 .td 2 14002 6 .template 19 1633 4 42 11 .test 2 24 25 4 -.testsettings 1 21 5 +.testsettings 1 21 1 5 .tf 21 1377 3 32 2 -.tfstate 4 307 18 11 4 +.tfstate 4 307 22 11 4 .tfvars 1 31 3 3 .tl 2 2161 165 2 .tmpl 5 336 3 9 .token 1 1 3 -.toml 83 2379 54 72 172 +.toml 83 2379 54 73 172 .tpl 1 43 1 .travis 1 34 4 3 1 .ts 583 106730 158 1935 203 .tsx 54 7914 1 124 5 -.ttar 2 6050 3 -.txt 443 78152 1775 14282 50 +.ttar 1 452 1 +.txt 440 78102 1861 14251 50 .utf8 1 77 2 .vsixmanifest 1 36 1 -.vsmdi 1 6 1 +.vsmdi 1 6 2 .vue 50 8736 1 183 1 -.xaml 21 8103 174 +.xaml 21 8103 175 .xcscheme 1 109 6 .xib 11 503 174 .xml 9 689 9 .xsl 1 311 1 -.yaml 137 19004 116 356 44 -.yml 418 36162 460 916 384 +.yaml 137 19004 128 356 44 +.yml 418 36162 515 910 384 .zsh 6 872 12 .zsh-theme 1 97 1 -TOTAL: 10281 16978521 7546 59932 5216 -credsweeper result_cnt : 6585, lost_cnt : 0, true_cnt : 6367, false_cnt : 218 +TOTAL: 10259 16348035 8706 59679 5182 +credsweeper result_cnt : 7664, lost_cnt : 0, true_cnt : 7472, false_cnt : 192 Rules Positives Negatives Templates Reported TP FP TN FN FPR FNR ACC PRC RCL F1 ------------------------------ ----------- ----------- ----------- ---------- ---- ---- ----- ---- -------- -------- -------- -------- -------- -------- -API 124 3162 185 112 109 3 3344 15 0.000896 0.120968 0.994814 0.973214 0.879032 0.923729 -AWS Client ID 168 13 0 160 160 0 13 8 0.000000 0.047619 0.955801 1.000000 0.952381 0.975610 -AWS Multi 75 12 0 87 75 11 1 0 0.916667 0.000000 0.873563 0.872093 1.000000 0.931677 -AWS S3 Bucket 61 25 0 92 61 25 0 0 1.000000 0.000000 0.709302 0.709302 1.000000 0.829932 -Atlassian Old PAT token 27 212 3 12 3 8 207 24 0.037209 0.888889 0.867769 0.272727 0.111111 0.157895 -Auth 408 2727 77 372 351 21 2783 57 0.007489 0.139706 0.975716 0.943548 0.860294 0.900000 +API 131 3126 185 111 109 2 3309 22 0.000604 0.167939 0.993027 0.981982 0.832061 0.900826 +AWS Client ID 167 18 0 160 160 0 18 7 0.000000 0.041916 0.962162 1.000000 0.958084 0.978593 +AWS Multi 75 14 0 87 75 11 3 0 0.785714 0.000000 0.876404 0.872093 1.000000 0.931677 +AWS S3 Bucket 66 24 0 92 66 24 0 0 1.000000 0.000000 0.733333 0.733333 1.000000 0.846154 +Atlassian Old PAT token 27 208 3 12 3 8 203 24 0.037915 0.888889 0.865546 0.272727 0.111111 0.157895 +Auth 412 2723 76 371 353 18 2781 59 0.006431 0.143204 0.976020 0.951482 0.856796 0.901660 Azure Access Token 19 0 0 12 12 0 0 7 0.368421 0.631579 1.000000 0.631579 0.774194 BASE64 Private Key 7 2 0 7 7 0 2 0 0.000000 0.000000 1.000000 1.000000 1.000000 1.000000 BASE64 encoded PEM Private Key 7 0 0 5 5 0 0 2 0.285714 0.714286 1.000000 0.714286 0.833333 -Bitbucket Client ID 142 1813 9 46 27 18 1804 115 0.009879 0.809859 0.932281 0.600000 0.190141 0.288770 -Bitbucket Client Secret 230 535 10 44 33 11 534 197 0.020183 0.856522 0.731613 0.750000 0.143478 0.240876 -Certificate 25 459 1 21 20 1 459 5 0.002174 0.200000 0.987629 0.952381 0.800000 0.869565 -Credential 91 155 74 90 87 3 226 4 0.013100 0.043956 0.978125 0.966667 0.956044 0.961326 +Bitbucket Client ID 142 1807 9 46 27 18 1798 115 0.009912 0.809859 0.932074 0.600000 0.190141 0.288770 +Bitbucket Client Secret 230 527 10 44 33 11 526 197 0.020484 0.856522 0.728814 0.750000 0.143478 0.240876 +Certificate 25 460 1 21 20 1 460 5 0.002169 0.200000 0.987654 0.952381 0.800000 0.869565 +Credential 94 154 74 90 90 0 228 4 0.000000 0.042553 0.987578 1.000000 0.957447 0.978261 Docker Swarm Token 2 0 0 2 2 0 0 0 0.000000 1.000000 1.000000 1.000000 1.000000 Dropbox App secret 62 114 0 46 36 9 105 26 0.078947 0.419355 0.801136 0.800000 0.580645 0.672897 Facebook Access Token 0 1 0 0 0 1 0 0.000000 1.000000 Firebase Domain 6 1 0 7 6 1 0 0 1.000000 0.000000 0.857143 0.857143 1.000000 0.923077 Github Old Token 1 0 0 1 1 0 0 0 0.000000 1.000000 1.000000 1.000000 1.000000 -Gitlab Feed Token 188 460 88 60 47 12 536 141 0.021898 0.750000 0.792120 0.796610 0.250000 0.380567 +Gitlab Feed Token 188 451 87 60 47 12 526 141 0.022305 0.750000 0.789256 0.796610 0.250000 0.380567 Gitlab Incoming Email Token 37 3 0 21 19 2 1 18 0.666667 0.486486 0.500000 0.904762 0.513514 0.655172 Google API Key 12 0 0 12 12 0 0 0 0.000000 1.000000 1.000000 1.000000 1.000000 Google Multi 10 2 0 11 10 1 1 0 0.500000 0.000000 0.916667 0.909091 1.000000 0.952381 Google OAuth Access Token 3 0 0 3 3 0 0 0 0.000000 1.000000 1.000000 1.000000 1.000000 Grafana Provisioned API Key 22 1 0 5 5 0 1 17 0.000000 0.772727 0.260870 1.000000 0.227273 0.370370 -JSON Web Token 169 61 0 130 130 0 61 39 0.000000 0.230769 0.830435 1.000000 0.769231 0.869565 +JSON Web Token 170 61 0 131 131 0 61 39 0.000000 0.229412 0.831169 1.000000 0.770588 0.870432 Jira / Confluence PAT token 0 4 0 0 0 4 0 0.000000 1.000000 Jira 2FA 14 6 0 10 10 0 6 4 0.000000 0.285714 0.800000 1.000000 0.714286 0.833333 -Key 497 8483 464 448 439 9 8938 58 0.001006 0.116700 0.992906 0.979911 0.883300 0.929101 -Nonce 90 47 0 84 83 1 46 7 0.021277 0.077778 0.941606 0.988095 0.922222 0.954023 -Other 0 0 5 0 0 5 0 0.000000 1.000000 +Key 522 8453 464 452 447 5 8912 75 0.000561 0.143678 0.991525 0.988938 0.856322 0.917864 +Nonce 91 47 0 84 83 1 46 8 0.021277 0.087912 0.934783 0.988095 0.912088 0.948571 PEM Private Key 1019 1483 0 1023 1019 4 1479 0 0.002697 0.000000 0.998401 0.996090 1.000000 0.998041 -Password 1834 7473 2741 1691 1627 64 10150 207 0.006266 0.112868 0.977507 0.962153 0.887132 0.923121 +Password 1841 7468 2724 1691 1637 54 10138 204 0.005298 0.110809 0.978559 0.968066 0.889191 0.926954 Salt 45 73 2 39 39 0 75 6 0.000000 0.133333 0.950000 1.000000 0.866667 0.928571 -Secret 1362 28492 868 1236 1231 5 29355 131 0.000170 0.096182 0.995573 0.995955 0.903818 0.947652 +Secret 1365 28359 868 1237 1233 4 29223 132 0.000137 0.096703 0.995554 0.996766 0.903297 0.947733 Seed 1 6 0 0 0 6 1 0.000000 1.000000 0.857143 0.000000 Slack Token 4 1 0 4 4 0 1 0 0.000000 0.000000 1.000000 1.000000 1.000000 1.000000 -Token 586 3974 438 513 504 9 4403 82 0.002040 0.139932 0.981793 0.982456 0.860068 0.917197 +Token 612 3949 437 516 511 5 4381 101 0.001140 0.165033 0.978792 0.990310 0.834967 0.906028 Twilio API Key 0 5 2 0 0 7 0 0.000000 1.000000 -URL Credentials 198 127 249 190 190 0 376 8 0.000000 0.040404 0.986063 1.000000 0.959596 0.979381 - 7546 59932 5216 6596 6367 218 59714 1179 0.003637 0.156242 0.979297 0.966894 0.843758 0.901139 +URL Credentials 209 127 240 200 200 0 367 9 0.000000 0.043062 0.984375 1.000000 0.956938 0.977995 +UUID 1068 1 0 1058 1057 1 0 11 1.000000 0.010300 0.988775 0.999055 0.989700 0.994356 + 8706 59679 5182 7671 7472 192 59487 1234 0.003217 0.141741 0.979147 0.974948 0.858259 0.912889 diff --git a/credsweeper/rules/config.yaml b/credsweeper/rules/config.yaml index afe66d028..29c266f8e 100644 --- a/credsweeper/rules/config.yaml +++ b/credsweeper/rules/config.yaml @@ -126,6 +126,23 @@ target: - code +- name: UUID + severity: info + confidence: strong + type: pattern + values: + - (?[0-9A-Fa-f]{8}(-[0-9A-Fa-f]{4}){3}-[0-9A-Fa-f]{12})(?![=0-9A-Za-z_+-]) + min_line_len: 36 + required_substrings: + - "-" + required_regex: "[0-9A-Za-z_/+-]{15}" + filter_type: + - ValuePatternCheck + use_ml: false + target: + - code + - doc + - name: AWS Client ID severity: high confidence: moderate diff --git a/tests/__init__.py b/tests/__init__.py index a501eae08..85a275175 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -1,20 +1,20 @@ from pathlib import Path # total number of files in test samples -SAMPLES_FILES_COUNT: int = 129 +SAMPLES_FILES_COUNT: int = 130 # the lowest value of ML threshold is used to display possible lowest values NEGLIGIBLE_ML_THRESHOLD = 0.0001 # credentials count after scan -SAMPLES_CRED_COUNT: int = 362 -SAMPLES_CRED_LINE_COUNT: int = 379 +SAMPLES_CRED_COUNT: int = 363 +SAMPLES_CRED_LINE_COUNT: int = 380 # credentials count after post-processing -SAMPLES_POST_CRED_COUNT: int = 321 +SAMPLES_POST_CRED_COUNT: int = 322 # with option --doc -SAMPLES_IN_DOC = 415 +SAMPLES_IN_DOC = 416 # archived credentials that are not found without --depth SAMPLES_IN_DEEP_1 = SAMPLES_POST_CRED_COUNT + 24 diff --git a/tests/data/depth_3.json b/tests/data/depth_3.json index 01cdc64ff..8fd52ab2c 100644 --- a/tests/data/depth_3.json +++ b/tests/data/depth_3.json @@ -10919,6 +10919,33 @@ } ] }, + { + "api_validation": "NOT_AVAILABLE", + "ml_validation": "NOT_AVAILABLE", + "ml_probability": null, + "rule": "UUID", + "severity": "info", + "confidence": "strong", + "line_data_list": [ + { + "line": "bace4d19-fa7e-beef-cafe-9129474bcd81 # tp", + "line_num": 1, + "path": "tests/samples/uuid", + "info": "tests/samples/uuid|RAW", + "value": "bace4d19-fa7e-beef-cafe-9129474bcd81", + "value_start": 0, + "value_end": 36, + "variable": null, + "variable_start": -2, + "variable_end": -2, + "entropy_validation": { + "iterator": "BASE36_CHARS", + "entropy": 3.2373263071270246, + "valid": true + } + } + ] + }, { "api_validation": "NOT_AVAILABLE", "ml_validation": "NOT_AVAILABLE", diff --git a/tests/data/doc.json b/tests/data/doc.json index 46c8cdb7d..0311797b5 100644 --- a/tests/data/doc.json +++ b/tests/data/doc.json @@ -13064,6 +13064,33 @@ } ] }, + { + "api_validation": "NOT_AVAILABLE", + "ml_validation": "NOT_AVAILABLE", + "ml_probability": null, + "rule": "UUID", + "severity": "info", + "confidence": "strong", + "line_data_list": [ + { + "line": "bace4d19-fa7e-beef-cafe-9129474bcd81 # tp", + "line_num": 1, + "path": "tests/samples/uuid", + "info": "tests/samples/uuid|RAW", + "value": "bace4d19-fa7e-beef-cafe-9129474bcd81", + "value_start": 0, + "value_end": 36, + "variable": null, + "variable_start": -2, + "variable_end": -2, + "entropy_validation": { + "iterator": "BASE36_CHARS", + "entropy": 3.2373263071270246, + "valid": true + } + } + ] + }, { "api_validation": "NOT_AVAILABLE", "ml_validation": "NOT_AVAILABLE", diff --git a/tests/data/ml_threshold.json b/tests/data/ml_threshold.json index d905cd4b5..01c1a0929 100644 --- a/tests/data/ml_threshold.json +++ b/tests/data/ml_threshold.json @@ -9981,6 +9981,33 @@ } ] }, + { + "api_validation": "NOT_AVAILABLE", + "ml_validation": "NOT_AVAILABLE", + "ml_probability": null, + "rule": "UUID", + "severity": "info", + "confidence": "strong", + "line_data_list": [ + { + "line": "bace4d19-fa7e-beef-cafe-9129474bcd81 # tp", + "line_num": 1, + "path": "tests/samples/uuid", + "info": "", + "value": "bace4d19-fa7e-beef-cafe-9129474bcd81", + "value_start": 0, + "value_end": 36, + "variable": null, + "variable_start": -2, + "variable_end": -2, + "entropy_validation": { + "iterator": "BASE36_CHARS", + "entropy": 3.2373263071270246, + "valid": true + } + } + ] + }, { "api_validation": "NOT_AVAILABLE", "ml_validation": "NOT_AVAILABLE", diff --git a/tests/data/output.json b/tests/data/output.json index 1d75227d5..f6dcde344 100644 --- a/tests/data/output.json +++ b/tests/data/output.json @@ -8901,6 +8901,33 @@ } ] }, + { + "api_validation": "NOT_AVAILABLE", + "ml_validation": "NOT_AVAILABLE", + "ml_probability": null, + "rule": "UUID", + "severity": "info", + "confidence": "strong", + "line_data_list": [ + { + "line": "bace4d19-fa7e-beef-cafe-9129474bcd81 # tp", + "line_num": 1, + "path": "tests/samples/uuid", + "info": "", + "value": "bace4d19-fa7e-beef-cafe-9129474bcd81", + "value_start": 0, + "value_end": 36, + "variable": null, + "variable_start": -2, + "variable_end": -2, + "entropy_validation": { + "iterator": "BASE36_CHARS", + "entropy": 3.2373263071270246, + "valid": true + } + } + ] + }, { "api_validation": "NOT_AVAILABLE", "ml_validation": "NOT_AVAILABLE", diff --git a/tests/ml_model/test_ml_validator.py b/tests/ml_model/test_ml_validator.py index ee7083ae9..3e61d2e1a 100644 --- a/tests/ml_model/test_ml_validator.py +++ b/tests/ml_model/test_ml_validator.py @@ -65,6 +65,32 @@ def validate(_candidate: Candidate) -> Tuple[bool, float]: decision, probability = validate(candidate) self.assertAlmostEqual(0.9980608820915222, probability, delta=NEGLIGIBLE_ML_THRESHOLD) + def test_ml_validator_auxiliary_p(self): + candidate = Candidate.get_dummy_candidate(self.config, "secret", "", "") + candidate.rule_name = "Secret" + candidate.line_data_list[0].line = "secret=bace4d19-dead-beef-cafe-9129474bcd81" + candidate.line_data_list[0].variable = "secret" + candidate.line_data_list[0].value_start = 7 + candidate.line_data_list[0].value_end = 43 + candidate.line_data_list[0].value = "bace4d19-dead-beef-cafe-9129474bcd81" + # auxiliary candidate for a pattern rule - without variable + aux_candidate = copy.deepcopy(candidate) + aux_candidate.line_data_list[0].variable = None + + # todo: the scores are low for current ML model - will be changed after train + + candidate_key = CandidateKey(candidate.line_data_list[0]) + sample_as_batch = [(candidate_key, [candidate])] + is_cred_batch, probability_batch = self.ml_validator.validate_groups(sample_as_batch, 2) + self.assertAlmostEqual(0.16333681344985962, probability_batch[0], delta=NEGLIGIBLE_ML_THRESHOLD) + + # auxiliary rule in train does not increase ML probability yet - will be used after next train + + aux_candidate.rule_name = "UUID" + sample_as_batch = [(candidate_key, [candidate, aux_candidate])] + is_cred_batch, probability_batch = self.ml_validator.validate_groups(sample_as_batch, 2) + self.assertAlmostEqual(0.16333681344985962, probability_batch[0], delta=NEGLIGIBLE_ML_THRESHOLD) + def test_extract_features_p(self): candidate1 = Candidate.get_dummy_candidate(self.config, "main.py", ".py", "info") candidate1.line_data_list[0].line = 'ABC123' diff --git a/tests/samples/uuid b/tests/samples/uuid new file mode 100644 index 000000000..0ce05f451 --- /dev/null +++ b/tests/samples/uuid @@ -0,0 +1,2 @@ +bace4d19-fa7e-beef-cafe-9129474bcd81 # tp +12345678-1234-1234-1234-1234567890ab # fp diff --git a/tests/test_main.py b/tests/test_main.py index 6c774c4f9..7e664004b 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -484,7 +484,7 @@ def test_pdf_p(self) -> None: # may be tested with # https://www.dcc.edu/documents/administration/offices/information-technology/password-examples.pdf content_provider: AbstractProvider = FilesProvider([SAMPLES_PATH / "sample.pdf"]) - cred_sweeper = CredSweeper(depth=33) + cred_sweeper = CredSweeper(depth=7) cred_sweeper.run(content_provider=content_provider) found_credentials = cred_sweeper.credential_manager.get_credentials() self.assertSetEqual({"AWS Client ID", "Password", "Github Classic Token", "Key"}, @@ -786,6 +786,7 @@ def test_param_n(self) -> None: ("pager.rs", b"token: impl AsRef,"), # ("pager.rs", b" let tokens = quote::quote! {"), # ("pager.rs", b" let cert_chain = x509_rx"), # + ("my.kt", b'val password: String? = null'), # ] content_provider: AbstractProvider = FilesProvider([(file_name, io.BytesIO(data_line)) for file_name, data_line in items]) @@ -819,7 +820,7 @@ def test_param_p(self) -> None: ("accept.py", b"password='Ahga%$FiQ@Ei8'", "password", "Ahga%$FiQ@Ei8"), # ("test.template", b" NAMED_API_KEY=qii7t1m6423127xto389xc914l34451qz5135865564sg ", "NAMED_API_KEY", "qii7t1m6423127xto389xc914l34451qz5135865564sg"), # - ("my.kt", b'val password: String? = "Ahga%$FiQ@Ei8"', "password", "Ahga%$FiQ@Ei8"), # + ("my.kt", b'val password: String = "Ahga%$FiQ@Ei8"', "password", "Ahga%$FiQ@Ei8"), # ] for file_name, data_line, variable, value in items: content_provider: AbstractProvider = FilesProvider([ From 061d0d52b6cf7efcd3a34d313f751f4cb7beb447 Mon Sep 17 00:00:00 2001 From: Roman Babenko Date: Fri, 9 Aug 2024 12:38:44 +0300 Subject: [PATCH 7/7] Version Up to v1.8.3 (#594) * UUID pattern added * [no ci] version up and BM rollback --- .github/workflows/benchmark.yml | 12 ++++-------- credsweeper/__init__.py | 2 +- 2 files changed, 5 insertions(+), 9 deletions(-) diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index 90dfa8f8d..78556d277 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -22,8 +22,7 @@ jobs: - name: Checkout CredData uses: actions/checkout@v4 with: - repository: babenek/CredData - ref: uuid + repository: Samsung/CredData - name: Markup hashing run: | @@ -73,8 +72,7 @@ jobs: - name: Checkout CredData uses: actions/checkout@v4 with: - repository: babenek/CredData - ref: uuid + repository: Samsung/CredData - name: Markup hashing run: | @@ -171,8 +169,7 @@ jobs: - name: Checkout CredData uses: actions/checkout@v4 with: - repository: babenek/CredData - ref: uuid + repository: Samsung/CredData - name: Markup hashing run: | @@ -354,8 +351,7 @@ jobs: - name: Checkout CredData uses: actions/checkout@v4 with: - repository: babenek/CredData - ref: uuid + repository: Samsung/CredData - name: Markup hashing run: | diff --git a/credsweeper/__init__.py b/credsweeper/__init__.py index f591d7706..6355a1d79 100644 --- a/credsweeper/__init__.py +++ b/credsweeper/__init__.py @@ -20,4 +20,4 @@ '__version__' ] -__version__ = "1.8.2" +__version__ = "1.8.3"