diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index 5797f1acb..78556d277 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -287,7 +287,7 @@ jobs: exit_code=0 LOW_DELTA=10 THRESHOLD=250 - + # RELEASE if [ ${RELEASE_TIME} -le ${HEAD_TIME} ]; then d=$(( 1000 * ( ${HEAD_TIME} - ${RELEASE_TIME} ) / ${RELEASE_TIME} )) @@ -311,7 +311,7 @@ jobs: echo "Speed-up." fi fi - + # BASE if [ ${BASE_TIME} -le ${HEAD_TIME} ]; then d=$(( 1000 * ( ${HEAD_TIME} - ${BASE_TIME} ) / ${BASE_TIME} )) @@ -335,10 +335,11 @@ jobs: echo "Speed-up." fi fi - + exit ${exit_code} # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # + experiment: # the ml train test is placed here to use cached data set needs: [ download_data ] @@ -428,24 +429,34 @@ jobs: exit 1 fi -# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # + # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # run_doc_benchmark: runs-on: ubuntu-latest - if: ${{ 'Samsung/CredSweeper' == github.event.pull_request.head.repo.full_name }} + if: ${{ 'push' == github.event_name }} or ${{ 'Samsung/CredSweeper' == github.event.pull_request.head.repo.full_name }} steps: - - name: Checkout CredSweeper + - name: Checkout CredSweeper PR if: ${{ 'pull_request' == github.event_name }} uses: actions/checkout@v4 with: ref: ${{ github.event.pull_request.head.sha }} + - name: Checkout CredSweeper HEAD + if: ${{ 'push' == github.event_name }} + uses: actions/checkout@v4 + with: + ref: ${{ github.event.head }} + - name: Send cURL request with the commit SHA - if: ${{ 'pull_request' == github.event_name }} run: | - COMMIT_SHA=$(git rev-parse HEAD) - curl -X POST ${{ secrets.SLACK_URL }} \ - --data-urlencode \ - "payload={'text':'[BMT Request] ${{ github.event.repository.html_url }}/commit/${COMMIT_SHA}'}" + if [[ "${{ secrets.SLACK_URL }}" =~ http.*/.*/.* ]]; then + COMMIT_SHA=$(git rev-parse HEAD) + echo ${COMMIT_SHA} + curl -X POST ${{ secrets.SLACK_URL }} \ + --data-urlencode \ + "payload={'text':'[BMT Request] ${{ github.event.repository.html_url }}/commit/${COMMIT_SHA}'}" + else + echo "secrets.SLACK_URL is not available" + fi -# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # + # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # diff --git a/cicd/benchmark.txt b/cicd/benchmark.txt index c8d8fa7f7..72246106f 100644 --- a/cicd/benchmark.txt +++ b/cicd/benchmark.txt @@ -1,23 +1,23 @@ -DATA: 16988573 interested lines. MARKUP: 62869 items +DATA: 16348035 interested lines. MARKUP: 62567 items FileType FileNumber ValidLines Positives Negatives Templates --------------- ------------ ------------ ----------- ----------- ----------- - 194 28318 65 430 89 + 194 28318 66 427 87 .1 2 641 2 5 .admx 1 26 1 -.adoc 1 158 11 6 1 +.adoc 1 158 13 6 1 .api 2 118 4 -.asciidoc 96 14471 53 348 27 -.axaml 5 286 6 -.backup 1 62 1 1 +.asciidoc 96 14471 51 348 27 +.axaml 5 286 5 +.backup 1 62 2 1 .bash 2 2158 2 1 -.bat 4 233 1 13 2 -.bats 15 2804 12 52 9 +.bat 4 233 14 2 +.bats 15 2804 14 50 9 .bazel 3 424 8 .build 2 40 3 .bundle 4 1512 570 .bzl 3 2503 11 -.c 179 284009 16 942 5 -.cc 30 30656 624 1 +.c 179 284009 9 943 5 +.cc 29 30562 622 1 .cf 3 126 2 1 .cfg 1 385 1 1 .cjs 1 725 3 6 @@ -25,20 +25,20 @@ FileType FileNumber ValidLines Positives Negatives Templat .cljc 5 2421 12 .cls 1 657 1 .cmd 4 401 2 3 -.cnf 8 858 18 46 18 +.cnf 8 858 18 45 18 .coffee 1 585 2 -.conf 61 4954 54 74 54 +.conf 60 4945 54 71 53 .config 20 492 16 33 1 .cpp 15 5688 2 61 .creds 1 10 1 1 .crlf 1 27 1 .crt 2 4979 253 -.cs 269 82442 121 910 99 +.cs 268 82410 158 907 94 .cshtml 5 180 12 .csp 3 379 11 .csproj 1 14 1 .css 6 13564 10 -.csv 1 109 81 +.csv 1 109 78 .dart 2 22 2 .deprecated 1 126 1 .development 1 5 1 @@ -52,51 +52,51 @@ FileType FileNumber ValidLines Positives Negatives Templat .env 10 136 11 3 17 .erb 13 323 27 .erl 4 96 8 -.ex 25 4968 3 105 5 -.example 17 1838 74 37 55 -.exs 24 4842 3 188 4 +.ex 25 4968 5 105 5 +.example 17 1838 75 38 54 +.exs 24 4842 8 190 4 .ext 5 211 1 4 2 -.fsproj 1 75 1 +.fsproj 1 75 1 1 .g4 2 201 2 .gd 1 37 1 .gml 3 3075 26 .gni 3 5017 18 -.go 1084 569469 666 4347 742 +.go 1080 566476 673 4319 741 .golden 5 1168 1 14 29 .gradle 45 3265 4 91 100 -.graphql 8 445 1 13 +.graphql 7 420 13 .graphqls 1 30 1 -.groovy 23 5011 25 211 1 -.h 11 2038 38 +.groovy 22 4986 24 215 1 +.h 9 1958 36 .haml 9 191 16 .hbs 2 54 3 -.hs 17 4509 37 71 5 -.html 53 15327 14 115 18 -.idl 2 777 4 -.iml 6 699 38 -.in 6 2130 4 81 12 +.hs 14 4140 31 72 5 +.html 53 15327 22 115 18 +.idl 2 777 1 4 +.iml 6 699 36 +.in 6 2130 3 78 12 .inc 2 56 2 1 -.ini 11 1437 24 12 18 +.ini 11 1437 25 12 18 .ipynb 1 134 5 -.j 1 241 4 -.j2 31 5601 8 214 10 -.java 621 134132 328 1356 170 -.jenkinsfile 1 58 1 7 +.j 1 241 2 2 +.j2 30 5530 6 213 10 +.java 621 134132 359 1360 170 +.jenkinsfile 1 58 2 7 .jinja2 1 64 2 -.js 659 536413 541 2645 336 -.json 861 13670751 917 11012 143 +.js 659 536413 536 2635 330 +.json 850 13046270 1074 10778 140 .jsp 13 3202 1 42 .jsx 7 857 19 -.jwt 6 8 7 +.jwt 1 1 2 .key 83 2737 70 14 -.kt 123 20774 53 383 3 -.l 1 982 2 -.las 1 6656 48 -.lasso 1 230 7 +.kt 123 20774 67 384 3 +.l 1 982 1 +.las 1 6656 46 +.lasso 1 230 6 .lasso9 1 164 5 .ldif 2 286 20 .ldiff 1 20 1 -.ldml 1 6656 48 +.ldml 1 6656 46 .leex 1 9 2 .less 4 3023 12 .libsonnet 2 210 1 11 @@ -104,16 +104,16 @@ FileType FileNumber ValidLines Positives Negatives Templat .lkml 1 43 1 .lock 24 160912 144 .log 2 199 38 52 -.lua 10 1924 3 37 3 -.m 16 13358 11 151 3 -.manifest 3 102 3 +.lua 10 1924 37 3 +.m 16 13358 11 152 3 +.manifest 3 102 9 3 .markdown 3 139 3 1 .markerb 3 12 3 .marko 1 21 2 -.md 679 149755 784 2584 671 +.md 674 149399 722 2365 662 .mdx 3 549 7 .mjml 1 18 1 -.mjs 22 4424 108 310 +.mjs 22 4424 78 343 .mk 1 5878 16 .ml 1 1856 24 .mlir 2 1596 19 @@ -122,41 +122,41 @@ FileType FileNumber ValidLines Positives Negatives Templat .mqh 1 1023 2 .msg 1 26644 1 1 .mysql 1 36 2 -.ndjson 2 5006 73 350 2 +.ndjson 2 5006 70 266 2 .nix 4 211 12 .nolint 1 2 1 .odd 1 1281 57 .oracle 1 9 1 .p8 4 64 4 .pan 2 48 4 -.patch 4 109405 27 +.patch 4 109405 4 27 .pbxproj 1 941 1 .pem 48 1169 47 8 -.php 371 75710 135 1833 80 +.php 371 75710 131 1769 80 .pl 16 14727 6 47 .pm 3 744 8 .po 3 2994 15 -.pod 9 1859 8 26 +.pod 9 1859 2 24 .pony 1 83 4 .postinst 2 354 4 16 .pp 10 563 23 .ppk 1 45 37 .private 1 15 1 .proj 1 85 3 -.properties 48 1621 56 28 34 -.proto 5 5768 58 -.ps1 17 8618 69 2 +.properties 48 1621 53 28 33 +.proto 5 5768 2 58 +.ps1 16 8509 15 67 2 .ps1xml 1 5022 1 .pug 2 193 2 .purs 1 69 4 .pxd 1 150 5 2 -.py 896 292902 676 3468 748 +.py 890 291553 685 3456 729 .pyi 4 1361 9 .pyp 1 167 1 .pyx 2 1094 21 .r 4 62 6 3 1 .rake 2 51 2 -.rb 862 131917 253 3465 615 +.rb 860 131838 259 3451 612 .re 1 31 1 .red 1 159 1 .release 1 13 4 @@ -168,103 +168,99 @@ FileType FileNumber ValidLines Positives Negatives Templat .rrc 39 1404 514 .rs 31 9855 2 238 11 .rsc 1 691 1 -.rsp 16 7101 23 11 28 -.rst 87 34151 64 360 68 +.rsp 16 7101 19 10 28 +.rst 86 33980 69 358 68 .rules 1 6 2 -.sample 2 25 1 7 4 -.sbt 3 570 7 2 -.scala 40 5071 13 102 +.sample 2 25 3 4 4 +.sbt 3 570 6 2 +.scala 40 5071 22 102 .scss 16 8553 32 1 .secrets 1 11 1 -.sh 144 21573 63 476 30 +.sh 143 21525 51 474 30 .slim 1 153 2 2 -.sln 1 306 2 .smali 1 775 12 -.snap 3 1708 1 34 2 +.snap 3 1708 9 30 2 .spec 2 332 2 .spin 1 565 1 -.sql 27 6606 29 69 4 +.sql 27 6606 126 62 4 .storyboard 20 1802 401 .strings 20 1240 184 .stub 3 84 6 .sublime-keymap 1 3 1 .sum 37 22854 283 .svg 1 638 12 -.swift 6 278 16 .t 9 1767 28 56 14 .td 2 14002 6 -.template 19 1633 5 42 11 +.template 19 1633 4 42 11 .test 2 24 25 4 -.testsettings 1 21 5 +.testsettings 1 21 1 5 .tf 21 1377 3 32 2 -.tfstate 4 307 21 10 4 +.tfstate 4 307 22 11 4 .tfvars 1 31 3 3 .tl 2 2161 165 2 .tmpl 5 336 3 9 .token 1 1 3 -.toml 83 2379 55 72 172 +.toml 83 2379 54 73 172 .tpl 1 43 1 .travis 1 34 4 3 1 -.ts 585 106846 172 1930 203 -.tsx 55 9846 1 128 5 -.ttar 2 6050 8 3 -.txt 444 78553 1830 14283 50 +.ts 583 106730 158 1935 203 +.tsx 54 7914 1 124 5 +.ttar 1 452 1 +.txt 440 78102 1861 14251 50 .utf8 1 77 2 .vsixmanifest 1 36 1 -.vsmdi 1 6 1 +.vsmdi 1 6 2 .vue 50 8736 1 183 1 -.xaml 21 8103 174 +.xaml 21 8103 175 .xcscheme 1 109 6 .xib 11 503 174 .xml 9 689 9 .xsl 1 311 1 -.yaml 149 20563 140 383 44 -.yml 418 36162 467 920 384 +.yaml 137 19004 128 356 44 +.yml 418 36162 515 910 384 .zsh 6 872 12 .zsh-theme 1 97 1 -TOTAL: 10333 16988573 8377 60439 5233 -credsweeper result_cnt : 7800, lost_cnt : 0, true_cnt : 7231, false_cnt : 569 +TOTAL: 10259 16348035 8706 59679 5182 +credsweeper result_cnt : 7664, lost_cnt : 0, true_cnt : 7472, false_cnt : 192 Rules Positives Negatives Templates Reported TP FP TN FN FPR FNR ACC PRC RCL F1 ------------------------------ ----------- ----------- ----------- ---------- ---- ---- ----- ---- -------- -------- -------- -------- -------- -------- -API 123 3163 185 112 109 3 3345 14 0.000896 0.113821 0.995102 0.973214 0.886179 0.927660 -AWS Client ID 168 13 0 160 160 0 13 8 0.000000 0.047619 0.955801 1.000000 0.952381 0.975610 -AWS Multi 75 12 0 87 75 11 1 0 0.916667 0.000000 0.873563 0.872093 1.000000 0.931677 -AWS S3 Bucket 61 25 0 87 61 24 1 0 0.960000 0.000000 0.720930 0.717647 1.000000 0.835616 -Atlassian Old PAT token 27 212 3 12 3 8 207 24 0.037209 0.888889 0.867769 0.272727 0.111111 0.157895 -Auth 407 2725 77 372 351 21 2781 56 0.007495 0.137592 0.976005 0.943548 0.862408 0.901155 -Azure Access Token 19 0 0 0 0 0 19 1.000000 0.000000 0.000000 +API 131 3126 185 111 109 2 3309 22 0.000604 0.167939 0.993027 0.981982 0.832061 0.900826 +AWS Client ID 167 18 0 160 160 0 18 7 0.000000 0.041916 0.962162 1.000000 0.958084 0.978593 +AWS Multi 75 14 0 87 75 11 3 0 0.785714 0.000000 0.876404 0.872093 1.000000 0.931677 +AWS S3 Bucket 66 24 0 92 66 24 0 0 1.000000 0.000000 0.733333 0.733333 1.000000 0.846154 +Atlassian Old PAT token 27 208 3 12 3 8 203 24 0.037915 0.888889 0.865546 0.272727 0.111111 0.157895 +Auth 412 2723 76 371 353 18 2781 59 0.006431 0.143204 0.976020 0.951482 0.856796 0.901660 +Azure Access Token 19 0 0 12 12 0 0 7 0.368421 0.631579 1.000000 0.631579 0.774194 BASE64 Private Key 7 2 0 7 7 0 2 0 0.000000 0.000000 1.000000 1.000000 1.000000 1.000000 BASE64 encoded PEM Private Key 7 0 0 5 5 0 0 2 0.285714 0.714286 1.000000 0.714286 0.833333 -Bitbucket Client ID 142 1813 9 46 27 18 1804 115 0.009879 0.809859 0.932281 0.600000 0.190141 0.288770 -Bitbucket Client Secret 230 535 10 44 33 11 534 197 0.020183 0.856522 0.731613 0.750000 0.143478 0.240876 -Certificate 25 459 1 21 20 1 459 5 0.002174 0.200000 0.987629 0.952381 0.800000 0.869565 -Credential 91 155 74 90 87 3 226 4 0.013100 0.043956 0.978125 0.966667 0.956044 0.961326 +Bitbucket Client ID 142 1807 9 46 27 18 1798 115 0.009912 0.809859 0.932074 0.600000 0.190141 0.288770 +Bitbucket Client Secret 230 527 10 44 33 11 526 197 0.020484 0.856522 0.728814 0.750000 0.143478 0.240876 +Certificate 25 460 1 21 20 1 460 5 0.002169 0.200000 0.987654 0.952381 0.800000 0.869565 +Credential 94 154 74 90 90 0 228 4 0.000000 0.042553 0.987578 1.000000 0.957447 0.978261 Docker Swarm Token 2 0 0 2 2 0 0 0 0.000000 1.000000 1.000000 1.000000 1.000000 Dropbox App secret 62 114 0 46 36 9 105 26 0.078947 0.419355 0.801136 0.800000 0.580645 0.672897 Facebook Access Token 0 1 0 0 0 1 0 0.000000 1.000000 Firebase Domain 6 1 0 7 6 1 0 0 1.000000 0.000000 0.857143 0.857143 1.000000 0.923077 Github Old Token 1 0 0 1 1 0 0 0 0.000000 1.000000 1.000000 1.000000 1.000000 -Gitlab Feed Token 188 460 88 60 47 12 536 141 0.021898 0.750000 0.792120 0.796610 0.250000 0.380567 +Gitlab Feed Token 188 451 87 60 47 12 526 141 0.022305 0.750000 0.789256 0.796610 0.250000 0.380567 Gitlab Incoming Email Token 37 3 0 21 19 2 1 18 0.666667 0.486486 0.500000 0.904762 0.513514 0.655172 Google API Key 12 0 0 12 12 0 0 0 0.000000 1.000000 1.000000 1.000000 1.000000 Google Multi 10 2 0 11 10 1 1 0 0.500000 0.000000 0.916667 0.909091 1.000000 0.952381 Google OAuth Access Token 3 0 0 3 3 0 0 0 0.000000 1.000000 1.000000 1.000000 1.000000 -Grafana Provisioned API Key 22 1 0 1 1 0 1 21 0.000000 0.954545 0.086957 1.000000 0.045455 0.086957 -IPv4 729 405 0 1205 728 342 63 1 0.844444 0.001372 0.697531 0.680374 0.998628 0.809339 -IPv6 33 131 0 33 33 0 131 0 0.000000 0.000000 1.000000 1.000000 1.000000 1.000000 -JSON Web Token 284 11 2 274 271 3 10 13 0.230769 0.045775 0.946128 0.989051 0.954225 0.971326 +Grafana Provisioned API Key 22 1 0 5 5 0 1 17 0.000000 0.772727 0.260870 1.000000 0.227273 0.370370 +JSON Web Token 170 61 0 131 131 0 61 39 0.000000 0.229412 0.831169 1.000000 0.770588 0.870432 Jira / Confluence PAT token 0 4 0 0 0 4 0 0.000000 1.000000 Jira 2FA 14 6 0 10 10 0 6 4 0.000000 0.285714 0.800000 1.000000 0.714286 0.833333 -Key 483 8494 464 445 436 9 8949 47 0.001005 0.097308 0.994068 0.979775 0.902692 0.939655 -Nonce 83 53 0 85 79 6 47 4 0.113208 0.048193 0.926471 0.929412 0.951807 0.940476 -Other 0 0 5 0 0 5 0 0.000000 1.000000 +Key 522 8453 464 452 447 5 8912 75 0.000561 0.143678 0.991525 0.988938 0.856322 0.917864 +Nonce 91 47 0 84 83 1 46 8 0.021277 0.087912 0.934783 0.988095 0.912088 0.948571 PEM Private Key 1019 1483 0 1023 1019 4 1479 0 0.002697 0.000000 0.998401 0.996090 1.000000 0.998041 -Password 1823 7474 2752 1681 1614 67 10159 209 0.006552 0.114646 0.977094 0.960143 0.885354 0.921233 -Salt 42 76 2 38 38 0 78 4 0.000000 0.095238 0.966667 1.000000 0.904762 0.950000 -Secret 1358 28497 869 1234 1229 5 29361 129 0.000170 0.094993 0.995639 0.995948 0.905007 0.948302 +Password 1841 7468 2724 1691 1637 54 10138 204 0.005298 0.110809 0.978559 0.968066 0.889191 0.926954 +Salt 45 73 2 39 39 0 75 6 0.000000 0.133333 0.950000 1.000000 0.866667 0.928571 +Secret 1365 28359 868 1237 1233 4 29223 132 0.000137 0.096703 0.995554 0.996766 0.903297 0.947733 Seed 1 6 0 0 0 6 1 0.000000 1.000000 0.857143 0.000000 Slack Token 4 1 0 4 4 0 1 0 0.000000 0.000000 1.000000 1.000000 1.000000 1.000000 -Token 585 3972 439 519 511 8 4403 74 0.001814 0.126496 0.983587 0.984586 0.873504 0.925725 +Token 612 3949 437 516 511 5 4381 101 0.001140 0.165033 0.978792 0.990310 0.834967 0.906028 Twilio API Key 0 5 2 0 0 7 0 0.000000 1.000000 -URL Credentials 194 125 251 184 184 0 376 10 0.000000 0.051546 0.982456 1.000000 0.948454 0.973545 - 8377 60439 5233 7942 7231 569 59870 1146 0.009414 0.136803 0.975078 0.927051 0.863197 0.893985 +URL Credentials 209 127 240 200 200 0 367 9 0.000000 0.043062 0.984375 1.000000 0.956938 0.977995 +UUID 1068 1 0 1058 1057 1 0 11 1.000000 0.010300 0.988775 0.999055 0.989700 0.994356 + 8706 59679 5182 7671 7472 192 59487 1234 0.003217 0.141741 0.979147 0.974948 0.858259 0.912889 diff --git a/credsweeper/__init__.py b/credsweeper/__init__.py index f591d7706..6355a1d79 100644 --- a/credsweeper/__init__.py +++ b/credsweeper/__init__.py @@ -20,4 +20,4 @@ '__version__' ] -__version__ = "1.8.2" +__version__ = "1.8.3" diff --git a/credsweeper/credentials/line_data.py b/credsweeper/credentials/line_data.py index 8007ec604..2d857c699 100644 --- a/credsweeper/credentials/line_data.py +++ b/credsweeper/credentials/line_data.py @@ -194,6 +194,8 @@ def sanitize_variable(self) -> None: while self.variable and sanitized_var_len != len(self.variable): sanitized_var_len = len(self.variable) self.variable = self.variable.strip(self.variable_strip_pattern) + if self.variable.endswith('\\'): + self.variable = self.variable[:-1] if variable and len(self.variable) < len(variable) and 0 <= self.variable_start and 0 <= self.variable_end: start = variable.find(self.variable) self.variable_start += start diff --git a/credsweeper/filters/__init__.py b/credsweeper/filters/__init__.py index 66de37b97..a91119e63 100644 --- a/credsweeper/filters/__init__.py +++ b/credsweeper/filters/__init__.py @@ -5,10 +5,12 @@ from credsweeper.filters.value_allowlist_check import ValueAllowlistCheck from credsweeper.filters.value_array_dictionary_check import ValueArrayDictionaryCheck from credsweeper.filters.value_atlassian_token_check import ValueAtlassianTokenCheck +from credsweeper.filters.value_azure_token_check import ValueAzureTokenCheck from credsweeper.filters.value_base32_data_check import ValueBase32DataCheck from credsweeper.filters.value_base64_data_check import ValueBase64DataCheck from credsweeper.filters.value_base64_encoded_pem_check import ValueBase64EncodedPem from credsweeper.filters.value_base64_key_check import ValueBase64KeyCheck +from credsweeper.filters.value_base64_part_check import ValueBase64PartCheck from credsweeper.filters.value_blocklist_check import ValueBlocklistCheck from credsweeper.filters.value_camel_case_check import ValueCamelCaseCheck from credsweeper.filters.value_couple_keyword_check import ValueCoupleKeywordCheck @@ -24,17 +26,14 @@ from credsweeper.filters.value_grafana_check import ValueGrafanaCheck from credsweeper.filters.value_grafana_service_check import ValueGrafanaServiceCheck from credsweeper.filters.value_hex_number_check import ValueHexNumberCheck -from credsweeper.filters.value_ip_check import ValueIPCheck from credsweeper.filters.value_jfrog_token_check import ValueJfrogTokenCheck from credsweeper.filters.value_json_web_token_check import ValueJsonWebTokenCheck from credsweeper.filters.value_last_word_check import ValueLastWordCheck -from credsweeper.filters.value_length_check import ValueLengthCheck from credsweeper.filters.value_method_check import ValueMethodCheck from credsweeper.filters.value_not_allowed_pattern_check import ValueNotAllowedPatternCheck from credsweeper.filters.value_not_part_encoded_check import ValueNotPartEncodedCheck from credsweeper.filters.value_number_check import ValueNumberCheck from credsweeper.filters.value_pattern_check import ValuePatternCheck -from credsweeper.filters.value_pattern_length_check import ValuePatternLengthCheck from credsweeper.filters.value_similarity_check import ValueSimilarityCheck from credsweeper.filters.value_split_keyword_check import ValueSplitKeywordCheck from credsweeper.filters.value_string_type_check import ValueStringTypeCheck diff --git a/credsweeper/filters/group/general_pattern.py b/credsweeper/filters/group/general_pattern.py index b340055fd..96018a5f4 100644 --- a/credsweeper/filters/group/general_pattern.py +++ b/credsweeper/filters/group/general_pattern.py @@ -1,5 +1,6 @@ from credsweeper.common.constants import GroupType from credsweeper.config import Config +from credsweeper.filters import ValueUselessWordCheck from credsweeper.filters.group import Group @@ -8,3 +9,4 @@ class GeneralPattern(Group): def __init__(self, config: Config) -> None: super().__init__(config, GroupType.PATTERN) + self.filters.extend([ValueUselessWordCheck()]) diff --git a/credsweeper/filters/group/group.py b/credsweeper/filters/group/group.py index 6ee25387d..37cea6948 100644 --- a/credsweeper/filters/group/group.py +++ b/credsweeper/filters/group/group.py @@ -5,9 +5,9 @@ from credsweeper.config import Config from credsweeper.filters import (Filter, LineSpecificKeyCheck, ValueAllowlistCheck, ValueArrayDictionaryCheck, ValueBlocklistCheck, ValueCamelCaseCheck, ValueFilePathCheck, ValueFirstWordCheck, - ValueLastWordCheck, ValueLengthCheck, ValueMethodCheck, ValueNotAllowedPatternCheck, - ValuePatternCheck, ValueSimilarityCheck, ValueStringTypeCheck, ValueTokenCheck, - VariableNotAllowedPatternCheck, ValuePatternLengthCheck, ValueHexNumberCheck) + ValueLastWordCheck, ValueMethodCheck, ValueNotAllowedPatternCheck, ValuePatternCheck, + ValueSimilarityCheck, ValueStringTypeCheck, ValueTokenCheck, + VariableNotAllowedPatternCheck, ValueHexNumberCheck) class Group(ABC): @@ -43,7 +43,6 @@ def get_keyword_base_filters(config: Config) -> List[Filter]: ValueFirstWordCheck(), ValueHexNumberCheck(), ValueLastWordCheck(), - ValueLengthCheck(config), ValueMethodCheck(), ValueSimilarityCheck(), ValueStringTypeCheck(config), @@ -60,5 +59,4 @@ def get_pattern_base_filters(config: Config) -> List[Filter]: return [ # LineSpecificKeyCheck(), # ValuePatternCheck(config), # - ValuePatternLengthCheck(config) ] diff --git a/credsweeper/filters/group/url_credentials_group.py b/credsweeper/filters/group/url_credentials_group.py index 9a7477191..23aba1d3b 100644 --- a/credsweeper/filters/group/url_credentials_group.py +++ b/credsweeper/filters/group/url_credentials_group.py @@ -2,8 +2,8 @@ from credsweeper.config import Config from credsweeper.filters import (ValueAllowlistCheck, ValueArrayDictionaryCheck, ValueBlocklistCheck, ValueCamelCaseCheck, ValueDictionaryValueLengthCheck, ValueFilePathCheck, - ValueFirstWordCheck, ValueLastWordCheck, ValueLengthCheck, ValueMethodCheck, - ValueNotAllowedPatternCheck, ValuePatternCheck, ValueStringTypeCheck, ValueTokenCheck) + ValueFirstWordCheck, ValueLastWordCheck, ValueMethodCheck, ValueNotAllowedPatternCheck, + ValuePatternCheck, ValueStringTypeCheck, ValueTokenCheck) from credsweeper.filters.group import Group @@ -25,11 +25,10 @@ def __init__(self, config: Config) -> None: ValueFilePathCheck(), ValueFirstWordCheck(), ValueLastWordCheck(), - ValueLengthCheck(config), ValueMethodCheck(), ValueStringTypeCheck(config), ValueNotAllowedPatternCheck(), ValueTokenCheck(), - ValueDictionaryValueLengthCheck(), + ValueDictionaryValueLengthCheck(min_len=4, max_len=80), ValuePatternCheck(config) ] diff --git a/credsweeper/filters/line_specific_key_check.py b/credsweeper/filters/line_specific_key_check.py index 8bbfa15a1..71fec9dc0 100644 --- a/credsweeper/filters/line_specific_key_check.py +++ b/credsweeper/filters/line_specific_key_check.py @@ -1,5 +1,6 @@ import re +from credsweeper.common.constants import ML_HUNK from credsweeper.config import Config from credsweeper.credentials import LineData from credsweeper.file_handler.analysis_target import AnalysisTarget @@ -10,8 +11,8 @@ class LineSpecificKeyCheck(Filter): """Check that values from list below is not in candidate line.""" - NOT_ALLOWED = [r"example", r"enc\(", r"enc\[", r"true", r"false"] - NOT_ALLOWED_PATTERN = re.compile(Util.get_regex_combine_or(NOT_ALLOWED)) + NOT_ALLOWED = [r"example", r"\benc[\(\[]", r"\btrue\b", r"\bfalse\b"] + NOT_ALLOWED_PATTERN = re.compile(Util.get_regex_combine_or(NOT_ALLOWED), re.IGNORECASE) def __init__(self, config: Config = None) -> None: pass @@ -29,8 +30,13 @@ def run(self, line_data: LineData, target: AnalysisTarget) -> bool: """ if line_data.line is None: return True + if 0 <= line_data.variable_start: + # variable may be defined too + sub_line_start = 0 if ML_HUNK >= line_data.variable_start else line_data.variable_start - ML_HUNK + else: + sub_line_start = 0 if ML_HUNK >= line_data.value_start else line_data.value_start - ML_HUNK - if self.NOT_ALLOWED_PATTERN.search(target.line_lower): + if self.NOT_ALLOWED_PATTERN.search(line_data.line, sub_line_start, line_data.value_end + ML_HUNK): return True return False diff --git a/credsweeper/filters/separator_unusual_check.py b/credsweeper/filters/separator_unusual_check.py deleted file mode 100644 index b05da326b..000000000 --- a/credsweeper/filters/separator_unusual_check.py +++ /dev/null @@ -1,49 +0,0 @@ -import logging - -from credsweeper.config import Config -from credsweeper.credentials import LineData -from credsweeper.file_handler.analysis_target import AnalysisTarget -from credsweeper.filters import Filter - -logger = logging.getLogger(__name__) - - -class SeparatorUnusualCheck(Filter): - """Check that candidate have no double symbol ops (like ++, --, <<) or comparison ops (like != or ==) as separator. - - Example: - `pwd == 'value'` - `pwd != 'value'` - `pwd << value` - - """ - - def __init__(self, config: Config = None) -> None: - pass - - def run(self, line_data: LineData, target: AnalysisTarget) -> bool: - """Run filter checks on received credential candidate data 'line_data'. - - Args: - line_data: credential candidate data - target: multiline target from which line data was obtained - - Return: - True, if need to filter candidate and False if left - - """ - if line_data.separator is None: - return True - - if 1 > line_data.separator_start: - logger.warning(f"Wrong separator start position {line_data}") - return True - - try: - if line_data.separator == line_data.line[line_data.separator_start + 1] or \ - (line_data.separator == "=" and line_data.line[line_data.separator_start - 1] == "!"): - return True - except IndexError: - return True - - return False diff --git a/credsweeper/filters/value_azure_token_check.py b/credsweeper/filters/value_azure_token_check.py new file mode 100644 index 000000000..d62293a54 --- /dev/null +++ b/credsweeper/filters/value_azure_token_check.py @@ -0,0 +1,52 @@ +import contextlib +import json + +from credsweeper.common.constants import Chars +from credsweeper.config import Config +from credsweeper.credentials import LineData +from credsweeper.file_handler.analysis_target import AnalysisTarget +from credsweeper.filters import Filter +from credsweeper.filters.value_entropy_base64_check import ValueEntropyBase64Check +from credsweeper.utils import Util + + +class ValueAzureTokenCheck(Filter): + """ + Azure tokens contains header, payload and signature + https://learn.microsoft.com/en-us/azure/active-directory-b2c/access-tokens + """ + + def __init__(self, config: Config = None) -> None: + pass + + def run(self, line_data: LineData, target: AnalysisTarget) -> bool: + """Run filter checks on received token which might be structured. + + Args: + line_data: credential candidate data + target: multiline target from which line data was obtained + + Return: + True, when need to filter candidate and False if left + + """ + with contextlib.suppress(Exception): + parts = line_data.value.split('.') + if 3 != len(parts): + return True + hdr = Util.decode_base64(parts[0], padding_safe=True, urlsafe_detect=True) + header = json.loads(hdr) + if not ("alg" in header and "typ" in header and "kid" in header): + # must be all parts in header + return True + pld = Util.decode_base64(parts[1], padding_safe=True, urlsafe_detect=True) + payload = json.loads(pld) + if not ("iss" in payload and "exp" in payload and "iat" in payload): + # must be all parts in payload + return True + min_entropy = ValueEntropyBase64Check.get_min_data_entropy(len(parts[2])) + entropy = Util.get_shannon_entropy(parts[2], Chars.BASE64URL_CHARS.value) + # good signature has to be like random bytes + return entropy < min_entropy + + return True diff --git a/credsweeper/filters/value_base64_part_check.py b/credsweeper/filters/value_base64_part_check.py new file mode 100644 index 000000000..d41a7ed8c --- /dev/null +++ b/credsweeper/filters/value_base64_part_check.py @@ -0,0 +1,56 @@ +import contextlib +import statistics + +from credsweeper.common.constants import Chars +from credsweeper.config import Config +from credsweeper.credentials import LineData +from credsweeper.file_handler.analysis_target import AnalysisTarget +from credsweeper.filters import Filter +from credsweeper.utils import Util + + +class ValueBase64PartCheck(Filter): + """ + Check that candidate is NOT a part of base64 long line + """ + + def __init__(self, config: Config = None) -> None: + pass + + def run(self, line_data: LineData, target: AnalysisTarget) -> bool: + """Run filter checks on received weird base64 token which must be a random string + + Args: + line_data: credential candidate data + target: multiline target from which line data was obtained + + Return: + True, when need to filter candidate and False if left + + """ + + with contextlib.suppress(Exception): + if line_data.value_start and '/' == line_data.line[line_data.value_start - 1]: + if '-' in line_data.value or '_' in line_data.value: + # the value contains url-safe chars, so '/' is a delimiter + return False + value_entropy = Util.get_shannon_entropy(line_data.value, Chars.BASE64STD_CHARS.value) + left_start = line_data.value_start - len(line_data.value) + if 0 > left_start: + left_start = 0 + left_entropy = Util.get_shannon_entropy(line_data.line[left_start:line_data.value_start], + Chars.BASE64STD_CHARS.value) + right_end = line_data.value_end + len(line_data.value) + if len(line_data.line) < right_end: + right_end = len(line_data.line) + right_entropy = Util.get_shannon_entropy(line_data.line[line_data.value_end:right_end], + Chars.BASE64STD_CHARS.value) + data = [value_entropy, left_entropy, right_entropy] + avg = statistics.mean(data) + stdev = statistics.stdev(data, avg) + avg_min = avg - stdev + if avg_min < left_entropy and avg_min < right_entropy: + # high entropy of bound parts looks like a part of base64 long line + return True + + return False diff --git a/credsweeper/filters/value_dictionary_value_length_check.py b/credsweeper/filters/value_dictionary_value_length_check.py index c0b92a846..8186f8229 100644 --- a/credsweeper/filters/value_dictionary_value_length_check.py +++ b/credsweeper/filters/value_dictionary_value_length_check.py @@ -7,8 +7,9 @@ class ValueDictionaryValueLengthCheck(Filter): """Check that candidate length is between 5 and 30.""" - def __init__(self, config: Config = None) -> None: - pass + def __init__(self, config: Config = None, min_len: int = 4, max_len: int = 31) -> None: + self.min_len = min_len + self.max_len = max_len def run(self, line_data: LineData, target: AnalysisTarget) -> bool: """Run filter checks on received credential candidate data 'line_data'. @@ -21,7 +22,7 @@ def run(self, line_data: LineData, target: AnalysisTarget) -> bool: True, if need to filter candidate and False if left """ - if 4 <= len(line_data.value) <= 31: + if self.min_len <= len(line_data.value) <= self.max_len: return False else: return True diff --git a/credsweeper/filters/value_ip_check.py b/credsweeper/filters/value_ip_check.py deleted file mode 100644 index eae2d6dec..000000000 --- a/credsweeper/filters/value_ip_check.py +++ /dev/null @@ -1,51 +0,0 @@ -import contextlib -import ipaddress -import re - -from credsweeper.common.constants import ML_HUNK -from credsweeper.config import Config -from credsweeper.credentials import LineData -from credsweeper.file_handler.analysis_target import AnalysisTarget -from credsweeper.filters import Filter -from credsweeper.utils import Util - - -class ValueIPCheck(Filter): - """Filter out some of insensible IP""" - - TRUE_POSITIVE_MARKERS = [r"\bip\b", "server", "addr", "login"] - TRUE_POSITIVE_PATTERN = re.compile(Util.get_regex_combine_or(TRUE_POSITIVE_MARKERS), flags=re.IGNORECASE) - - FALSE_POSITIVE_MARKERS = ["version", "oid", "section", "rfc"] - FALSE_POSITIVE_PATTERN = re.compile(Util.get_regex_combine_or(FALSE_POSITIVE_MARKERS), flags=re.IGNORECASE) - - def __init__(self, config: Config = None) -> None: - pass - - def run(self, line_data: LineData, target: AnalysisTarget) -> bool: - """Run filter checks on received credential candidate data 'line_data'. - - Args: - line_data: credential candidate data - target: multiline target from which line data was obtained - - Return: - True, if need to filter candidate and False if left - - """ - - with contextlib.suppress(Exception): - ip = ipaddress.ip_address(line_data.value) - if 4 == ip.version: - byte_sum = sum(x for x in ip.packed) - if 100 > (byte_sum >> 2): - # versions usually have low average of sum the bytes - search_text = Util.subtext(line_data.line, line_data.value_start, ML_HUNK) - if self.FALSE_POSITIVE_PATTERN.search(search_text) \ - and not self.TRUE_POSITIVE_PATTERN.search(search_text): - return True - if ip.is_loopback or ip.is_private or ip.is_reserved or ip.is_link_local or ip.is_multicast: - return True - return False - - return True diff --git a/credsweeper/filters/value_json_web_token_check.py b/credsweeper/filters/value_json_web_token_check.py index 7f6048a80..ed6a2e2e1 100644 --- a/credsweeper/filters/value_json_web_token_check.py +++ b/credsweeper/filters/value_json_web_token_check.py @@ -11,9 +11,19 @@ class ValueJsonWebTokenCheck(Filter): """ Check that candidate is JWT which starts usually from 'eyJ' - only header is parsed with "typ" or "alg" member from example of RFC7519 - https://datatracker.ietf.org/doc/html/rfc7519 + registered keys are checked to be in the JWT parts + https://www.iana.org/assignments/jose/jose.xhtml """ + header_keys = { + "alg", "jku", "jwk", "kid", "x5u", "x5c", "x5t", "x5t#S256", "typ", "cty", "crit", "alg", "enc", "zip", "jku", + "jwk", "kid", "x5u", "x5c", "x5t", "x5t#S256", "typ", "cty", "crit", "epk", "apu", "apv", "iv", "tag", "p2s", + "p2c", "iss", "sub", "aud", "b64", "ppt", "url", "nonce", "svt" + } + payload_keys = { + "iss", "sub", "aud", "exp", "nbf", "iat", "jti", "kty", "use", "key_ops", "alg", "enc", "zip", "jku", "jwk", + "kid", "x5u", "x5c", "x5t", "x5t#S256", "crv", "x", "y", "d", "n", "e", "d", "p", "q", "dp", "dq", "qi", "oth", + "k", "crv", "d", "x", "ext", "crit", "keys", "id", "role", "token", "secret", "password", "nonce" + } def __init__(self, config: Config = None) -> None: pass @@ -29,12 +39,29 @@ def run(self, line_data: LineData, target: AnalysisTarget) -> bool: True, when need to filter candidate and False if left """ + header_check = False + payload_check = False + signature_check = False with contextlib.suppress(Exception): - delimiter_pos = line_data.value.find(".") - # jwt token. '.' must be always in given data, according regex in rule - value = line_data.value[:delimiter_pos] - decoded = Util.decode_base64(value, padding_safe=True, urlsafe_detect=True) - if header := json.loads(decoded): - if "alg" in header or "typ" in header: - return False - return True + jwt_parts = line_data.value.split('.') + for part in jwt_parts: + data = Util.decode_base64(part, padding_safe=True, urlsafe_detect=True) + if part.startswith("eyJ"): + # open part - just base64 encoded + json_keys = json.loads(data).keys() + # header will be checked first + if not header_check: + header_check = bool(ValueJsonWebTokenCheck.header_keys.intersection(json_keys)) + # payload follows the header + elif not payload_check: + payload_check = bool(ValueJsonWebTokenCheck.payload_keys.intersection(json_keys)) + # any other payloads are allowed + elif header_check and payload_check and not signature_check: + # signature check or skip encrypted part + signature_check = not Util.is_ascii_entropy_validate(data) + else: + break + if header_check and payload_check and signature_check: + return False + else: + return True diff --git a/credsweeper/filters/value_length_check.py b/credsweeper/filters/value_length_check.py deleted file mode 100644 index 57596f35e..000000000 --- a/credsweeper/filters/value_length_check.py +++ /dev/null @@ -1,26 +0,0 @@ -from credsweeper.config import Config -from credsweeper.credentials import LineData -from credsweeper.file_handler.analysis_target import AnalysisTarget -from credsweeper.filters import Filter - - -class ValueLengthCheck(Filter): - """Check if potential candidate value is not too short (longer or equal to `min_len`).""" - - def __init__(self, config: Config) -> None: - self.min_len = config.min_keyword_value_length - - def run(self, line_data: LineData, target: AnalysisTarget) -> bool: - """Run filter checks on received credential candidate data 'line_data'. - - Args: - line_data: credential candidate data - target: multiline target from which line data was obtained - - Return: - True, if need to filter candidate and False if left - - """ - if len(line_data.value) < self.min_len: - return True - return False diff --git a/credsweeper/filters/value_not_allowed_pattern_check.py b/credsweeper/filters/value_not_allowed_pattern_check.py index 944c9c34e..a0cc89aa1 100644 --- a/credsweeper/filters/value_not_allowed_pattern_check.py +++ b/credsweeper/filters/value_not_allowed_pattern_check.py @@ -10,7 +10,7 @@ class ValueNotAllowedPatternCheck(Filter): """Check that secret doesn't open or closes brackets or a new line.""" - NOT_ALLOWED = [r"[<>\[\]{}]\s+", r"^\s*\\", r"^\s*\\n\s*"] + NOT_ALLOWED = [r"[<>\[\]{}]\s+", r"\\u00(26|3c)gt;?(\s|\\+[nrt])?", r"^\s*\\", r"^\s*\\n\s*"] NOT_ALLOWED_PATTERN = re.compile( # f"{Util.get_regex_combine_or(NOT_ALLOWED)}$", # flags=re.IGNORECASE) diff --git a/credsweeper/filters/value_pattern_length_check.py b/credsweeper/filters/value_pattern_length_check.py deleted file mode 100644 index dd4531bf1..000000000 --- a/credsweeper/filters/value_pattern_length_check.py +++ /dev/null @@ -1,10 +0,0 @@ -from credsweeper.config import Config -from credsweeper.filters import ValueLengthCheck - - -class ValuePatternLengthCheck(ValueLengthCheck): - """Check if potential candidate value is not too short like ValueLengthCheck but with different min_len""" - - def __init__(self, config: Config) -> None: - super().__init__(config) - self.min_len = config.min_pattern_value_length diff --git a/credsweeper/filters/value_useless_word_check.py b/credsweeper/filters/value_useless_word_check.py index c921c937d..6182d1e3f 100644 --- a/credsweeper/filters/value_useless_word_check.py +++ b/credsweeper/filters/value_useless_word_check.py @@ -11,11 +11,10 @@ class ValueUselessWordCheck(Filter): """Check is candidate value contains sub-rows with operators (like ->).""" NOT_ALLOWED = [ - "((\\{)?(0x)+([0-9a-f]|\\%){1}.*)", # Check is contain \{0x or 0x - "(\\-\\>.*)", # Check if contain -> - "(xxxx.*)", # Check if contain xxxxx + "((\\{)?(0x)+([0-9a-f]|\\%){1})", # Check is contain \{0x or 0x + r"((\w+)?->)", # Check if contain -> + "(.*example)", # Check if contain `example` word "(\\$\\w+)", # Check whether it looks like a variable e.g. $word - "(\\s).*" # Check if contain \s ] NOT_ALLOWED_PATTERN = re.compile( # Util.get_regex_combine_or(NOT_ALLOWED), # diff --git a/credsweeper/rules/config.yaml b/credsweeper/rules/config.yaml index fda7112eb..29c266f8e 100644 --- a/credsweeper/rules/config.yaml +++ b/credsweeper/rules/config.yaml @@ -126,33 +126,22 @@ target: - code -- name: IPv4 - severity: info - confidence: weak - type: pattern - values: - - (?[0-2]?[0-9]{1,2}\.[0-2]?[0-9]{1,2}\.[0-2]?[0-9]{1,2}\.[0-2]?[0-9]{1,2})(?![.0-9a-zA-Z]) - filter_type: - - ValueIPCheck - min_line_len: 10 - required_substrings: - - "." - target: - - code - -- name: IPv6 +- name: UUID severity: info confidence: strong type: pattern values: - - (?[0-9A-Fa-f]{0,4}:(:?[0-9A-Fa-f]{1,4}:?){0,6}:[0-9A-Fa-f]{1,4})(?![:0-9A-Za-z]) - filter_type: - - ValueIPCheck - min_line_len: 10 + - (?[0-9A-Fa-f]{8}(-[0-9A-Fa-f]{4}){3}-[0-9A-Fa-f]{12})(?![=0-9A-Za-z_+-]) + min_line_len: 36 required_substrings: - - ":" + - "-" + required_regex: "[0-9A-Za-z_/+-]{15}" + filter_type: + - ValuePatternCheck + use_ml: false target: - code + - doc - name: AWS Client ID severity: high @@ -233,7 +222,9 @@ type: pattern values: - (?EAA[0-9A-Za-z]{80,800}) - filter_type: GeneralPattern + filter_type: + - ValuePatternCheck + - ValueBase64PartCheck required_substrings: - EAA min_line_len: 80 @@ -364,17 +355,18 @@ - name: JSON Web Token severity: medium - confidence: moderate + confidence: strong type: pattern values: - - (?eyJ[0-9A-Za-z_=-]{15,8000}([.0-9A-Za-z_=-]{1,8000})?) - filter_type: GeneralPattern - use_ml: true + - (?eyJ[0-9A-Za-z_+/=-]{15,8000}(\.[0-9A-Za-z_+/=-]{0,8000}){2,16}) + filter_type: + - ValueJsonWebTokenCheck required_substrings: - eyJ min_line_len: 18 target: - code + - doc - name: MailChimp API Key severity: high @@ -673,7 +665,7 @@ confidence: moderate type: pattern values: - - (?P[\"'])?(?P[+0-9A-Za-z-]{2,80}://)([^\s\'"<>\[\]^~`{|}@:/]{0,80}:){1,3}(?P[^\s\'"<>\[\]^~`{|}@:/]{3,80})@[^\s\'"<>\[\]^~`{|}@:/]{1,800}\\{0,8}(?P[\"'])? + - (?P[\"'])?(?P[+0-9A-Za-z-]{2,80}://)([^\s\'"<>\[\]^~`{|}:/]{0,80}:){1,3}(?P[^\s\'"<>\[\]^~`{|}@:/]{3,80})@[^\s\'"<>\[\]^~`{|}@:/]{1,800}\\{0,8}(?P[\"'])? filter_type: UrlCredentialsGroup use_ml: true required_substrings: @@ -867,9 +859,9 @@ confidence: strong type: pattern values: - - (?eyJ[A-Za-z0-9_=-]{50,500}\.eyJ[A-Za-z0-9_=-]{1,8000}\.[A-Za-z0-9_=-]{1,8000}) + - (?eyJ[A-Za-z0-9_=-]{50,500}\.eyJ[A-Za-z0-9_=-]{8,8000}\.[A-Za-z0-9_=-]{18,800}) filter_type: - - ValueJsonWebTokenCheck + - ValueAzureTokenCheck required_substrings: - eyJ min_line_len: 148 diff --git a/credsweeper/scanner/scan_type/scan_type.py b/credsweeper/scanner/scan_type/scan_type.py index 8a95df687..ffcec526c 100644 --- a/credsweeper/scanner/scan_type/scan_type.py +++ b/credsweeper/scanner/scan_type/scan_type.py @@ -164,24 +164,23 @@ def _get_candidates(cls, config: Config, rule: Rule, target: AnalysisTarget) -> if config.exclude_lines and target.line_strip in config.exclude_lines: return candidates - line_data_list = cls.get_line_data_list(config=config, - target=target, - pattern=rule.patterns[0], - filters=rule.filters) - - for line_data in line_data_list: - if config.exclude_values and line_data.value.strip() in config.exclude_values: - continue - - candidate = Candidate([line_data], rule.patterns, rule.rule_name, rule.severity, config, rule.validations, - rule.use_ml, rule.confidence) - # single pattern with multiple values means all the patterns must matched in target - if 1 < len(rule.patterns) and rule.rule_type in (RuleType.PATTERN, RuleType.KEYWORD): - # additional check whether all patterns match - if not cls._aux_scan(config, rule, target, candidate): - # cannot find secondary values for the candidate + if line_data_list := cls.get_line_data_list(config=config, + target=target, + pattern=rule.patterns[0], + filters=rule.filters): + for line_data in line_data_list: + if config.exclude_values and line_data.value.strip() in config.exclude_values: continue - candidates.append(candidate) + + candidate = Candidate([line_data], rule.patterns, rule.rule_name, rule.severity, config, + rule.validations, rule.use_ml, rule.confidence) + # single pattern with multiple values means all the patterns must matched in target + if 1 < len(rule.patterns) and rule.rule_type in (RuleType.PATTERN, RuleType.KEYWORD): + # additional check whether all patterns match + if not cls._aux_scan(config, rule, target, candidate): + # cannot find secondary values for the candidate + continue + candidates.append(candidate) return candidates @classmethod diff --git a/credsweeper/utils/util.py b/credsweeper/utils/util.py index 1fa4d8188..80affdb29 100644 --- a/credsweeper/utils/util.py +++ b/credsweeper/utils/util.py @@ -85,6 +85,8 @@ def get_shannon_entropy(data: str, iterator: str) -> float: 32: 3.25392803184602, 40: 3.64853567064867, 64: 4.57756933688035, + 384: 7.39, + 512: 7.55, } @staticmethod @@ -96,10 +98,13 @@ def get_min_data_entropy(x: int) -> float: # approximated for range 12 - 64 _x = x - 8 y = ((0.000016617804 * _x - 0.002695077) * _x + 0.170393) * _x + 0.4 - elif 64 < x: + elif 64 < x < 384: # logarithm base 2 - slow, but precise _x = x - 8 - y = 1.581026279659 * math.log2(_x) - 1.90156 + y = 1.095884 * math.log2(_x) - 1.90156 + elif 384 < x < 512: + # solved for 384 - 512 + y = -0.11215851 * math.log2(x)**2 + 2.34303484 * math.log2(x) - 4.4466237 else: # less or equal to 8 bytes might have 0 entropy y = 0 diff --git a/docs/source/credsweeper.filters.rst b/docs/source/credsweeper.filters.rst index f2da332f5..102580f39 100644 --- a/docs/source/credsweeper.filters.rst +++ b/docs/source/credsweeper.filters.rst @@ -20,18 +20,18 @@ credsweeper.filters.filter module :undoc-members: :show-inheritance: -credsweeper.filters.line\_specific\_key\_check module ------------------------------------------------------ +credsweeper.filters.line\_git\_binary\_check module +--------------------------------------------------- -.. automodule:: credsweeper.filters.line_specific_key_check +.. automodule:: credsweeper.filters.line_git_binary_check :members: :undoc-members: :show-inheritance: -credsweeper.filters.separator\_unusual\_check module ----------------------------------------------------- +credsweeper.filters.line\_specific\_key\_check module +----------------------------------------------------- -.. automodule:: credsweeper.filters.separator_unusual_check +.. automodule:: credsweeper.filters.line_specific_key_check :members: :undoc-members: :show-inheritance: @@ -60,6 +60,14 @@ credsweeper.filters.value\_atlassian\_token\_check module :undoc-members: :show-inheritance: +credsweeper.filters.value\_azure\_token\_check module +----------------------------------------------------- + +.. automodule:: credsweeper.filters.value_azure_token_check + :members: + :undoc-members: + :show-inheritance: + credsweeper.filters.value\_base32\_data\_check module ----------------------------------------------------- @@ -92,6 +100,14 @@ credsweeper.filters.value\_base64\_key\_check module :undoc-members: :show-inheritance: +credsweeper.filters.value\_base64\_part\_check module +----------------------------------------------------- + +.. automodule:: credsweeper.filters.value_base64_part_check + :members: + :undoc-members: + :show-inheritance: + credsweeper.filters.value\_blocklist\_check module -------------------------------------------------- @@ -132,6 +148,14 @@ credsweeper.filters.value\_dictionary\_value\_length\_check module :undoc-members: :show-inheritance: +credsweeper.filters.value\_discord\_bot\_check module +----------------------------------------------------- + +.. automodule:: credsweeper.filters.value_discord_bot_check + :members: + :undoc-members: + :show-inheritance: + credsweeper.filters.value\_entropy\_base32\_check module -------------------------------------------------------- @@ -188,10 +212,18 @@ credsweeper.filters.value\_grafana\_check module :undoc-members: :show-inheritance: -credsweeper.filters.value\_ip\_check module -------------------------------------------- +credsweeper.filters.value\_grafana\_service\_check module +--------------------------------------------------------- -.. automodule:: credsweeper.filters.value_ip_check +.. automodule:: credsweeper.filters.value_grafana_service_check + :members: + :undoc-members: + :show-inheritance: + +credsweeper.filters.value\_hex\_number\_check module +---------------------------------------------------- + +.. automodule:: credsweeper.filters.value_hex_number_check :members: :undoc-members: :show-inheritance: @@ -220,14 +252,6 @@ credsweeper.filters.value\_last\_word\_check module :undoc-members: :show-inheritance: -credsweeper.filters.value\_length\_check module ------------------------------------------------ - -.. automodule:: credsweeper.filters.value_length_check - :members: - :undoc-members: - :show-inheritance: - credsweeper.filters.value\_method\_check module ----------------------------------------------- @@ -268,14 +292,6 @@ credsweeper.filters.value\_pattern\_check module :undoc-members: :show-inheritance: -credsweeper.filters.value\_pattern\_length\_check module --------------------------------------------------------- - -.. automodule:: credsweeper.filters.value_pattern_length_check - :members: - :undoc-members: - :show-inheritance: - credsweeper.filters.value\_similarity\_check module --------------------------------------------------- diff --git a/tests/__init__.py b/tests/__init__.py index bff4d614a..85a275175 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -1,24 +1,24 @@ from pathlib import Path # total number of files in test samples -SAMPLES_FILES_COUNT: int = 131 +SAMPLES_FILES_COUNT: int = 130 # the lowest value of ML threshold is used to display possible lowest values NEGLIGIBLE_ML_THRESHOLD = 0.0001 # credentials count after scan -SAMPLES_CRED_COUNT: int = 430 -SAMPLES_CRED_LINE_COUNT: int = 447 +SAMPLES_CRED_COUNT: int = 363 +SAMPLES_CRED_LINE_COUNT: int = 380 # credentials count after post-processing -SAMPLES_POST_CRED_COUNT: int = 388 +SAMPLES_POST_CRED_COUNT: int = 322 # with option --doc -SAMPLES_IN_DOC = 411 +SAMPLES_IN_DOC = 416 # archived credentials that are not found without --depth -SAMPLES_IN_DEEP_1 = SAMPLES_POST_CRED_COUNT + 25 -SAMPLES_IN_DEEP_2 = SAMPLES_IN_DEEP_1 + 18 +SAMPLES_IN_DEEP_1 = SAMPLES_POST_CRED_COUNT + 24 +SAMPLES_IN_DEEP_2 = SAMPLES_IN_DEEP_1 + 17 SAMPLES_IN_DEEP_3 = SAMPLES_IN_DEEP_2 + 1 # well known string with all latin letters diff --git a/tests/data/depth_3.json b/tests/data/depth_3.json index 369eeee14..8fd52ab2c 100644 --- a/tests/data/depth_3.json +++ b/tests/data/depth_3.json @@ -394,7 +394,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.941, + "ml_probability": 0.963, "rule": "Auth", "severity": "medium", "confidence": "moderate", @@ -418,33 +418,6 @@ } ] }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.941, - "rule": "JSON Web Token", - "severity": "medium", - "confidence": "moderate", - "line_data_list": [ - { - "line": "curl -H \"Authorization: Bearer eyJGRpVu1c2VzY2-823r_db32hbf4W1lbj\" http://localhost:8080/.", - "line_num": 9, - "path": "tests/samples/auth_n.template", - "info": "tests/samples/auth_n.template|RAW", - "value": "eyJGRpVu1c2VzY2-823r_db32hbf4W1lbj", - "value_start": 31, - "value_end": 65, - "variable": null, - "variable_start": -2, - "variable_end": -2, - "entropy_validation": { - "iterator": "BASE36_CHARS", - "entropy": 3.2479906920322064, - "valid": true - } - } - ] - }, { "api_validation": "NOT_AVAILABLE", "ml_validation": "NOT_AVAILABLE", @@ -805,11 +778,11 @@ }, { "api_validation": "NOT_AVAILABLE", - "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.93, + "ml_validation": "NOT_AVAILABLE", + "ml_probability": null, "rule": "JSON Web Token", "severity": "medium", - "confidence": "moderate", + "confidence": "strong", "line_data_list": [ { "line": "eyJ0eXAiOiJKV1QiLCJhbGciOiJSUzI1NiIsIng1dCI6Imk2bEdrM0ZaenhSY1ViMkMzbkVRN3N5SEpsWSIsImtpZCI6Imk2bEdrM0ZaenhSY1ViMkMzbkVRN3N5SEpsWSJ9.eyJhdWQiOiJlZjFkYTlkNC1mZjc3LTRjM2UtYTAwNS04NDBjM2Y4MzA3NDUiLCJpc3MiOiJodHRwczovL3N0cy53aW5kb3dzLm5ldC9mYTE1ZDY5Mi1lOWM3LTQ0NjAtYTc0My0yOWYyOTUyMjIyOS8iLCJpYXQiOjE1MzcyMzMxMDYsIm5iZiI6MTUzNzIzMzEwNiwiZXhwIjoxNTM3MjM3MDA2LCJhY3IiOiIxIiwiYWlvIjoiQVhRQWkvOElBQUFBRm0rRS9RVEcrZ0ZuVnhMaldkdzhLKzYxQUdyU091TU1GNmViYU1qN1hPM0libUQzZkdtck95RCtOdlp5R24yVmFUL2tES1h3NE1JaHJnR1ZxNkJuOHdMWG9UMUxrSVorRnpRVmtKUFBMUU9WNEtjWHFTbENWUERTL0RpQ0RnRTIyMlRJbU12V05hRU1hVU9Uc0lHdlRRPT0iLCJhbXIiOlsid2lhIl0sImFwcGlkIjoiNzVkYmU3N2YtMTBhMy00ZTU5LTg1ZmQtOGMxMjc1NDRmMTdjIiwiYXBwaWRhY3IiOiIwIiwiZW1haWwiOiJBYmVMaUBtaWNyb3NvZnQuY29tIiwiZmFtaWx5X25hbWUiOiJMaW5jb2xuIiwiZ2l2ZW5fbmFtZSI6IkFiZSAoTVNGVCkiLCJpZHAiOiJodHRwczovL3N0cy53aW5kb3dzLm5ldC83MmY5ODhiZi04NmYxLTQxYWYtOTFhYi0yZDdjZDAxMjIyNDcvIiwiaXBhZGRyIjoiMjIyLjIyMi4yMjIuMjIiLCJuYW1lIjoiYWJlbGkiLCJvaWQiOiIwMjIyM2I2Yi1hYTFkLTQyZDQtOWVjMC0xYjJiYjkxOTQ0MzgiLCJyaCI6IkkiLCJzY3AiOiJ1c2VyX2ltcGVyc29uYXRpb24iLCJzdWIiOiJsM19yb0lTUVUyMjJiVUxTOXlpMmswWHBxcE9pTXo1SDNaQUNvMUdlWEEiLCJ0aWQiOiJmYTE1ZDY5Mi1lOWM3LTQ0NjAtYTc0My0yOWYyOTU2ZmQ0MjkiLCJ1bmlxdWVfbmFtZSI6ImFiZWxpQG1pY3Jvc29mdC5jb20iLCJ1dGkiOiJGVnNHeFlYSTMwLVR1aWt1dVVvRkFBIiwidmVyIjoiMS4wIn0.D3H6pMUtQnoJAGq6AHd", @@ -830,6 +803,60 @@ } ] }, + { + "api_validation": "NOT_AVAILABLE", + "ml_validation": "NOT_AVAILABLE", + "ml_probability": null, + "rule": "Azure Access Token", + "severity": "high", + "confidence": "strong", + "line_data_list": [ + { + "line": "eyJ0eXAiOiJKV1QiLCJhbGciOiJSUzI1NiIsImtpZCI6Imk2bEdrM0ZaenhSY1ViMkMzbkVRN3N5SEpsWSJ9.eyJhdWQiOiI2ZTc0MTcyYi1iZTU2LTQ4NDMtOWZmNC1lNjZhMzliYjEyZTMiLCJpc3MiOiJodHRwczovL2xvZ2luLm1pY3Jvc29mdG9ubGluZS5jb20vNzJmOTg4YmYtODZmMS00MWFmLTkxYWItMmQ3Y2QwMTFkYjQ3L3YyLjAiLCJpYXQiOjE1MzcyMzEwNDgsIm5iZiI6MTUzNzIzMTA0OCwiZXhwIjoxNTM3MjM0OTQ4LCJhaW8iOiJBWFFBaS84SUFBQUF0QWFaTG8zQ2hNaWY2S09udHRSQjdlQnE0L0RjY1F6amNKR3hQWXkvQzNqRGFOR3hYZDZ3TklJVkdSZ2hOUm53SjFsT2NBbk5aY2p2a295ckZ4Q3R0djMzMTQwUmlvT0ZKNGJDQ0dWdW9DYWcxdU9UVDIyMjIyZ0h3TFBZUS91Zjc5UVgrMEtJaWpkcm1wNjlSY3R6bVE9PSIsImF6cCI6IjZlNzQxNzJiLWJlNTYtNDg0My05ZmY0LWU2NmEzOWJiMTJlMyIsImF6cGFjciI6IjAiLCJuYW1lIjoiQWJlIExpbmNvbG4iLCJvaWQiOiI2OTAyMjJiZS1mZjFhLTRkNTYtYWJkMS03ZTRmN2QzOGU0NzQiLCJwcmVmZXJyZWRfdXNlcm5hbWUiOiJhYmVsaUBtaWNyb3NvZnQuY29tIiwicmgiOiJJIiwic2NwIjoiYWNjZXNzX2FzX3VzZXIiLCJzdWIiOiJIS1pwZmFIeVdhZGVPb3VZbGl0anJJLUtmZlRtMjIyWDVyclYzeERxZktRIiwidGlkIjoiNzJmOTg4YmYtODZmMS00MWFmLTkxYWItMmQ3Y2QwMTFkYjQ3IiwidXRpIjoiZnFpQnFYTFBqMGVRYTgyUy1JWUZBQSIsInZlciI6IjIuMCJ9.pj4N-w_3Us9DrBLfpCt", + "line_num": 2, + "path": "tests/samples/azure_access_token", + "info": "tests/samples/azure_access_token|RAW", + "value": "eyJ0eXAiOiJKV1QiLCJhbGciOiJSUzI1NiIsImtpZCI6Imk2bEdrM0ZaenhSY1ViMkMzbkVRN3N5SEpsWSJ9.eyJhdWQiOiI2ZTc0MTcyYi1iZTU2LTQ4NDMtOWZmNC1lNjZhMzliYjEyZTMiLCJpc3MiOiJodHRwczovL2xvZ2luLm1pY3Jvc29mdG9ubGluZS5jb20vNzJmOTg4YmYtODZmMS00MWFmLTkxYWItMmQ3Y2QwMTFkYjQ3L3YyLjAiLCJpYXQiOjE1MzcyMzEwNDgsIm5iZiI6MTUzNzIzMTA0OCwiZXhwIjoxNTM3MjM0OTQ4LCJhaW8iOiJBWFFBaS84SUFBQUF0QWFaTG8zQ2hNaWY2S09udHRSQjdlQnE0L0RjY1F6amNKR3hQWXkvQzNqRGFOR3hYZDZ3TklJVkdSZ2hOUm53SjFsT2NBbk5aY2p2a295ckZ4Q3R0djMzMTQwUmlvT0ZKNGJDQ0dWdW9DYWcxdU9UVDIyMjIyZ0h3TFBZUS91Zjc5UVgrMEtJaWpkcm1wNjlSY3R6bVE9PSIsImF6cCI6IjZlNzQxNzJiLWJlNTYtNDg0My05ZmY0LWU2NmEzOWJiMTJlMyIsImF6cGFjciI6IjAiLCJuYW1lIjoiQWJlIExpbmNvbG4iLCJvaWQiOiI2OTAyMjJiZS1mZjFhLTRkNTYtYWJkMS03ZTRmN2QzOGU0NzQiLCJwcmVmZXJyZWRfdXNlcm5hbWUiOiJhYmVsaUBtaWNyb3NvZnQuY29tIiwicmgiOiJJIiwic2NwIjoiYWNjZXNzX2FzX3VzZXIiLCJzdWIiOiJIS1pwZmFIeVdhZGVPb3VZbGl0anJJLUtmZlRtMjIyWDVyclYzeERxZktRIiwidGlkIjoiNzJmOTg4YmYtODZmMS00MWFmLTkxYWItMmQ3Y2QwMTFkYjQ3IiwidXRpIjoiZnFpQnFYTFBqMGVRYTgyUy1JWUZBQSIsInZlciI6IjIuMCJ9.pj4N-w_3Us9DrBLfpCt", + "value_start": 0, + "value_end": 1029, + "variable": null, + "variable_start": -2, + "variable_end": -2, + "entropy_validation": { + "iterator": "BASE64_CHARS", + "entropy": 5.6044494049575055, + "valid": true + } + } + ] + }, + { + "api_validation": "NOT_AVAILABLE", + "ml_validation": "NOT_AVAILABLE", + "ml_probability": null, + "rule": "JSON Web Token", + "severity": "medium", + "confidence": "strong", + "line_data_list": [ + { + "line": "eyJ0eXAiOiJKV1QiLCJhbGciOiJSUzI1NiIsImtpZCI6Imk2bEdrM0ZaenhSY1ViMkMzbkVRN3N5SEpsWSJ9.eyJhdWQiOiI2ZTc0MTcyYi1iZTU2LTQ4NDMtOWZmNC1lNjZhMzliYjEyZTMiLCJpc3MiOiJodHRwczovL2xvZ2luLm1pY3Jvc29mdG9ubGluZS5jb20vNzJmOTg4YmYtODZmMS00MWFmLTkxYWItMmQ3Y2QwMTFkYjQ3L3YyLjAiLCJpYXQiOjE1MzcyMzEwNDgsIm5iZiI6MTUzNzIzMTA0OCwiZXhwIjoxNTM3MjM0OTQ4LCJhaW8iOiJBWFFBaS84SUFBQUF0QWFaTG8zQ2hNaWY2S09udHRSQjdlQnE0L0RjY1F6amNKR3hQWXkvQzNqRGFOR3hYZDZ3TklJVkdSZ2hOUm53SjFsT2NBbk5aY2p2a295ckZ4Q3R0djMzMTQwUmlvT0ZKNGJDQ0dWdW9DYWcxdU9UVDIyMjIyZ0h3TFBZUS91Zjc5UVgrMEtJaWpkcm1wNjlSY3R6bVE9PSIsImF6cCI6IjZlNzQxNzJiLWJlNTYtNDg0My05ZmY0LWU2NmEzOWJiMTJlMyIsImF6cGFjciI6IjAiLCJuYW1lIjoiQWJlIExpbmNvbG4iLCJvaWQiOiI2OTAyMjJiZS1mZjFhLTRkNTYtYWJkMS03ZTRmN2QzOGU0NzQiLCJwcmVmZXJyZWRfdXNlcm5hbWUiOiJhYmVsaUBtaWNyb3NvZnQuY29tIiwicmgiOiJJIiwic2NwIjoiYWNjZXNzX2FzX3VzZXIiLCJzdWIiOiJIS1pwZmFIeVdhZGVPb3VZbGl0anJJLUtmZlRtMjIyWDVyclYzeERxZktRIiwidGlkIjoiNzJmOTg4YmYtODZmMS00MWFmLTkxYWItMmQ3Y2QwMTFkYjQ3IiwidXRpIjoiZnFpQnFYTFBqMGVRYTgyUy1JWUZBQSIsInZlciI6IjIuMCJ9.pj4N-w_3Us9DrBLfpCt", + "line_num": 2, + "path": "tests/samples/azure_access_token", + "info": "tests/samples/azure_access_token|RAW", + "value": "eyJ0eXAiOiJKV1QiLCJhbGciOiJSUzI1NiIsImtpZCI6Imk2bEdrM0ZaenhSY1ViMkMzbkVRN3N5SEpsWSJ9.eyJhdWQiOiI2ZTc0MTcyYi1iZTU2LTQ4NDMtOWZmNC1lNjZhMzliYjEyZTMiLCJpc3MiOiJodHRwczovL2xvZ2luLm1pY3Jvc29mdG9ubGluZS5jb20vNzJmOTg4YmYtODZmMS00MWFmLTkxYWItMmQ3Y2QwMTFkYjQ3L3YyLjAiLCJpYXQiOjE1MzcyMzEwNDgsIm5iZiI6MTUzNzIzMTA0OCwiZXhwIjoxNTM3MjM0OTQ4LCJhaW8iOiJBWFFBaS84SUFBQUF0QWFaTG8zQ2hNaWY2S09udHRSQjdlQnE0L0RjY1F6amNKR3hQWXkvQzNqRGFOR3hYZDZ3TklJVkdSZ2hOUm53SjFsT2NBbk5aY2p2a295ckZ4Q3R0djMzMTQwUmlvT0ZKNGJDQ0dWdW9DYWcxdU9UVDIyMjIyZ0h3TFBZUS91Zjc5UVgrMEtJaWpkcm1wNjlSY3R6bVE9PSIsImF6cCI6IjZlNzQxNzJiLWJlNTYtNDg0My05ZmY0LWU2NmEzOWJiMTJlMyIsImF6cGFjciI6IjAiLCJuYW1lIjoiQWJlIExpbmNvbG4iLCJvaWQiOiI2OTAyMjJiZS1mZjFhLTRkNTYtYWJkMS03ZTRmN2QzOGU0NzQiLCJwcmVmZXJyZWRfdXNlcm5hbWUiOiJhYmVsaUBtaWNyb3NvZnQuY29tIiwicmgiOiJJIiwic2NwIjoiYWNjZXNzX2FzX3VzZXIiLCJzdWIiOiJIS1pwZmFIeVdhZGVPb3VZbGl0anJJLUtmZlRtMjIyWDVyclYzeERxZktRIiwidGlkIjoiNzJmOTg4YmYtODZmMS00MWFmLTkxYWItMmQ3Y2QwMTFkYjQ3IiwidXRpIjoiZnFpQnFYTFBqMGVRYTgyUy1JWUZBQSIsInZlciI6IjIuMCJ9.pj4N-w_3Us9DrBLfpCt", + "value_start": 0, + "value_end": 1029, + "variable": null, + "variable_start": -2, + "variable_end": -2, + "entropy_validation": { + "iterator": "BASE64_CHARS", + "entropy": 5.6044494049575055, + "valid": true + } + } + ] + }, { "api_validation": "NOT_AVAILABLE", "ml_validation": "NOT_AVAILABLE", @@ -2369,33 +2396,6 @@ } ] }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "IPv4", - "severity": "info", - "confidence": "weak", - "line_data_list": [ - { - "line": "(98.76.54.32)ID:master PW:dipPr149Gg!", - "line_num": 50, - "path": "tests/samples/doc_id_pair_passwd_pair", - "info": "tests/samples/doc_id_pair_passwd_pair|RAW", - "value": "98.76.54.32", - "value_start": 1, - "value_end": 12, - "variable": null, - "variable_start": -2, - "variable_end": -2, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 2.515950268099853, - "valid": false - } - } - ] - }, { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", @@ -3179,33 +3179,6 @@ } ] }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "IPv4", - "severity": "info", - "confidence": "weak", - "line_data_list": [ - { - "line": "ANYid:master pw:dipPr194Gg! ip:98.76.54.32", - "line_num": 95, - "path": "tests/samples/doc_id_pair_passwd_pair", - "info": "tests/samples/doc_id_pair_passwd_pair|RAW", - "value": "98.76.54.32", - "value_start": 31, - "value_end": 42, - "variable": null, - "variable_start": -2, - "variable_end": -2, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 2.515950268099853, - "valid": false - } - } - ] - }, { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", @@ -3233,33 +3206,6 @@ } ] }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "IPv4", - "severity": "info", - "confidence": "weak", - "line_data_list": [ - { - "line": "master@98.76.54.32 mailto:{1} (password-dipPr196Gg!) # skip", - "line_num": 97, - "path": "tests/samples/doc_id_pair_passwd_pair", - "info": "tests/samples/doc_id_pair_passwd_pair|RAW", - "value": "98.76.54.32", - "value_start": 7, - "value_end": 18, - "variable": null, - "variable_start": -2, - "variable_end": -2, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 2.515950268099853, - "valid": false - } - } - ] - }, { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", @@ -3341,60 +3287,6 @@ } ] }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "IPv4", - "severity": "info", - "confidence": "weak", - "line_data_list": [ - { - "line": "98.76.54.32(ID:master/PW:iPp10@GRq) # todo: move into other sample ?", - "line_num": 11, - "path": "tests/samples/doc_id_passwd_pair", - "info": "tests/samples/doc_id_passwd_pair|RAW", - "value": "98.76.54.32", - "value_start": 0, - "value_end": 11, - "variable": null, - "variable_start": -2, - "variable_end": -2, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 2.515950268099853, - "valid": false - } - } - ] - }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "IPv4", - "severity": "info", - "confidence": "weak", - "line_data_list": [ - { - "line": "98.76.54.32 id/pw:master/iPp19@GRq", - "line_num": 20, - "path": "tests/samples/doc_id_passwd_pair", - "info": "tests/samples/doc_id_passwd_pair|RAW", - "value": "98.76.54.32", - "value_start": 0, - "value_end": 11, - "variable": null, - "variable_start": -2, - "variable_end": -2, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 2.515950268099853, - "valid": false - } - } - ] - }, { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", @@ -3638,33 +3530,6 @@ } ] }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "IPv4", - "severity": "info", - "confidence": "weak", - "line_data_list": [ - { - "line": "master@98.76.54.32 password:Prl23Db#@", - "line_num": 14, - "path": "tests/samples/doc_passwd_pair", - "info": "tests/samples/doc_passwd_pair|RAW", - "value": "98.76.54.32", - "value_start": 7, - "value_end": 18, - "variable": null, - "variable_start": -2, - "variable_end": -2, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 2.515950268099853, - "valid": false - } - } - ] - }, { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", @@ -4855,26 +4720,26 @@ }, { "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "IPv4", - "severity": "info", - "confidence": "weak", + "ml_validation": "VALIDATED_KEY", + "ml_probability": 0.956, + "rule": "Password", + "severity": "medium", + "confidence": "moderate", "line_data_list": [ { - "line": "98.76.54.32 (master/IhqSb1Gg)", - "line_num": 1, + "line": "master@98.76.54.32(pw:IhqSb1Gg)", + "line_num": 3, "path": "tests/samples/doc_various", "info": "tests/samples/doc_various|RAW", - "value": "98.76.54.32", - "value_start": 0, - "value_end": 11, - "variable": null, - "variable_start": -2, - "variable_end": -2, + "value": "IhqSb1Gg)", + "value_start": 22, + "value_end": 31, + "variable": "pw", + "variable_start": 19, + "variable_end": 21, "entropy_validation": { "iterator": "BASE64_CHARS", - "entropy": 2.515950268099853, + "entropy": 2.8177111123931664, "valid": false } } @@ -4882,96 +4747,15 @@ }, { "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "IPv4", - "severity": "info", - "confidence": "weak", + "ml_validation": "VALIDATED_KEY", + "ml_probability": 0.896, + "rule": "Password", + "severity": "medium", + "confidence": "moderate", "line_data_list": [ { - "line": "98.76.54.32(master/IhqSb1Gg)", - "line_num": 2, - "path": "tests/samples/doc_various", - "info": "tests/samples/doc_various|RAW", - "value": "98.76.54.32", - "value_start": 0, - "value_end": 11, - "variable": null, - "variable_start": -2, - "variable_end": -2, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 2.515950268099853, - "valid": false - } - } - ] - }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "IPv4", - "severity": "info", - "confidence": "weak", - "line_data_list": [ - { - "line": "master@98.76.54.32(pw:IhqSb1Gg)", - "line_num": 3, - "path": "tests/samples/doc_various", - "info": "tests/samples/doc_various|RAW", - "value": "98.76.54.32", - "value_start": 7, - "value_end": 18, - "variable": null, - "variable_start": -2, - "variable_end": -2, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 2.515950268099853, - "valid": false - } - } - ] - }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.956, - "rule": "Password", - "severity": "medium", - "confidence": "moderate", - "line_data_list": [ - { - "line": "master@98.76.54.32(pw:IhqSb1Gg)", - "line_num": 3, - "path": "tests/samples/doc_various", - "info": "tests/samples/doc_various|RAW", - "value": "IhqSb1Gg)", - "value_start": 22, - "value_end": 31, - "variable": "pw", - "variable_start": 19, - "variable_end": 21, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 2.8177111123931664, - "valid": false - } - } - ] - }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.896, - "rule": "Password", - "severity": "medium", - "confidence": "moderate", - "line_data_list": [ - { - "line": "ID:gildong.hong@example.com mailto:{1} PW:IhqSb1Gg", - "line_num": 4, + "line": "ID:gildong.hong@example.com mailto:{1} PW:IhqSb1Gg", + "line_num": 4, "path": "tests/samples/doc_various", "info": "tests/samples/doc_various|RAW", "value": "IhqSb1Gg", @@ -5015,33 +4799,6 @@ } ] }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "IPv4", - "severity": "info", - "confidence": "weak", - "line_data_list": [ - { - "line": "ssh gildong.hong@98.76.54.32 mailto:{1} (PW:IhqSb1Gg)", - "line_num": 6, - "path": "tests/samples/doc_various", - "info": "tests/samples/doc_various|RAW", - "value": "98.76.54.32", - "value_start": 17, - "value_end": 28, - "variable": null, - "variable_start": -2, - "variable_end": -2, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 2.515950268099853, - "valid": false - } - } - ] - }, { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", @@ -5069,33 +4826,6 @@ } ] }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "IPv4", - "severity": "info", - "confidence": "weak", - "line_data_list": [ - { - "line": "ssh gildong.hong@98.76.54.32 mailto:{1} password:IhqSb1Gg", - "line_num": 7, - "path": "tests/samples/doc_various", - "info": "tests/samples/doc_various|RAW", - "value": "98.76.54.32", - "value_start": 17, - "value_end": 28, - "variable": null, - "variable_start": -2, - "variable_end": -2, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 2.515950268099853, - "valid": false - } - } - ] - }, { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", @@ -5123,33 +4853,6 @@ } ] }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "IPv4", - "severity": "info", - "confidence": "weak", - "line_data_list": [ - { - "line": "ssh -P IhqSb1Gg gildong.hong@98.76.54.32 mailto:{1} (password:IhqSb1Gg)", - "line_num": 11, - "path": "tests/samples/doc_various", - "info": "tests/samples/doc_various|RAW", - "value": "98.76.54.32", - "value_start": 29, - "value_end": 40, - "variable": null, - "variable_start": -2, - "variable_end": -2, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 2.515950268099853, - "valid": false - } - } - ] - }, { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", @@ -5177,60 +4880,6 @@ } ] }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "IPv4", - "severity": "info", - "confidence": "weak", - "line_data_list": [ - { - "line": "98.76.54.32 xxxx (master/IhqSb1Gg)", - "line_num": 14, - "path": "tests/samples/doc_various", - "info": "tests/samples/doc_various|RAW", - "value": "98.76.54.32", - "value_start": 0, - "value_end": 11, - "variable": null, - "variable_start": -2, - "variable_end": -2, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 2.515950268099853, - "valid": false - } - } - ] - }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "IPv4", - "severity": "info", - "confidence": "weak", - "line_data_list": [ - { - "line": "gildong.hong@98.76.54.32 pwd:IhqSb1Gg", - "line_num": 15, - "path": "tests/samples/doc_various", - "info": "tests/samples/doc_various|RAW", - "value": "98.76.54.32", - "value_start": 13, - "value_end": 24, - "variable": null, - "variable_start": -2, - "variable_end": -2, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 2.515950268099853, - "valid": false - } - } - ] - }, { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", @@ -5285,33 +4934,6 @@ } ] }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "IPv4", - "severity": "info", - "confidence": "weak", - "line_data_list": [ - { - "line": "98.76.54.32(pw:IhqSb1Gg)", - "line_num": 17, - "path": "tests/samples/doc_various", - "info": "tests/samples/doc_various|RAW", - "value": "98.76.54.32", - "value_start": 0, - "value_end": 11, - "variable": null, - "variable_start": -2, - "variable_end": -2, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 2.515950268099853, - "valid": false - } - } - ] - }, { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", @@ -5339,33 +4961,6 @@ } ] }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "IPv4", - "severity": "info", - "confidence": "weak", - "line_data_list": [ - { - "line": "98.76.54.32/pw:IhqSb1Gg", - "line_num": 19, - "path": "tests/samples/doc_various", - "info": "tests/samples/doc_various|RAW", - "value": "98.76.54.32", - "value_start": 0, - "value_end": 11, - "variable": null, - "variable_start": -2, - "variable_end": -2, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 2.515950268099853, - "valid": false - } - } - ] - }, { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", @@ -5449,50 +5044,23 @@ }, { "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "IPv4", - "severity": "info", - "confidence": "weak", + "ml_validation": "VALIDATED_KEY", + "ml_probability": 0.982, + "rule": "Password", + "severity": "medium", + "confidence": "moderate", "line_data_list": [ { "line": "sftp gildong.hong@98.76.54.32 mailto:{1} (pw:IhqSb1Gg)", "line_num": 22, "path": "tests/samples/doc_various", "info": "tests/samples/doc_various|RAW", - "value": "98.76.54.32", - "value_start": 18, - "value_end": 29, - "variable": null, - "variable_start": -2, - "variable_end": -2, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 2.515950268099853, - "valid": false - } - } - ] - }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.982, - "rule": "Password", - "severity": "medium", - "confidence": "moderate", - "line_data_list": [ - { - "line": "sftp gildong.hong@98.76.54.32 mailto:{1} (pw:IhqSb1Gg)", - "line_num": 22, - "path": "tests/samples/doc_various", - "info": "tests/samples/doc_various|RAW", - "value": "IhqSb1Gg)", - "value_start": 45, - "value_end": 54, - "variable": "pw", - "variable_start": 42, - "variable_end": 44, + "value": "IhqSb1Gg)", + "value_start": 45, + "value_end": 54, + "variable": "pw", + "variable_start": 42, + "variable_end": 44, "entropy_validation": { "iterator": "BASE64_CHARS", "entropy": 2.8177111123931664, @@ -5501,33 +5069,6 @@ } ] }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "IPv4", - "severity": "info", - "confidence": "weak", - "line_data_list": [ - { - "line": "gildong.hong@98.76.54.32 mailto:{1} (pw:IhqSb1Gg)", - "line_num": 23, - "path": "tests/samples/doc_various", - "info": "tests/samples/doc_various|RAW", - "value": "98.76.54.32", - "value_start": 13, - "value_end": 24, - "variable": null, - "variable_start": -2, - "variable_end": -2, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 2.515950268099853, - "valid": false - } - } - ] - }, { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", @@ -5582,33 +5123,6 @@ } ] }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "IPv4", - "severity": "info", - "confidence": "weak", - "line_data_list": [ - { - "line": "ssh gildong.hong@98.76.54.32 mailto:{1} (password:IhqSb1Gg)", - "line_num": 25, - "path": "tests/samples/doc_various", - "info": "tests/samples/doc_various|RAW", - "value": "98.76.54.32", - "value_start": 17, - "value_end": 28, - "variable": null, - "variable_start": -2, - "variable_end": -2, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 2.515950268099853, - "valid": false - } - } - ] - }, { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", @@ -5636,33 +5150,6 @@ } ] }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "IPv4", - "severity": "info", - "confidence": "weak", - "line_data_list": [ - { - "line": "ssh gildong.hong@98.76.54.32 mailto:{1} master/IhqSb1Gg", - "line_num": 26, - "path": "tests/samples/doc_various", - "info": "tests/samples/doc_various|RAW", - "value": "98.76.54.32", - "value_start": 17, - "value_end": 28, - "variable": null, - "variable_start": -2, - "variable_end": -2, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 2.515950268099853, - "valid": false - } - } - ] - }, { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", @@ -5771,33 +5258,6 @@ } ] }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "IPv4", - "severity": "info", - "confidence": "weak", - "line_data_list": [ - { - "line": "98.76.54.32 ANY_PW:IhqSb1Gg", - "line_num": 34, - "path": "tests/samples/doc_various", - "info": "tests/samples/doc_various|RAW", - "value": "98.76.54.32", - "value_start": 0, - "value_end": 11, - "variable": null, - "variable_start": -2, - "variable_end": -2, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 2.515950268099853, - "valid": false - } - } - ] - }, { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", @@ -5825,33 +5285,6 @@ } ] }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "IPv4", - "severity": "info", - "confidence": "weak", - "line_data_list": [ - { - "line": "98.76.54.32(ID/PW:IhqSb1Gg)", - "line_num": 36, - "path": "tests/samples/doc_various", - "info": "tests/samples/doc_various|RAW", - "value": "98.76.54.32", - "value_start": 0, - "value_end": 11, - "variable": null, - "variable_start": -2, - "variable_end": -2, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 2.515950268099853, - "valid": false - } - } - ] - }, { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", @@ -5879,33 +5312,6 @@ } ] }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "IPv4", - "severity": "info", - "confidence": "weak", - "line_data_list": [ - { - "line": "gildong.hong@98.76.54.32 (pwd:IhqSb1Gg)", - "line_num": 38, - "path": "tests/samples/doc_various", - "info": "tests/samples/doc_various|RAW", - "value": "98.76.54.32", - "value_start": 13, - "value_end": 24, - "variable": null, - "variable_start": -2, - "variable_end": -2, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 2.515950268099853, - "valid": false - } - } - ] - }, { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", @@ -5960,33 +5366,6 @@ } ] }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "IPv4", - "severity": "info", - "confidence": "weak", - "line_data_list": [ - { - "line": "gildong.hong@98.76.54.32(master/IhqSb1Gg)", - "line_num": 44, - "path": "tests/samples/doc_various", - "info": "tests/samples/doc_various|RAW", - "value": "98.76.54.32", - "value_start": 13, - "value_end": 24, - "variable": null, - "variable_start": -2, - "variable_end": -2, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 2.515950268099853, - "valid": false - } - } - ] - }, { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", @@ -6016,26 +5395,26 @@ }, { "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "IPv4", - "severity": "info", - "confidence": "weak", + "ml_validation": "VALIDATED_KEY", + "ml_probability": 0.896, + "rule": "Password", + "severity": "medium", + "confidence": "moderate", "line_data_list": [ { - "line": "gildong.hong@98.76.54.32,pw:IhqSb1Gg", - "line_num": 47, + "line": "98.76.54.32:xxxx(PW:IhqSb1Gg)", + "line_num": 51, "path": "tests/samples/doc_various", "info": "tests/samples/doc_various|RAW", - "value": "98.76.54.32", - "value_start": 13, - "value_end": 24, - "variable": null, - "variable_start": -2, - "variable_end": -2, + "value": "IhqSb1Gg)", + "value_start": 20, + "value_end": 29, + "variable": "PW", + "variable_start": 17, + "variable_end": 19, "entropy_validation": { "iterator": "BASE64_CHARS", - "entropy": 2.515950268099853, + "entropy": 2.8177111123931664, "valid": false } } @@ -6043,26 +5422,26 @@ }, { "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "IPv4", - "severity": "info", - "confidence": "weak", + "ml_validation": "VALIDATED_KEY", + "ml_probability": 0.949, + "rule": "Password", + "severity": "medium", + "confidence": "moderate", "line_data_list": [ { - "line": "98.76.54.32(master/IhqSb1Gg,master/IhqSb1Gg)", - "line_num": 48, + "line": "gildong.hong@98.76.54.32 PW:IhqSb1Gg", + "line_num": 56, "path": "tests/samples/doc_various", "info": "tests/samples/doc_various|RAW", - "value": "98.76.54.32", - "value_start": 0, - "value_end": 11, - "variable": null, - "variable_start": -2, - "variable_end": -2, + "value": "IhqSb1Gg", + "value_start": 28, + "value_end": 36, + "variable": "PW", + "variable_start": 25, + "variable_end": 27, "entropy_validation": { "iterator": "BASE64_CHARS", - "entropy": 2.515950268099853, + "entropy": 3.0, "valid": false } } @@ -6070,204 +5449,15 @@ }, { "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "IPv4", - "severity": "info", - "confidence": "weak", + "ml_validation": "VALIDATED_KEY", + "ml_probability": 0.999, + "rule": "Password", + "severity": "medium", + "confidence": "moderate", "line_data_list": [ { - "line": "98.76.54.32(master/IhqSb1Gg master/IhqSb1Gg)", - "line_num": 49, - "path": "tests/samples/doc_various", - "info": "tests/samples/doc_various|RAW", - "value": "98.76.54.32", - "value_start": 0, - "value_end": 11, - "variable": null, - "variable_start": -2, - "variable_end": -2, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 2.515950268099853, - "valid": false - } - } - ] - }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "IPv4", - "severity": "info", - "confidence": "weak", - "line_data_list": [ - { - "line": "xxxx:98.76.54.32(master/IhqSb1Gg)", - "line_num": 50, - "path": "tests/samples/doc_various", - "info": "tests/samples/doc_various|RAW", - "value": "98.76.54.32", - "value_start": 5, - "value_end": 16, - "variable": null, - "variable_start": -2, - "variable_end": -2, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 2.515950268099853, - "valid": false - } - } - ] - }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "IPv4", - "severity": "info", - "confidence": "weak", - "line_data_list": [ - { - "line": "98.76.54.32:xxxx(PW:IhqSb1Gg)", - "line_num": 51, - "path": "tests/samples/doc_various", - "info": "tests/samples/doc_various|RAW", - "value": "98.76.54.32", - "value_start": 0, - "value_end": 11, - "variable": null, - "variable_start": -2, - "variable_end": -2, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 2.515950268099853, - "valid": false - } - } - ] - }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.896, - "rule": "Password", - "severity": "medium", - "confidence": "moderate", - "line_data_list": [ - { - "line": "98.76.54.32:xxxx(PW:IhqSb1Gg)", - "line_num": 51, - "path": "tests/samples/doc_various", - "info": "tests/samples/doc_various|RAW", - "value": "IhqSb1Gg)", - "value_start": 20, - "value_end": 29, - "variable": "PW", - "variable_start": 17, - "variable_end": 19, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 2.8177111123931664, - "valid": false - } - } - ] - }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "IPv4", - "severity": "info", - "confidence": "weak", - "line_data_list": [ - { - "line": "gildong.hong@98.76.54.32 PW:IhqSb1Gg", - "line_num": 56, - "path": "tests/samples/doc_various", - "info": "tests/samples/doc_various|RAW", - "value": "98.76.54.32", - "value_start": 13, - "value_end": 24, - "variable": null, - "variable_start": -2, - "variable_end": -2, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 2.515950268099853, - "valid": false - } - } - ] - }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.949, - "rule": "Password", - "severity": "medium", - "confidence": "moderate", - "line_data_list": [ - { - "line": "gildong.hong@98.76.54.32 PW:IhqSb1Gg", - "line_num": 56, - "path": "tests/samples/doc_various", - "info": "tests/samples/doc_various|RAW", - "value": "IhqSb1Gg", - "value_start": 28, - "value_end": 36, - "variable": "PW", - "variable_start": 25, - "variable_end": 27, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 3.0, - "valid": false - } - } - ] - }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "IPv4", - "severity": "info", - "confidence": "weak", - "line_data_list": [ - { - "line": "http|https://98.76.54.32/xxxx(master/IhqSb1Gg)", - "line_num": 59, - "path": "tests/samples/doc_various", - "info": "tests/samples/doc_various|RAW", - "value": "98.76.54.32", - "value_start": 13, - "value_end": 24, - "variable": null, - "variable_start": -2, - "variable_end": -2, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 2.515950268099853, - "valid": false - } - } - ] - }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.999, - "rule": "Password", - "severity": "medium", - "confidence": "moderate", - "line_data_list": [ - { - "line": "ANY_user:xxxx ANY_pwd:IhqSb1Gg", - "line_num": 61, + "line": "ANY_user:xxxx ANY_pwd:IhqSb1Gg", + "line_num": 61, "path": "tests/samples/doc_various", "info": "tests/samples/doc_various|RAW", "value": "IhqSb1Gg", @@ -6392,60 +5582,6 @@ } ] }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "IPv4", - "severity": "info", - "confidence": "weak", - "line_data_list": [ - { - "line": "98.76.54.32(ID/PW:master/IhqSb1Gg)", - "line_num": 70, - "path": "tests/samples/doc_various", - "info": "tests/samples/doc_various|RAW", - "value": "98.76.54.32", - "value_start": 0, - "value_end": 11, - "variable": null, - "variable_start": -2, - "variable_end": -2, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 2.515950268099853, - "valid": false - } - } - ] - }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "IPv4", - "severity": "info", - "confidence": "weak", - "line_data_list": [ - { - "line": "master@98.76.54.32(pw:IhqSb1Gg)", - "line_num": 73, - "path": "tests/samples/doc_various", - "info": "tests/samples/doc_various|RAW", - "value": "98.76.54.32", - "value_start": 7, - "value_end": 18, - "variable": null, - "variable_start": -2, - "variable_end": -2, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 2.515950268099853, - "valid": false - } - } - ] - }, { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", @@ -6473,60 +5609,6 @@ } ] }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "IPv4", - "severity": "info", - "confidence": "weak", - "line_data_list": [ - { - "line": "master@98.76.54.32,PW:IhqSb1Gg", - "line_num": 74, - "path": "tests/samples/doc_various", - "info": "tests/samples/doc_various|RAW", - "value": "98.76.54.32", - "value_start": 7, - "value_end": 18, - "variable": null, - "variable_start": -2, - "variable_end": -2, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 2.515950268099853, - "valid": false - } - } - ] - }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "IPv4", - "severity": "info", - "confidence": "weak", - "line_data_list": [ - { - "line": "98.76.54.32 pw:IhqSb1Gg", - "line_num": 75, - "path": "tests/samples/doc_various", - "info": "tests/samples/doc_various|RAW", - "value": "98.76.54.32", - "value_start": 0, - "value_end": 11, - "variable": null, - "variable_start": -2, - "variable_end": -2, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 2.515950268099853, - "valid": false - } - } - ] - }, { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", @@ -6581,60 +5663,6 @@ } ] }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "IPv4", - "severity": "info", - "confidence": "weak", - "line_data_list": [ - { - "line": "98.76.54.32-->master/IhqSb1Gg", - "line_num": 81, - "path": "tests/samples/doc_various", - "info": "tests/samples/doc_various|RAW", - "value": "98.76.54.32", - "value_start": 0, - "value_end": 11, - "variable": null, - "variable_start": -2, - "variable_end": -2, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 2.515950268099853, - "valid": false - } - } - ] - }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "IPv4", - "severity": "info", - "confidence": "weak", - "line_data_list": [ - { - "line": "scp gildong.hong@98.76.54.32 mailto:{1} pw:IhqSb1Gg", - "line_num": 82, - "path": "tests/samples/doc_various", - "info": "tests/samples/doc_various|RAW", - "value": "98.76.54.32", - "value_start": 17, - "value_end": 28, - "variable": null, - "variable_start": -2, - "variable_end": -2, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 2.515950268099853, - "valid": false - } - } - ] - }, { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", @@ -6691,26 +5719,26 @@ }, { "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "IPv4", - "severity": "info", - "confidence": "weak", + "ml_validation": "VALIDATED_KEY", + "ml_probability": 0.943, + "rule": "Password", + "severity": "medium", + "confidence": "moderate", "line_data_list": [ { "line": "gildong.hong@98.76.54.32 mailto:{1} pw:IhqSb1Gg", "line_num": 84, "path": "tests/samples/doc_various", "info": "tests/samples/doc_various|RAW", - "value": "98.76.54.32", - "value_start": 13, - "value_end": 24, - "variable": null, - "variable_start": -2, - "variable_end": -2, + "value": "IhqSb1Gg", + "value_start": 39, + "value_end": 47, + "variable": "pw", + "variable_start": 36, + "variable_end": 38, "entropy_validation": { "iterator": "BASE64_CHARS", - "entropy": 2.515950268099853, + "entropy": 3.0, "valid": false } } @@ -6719,22 +5747,22 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.943, + "ml_probability": 0.996, "rule": "Password", "severity": "medium", "confidence": "moderate", "line_data_list": [ { - "line": "gildong.hong@98.76.54.32 mailto:{1} pw:IhqSb1Gg", - "line_num": 84, + "line": "ssh gildong.hong@98.76.54.32 mailto:{1},pw:IhqSb1Gg", + "line_num": 85, "path": "tests/samples/doc_various", "info": "tests/samples/doc_various|RAW", "value": "IhqSb1Gg", - "value_start": 39, - "value_end": 47, + "value_start": 43, + "value_end": 51, "variable": "pw", - "variable_start": 36, - "variable_end": 38, + "variable_start": 40, + "variable_end": 42, "entropy_validation": { "iterator": "BASE64_CHARS", "entropy": 3.0, @@ -6745,26 +5773,26 @@ }, { "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "IPv4", - "severity": "info", - "confidence": "weak", + "ml_validation": "VALIDATED_KEY", + "ml_probability": 0.995, + "rule": "Password", + "severity": "medium", + "confidence": "moderate", "line_data_list": [ { - "line": "ssh gildong.hong@98.76.54.32 mailto:{1},pw:IhqSb1Gg", - "line_num": 85, + "line": "(ssh gildong.hong@98.76.54.32 mailto{1}) pwd:IhqSb1Gg", + "line_num": 87, "path": "tests/samples/doc_various", "info": "tests/samples/doc_various|RAW", - "value": "98.76.54.32", - "value_start": 17, - "value_end": 28, - "variable": null, - "variable_start": -2, - "variable_end": -2, + "value": "IhqSb1Gg", + "value_start": 45, + "value_end": 53, + "variable": "pwd", + "variable_start": 41, + "variable_end": 44, "entropy_validation": { "iterator": "BASE64_CHARS", - "entropy": 2.515950268099853, + "entropy": 3.0, "valid": false } } @@ -6779,16 +5807,16 @@ "confidence": "moderate", "line_data_list": [ { - "line": "ssh gildong.hong@98.76.54.32 mailto:{1},pw:IhqSb1Gg", - "line_num": 85, + "line": "ssh gildong.hong@98.76.54.32 mailto:{1}, pw:IhqSb1Gg", + "line_num": 90, "path": "tests/samples/doc_various", "info": "tests/samples/doc_various|RAW", "value": "IhqSb1Gg", - "value_start": 43, - "value_end": 51, + "value_start": 44, + "value_end": 52, "variable": "pw", - "variable_start": 40, - "variable_end": 42, + "variable_start": 41, + "variable_end": 43, "entropy_validation": { "iterator": "BASE64_CHARS", "entropy": 3.0, @@ -6799,26 +5827,26 @@ }, { "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "IPv4", - "severity": "info", - "confidence": "weak", + "ml_validation": "VALIDATED_KEY", + "ml_probability": 0.998, + "rule": "Password", + "severity": "medium", + "confidence": "moderate", "line_data_list": [ { - "line": "gildong.hong@98.76.54.32 mailto:{1} (master/IhqSb1Gg)", - "line_num": 86, + "line": "ssh gildong.hong@98.76.54.32 mailto:{1} (pwd:IhqSb1Gg)", + "line_num": 93, "path": "tests/samples/doc_various", "info": "tests/samples/doc_various|RAW", - "value": "98.76.54.32", - "value_start": 13, - "value_end": 24, - "variable": null, - "variable_start": -2, - "variable_end": -2, + "value": "IhqSb1Gg)", + "value_start": 45, + "value_end": 54, + "variable": "pwd", + "variable_start": 41, + "variable_end": 44, "entropy_validation": { "iterator": "BASE64_CHARS", - "entropy": 2.515950268099853, + "entropy": 2.8177111123931664, "valid": false } } @@ -6826,26 +5854,26 @@ }, { "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "IPv4", - "severity": "info", - "confidence": "weak", + "ml_validation": "VALIDATED_KEY", + "ml_probability": 0.997, + "rule": "Password", + "severity": "medium", + "confidence": "moderate", "line_data_list": [ { - "line": "(ssh gildong.hong@98.76.54.32 mailto{1}) pwd:IhqSb1Gg", - "line_num": 87, + "line": "gildong.hong@98.76.54.32 mailto:{1} (password:IhqSb1Gg)", + "line_num": 94, "path": "tests/samples/doc_various", "info": "tests/samples/doc_various|RAW", - "value": "98.76.54.32", - "value_start": 18, - "value_end": 29, - "variable": null, - "variable_start": -2, - "variable_end": -2, + "value": "IhqSb1Gg)", + "value_start": 46, + "value_end": 55, + "variable": "password", + "variable_start": 37, + "variable_end": 45, "entropy_validation": { "iterator": "BASE64_CHARS", - "entropy": 2.515950268099853, + "entropy": 2.8177111123931664, "valid": false } } @@ -6860,15 +5888,15 @@ "confidence": "moderate", "line_data_list": [ { - "line": "(ssh gildong.hong@98.76.54.32 mailto{1}) pwd:IhqSb1Gg", - "line_num": 87, + "line": "gildong.hong@98.76.54.32 mailto:{1} Password:IhqSb1Gg", + "line_num": 96, "path": "tests/samples/doc_various", "info": "tests/samples/doc_various|RAW", "value": "IhqSb1Gg", "value_start": 45, "value_end": 53, - "variable": "pwd", - "variable_start": 41, + "variable": "Password", + "variable_start": 36, "variable_end": 44, "entropy_validation": { "iterator": "BASE64_CHARS", @@ -6880,26 +5908,26 @@ }, { "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "IPv4", - "severity": "info", - "confidence": "weak", + "ml_validation": "VALIDATED_KEY", + "ml_probability": 0.979, + "rule": "Password", + "severity": "medium", + "confidence": "moderate", "line_data_list": [ { - "line": "ssh gildong.hong@98.76.54.32 (master/IhqSb1Gg)", - "line_num": 88, + "line": "gildong.hong@98.76.54.32 mailto:{1} (pass:IhqSb1Gg)", + "line_num": 97, "path": "tests/samples/doc_various", "info": "tests/samples/doc_various|RAW", - "value": "98.76.54.32", - "value_start": 17, - "value_end": 28, - "variable": null, - "variable_start": -2, - "variable_end": -2, + "value": "IhqSb1Gg)", + "value_start": 42, + "value_end": 51, + "variable": "pass", + "variable_start": 37, + "variable_end": 41, "entropy_validation": { "iterator": "BASE64_CHARS", - "entropy": 2.515950268099853, + "entropy": 2.8177111123931664, "valid": false } } @@ -6907,26 +5935,26 @@ }, { "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "IPv4", - "severity": "info", - "confidence": "weak", + "ml_validation": "VALIDATED_KEY", + "ml_probability": 0.994, + "rule": "Password", + "severity": "medium", + "confidence": "moderate", "line_data_list": [ { - "line": "ssh gildong.hong@98.76.54.32 mailto:[1} \uacc4\uc815master/IhqSb1Gg", - "line_num": 89, + "line": "ssh gildong.hong@98.76.54.32 mailto:{1} pw:IhqSb1Gg", + "line_num": 100, "path": "tests/samples/doc_various", "info": "tests/samples/doc_various|RAW", - "value": "98.76.54.32", - "value_start": 17, - "value_end": 28, - "variable": null, - "variable_start": -2, - "variable_end": -2, + "value": "IhqSb1Gg", + "value_start": 43, + "value_end": 51, + "variable": "pw", + "variable_start": 40, + "variable_end": 42, "entropy_validation": { "iterator": "BASE64_CHARS", - "entropy": 2.515950268099853, + "entropy": 3.0, "valid": false } } @@ -6934,26 +5962,26 @@ }, { "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "IPv4", - "severity": "info", - "confidence": "weak", + "ml_validation": "VALIDATED_KEY", + "ml_probability": 0.999, + "rule": "Password", + "severity": "medium", + "confidence": "moderate", "line_data_list": [ { - "line": "ssh gildong.hong@98.76.54.32 mailto:{1}, pw:IhqSb1Gg", - "line_num": 90, + "line": "ssh gildong.hong@98.76.54.32 mailto:{1} pass:IhqSb1Gg", + "line_num": 101, "path": "tests/samples/doc_various", "info": "tests/samples/doc_various|RAW", - "value": "98.76.54.32", - "value_start": 17, - "value_end": 28, - "variable": null, - "variable_start": -2, - "variable_end": -2, + "value": "IhqSb1Gg", + "value_start": 45, + "value_end": 53, + "variable": "pass", + "variable_start": 40, + "variable_end": 44, "entropy_validation": { "iterator": "BASE64_CHARS", - "entropy": 2.515950268099853, + "entropy": 3.0, "valid": false } } @@ -6962,21 +5990,21 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.996, + "ml_probability": 0.999, "rule": "Password", "severity": "medium", "confidence": "moderate", "line_data_list": [ { - "line": "ssh gildong.hong@98.76.54.32 mailto:{1}, pw:IhqSb1Gg", - "line_num": 90, + "line": "id:gildong.hong@xxx.com mailto:{1}/password:IhqSb1Gg", + "line_num": 102, "path": "tests/samples/doc_various", "info": "tests/samples/doc_various|RAW", "value": "IhqSb1Gg", "value_start": 44, "value_end": 52, - "variable": "pw", - "variable_start": 41, + "variable": "password", + "variable_start": 35, "variable_end": 43, "entropy_validation": { "iterator": "BASE64_CHARS", @@ -6988,26 +6016,26 @@ }, { "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "IPv4", - "severity": "info", - "confidence": "weak", + "ml_validation": "VALIDATED_KEY", + "ml_probability": 1.0, + "rule": "Password", + "severity": "medium", + "confidence": "moderate", "line_data_list": [ { - "line": "gildong.hong@98.76.54.32 mailto:{1} (\ube44\ubc88 IhqSb1Gg)", - "line_num": 91, + "line": "ssh gildong.hong@98.76.54.32 mailto:{1}/password:IhqSb1Gg", + "line_num": 104, "path": "tests/samples/doc_various", "info": "tests/samples/doc_various|RAW", - "value": "98.76.54.32", - "value_start": 13, - "value_end": 24, - "variable": null, - "variable_start": -2, - "variable_end": -2, + "value": "IhqSb1Gg", + "value_start": 49, + "value_end": 57, + "variable": "password", + "variable_start": 40, + "variable_end": 48, "entropy_validation": { "iterator": "BASE64_CHARS", - "entropy": 2.515950268099853, + "entropy": 3.0, "valid": false } } @@ -7015,26 +6043,26 @@ }, { "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "IPv4", - "severity": "info", - "confidence": "weak", + "ml_validation": "VALIDATED_KEY", + "ml_probability": 0.934, + "rule": "Password", + "severity": "medium", + "confidence": "moderate", "line_data_list": [ { - "line": "ssh gildong.hong@98.76.54.32 mailto:{1} password IhqSb1Gg", - "line_num": 92, + "line": "-ANYID:gildong.hong@example.com mailto:{1} -pw:IhqSb1Gg", + "line_num": 105, "path": "tests/samples/doc_various", "info": "tests/samples/doc_various|RAW", - "value": "98.76.54.32", - "value_start": 17, - "value_end": 28, - "variable": null, - "variable_start": -2, - "variable_end": -2, + "value": "IhqSb1Gg", + "value_start": 47, + "value_end": 55, + "variable": "pw", + "variable_start": 44, + "variable_end": 46, "entropy_validation": { "iterator": "BASE64_CHARS", - "entropy": 2.515950268099853, + "entropy": 3.0, "valid": false } } @@ -7042,26 +6070,26 @@ }, { "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "IPv4", - "severity": "info", - "confidence": "weak", + "ml_validation": "VALIDATED_KEY", + "ml_probability": 0.97, + "rule": "Password", + "severity": "medium", + "confidence": "moderate", "line_data_list": [ { - "line": "ssh gildong.hong@98.76.54.32 mailto:{1} (pwd:IhqSb1Gg)", - "line_num": 93, + "line": "ID:gildong.hong@xxxx.net mailto:{1} pw:IhqSb1Gg", + "line_num": 106, "path": "tests/samples/doc_various", "info": "tests/samples/doc_various|RAW", - "value": "98.76.54.32", - "value_start": 17, - "value_end": 28, - "variable": null, - "variable_start": -2, - "variable_end": -2, + "value": "IhqSb1Gg", + "value_start": 39, + "value_end": 47, + "variable": "pw", + "variable_start": 36, + "variable_end": 38, "entropy_validation": { "iterator": "BASE64_CHARS", - "entropy": 2.515950268099853, + "entropy": 3.0, "valid": false } } @@ -7070,22 +6098,22 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.998, + "ml_probability": 0.978, "rule": "Password", "severity": "medium", "confidence": "moderate", "line_data_list": [ { - "line": "ssh gildong.hong@98.76.54.32 mailto:{1} (pwd:IhqSb1Gg)", - "line_num": 93, + "line": "http://98.76.54.32:xxx(pw:IhqSb1Gg)", + "line_num": 108, "path": "tests/samples/doc_various", "info": "tests/samples/doc_various|RAW", "value": "IhqSb1Gg)", - "value_start": 45, - "value_end": 54, - "variable": "pwd", - "variable_start": 41, - "variable_end": 44, + "value_start": 26, + "value_end": 35, + "variable": "pw", + "variable_start": 23, + "variable_end": 25, "entropy_validation": { "iterator": "BASE64_CHARS", "entropy": 2.8177111123931664, @@ -7098,52 +6126,52 @@ "api_validation": "NOT_AVAILABLE", "ml_validation": "NOT_AVAILABLE", "ml_probability": null, - "rule": "IPv4", - "severity": "info", + "rule": "Dropbox API secret (long term)", + "severity": "high", "confidence": "weak", "line_data_list": [ { - "line": "gildong.hong@98.76.54.32 mailto:{1} (password:IhqSb1Gg)", - "line_num": 94, - "path": "tests/samples/doc_various", - "info": "tests/samples/doc_various|RAW", - "value": "98.76.54.32", - "value_start": 13, - "value_end": 24, + "line": "var g = '7rBynGo0b1cAAAAAAAAAAc72L3T6rQK51mB5a06ijnwRG91deTxvSqdZNAlxq8pZ'", + "line_num": 1, + "path": "tests/samples/dropbox_api_secret_long_term", + "info": "tests/samples/dropbox_api_secret_long_term|RAW", + "value": "7rBynGo0b1cAAAAAAAAAAc72L3T6rQK51mB5a06ijnwRG91deTxvSqdZNAlxq8pZ", + "value_start": 9, + "value_end": 73, "variable": null, "variable_start": -2, "variable_end": -2, "entropy_validation": { "iterator": "BASE64_CHARS", - "entropy": 2.515950268099853, - "valid": false + "entropy": 4.89361507332541, + "valid": true } } ] }, { "api_validation": "NOT_AVAILABLE", - "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.997, - "rule": "Password", - "severity": "medium", - "confidence": "moderate", + "ml_validation": "NOT_AVAILABLE", + "ml_probability": null, + "rule": "Dropbox App secret", + "severity": "info", + "confidence": "weak", "line_data_list": [ { - "line": "gildong.hong@98.76.54.32 mailto:{1} (password:IhqSb1Gg)", - "line_num": 94, - "path": "tests/samples/doc_various", - "info": "tests/samples/doc_various|RAW", - "value": "IhqSb1Gg)", - "value_start": 46, - "value_end": 55, - "variable": "password", - "variable_start": 37, - "variable_end": 45, + "line": "var app_unique_val_s = 'wpv1jq9xwanbn3n';", + "line_num": 1, + "path": "tests/samples/dropbox_app_secret", + "info": "tests/samples/dropbox_app_secret|RAW", + "value": "wpv1jq9xwanbn3n", + "value_start": 24, + "value_end": 39, + "variable": null, + "variable_start": -2, + "variable_end": -2, "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 2.8177111123931664, - "valid": false + "iterator": "BASE36_CHARS", + "entropy": 3.4565647621309536, + "valid": true } } ] @@ -7152,79 +6180,79 @@ "api_validation": "NOT_AVAILABLE", "ml_validation": "NOT_AVAILABLE", "ml_probability": null, - "rule": "IPv4", - "severity": "info", - "confidence": "weak", + "rule": "Dropbox OAuth2 API Access Token", + "severity": "high", + "confidence": "moderate", "line_data_list": [ { - "line": "gildong.hong@98.76.54.32 mailto:{1} Password:IhqSb1Gg", - "line_num": 96, - "path": "tests/samples/doc_various", - "info": "tests/samples/doc_various|RAW", - "value": "98.76.54.32", - "value_start": 13, - "value_end": 24, + "line": "var dropbox = 'sl.BdmpmC82mhhySscKk2oQGyE5l--8LdAQftLTXVGQhP39Z8FtAK1BhePhyevurA-Elt7ToIr6OpwzKAYE7RBqpu6VVyQU5WlCTL_Q7N4gElXahaWou6aPpOIwgGCIOq9aeC3YFoc';", + "line_num": 1, + "path": "tests/samples/dropbox_oauth_token", + "info": "tests/samples/dropbox_oauth_token|RAW", + "value": "sl.BdmpmC82mhhySscKk2oQGyE5l--8LdAQftLTXVGQhP39Z8FtAK1BhePhyevurA-Elt7ToIr6OpwzKAYE7RBqpu6VVyQU5WlCTL_Q7N4gElXahaWou6aPpOIwgGCIOq9aeC3YFoc", + "value_start": 15, + "value_end": 153, "variable": null, "variable_start": -2, "variable_end": -2, "entropy_validation": { "iterator": "BASE64_CHARS", - "entropy": 2.515950268099853, - "valid": false + "entropy": 5.395844179446957, + "valid": true } } ] }, { "api_validation": "NOT_AVAILABLE", - "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.995, - "rule": "Password", - "severity": "medium", + "ml_validation": "NOT_AVAILABLE", + "ml_probability": null, + "rule": "Dynatrace API Token", + "severity": "high", "confidence": "moderate", "line_data_list": [ { - "line": "gildong.hong@98.76.54.32 mailto:{1} Password:IhqSb1Gg", - "line_num": 96, - "path": "tests/samples/doc_various", - "info": "tests/samples/doc_various|RAW", - "value": "IhqSb1Gg", - "value_start": 45, - "value_end": 53, - "variable": "Password", - "variable_start": 36, - "variable_end": 44, + "line": "dt0c01.ST2EY72KQINMH574WMNVI7YN.G3DFPBEJYMODIDAEX454M7YWBUVEFOWKPRVMWFASS64NFH52PX6BNDVFFM572RZM", + "line_num": 1, + "path": "tests/samples/dynatrace_api.hs", + "info": "tests/samples/dynatrace_api.hs|RAW", + "value": "dt0c01.ST2EY72KQINMH574WMNVI7YN.G3DFPBEJYMODIDAEX454M7YWBUVEFOWKPRVMWFASS64NFH52PX6BNDVFFM572RZM", + "value_start": 0, + "value_end": 96, + "variable": null, + "variable_start": -2, + "variable_end": -2, "entropy_validation": { "iterator": "BASE64_CHARS", - "entropy": 3.0, - "valid": false + "entropy": 4.808191506786782, + "valid": true } } ] }, { "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "IPv4", - "severity": "info", - "confidence": "weak", + "ml_validation": "VALIDATED_KEY", + "ml_probability": 0.756, + "rule": "Github Old Token", + "severity": "high", + "confidence": "moderate", "line_data_list": [ { - "line": "gildong.hong@98.76.54.32 mailto:{1} (pass:IhqSb1Gg)", - "line_num": 97, - "path": "tests/samples/doc_various", - "info": "tests/samples/doc_various|RAW", - "value": "98.76.54.32", - "value_start": 13, - "value_end": 24, + "line": "\\ngit_token = \"gireogicracklecrackle1231567190113413981\"\\n\\n", + "line_num": 1, + "path": "tests/samples/encoded_data", + "info": "tests/samples/encoded_data|ENCODED|RAW", + "value": "gireogicracklecrackle1231567190113413981", + "value_start": 15, + "value_end": 55, "variable": null, "variable_start": -2, "variable_end": -2, "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 2.515950268099853, - "valid": false + "iterator": "BASE36_CHARS", + "entropy": 3.97402442086502, + "valid": true } } ] @@ -7232,26 +6260,26 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.979, - "rule": "Password", + "ml_probability": 0.756, + "rule": "Token", "severity": "medium", "confidence": "moderate", "line_data_list": [ { - "line": "gildong.hong@98.76.54.32 mailto:{1} (pass:IhqSb1Gg)", - "line_num": 97, - "path": "tests/samples/doc_various", - "info": "tests/samples/doc_various|RAW", - "value": "IhqSb1Gg)", - "value_start": 42, - "value_end": 51, - "variable": "pass", - "variable_start": 37, - "variable_end": 41, + "line": "\\ngit_token = \"gireogicracklecrackle1231567190113413981\"\\n\\n", + "line_num": 1, + "path": "tests/samples/encoded_data", + "info": "tests/samples/encoded_data|ENCODED|RAW", + "value": "gireogicracklecrackle1231567190113413981", + "value_start": 15, + "value_end": 55, + "variable": "git_token", + "variable_start": 2, + "variable_end": 11, "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 2.8177111123931664, - "valid": false + "iterator": "BASE36_CHARS", + "entropy": 3.97402442086502, + "valid": true } } ] @@ -7260,861 +6288,24 @@ "api_validation": "NOT_AVAILABLE", "ml_validation": "NOT_AVAILABLE", "ml_probability": null, - "rule": "IPv4", - "severity": "info", - "confidence": "weak", - "line_data_list": [ - { - "line": "ssh gildong.hong@98.76.54.32 mailto:{1} (xxxx//IhqSb1Gg)", - "line_num": 99, - "path": "tests/samples/doc_various", - "info": "tests/samples/doc_various|RAW", - "value": "98.76.54.32", - "value_start": 17, - "value_end": 28, - "variable": null, - "variable_start": -2, - "variable_end": -2, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 2.515950268099853, - "valid": false - } - } - ] - }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "IPv4", - "severity": "info", - "confidence": "weak", - "line_data_list": [ - { - "line": "ssh gildong.hong@98.76.54.32 mailto:{1} pw:IhqSb1Gg", - "line_num": 100, - "path": "tests/samples/doc_various", - "info": "tests/samples/doc_various|RAW", - "value": "98.76.54.32", - "value_start": 17, - "value_end": 28, - "variable": null, - "variable_start": -2, - "variable_end": -2, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 2.515950268099853, - "valid": false - } - } - ] - }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.994, - "rule": "Password", - "severity": "medium", - "confidence": "moderate", - "line_data_list": [ - { - "line": "ssh gildong.hong@98.76.54.32 mailto:{1} pw:IhqSb1Gg", - "line_num": 100, - "path": "tests/samples/doc_various", - "info": "tests/samples/doc_various|RAW", - "value": "IhqSb1Gg", - "value_start": 43, - "value_end": 51, - "variable": "pw", - "variable_start": 40, - "variable_end": 42, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 3.0, - "valid": false - } - } - ] - }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "IPv4", - "severity": "info", - "confidence": "weak", - "line_data_list": [ - { - "line": "ssh gildong.hong@98.76.54.32 mailto:{1} pass:IhqSb1Gg", - "line_num": 101, - "path": "tests/samples/doc_various", - "info": "tests/samples/doc_various|RAW", - "value": "98.76.54.32", - "value_start": 17, - "value_end": 28, - "variable": null, - "variable_start": -2, - "variable_end": -2, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 2.515950268099853, - "valid": false - } - } - ] - }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.999, - "rule": "Password", - "severity": "medium", - "confidence": "moderate", - "line_data_list": [ - { - "line": "ssh gildong.hong@98.76.54.32 mailto:{1} pass:IhqSb1Gg", - "line_num": 101, - "path": "tests/samples/doc_various", - "info": "tests/samples/doc_various|RAW", - "value": "IhqSb1Gg", - "value_start": 45, - "value_end": 53, - "variable": "pass", - "variable_start": 40, - "variable_end": 44, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 3.0, - "valid": false - } - } - ] - }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.999, - "rule": "Password", - "severity": "medium", - "confidence": "moderate", - "line_data_list": [ - { - "line": "id:gildong.hong@xxx.com mailto:{1}/password:IhqSb1Gg", - "line_num": 102, - "path": "tests/samples/doc_various", - "info": "tests/samples/doc_various|RAW", - "value": "IhqSb1Gg", - "value_start": 44, - "value_end": 52, - "variable": "password", - "variable_start": 35, - "variable_end": 43, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 3.0, - "valid": false - } - } - ] - }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "IPv4", - "severity": "info", - "confidence": "weak", - "line_data_list": [ - { - "line": "ssh gildong.hong@98.76.54.32 mailto:{1}/password:IhqSb1Gg", - "line_num": 104, - "path": "tests/samples/doc_various", - "info": "tests/samples/doc_various|RAW", - "value": "98.76.54.32", - "value_start": 17, - "value_end": 28, - "variable": null, - "variable_start": -2, - "variable_end": -2, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 2.515950268099853, - "valid": false - } - } - ] - }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "VALIDATED_KEY", - "ml_probability": 1.0, - "rule": "Password", - "severity": "medium", - "confidence": "moderate", - "line_data_list": [ - { - "line": "ssh gildong.hong@98.76.54.32 mailto:{1}/password:IhqSb1Gg", - "line_num": 104, - "path": "tests/samples/doc_various", - "info": "tests/samples/doc_various|RAW", - "value": "IhqSb1Gg", - "value_start": 49, - "value_end": 57, - "variable": "password", - "variable_start": 40, - "variable_end": 48, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 3.0, - "valid": false - } - } - ] - }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.934, - "rule": "Password", - "severity": "medium", - "confidence": "moderate", - "line_data_list": [ - { - "line": "-ANYID:gildong.hong@example.com mailto:{1} -pw:IhqSb1Gg", - "line_num": 105, - "path": "tests/samples/doc_various", - "info": "tests/samples/doc_various|RAW", - "value": "IhqSb1Gg", - "value_start": 47, - "value_end": 55, - "variable": "pw", - "variable_start": 44, - "variable_end": 46, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 3.0, - "valid": false - } - } - ] - }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.97, - "rule": "Password", - "severity": "medium", - "confidence": "moderate", - "line_data_list": [ - { - "line": "ID:gildong.hong@xxxx.net mailto:{1} pw:IhqSb1Gg", - "line_num": 106, - "path": "tests/samples/doc_various", - "info": "tests/samples/doc_various|RAW", - "value": "IhqSb1Gg", - "value_start": 39, - "value_end": 47, - "variable": "pw", - "variable_start": 36, - "variable_end": 38, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 3.0, - "valid": false - } - } - ] - }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "IPv4", - "severity": "info", - "confidence": "weak", - "line_data_list": [ - { - "line": "ID/PW 98.76.54.32:xxx master/IhqSb1Gg", - "line_num": 107, - "path": "tests/samples/doc_various", - "info": "tests/samples/doc_various|RAW", - "value": "98.76.54.32", - "value_start": 6, - "value_end": 17, - "variable": null, - "variable_start": -2, - "variable_end": -2, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 2.515950268099853, - "valid": false - } - } - ] - }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "IPv4", - "severity": "info", - "confidence": "weak", - "line_data_list": [ - { - "line": "http://98.76.54.32:xxx(pw:IhqSb1Gg)", - "line_num": 108, - "path": "tests/samples/doc_various", - "info": "tests/samples/doc_various|RAW", - "value": "98.76.54.32", - "value_start": 7, - "value_end": 18, - "variable": null, - "variable_start": -2, - "variable_end": -2, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 2.515950268099853, - "valid": false - } - } - ] - }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.978, - "rule": "Password", - "severity": "medium", - "confidence": "moderate", - "line_data_list": [ - { - "line": "http://98.76.54.32:xxx(pw:IhqSb1Gg)", - "line_num": 108, - "path": "tests/samples/doc_various", - "info": "tests/samples/doc_various|RAW", - "value": "IhqSb1Gg)", - "value_start": 26, - "value_end": 35, - "variable": "pw", - "variable_start": 23, - "variable_end": 25, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 2.8177111123931664, - "valid": false - } - } - ] - }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "Dropbox API secret (long term)", - "severity": "high", - "confidence": "weak", - "line_data_list": [ - { - "line": "var g = '7rBynGo0b1cAAAAAAAAAAc72L3T6rQK51mB5a06ijnwRG91deTxvSqdZNAlxq8pZ'", - "line_num": 1, - "path": "tests/samples/dropbox_api_secret_long_term", - "info": "tests/samples/dropbox_api_secret_long_term|RAW", - "value": "7rBynGo0b1cAAAAAAAAAAc72L3T6rQK51mB5a06ijnwRG91deTxvSqdZNAlxq8pZ", - "value_start": 9, - "value_end": 73, - "variable": null, - "variable_start": -2, - "variable_end": -2, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 4.89361507332541, - "valid": true - } - } - ] - }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "Dropbox App secret", - "severity": "info", - "confidence": "weak", - "line_data_list": [ - { - "line": "var app_unique_val_s = 'wpv1jq9xwanbn3n';", - "line_num": 1, - "path": "tests/samples/dropbox_app_secret", - "info": "tests/samples/dropbox_app_secret|RAW", - "value": "wpv1jq9xwanbn3n", - "value_start": 24, - "value_end": 39, - "variable": null, - "variable_start": -2, - "variable_end": -2, - "entropy_validation": { - "iterator": "BASE36_CHARS", - "entropy": 3.4565647621309536, - "valid": true - } - } - ] - }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "Dropbox OAuth2 API Access Token", - "severity": "high", - "confidence": "moderate", - "line_data_list": [ - { - "line": "var dropbox = 'sl.BdmpmC82mhhySscKk2oQGyE5l--8LdAQftLTXVGQhP39Z8FtAK1BhePhyevurA-Elt7ToIr6OpwzKAYE7RBqpu6VVyQU5WlCTL_Q7N4gElXahaWou6aPpOIwgGCIOq9aeC3YFoc';", - "line_num": 1, - "path": "tests/samples/dropbox_oauth_token", - "info": "tests/samples/dropbox_oauth_token|RAW", - "value": "sl.BdmpmC82mhhySscKk2oQGyE5l--8LdAQftLTXVGQhP39Z8FtAK1BhePhyevurA-Elt7ToIr6OpwzKAYE7RBqpu6VVyQU5WlCTL_Q7N4gElXahaWou6aPpOIwgGCIOq9aeC3YFoc", - "value_start": 15, - "value_end": 153, - "variable": null, - "variable_start": -2, - "variable_end": -2, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 5.395844179446957, - "valid": true - } - } - ] - }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "Dynatrace API Token", - "severity": "high", - "confidence": "moderate", - "line_data_list": [ - { - "line": "dt0c01.ST2EY72KQINMH574WMNVI7YN.G3DFPBEJYMODIDAEX454M7YWBUVEFOWKPRVMWFASS64NFH52PX6BNDVFFM572RZM", - "line_num": 1, - "path": "tests/samples/dynatrace_api.hs", - "info": "tests/samples/dynatrace_api.hs|RAW", - "value": "dt0c01.ST2EY72KQINMH574WMNVI7YN.G3DFPBEJYMODIDAEX454M7YWBUVEFOWKPRVMWFASS64NFH52PX6BNDVFFM572RZM", - "value_start": 0, - "value_end": 96, - "variable": null, - "variable_start": -2, - "variable_end": -2, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 4.808191506786782, - "valid": true - } - } - ] - }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.756, - "rule": "Github Old Token", - "severity": "high", - "confidence": "moderate", - "line_data_list": [ - { - "line": "\\ngit_token = \"gireogicracklecrackle1231567190113413981\"\\n\\n", - "line_num": 1, - "path": "tests/samples/encoded_data", - "info": "tests/samples/encoded_data|ENCODED|RAW", - "value": "gireogicracklecrackle1231567190113413981", - "value_start": 15, - "value_end": 55, - "variable": null, - "variable_start": -2, - "variable_end": -2, - "entropy_validation": { - "iterator": "BASE36_CHARS", - "entropy": 3.97402442086502, - "valid": true - } - } - ] - }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.756, - "rule": "Token", - "severity": "medium", - "confidence": "moderate", - "line_data_list": [ - { - "line": "\\ngit_token = \"gireogicracklecrackle1231567190113413981\"\\n\\n", - "line_num": 1, - "path": "tests/samples/encoded_data", - "info": "tests/samples/encoded_data|ENCODED|RAW", - "value": "gireogicracklecrackle1231567190113413981", - "value_start": 15, - "value_end": 55, - "variable": "git_token", - "variable_start": 2, - "variable_end": 11, - "entropy_validation": { - "iterator": "BASE36_CHARS", - "entropy": 3.97402442086502, - "valid": true - } - } - ] - }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "Facebook Access Token", - "severity": "high", - "confidence": "moderate", - "line_data_list": [ - { - "line": "GI_REO_GI_FACEBOOK_TOKEN = \"EAACEb00Kse0BAlGy7KeQ5YnaCEd09Eose0cBAlGy7KeQ5Yna9CoDsup39tiYdoQ4jH9Coup39tiYdWoQ4jHFZD\"", - "line_num": 1, - "path": "tests/samples/facebook_key", - "info": "tests/samples/facebook_key|RAW", - "value": "EAACEb00Kse0BAlGy7KeQ5YnaCEd09Eose0cBAlGy7KeQ5Yna9CoDsup39tiYdoQ4jH9Coup39tiYdWoQ4jHFZD", - "value_start": 28, - "value_end": 115, - "variable": null, - "variable_start": -2, - "variable_end": -2, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 4.936120692057916, - "valid": true - } - } - ] - }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.999, - "rule": "Token", - "severity": "medium", - "confidence": "moderate", - "line_data_list": [ - { - "line": "GI_REO_GI_FACEBOOK_TOKEN = \"EAACEb00Kse0BAlGy7KeQ5YnaCEd09Eose0cBAlGy7KeQ5Yna9CoDsup39tiYdoQ4jH9Coup39tiYdWoQ4jHFZD\"", - "line_num": 1, - "path": "tests/samples/facebook_key", - "info": "tests/samples/facebook_key|RAW", - "value": "EAACEb00Kse0BAlGy7KeQ5YnaCEd09Eose0cBAlGy7KeQ5Yna9CoDsup39tiYdoQ4jH9Coup39tiYdWoQ4jHFZD", - "value_start": 28, - "value_end": 115, - "variable": "GI_REO_GI_FACEBOOK_TOKEN", - "variable_start": 0, - "variable_end": 24, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 4.936120692057916, - "valid": true - } - } - ] - }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "Facebook App Token", - "severity": "high", - "confidence": "moderate", - "line_data_list": [ - { - "line": "1527194624358273|qbBf2-fdB9zZpqLA0_2nNzZDw2M", - "line_num": 2, - "path": "tests/samples/facebook_key", - "info": "tests/samples/facebook_key|RAW", - "value": "1527194624358273|qbBf2-fdB9zZpqLA0_2nNzZDw2M", - "value_start": 0, - "value_end": 44, - "variable": null, - "variable_start": -2, - "variable_end": -2, - "entropy_validation": { - "iterator": "BASE36_CHARS", - "entropy": 3.2089099270924217, - "valid": true - } - } - ] - }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "Firebase Domain", - "severity": "info", - "confidence": "moderate", - "line_data_list": [ - { - "line": "test-app-domain-42.firebaseapp.com", - "line_num": 1, - "path": "tests/samples/firebase_domain", - "info": "tests/samples/firebase_domain|RAW", - "value": "test-app-domain-42.firebaseapp.com", - "value_start": 0, - "value_end": 34, - "variable": null, - "variable_start": -2, - "variable_end": -2, - "entropy_validation": { - "iterator": "BASE36_CHARS", - "entropy": 3.4347510262969525, - "valid": true - } - } - ] - }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "Firebase Domain", - "severity": "info", - "confidence": "moderate", - "line_data_list": [ - { - "line": "test2.io.firebaseio.com", - "line_num": 2, - "path": "tests/samples/firebase_domain", - "info": "tests/samples/firebase_domain|RAW", - "value": "test2.io.firebaseio.com", - "value_start": 0, - "value_end": 23, - "variable": null, - "variable_start": -2, - "variable_end": -2, - "entropy_validation": { - "iterator": "BASE36_CHARS", - "entropy": 3.1394163745499943, - "valid": true - } - } - ] - }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "Github Classic Token", - "severity": "high", - "confidence": "strong", - "line_data_list": [ - { - "line": "ghp_00000000000000000000000000000004WZ4EQ # classic", - "line_num": 1, - "path": "tests/samples/github_classic_token", - "info": "tests/samples/github_classic_token|RAW", - "value": "ghp_00000000000000000000000000000004WZ4EQ", - "value_start": 0, - "value_end": 41, - "variable": null, - "variable_start": -2, - "variable_end": -2, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 1.4322437698226884, - "valid": false - } - } - ] - }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "Github Fine-granted Token", - "severity": "high", - "confidence": "strong", - "line_data_list": [ - { - "line": "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", - "line_num": 2, - "path": "tests/samples/github_fine_granted_token", - "info": "tests/samples/github_fine_granted_token|RAW", - "value": "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", - "value_start": 0, - "value_end": 93, - "variable": null, - "variable_start": -2, - "variable_end": -2, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 5.255374790203285, - "valid": true - } - } - ] - }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "VALIDATED_KEY", - "ml_probability": 1.0, - "rule": "Github Old Token", + "rule": "Facebook Access Token", "severity": "high", "confidence": "moderate", "line_data_list": [ - { - "line": "GITHUB_ACCESS_TOKEN = \"lbyxnhqjfnzhlpnvcvaxjumgvdlnktgugdofmzyy\"", - "line_num": 1, - "path": "tests/samples/github_key.groovy", - "info": "tests/samples/github_key.groovy|RAW", - "value": "lbyxnhqjfnzhlpnvcvaxjumgvdlnktgugdofmzyy", - "value_start": 23, - "value_end": 63, - "variable": null, - "variable_start": -2, - "variable_end": -2, - "entropy_validation": { - "iterator": "BASE36_CHARS", - "entropy": 4.246439344671015, - "valid": true - } - } - ] - }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "VALIDATED_KEY", - "ml_probability": 1.0, - "rule": "Token", - "severity": "medium", - "confidence": "moderate", - "line_data_list": [ - { - "line": "GITHUB_ACCESS_TOKEN = \"lbyxnhqjfnzhlpnvcvaxjumgvdlnktgugdofmzyy\"", - "line_num": 1, - "path": "tests/samples/github_key.groovy", - "info": "tests/samples/github_key.groovy|RAW", - "value": "lbyxnhqjfnzhlpnvcvaxjumgvdlnktgugdofmzyy", - "value_start": 23, - "value_end": 63, - "variable": "GITHUB_ACCESS_TOKEN", - "variable_start": 0, - "variable_end": 19, - "entropy_validation": { - "iterator": "BASE36_CHARS", - "entropy": 4.246439344671015, - "valid": true - } - } - ] - }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "Gitlab Incoming Email Token", - "severity": "info", - "confidence": "weak", - "line_data_list": [ - { - "line": "var email_t = '7e4v6v5j2nepcc8f5zvatgl9g';", - "line_num": 1, - "path": "tests/samples/gitlab_email_token", - "info": "tests/samples/gitlab_email_token|RAW", - "value": "7e4v6v5j2nepcc8f5zvatgl9g", - "value_start": 15, - "value_end": 40, - "variable": null, - "variable_start": -2, - "variable_end": -2, - "entropy_validation": { - "iterator": "BASE36_CHARS", - "entropy": 4.133660689688185, - "valid": true - } - } - ] - }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "Gitlab Feed Token", - "severity": "info", - "confidence": "weak", - "line_data_list": [ - { - "line": "feed_n = 'o9aEaH32LN618KhF7e_L'", - "line_num": 1, - "path": "tests/samples/gitlab_feed_token", - "info": "tests/samples/gitlab_feed_token|RAW", - "value": "o9aEaH32LN618KhF7e_L", - "value_start": 10, - "value_end": 30, - "variable": null, - "variable_start": -2, - "variable_end": -2, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 3.9058316901429944, - "valid": false - } - } - ] - }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "Gitlab PAT", - "severity": "high", - "confidence": "strong", - "line_data_list": [ - { - "line": "var pat = 'glpat-a6N2pFAr2L2A6iRsA_mw';", - "line_num": 1, - "path": "tests/samples/gitlab_pat_api", - "info": "tests/samples/gitlab_pat_api|RAW", - "value": "glpat-a6N2pFAr2L2A6iRsA_mw", - "value_start": 11, - "value_end": 37, - "variable": null, - "variable_start": -2, - "variable_end": -2, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 3.7423376242715105, - "valid": false - } - } - ] - }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "Gitlab Pipeline Trigger Token", - "severity": "high", - "confidence": "strong", - "line_data_list": [ - { - "line": "trigger = 'glptt-33276248c9748113e978392e5c074b7f974f8683';", - "line_num": 1, - "path": "tests/samples/gitlab_pipeline_trigger_token", - "info": "tests/samples/gitlab_pipeline_trigger_token|RAW", - "value": "glptt-33276248c9748113e978392e5c074b7f974f8683", - "value_start": 11, - "value_end": 57, + { + "line": "GI_REO_GI_FACEBOOK_TOKEN = \"EAACEb00Kse0BAlGy7KeQ5YnaCEd09Eose0cBAlGy7KeQ5Yna9CoDsup39tiYdoQ4jH9Coup39tiYdWoQ4jHFZD\"", + "line_num": 1, + "path": "tests/samples/facebook_key", + "info": "tests/samples/facebook_key|RAW", + "value": "EAACEb00Kse0BAlGy7KeQ5YnaCEd09Eose0cBAlGy7KeQ5Yna9CoDsup39tiYdoQ4jH9Coup39tiYdWoQ4jHFZD", + "value_start": 28, + "value_end": 115, "variable": null, "variable_start": -2, "variable_end": -2, "entropy_validation": { - "iterator": "BASE36_CHARS", - "entropy": 3.8494857514609038, + "iterator": "BASE64_CHARS", + "entropy": 4.936120692057916, "valid": true } } @@ -8122,26 +6313,26 @@ }, { "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "Gitlab Registration Runner Token", - "severity": "high", - "confidence": "strong", + "ml_validation": "VALIDATED_KEY", + "ml_probability": 0.999, + "rule": "Token", + "severity": "medium", + "confidence": "moderate", "line_data_list": [ { - "line": "gitlab_runner = 'GR1348941jG6xeSsmN8DFVKoyBYu2';", + "line": "GI_REO_GI_FACEBOOK_TOKEN = \"EAACEb00Kse0BAlGy7KeQ5YnaCEd09Eose0cBAlGy7KeQ5Yna9CoDsup39tiYdoQ4jH9Coup39tiYdWoQ4jHFZD\"", "line_num": 1, - "path": "tests/samples/gitlab_registration_runner", - "info": "tests/samples/gitlab_registration_runner|RAW", - "value": "GR1348941jG6xeSsmN8DFVKoyBYu2", - "value_start": 17, - "value_end": 46, - "variable": null, - "variable_start": -2, - "variable_end": -2, + "path": "tests/samples/facebook_key", + "info": "tests/samples/facebook_key|RAW", + "value": "EAACEb00Kse0BAlGy7KeQ5YnaCEd09Eose0cBAlGy7KeQ5Yna9CoDsup39tiYdoQ4jH9Coup39tiYdWoQ4jHFZD", + "value_start": 28, + "value_end": 115, + "variable": "GI_REO_GI_FACEBOOK_TOKEN", + "variable_start": 0, + "variable_end": 24, "entropy_validation": { "iterator": "BASE64_CHARS", - "entropy": 4.582118926162056, + "entropy": 4.936120692057916, "valid": true } } @@ -8151,25 +6342,25 @@ "api_validation": "NOT_AVAILABLE", "ml_validation": "NOT_AVAILABLE", "ml_probability": null, - "rule": "Gitlab Registration Runner Token 2023", + "rule": "Facebook App Token", "severity": "high", - "confidence": "strong", + "confidence": "moderate", "line_data_list": [ { - "line": "return \"glrt-2CR8_eVxiio-1QmzPZwa\"", - "line_num": 1, - "path": "tests/samples/gitlab_registration_runner_2023", - "info": "tests/samples/gitlab_registration_runner_2023|RAW", - "value": "glrt-2CR8_eVxiio-1QmzPZwa", - "value_start": 8, - "value_end": 33, + "line": "1527194624358273|qbBf2-fdB9zZpqLA0_2nNzZDw2M", + "line_num": 2, + "path": "tests/samples/facebook_key", + "info": "tests/samples/facebook_key|RAW", + "value": "1527194624358273|qbBf2-fdB9zZpqLA0_2nNzZDw2M", + "value_start": 0, + "value_end": 44, "variable": null, "variable_start": -2, "variable_end": -2, "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 4.006593447001756, - "valid": false + "iterator": "BASE36_CHARS", + "entropy": 3.2089099270924217, + "valid": true } } ] @@ -8178,24 +6369,24 @@ "api_validation": "NOT_AVAILABLE", "ml_validation": "NOT_AVAILABLE", "ml_probability": null, - "rule": "Google API Key", - "severity": "high", + "rule": "Firebase Domain", + "severity": "info", "confidence": "moderate", "line_data_list": [ { - "line": "AIzaGiReoG-CrackleCrackle12315618_12315", + "line": "test-app-domain-42.firebaseapp.com", "line_num": 1, - "path": "tests/samples/google_api_key.toml", - "info": "tests/samples/google_api_key.toml|RAW", - "value": "AIzaGiReoG-CrackleCrackle12315618_12315", + "path": "tests/samples/firebase_domain", + "info": "tests/samples/firebase_domain|RAW", + "value": "test-app-domain-42.firebaseapp.com", "value_start": 0, - "value_end": 39, + "value_end": 34, "variable": null, "variable_start": -2, "variable_end": -2, "entropy_validation": { "iterator": "BASE36_CHARS", - "entropy": 3.165196181720608, + "entropy": 3.4347510262969525, "valid": true } } @@ -8205,41 +6396,51 @@ "api_validation": "NOT_AVAILABLE", "ml_validation": "NOT_AVAILABLE", "ml_probability": null, - "rule": "Google Multi", - "severity": "high", + "rule": "Firebase Domain", + "severity": "info", "confidence": "moderate", "line_data_list": [ { - "line": "194206074328-qp89pdv6fi35vsi71258g1eh31q6h7c3.apps.googleusercontent.com", + "line": "test2.io.firebaseio.com", "line_num": 2, - "path": "tests/samples/google_multi", - "info": "tests/samples/google_multi|RAW", - "value": "194206074328-qp89pdv6fi35vsi71258g1eh31q6h7c3.apps.googleusercontent.com", + "path": "tests/samples/firebase_domain", + "info": "tests/samples/firebase_domain|RAW", + "value": "test2.io.firebaseio.com", "value_start": 0, - "value_end": 72, + "value_end": 23, "variable": null, "variable_start": -2, "variable_end": -2, "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 4.531537327540733, + "iterator": "BASE36_CHARS", + "entropy": 3.1394163745499943, "valid": true } - }, + } + ] + }, + { + "api_validation": "NOT_AVAILABLE", + "ml_validation": "NOT_AVAILABLE", + "ml_probability": null, + "rule": "Github Classic Token", + "severity": "high", + "confidence": "strong", + "line_data_list": [ { - "line": "4L2QMyTm6Rr0o46ytGiReoG1", - "line_num": 4, - "path": "tests/samples/google_multi", - "info": "tests/samples/google_multi|RAW", - "value": "4L2QMyTm6Rr0o46ytGiReoG1", + "line": "ghp_00000000000000000000000000000004WZ4EQ # classic", + "line_num": 1, + "path": "tests/samples/github_classic_token", + "info": "tests/samples/github_classic_token|RAW", + "value": "ghp_00000000000000000000000000000004WZ4EQ", "value_start": 0, - "value_end": 24, + "value_end": 41, "variable": null, "variable_start": -2, "variable_end": -2, "entropy_validation": { "iterator": "BASE64_CHARS", - "entropy": 4.084962500721157, + "entropy": 1.4322437698226884, "valid": false } } @@ -8249,69 +6450,79 @@ "api_validation": "NOT_AVAILABLE", "ml_validation": "NOT_AVAILABLE", "ml_probability": null, - "rule": "Google Multi", + "rule": "Github Fine-granted Token", "severity": "high", - "confidence": "moderate", + "confidence": "strong", "line_data_list": [ { - "line": "\"id\":\"194206074328-qdv6fi3eh31q6h7c35vsi7p89p1258g1.apps.googleusercontent.com\",\"CEKPET\":\"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\",", - "line_num": 18, - "path": "tests/samples/google_multi", - "info": "tests/samples/google_multi|RAW", - "value": "194206074328-qdv6fi3eh31q6h7c35vsi7p89p1258g1.apps.googleusercontent.com", - "value_start": 6, - "value_end": 78, + "line": "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", + "line_num": 2, + "path": "tests/samples/github_fine_granted_token", + "info": "tests/samples/github_fine_granted_token|RAW", + "value": "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", + "value_start": 0, + "value_end": 93, "variable": null, "variable_start": -2, "variable_end": -2, "entropy_validation": { "iterator": "BASE64_CHARS", - "entropy": 4.531537327540733, + "entropy": 5.255374790203285, "valid": true } - }, + } + ] + }, + { + "api_validation": "NOT_AVAILABLE", + "ml_validation": "VALIDATED_KEY", + "ml_probability": 1.0, + "rule": "Github Old Token", + "severity": "high", + "confidence": "moderate", + "line_data_list": [ { - "line": "\"id\":\"194206074328-qdv6fi3eh31q6h7c35vsi7p89p1258g1.apps.googleusercontent.com\",\"CEKPET\":\"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\",", - "line_num": 18, - "path": "tests/samples/google_multi", - "info": "tests/samples/google_multi|RAW", - "value": "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", - "value_start": 90, - "value_end": 125, + "line": "GITHUB_ACCESS_TOKEN = \"lbyxnhqjfnzhlpnvcvaxjumgvdlnktgugdofmzyy\"", + "line_num": 1, + "path": "tests/samples/github_key.groovy", + "info": "tests/samples/github_key.groovy|RAW", + "value": "lbyxnhqjfnzhlpnvcvaxjumgvdlnktgugdofmzyy", + "value_start": 23, + "value_end": 63, "variable": null, "variable_start": -2, "variable_end": -2, "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 4.436181130262395, - "valid": false + "iterator": "BASE36_CHARS", + "entropy": 4.246439344671015, + "valid": true } } ] }, { "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "Google OAuth Secret", - "severity": "high", - "confidence": "strong", + "ml_validation": "VALIDATED_KEY", + "ml_probability": 1.0, + "rule": "Token", + "severity": "medium", + "confidence": "moderate", "line_data_list": [ { - "line": "\"id\":\"194206074328-qdv6fi3eh31q6h7c35vsi7p89p1258g1.apps.googleusercontent.com\",\"CEKPET\":\"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\",", - "line_num": 18, - "path": "tests/samples/google_multi", - "info": "tests/samples/google_multi|RAW", - "value": "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", - "value_start": 90, - "value_end": 125, - "variable": null, - "variable_start": -2, - "variable_end": -2, + "line": "GITHUB_ACCESS_TOKEN = \"lbyxnhqjfnzhlpnvcvaxjumgvdlnktgugdofmzyy\"", + "line_num": 1, + "path": "tests/samples/github_key.groovy", + "info": "tests/samples/github_key.groovy|RAW", + "value": "lbyxnhqjfnzhlpnvcvaxjumgvdlnktgugdofmzyy", + "value_start": 23, + "value_end": 63, + "variable": "GITHUB_ACCESS_TOKEN", + "variable_start": 0, + "variable_end": 19, "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 4.436181130262395, - "valid": false + "iterator": "BASE36_CHARS", + "entropy": 4.246439344671015, + "valid": true } } ] @@ -8320,24 +6531,24 @@ "api_validation": "NOT_AVAILABLE", "ml_validation": "NOT_AVAILABLE", "ml_probability": null, - "rule": "Google OAuth Access Token", - "severity": "high", - "confidence": "moderate", + "rule": "Gitlab Incoming Email Token", + "severity": "info", + "confidence": "weak", "line_data_list": [ { - "line": "google_oauth_key = \"ya29.gi_reo_gi_crackle_ln22\"", + "line": "var email_t = '7e4v6v5j2nepcc8f5zvatgl9g';", "line_num": 1, - "path": "tests/samples/google_oauth_key", - "info": "tests/samples/google_oauth_key|RAW", - "value": "ya29.gi_reo_gi_crackle_ln22", - "value_start": 20, - "value_end": 47, + "path": "tests/samples/gitlab_email_token", + "info": "tests/samples/gitlab_email_token|RAW", + "value": "7e4v6v5j2nepcc8f5zvatgl9g", + "value_start": 15, + "value_end": 40, "variable": null, "variable_start": -2, "variable_end": -2, "entropy_validation": { "iterator": "BASE36_CHARS", - "entropy": 3.1797273164975133, + "entropy": 4.133660689688185, "valid": true } } @@ -8347,25 +6558,25 @@ "api_validation": "NOT_AVAILABLE", "ml_validation": "NOT_AVAILABLE", "ml_probability": null, - "rule": "Grafana Access Policy Token", - "severity": "high", - "confidence": "strong", + "rule": "Gitlab Feed Token", + "severity": "info", + "confidence": "weak", "line_data_list": [ { - "line": "grafana_policy = 'glc_eyJvIjoiMjA0NjMwIiwibiI6InRlc3QtdG9rZW4iLCJrIjoidklnbjJ2WHc5MTVXOWtNOWxsNHcyZHEyIiwibSI6eyJyIjoicHJvZC0wIn19'", + "line": "feed_n = 'o9aEaH32LN618KhF7e_L'", "line_num": 1, - "path": "tests/samples/grafana_access_policy_token", - "info": "tests/samples/grafana_access_policy_token|RAW", - "value": "glc_eyJvIjoiMjA0NjMwIiwibiI6InRlc3QtdG9rZW4iLCJrIjoidklnbjJ2WHc5MTVXOWtNOWxsNHcyZHEyIiwibSI6eyJyIjoicHJvZC0wIn19", - "value_start": 18, - "value_end": 130, + "path": "tests/samples/gitlab_feed_token", + "info": "tests/samples/gitlab_feed_token|RAW", + "value": "o9aEaH32LN618KhF7e_L", + "value_start": 10, + "value_end": 30, "variable": null, "variable_start": -2, "variable_end": -2, "entropy_validation": { "iterator": "BASE64_CHARS", - "entropy": 5.097632476604024, - "valid": true + "entropy": 3.9058316901429944, + "valid": false } } ] @@ -8374,24 +6585,24 @@ "api_validation": "NOT_AVAILABLE", "ml_validation": "NOT_AVAILABLE", "ml_probability": null, - "rule": "Grafana Provisioned API Key", + "rule": "Gitlab PAT", "severity": "high", "confidence": "strong", "line_data_list": [ { - "line": "grafana = 'eyJrIjoiMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMCIsIm4iOiJ4eHh4IiwiaWQiOjIwNDM2MH0='", + "line": "var pat = 'glpat-a6N2pFAr2L2A6iRsA_mw';", "line_num": 1, - "path": "tests/samples/grafana_provisioned_api_key", - "info": "tests/samples/grafana_provisioned_api_key|RAW", - "value": "eyJrIjoiMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMCIsIm4iOiJ4eHh4IiwiaWQiOjIwNDM2MH0=", + "path": "tests/samples/gitlab_pat_api", + "info": "tests/samples/gitlab_pat_api|RAW", + "value": "glpat-a6N2pFAr2L2A6iRsA_mw", "value_start": 11, - "value_end": 107, + "value_end": 37, "variable": null, "variable_start": -2, "variable_end": -2, "entropy_validation": { "iterator": "BASE64_CHARS", - "entropy": 3.8153130511409934, + "entropy": 3.7423376242715105, "valid": false } } @@ -8399,27 +6610,27 @@ }, { "api_validation": "NOT_AVAILABLE", - "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.985, - "rule": "JSON Web Token", - "severity": "medium", - "confidence": "moderate", + "ml_validation": "NOT_AVAILABLE", + "ml_probability": null, + "rule": "Gitlab Pipeline Trigger Token", + "severity": "high", + "confidence": "strong", "line_data_list": [ { - "line": "grafana = 'eyJrIjoiMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMCIsIm4iOiJ4eHh4IiwiaWQiOjIwNDM2MH0='", + "line": "trigger = 'glptt-33276248c9748113e978392e5c074b7f974f8683';", "line_num": 1, - "path": "tests/samples/grafana_provisioned_api_key", - "info": "tests/samples/grafana_provisioned_api_key|RAW", - "value": "eyJrIjoiMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMCIsIm4iOiJ4eHh4IiwiaWQiOjIwNDM2MH0=", + "path": "tests/samples/gitlab_pipeline_trigger_token", + "info": "tests/samples/gitlab_pipeline_trigger_token|RAW", + "value": "glptt-33276248c9748113e978392e5c074b7f974f8683", "value_start": 11, - "value_end": 107, + "value_end": 57, "variable": null, "variable_start": -2, "variable_end": -2, "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 3.8153130511409934, - "valid": false + "iterator": "BASE36_CHARS", + "entropy": 3.8494857514609038, + "valid": true } } ] @@ -8428,24 +6639,24 @@ "api_validation": "NOT_AVAILABLE", "ml_validation": "NOT_AVAILABLE", "ml_probability": null, - "rule": "Grafana Service Account Token", + "rule": "Gitlab Registration Runner Token", "severity": "high", "confidence": "strong", "line_data_list": [ { - "line": "glsa_ThisI5NtTheTok3nYou8reLo0k1ngF0r_0a2a3df7", + "line": "gitlab_runner = 'GR1348941jG6xeSsmN8DFVKoyBYu2';", "line_num": 1, - "path": "tests/samples/grafana_service_accounts", - "info": "tests/samples/grafana_service_accounts|RAW", - "value": "glsa_ThisI5NtTheTok3nYou8reLo0k1ngF0r_0a2a3df7", - "value_start": 0, + "path": "tests/samples/gitlab_registration_runner", + "info": "tests/samples/gitlab_registration_runner|RAW", + "value": "GR1348941jG6xeSsmN8DFVKoyBYu2", + "value_start": 17, "value_end": 46, "variable": null, "variable_start": -2, "variable_end": -2, "entropy_validation": { "iterator": "BASE64_CHARS", - "entropy": 4.52211252299684, + "entropy": 4.582118926162056, "valid": true } } @@ -8455,25 +6666,25 @@ "api_validation": "NOT_AVAILABLE", "ml_validation": "NOT_AVAILABLE", "ml_probability": null, - "rule": "Hashicorp Terraform Token", + "rule": "Gitlab Registration Runner Token 2023", "severity": "high", "confidence": "strong", "line_data_list": [ { - "line": "Z28P3STmkBQi1Y.atlasv1.YE7RBqu6VVyQIOq9a1eC3YFU5Elt7ToIr6OwzKAWlCTQ7N4gElXaWou6aPpOIwGCoc0", + "line": "return \"glrt-2CR8_eVxiio-1QmzPZwa\"", "line_num": 1, - "path": "tests/samples/hashicorp_terraform", - "info": "tests/samples/hashicorp_terraform|RAW", - "value": "Z28P3STmkBQi1Y.atlasv1.YE7RBqu6VVyQIOq9a1eC3YFU5Elt7ToIr6OwzKAWlCTQ7N4gElXaWou6aPpOIwGCoc0", - "value_start": 0, - "value_end": 90, + "path": "tests/samples/gitlab_registration_runner_2023", + "info": "tests/samples/gitlab_registration_runner_2023|RAW", + "value": "glrt-2CR8_eVxiio-1QmzPZwa", + "value_start": 8, + "value_end": 33, "variable": null, "variable_start": -2, "variable_end": -2, "entropy_validation": { "iterator": "BASE64_CHARS", - "entropy": 5.348551883097512, - "valid": true + "entropy": 4.006593447001756, + "valid": false } } ] @@ -8482,24 +6693,24 @@ "api_validation": "NOT_AVAILABLE", "ml_validation": "NOT_AVAILABLE", "ml_probability": null, - "rule": "Heroku API Key", + "rule": "Google API Key", "severity": "high", "confidence": "moderate", "line_data_list": [ { - "line": "HerOkUa04b8c1d-A147-b252-3b6a8f9c2b16", + "line": "AIzaGiReoG-CrackleCrackle12315618_12315", "line_num": 1, - "path": "tests/samples/heroku_api.toml", - "info": "tests/samples/heroku_api.toml|RAW", - "value": "HerOkUa04b8c1d-A147-b252-3b6a8f9c2b16", + "path": "tests/samples/google_api_key.toml", + "info": "tests/samples/google_api_key.toml|RAW", + "value": "AIzaGiReoG-CrackleCrackle12315618_12315", "value_start": 0, - "value_end": 37, + "value_end": 39, "variable": null, "variable_start": -2, "variable_end": -2, "entropy_validation": { "iterator": "BASE36_CHARS", - "entropy": 3.48037367471734, + "entropy": 3.165196181720608, "valid": true } } @@ -8509,78 +6720,41 @@ "api_validation": "NOT_AVAILABLE", "ml_validation": "NOT_AVAILABLE", "ml_probability": null, - "rule": "Instagram Access Token", + "rule": "Google Multi", "severity": "high", - "confidence": "strong", + "confidence": "moderate", "line_data_list": [ { - "line": "IGQVJWS3pUNmZA2MUJDVlRwLW9ac1lrU05nZAmpzWHQtWHFJSEFRMF9tWVpRdEd70HQ5Wk8wSnY0R0VEQnVQdUU0MnpxNWxocUYyNmZAXSTUtVVNrMmh1ZAHZAQQno2ZA0VHR0lLWkk1N1R5RDFvM0dmVEpIYQZGZX", - "line_num": 1, - "path": "tests/samples/instagram_access_token", - "info": "tests/samples/instagram_access_token|RAW", - "value": "IGQVJWS3pUNmZA2MUJDVlRwLW9ac1lrU05nZAmpzWHQtWHFJSEFRMF9tWVpRdEd70HQ5Wk8wSnY0R0VEQnVQdUU0MnpxNWxocUYyNmZAXSTUtVVNrMmh1ZAHZAQQno2ZA0VHR0lLWkk1N1R5RDFvM0dmVEpIYQZGZX", + "line": "194206074328-qp89pdv6fi35vsi71258g1eh31q6h7c3.apps.googleusercontent.com", + "line_num": 2, + "path": "tests/samples/google_multi", + "info": "tests/samples/google_multi|RAW", + "value": "194206074328-qp89pdv6fi35vsi71258g1eh31q6h7c3.apps.googleusercontent.com", "value_start": 0, - "value_end": 162, + "value_end": 72, "variable": null, "variable_start": -2, "variable_end": -2, "entropy_validation": { "iterator": "BASE64_CHARS", - "entropy": 5.231644837540696, + "entropy": 4.531537327540733, "valid": true } - } - ] - }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "IPv4", - "severity": "info", - "confidence": "weak", - "line_data_list": [ + }, { - "line": "100.64.0.0/10", - "line_num": 13, - "path": "tests/samples/ipv4", - "info": "tests/samples/ipv4|RAW", - "value": "100.64.0.0", + "line": "4L2QMyTm6Rr0o46ytGiReoG1", + "line_num": 4, + "path": "tests/samples/google_multi", + "info": "tests/samples/google_multi|RAW", + "value": "4L2QMyTm6Rr0o46ytGiReoG1", "value_start": 0, - "value_end": 10, + "value_end": 24, "variable": null, "variable_start": -2, "variable_end": -2, "entropy_validation": { "iterator": "BASE64_CHARS", - "entropy": 1.5253496664211537, - "valid": false - } - } - ] - }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "IPv4", - "severity": "info", - "confidence": "weak", - "line_data_list": [ - { - "line": "100.64.0.0\u2013100.127.255.255", - "line_num": 14, - "path": "tests/samples/ipv4", - "info": "tests/samples/ipv4|RAW", - "value": "100.127.255.255", - "value_start": 11, - "value_end": 26, - "variable": null, - "variable_start": -2, - "variable_end": -2, - "entropy_validation": { - "iterator": "BASE36_CHARS", - "entropy": 2.008519976342584, + "entropy": 4.084962500721157, "valid": false } } @@ -8590,51 +6764,41 @@ "api_validation": "NOT_AVAILABLE", "ml_validation": "NOT_AVAILABLE", "ml_probability": null, - "rule": "IPv4", - "severity": "info", - "confidence": "weak", + "rule": "Google Multi", + "severity": "high", + "confidence": "moderate", "line_data_list": [ { - "line": "100.64.0.0\u2013100.127.255.255", - "line_num": 14, - "path": "tests/samples/ipv4", - "info": "tests/samples/ipv4|RAW", - "value": "100.64.0.0", - "value_start": 0, - "value_end": 10, + "line": "\"id\":\"194206074328-qdv6fi3eh31q6h7c35vsi7p89p1258g1.apps.googleusercontent.com\",\"CEKPET\":\"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\",", + "line_num": 18, + "path": "tests/samples/google_multi", + "info": "tests/samples/google_multi|RAW", + "value": "194206074328-qdv6fi3eh31q6h7c35vsi7p89p1258g1.apps.googleusercontent.com", + "value_start": 6, + "value_end": 78, "variable": null, "variable_start": -2, "variable_end": -2, "entropy_validation": { "iterator": "BASE64_CHARS", - "entropy": 1.5253496664211537, - "valid": false + "entropy": 4.531537327540733, + "valid": true } - } - ] - }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "IPv4", - "severity": "info", - "confidence": "weak", - "line_data_list": [ + }, { - "line": "192.0.0.0\u2013192.0.0.255", - "line_num": 22, - "path": "tests/samples/ipv4", - "info": "tests/samples/ipv4|RAW", - "value": "192.0.0.255", - "value_start": 10, - "value_end": 21, + "line": "\"id\":\"194206074328-qdv6fi3eh31q6h7c35vsi7p89p1258g1.apps.googleusercontent.com\",\"CEKPET\":\"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\",", + "line_num": 18, + "path": "tests/samples/google_multi", + "info": "tests/samples/google_multi|RAW", + "value": "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", + "value_start": 90, + "value_end": 125, "variable": null, "variable_start": -2, "variable_end": -2, "entropy_validation": { "iterator": "BASE64_CHARS", - "entropy": 1.9704957226453073, + "entropy": 4.436181130262395, "valid": false } } @@ -8644,24 +6808,24 @@ "api_validation": "NOT_AVAILABLE", "ml_validation": "NOT_AVAILABLE", "ml_probability": null, - "rule": "IPv4", - "severity": "info", - "confidence": "weak", + "rule": "Google OAuth Secret", + "severity": "high", + "confidence": "strong", "line_data_list": [ { - "line": "192.88.99.0/24", - "line_num": 25, - "path": "tests/samples/ipv4", - "info": "tests/samples/ipv4|RAW", - "value": "192.88.99.0", - "value_start": 0, - "value_end": 11, + "line": "\"id\":\"194206074328-qdv6fi3eh31q6h7c35vsi7p89p1258g1.apps.googleusercontent.com\",\"CEKPET\":\"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\",", + "line_num": 18, + "path": "tests/samples/google_multi", + "info": "tests/samples/google_multi|RAW", + "value": "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", + "value_start": 90, + "value_end": 125, "variable": null, "variable_start": -2, "variable_end": -2, "entropy_validation": { - "iterator": "BASE36_CHARS", - "entropy": 1.9018695860849921, + "iterator": "BASE64_CHARS", + "entropy": 4.436181130262395, "valid": false } } @@ -8671,25 +6835,25 @@ "api_validation": "NOT_AVAILABLE", "ml_validation": "NOT_AVAILABLE", "ml_probability": null, - "rule": "IPv4", - "severity": "info", - "confidence": "weak", + "rule": "Google OAuth Access Token", + "severity": "high", + "confidence": "moderate", "line_data_list": [ { - "line": "192.88.99.0\u2013192.88.99.255", - "line_num": 26, - "path": "tests/samples/ipv4", - "info": "tests/samples/ipv4|RAW", - "value": "192.88.99.0", - "value_start": 0, - "value_end": 11, + "line": "google_oauth_key = \"ya29.gi_reo_gi_crackle_ln22\"", + "line_num": 1, + "path": "tests/samples/google_oauth_key", + "info": "tests/samples/google_oauth_key|RAW", + "value": "ya29.gi_reo_gi_crackle_ln22", + "value_start": 20, + "value_end": 47, "variable": null, "variable_start": -2, "variable_end": -2, "entropy_validation": { "iterator": "BASE36_CHARS", - "entropy": 1.9018695860849921, - "valid": false + "entropy": 3.1797273164975133, + "valid": true } } ] @@ -8698,25 +6862,25 @@ "api_validation": "NOT_AVAILABLE", "ml_validation": "NOT_AVAILABLE", "ml_probability": null, - "rule": "IPv4", - "severity": "info", - "confidence": "weak", + "rule": "Grafana Access Policy Token", + "severity": "high", + "confidence": "strong", "line_data_list": [ { - "line": "192.88.99.0\u2013192.88.99.255", - "line_num": 26, - "path": "tests/samples/ipv4", - "info": "tests/samples/ipv4|RAW", - "value": "192.88.99.255", - "value_start": 12, - "value_end": 25, + "line": "grafana_policy = 'glc_eyJvIjoiMjA0NjMwIiwibiI6InRlc3QtdG9rZW4iLCJrIjoidklnbjJ2WHc5MTVXOWtNOWxsNHcyZHEyIiwibSI6eyJyIjoicHJvZC0wIn19'", + "line_num": 1, + "path": "tests/samples/grafana_access_policy_token", + "info": "tests/samples/grafana_access_policy_token|RAW", + "value": "glc_eyJvIjoiMjA0NjMwIiwibiI6InRlc3QtdG9rZW4iLCJrIjoidklnbjJ2WHc5MTVXOWtNOWxsNHcyZHEyIiwibSI6eyJyIjoicHJvZC0wIn19", + "value_start": 18, + "value_end": 130, "variable": null, "variable_start": -2, "variable_end": -2, "entropy_validation": { "iterator": "BASE64_CHARS", - "entropy": 2.019193052249804, - "valid": false + "entropy": 5.097632476604024, + "valid": true } } ] @@ -8725,24 +6889,24 @@ "api_validation": "NOT_AVAILABLE", "ml_validation": "NOT_AVAILABLE", "ml_probability": null, - "rule": "IPv6", - "severity": "info", + "rule": "Grafana Provisioned API Key", + "severity": "high", "confidence": "strong", "line_data_list": [ { - "line": "2004:5678::9324", - "line_num": 12, - "path": "tests/samples/ipv6", - "info": "tests/samples/ipv6|RAW", - "value": "2004:5678::9324", - "value_start": 0, - "value_end": 15, + "line": "grafana = 'eyJrIjoiMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMCIsIm4iOiJ4eHh4IiwiaWQiOjIwNDM2MH0='", + "line_num": 1, + "path": "tests/samples/grafana_provisioned_api_key", + "info": "tests/samples/grafana_provisioned_api_key|RAW", + "value": "eyJrIjoiMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMCIsIm4iOiJ4eHh4IiwiaWQiOjIwNDM2MH0=", + "value_start": 11, + "value_end": 107, "variable": null, "variable_start": -2, "variable_end": -2, "entropy_validation": { - "iterator": "BASE36_CHARS", - "entropy": 2.725512476486815, + "iterator": "BASE64_CHARS", + "entropy": 3.8153130511409934, "valid": false } } @@ -8752,25 +6916,25 @@ "api_validation": "NOT_AVAILABLE", "ml_validation": "NOT_AVAILABLE", "ml_probability": null, - "rule": "IPv6", - "severity": "info", + "rule": "Grafana Service Account Token", + "severity": "high", "confidence": "strong", "line_data_list": [ { - "line": "2004::5678:9", - "line_num": 13, - "path": "tests/samples/ipv6", - "info": "tests/samples/ipv6|RAW", - "value": "2004::5678:9", + "line": "glsa_ThisI5NtTheTok3nYou8reLo0k1ngF0r_0a2a3df7", + "line_num": 1, + "path": "tests/samples/grafana_service_accounts", + "info": "tests/samples/grafana_service_accounts|RAW", + "value": "glsa_ThisI5NtTheTok3nYou8reLo0k1ngF0r_0a2a3df7", "value_start": 0, - "value_end": 12, + "value_end": 46, "variable": null, "variable_start": -2, "variable_end": -2, "entropy_validation": { - "iterator": "BASE36_CHARS", - "entropy": 2.5220552088742005, - "valid": false + "iterator": "BASE64_CHARS", + "entropy": 4.52211252299684, + "valid": true } } ] @@ -8779,25 +6943,25 @@ "api_validation": "NOT_AVAILABLE", "ml_validation": "NOT_AVAILABLE", "ml_probability": null, - "rule": "IPv6", - "severity": "info", + "rule": "Hashicorp Terraform Token", + "severity": "high", "confidence": "strong", "line_data_list": [ { - "line": "2041:0000:140F::875B:131B", - "line_num": 14, - "path": "tests/samples/ipv6", - "info": "tests/samples/ipv6|RAW", - "value": "2041:0000:140F::875B:131B", + "line": "Z28P3STmkBQi1Y.atlasv1.YE7RBqu6VVyQIOq9a1eC3YFU5Elt7ToIr6OwzKAWlCTQ7N4gElXaWou6aPpOIwGCoc0", + "line_num": 1, + "path": "tests/samples/hashicorp_terraform", + "info": "tests/samples/hashicorp_terraform|RAW", + "value": "Z28P3STmkBQi1Y.atlasv1.YE7RBqu6VVyQIOq9a1eC3YFU5Elt7ToIr6OwzKAWlCTQ7N4gElXaWou6aPpOIwGCoc0", "value_start": 0, - "value_end": 25, + "value_end": 90, "variable": null, "variable_start": -2, "variable_end": -2, "entropy_validation": { - "iterator": "HEX_CHARS", - "entropy": 2.6146939516467023, - "valid": false + "iterator": "BASE64_CHARS", + "entropy": 5.348551883097512, + "valid": true } } ] @@ -8806,25 +6970,25 @@ "api_validation": "NOT_AVAILABLE", "ml_validation": "NOT_AVAILABLE", "ml_probability": null, - "rule": "IPv6", - "severity": "info", - "confidence": "strong", + "rule": "Heroku API Key", + "severity": "high", + "confidence": "moderate", "line_data_list": [ { - "line": "LONG LINE TEST: 2041:0:140F::875B:131B 2041:0:140F::875B:131B ;", - "line_num": 15, - "path": "tests/samples/ipv6", - "info": "tests/samples/ipv6|RAW", - "value": "2041:0:140F::875B:131B", - "value_start": 7010, - "value_end": 7032, + "line": "HerOkUa04b8c1d-A147-b252-3b6a8f9c2b16", + "line_num": 1, + "path": "tests/samples/heroku_api.toml", + "info": "tests/samples/heroku_api.toml|RAW", + "value": "HerOkUa04b8c1d-A147-b252-3b6a8f9c2b16", + "value_start": 0, + "value_end": 37, "variable": null, "variable_start": -2, "variable_end": -2, "entropy_validation": { - "iterator": "HEX_CHARS", - "entropy": 2.684338637030481, - "valid": false + "iterator": "BASE36_CHARS", + "entropy": 3.48037367471734, + "valid": true } } ] @@ -8833,25 +6997,25 @@ "api_validation": "NOT_AVAILABLE", "ml_validation": "NOT_AVAILABLE", "ml_probability": null, - "rule": "IPv6", - "severity": "info", + "rule": "Instagram Access Token", + "severity": "high", "confidence": "strong", "line_data_list": [ { - "line": "LONG LINE TEST: 2041:0:140F::875B:131B 2041:0:140F::875B:131B ;", - "line_num": 15, - "path": "tests/samples/ipv6", - "info": "tests/samples/ipv6|RAW", - "value": "2041:0:140F::875B:131B", - "value_start": 7989, - "value_end": 8011, + "line": "IGQVJWS3pUNmZA2MUJDVlRwLW9ac1lrU05nZAmpzWHQtWHFJSEFRMF9tWVpRdEd70HQ5Wk8wSnY0R0VEQnVQdUU0MnpxNWxocUYyNmZAXSTUtVVNrMmh1ZAHZAQQno2ZA0VHR0lLWkk1N1R5RDFvM0dmVEpIYQZGZX", + "line_num": 1, + "path": "tests/samples/instagram_access_token", + "info": "tests/samples/instagram_access_token|RAW", + "value": "IGQVJWS3pUNmZA2MUJDVlRwLW9ac1lrU05nZAmpzWHQtWHFJSEFRMF9tWVpRdEd70HQ5Wk8wSnY0R0VEQnVQdUU0MnpxNWxocUYyNmZAXSTUtVVNrMmh1ZAHZAQQno2ZA0VHR0lLWkk1N1R5RDFvM0dmVEpIYQZGZX", + "value_start": 0, + "value_end": 162, "variable": null, "variable_start": -2, "variable_end": -2, "entropy_validation": { - "iterator": "HEX_CHARS", - "entropy": 2.684338637030481, - "valid": false + "iterator": "BASE64_CHARS", + "entropy": 5.231644837540696, + "valid": true } } ] @@ -8993,26 +7157,26 @@ }, { "api_validation": "NOT_AVAILABLE", - "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.931, + "ml_validation": "NOT_AVAILABLE", + "ml_probability": null, "rule": "JSON Web Token", "severity": "medium", - "confidence": "moderate", + "confidence": "strong", "line_data_list": [ { - "line": "$payload = 'eyJgsIZgeJhvNgFpSmlP.eyJcaaF9xCe7shE0ENPiBlEJOpS'", + "line": "detected: eyJhbGciOiJSUzI1NiJ9Cg.eyJleHAiOjY1NTM2fQo.Ce7sh0ENPiBlE_dose0cBA", "line_num": 1, - "path": "tests/samples/json_web_token.hs", - "info": "tests/samples/json_web_token.hs|RAW", - "value": "eyJgsIZgeJhvNgFpSmlP.eyJcaaF9xCe7shE0ENPiBlEJOpS", - "value_start": 12, - "value_end": 60, + "path": "tests/samples/json_web_token", + "info": "tests/samples/json_web_token|RAW", + "value": "eyJhbGciOiJSUzI1NiJ9Cg.eyJleHAiOjY1NTM2fQo.Ce7sh0ENPiBlE_dose0cBA", + "value_start": 10, + "value_end": 75, "variable": null, "variable_start": -2, "variable_end": -2, "entropy_validation": { "iterator": "BASE64_CHARS", - "entropy": 4.520488802699322, + "entropy": 4.790963630103494, "valid": true } } @@ -9102,17 +7266,17 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.986, + "ml_probability": 0.999, "rule": "Secret", "severity": "medium", "confidence": "moderate", "line_data_list": [ { - "line": "secret_looks_like_linux_path__=\"VnpmUGWxhQW/9KQAwrL2ZYd/DJPNG1PQjEXAMPLE\"", + "line": "secret_looks_like_linux_path__=\"VnpmUGWxhQW/9KQAwrL2ZYd/DJPNG1PQjEXAMbLE\"", "line_num": 5, "path": "tests/samples/key.hs", "info": "tests/samples/key.hs|RAW", - "value": "VnpmUGWxhQW/9KQAwrL2ZYd/DJPNG1PQjEXAMPLE", + "value": "VnpmUGWxhQW/9KQAwrL2ZYd/DJPNG1PQjEXAMbLE", "value_start": 32, "value_end": 72, "variable": "secret_looks_like_linux_path__", @@ -9120,7 +7284,7 @@ "variable_end": 30, "entropy_validation": { "iterator": "BASE64_CHARS", - "entropy": 4.784183719779189, + "entropy": 4.8530559073332755, "valid": true } } @@ -11189,6 +9353,33 @@ } ] }, + { + "api_validation": "NOT_AVAILABLE", + "ml_validation": "VALIDATED_KEY", + "ml_probability": 0.911, + "rule": "Salt", + "severity": "medium", + "confidence": "moderate", + "line_data_list": [ + { + "line": "salt3 = \" 827634876\"; ", + "line_num": 1, + "path": "tests/samples/salt.py", + "info": "tests/samples/salt.py|STRUCT|STRUCT:2|KEYWORD:`salt3 = \" 827634876\"; `", + "value": " 827634876", + "value_start": 9, + "value_end": 19, + "variable": "salt3", + "variable_start": 0, + "variable_end": 5, + "entropy_validation": { + "iterator": "BASE64_CHARS", + "entropy": 2.389735285398626, + "valid": false + } + } + ] + }, { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", @@ -11270,6 +9461,33 @@ } ] }, + { + "api_validation": "NOT_AVAILABLE", + "ml_validation": "VALIDATED_KEY", + "ml_probability": 0.665, + "rule": "Salt", + "severity": "medium", + "confidence": "moderate", + "line_data_list": [ + { + "line": "json_escaped = \"{\\\\\\\"salt8\\\\\\\":\\\\\\\"4b9a6d8b638eb0c6\\\\\\\"}\"", + "line_num": 5, + "path": "tests/samples/salt.py", + "info": "tests/samples/salt.py|RAW", + "value": "4b9a6d8b638eb0c6", + "value_start": 35, + "value_end": 51, + "variable": "salt8", + "variable_start": 21, + "variable_end": 26, + "entropy_validation": { + "iterator": "BASE36_CHARS", + "entropy": 3.2806390622295662, + "valid": true + } + } + ] + }, { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", @@ -11324,60 +9542,6 @@ } ] }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "IPv4", - "severity": "info", - "confidence": "weak", - "line_data_list": [ - { - "line": "18LibreOffice/7.6.2.1$Linux_X86_64 LibreOffice_project/9d0b4c0791fc17bc4181a67fd90c5aaed576d1c015.00001459612", - "line_num": 2, - "path": "tests/samples/sample.docx", - "info": "tests/samples/sample.docx.gz|GZIP|tests/samples/sample.docx|ZIP|docProps/app.xml|RAW", - "value": "7.6.2.1", - "value_start": 245, - "value_end": 252, - "variable": null, - "variable_start": -2, - "variable_end": -2, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 1.6042028126043455, - "valid": false - } - } - ] - }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "IPv4", - "severity": "info", - "confidence": "weak", - "line_data_list": [ - { - "line": "18LibreOffice/7.6.2.1$Linux_X86_64 LibreOffice_project/9d0b4c0791fc17bc4181a67fd90c5aaed576d1c015.00001460622", - "line_num": 2, - "path": "tests/samples/sample.docx", - "info": "tests/samples/sample.docx|ZIP|docProps/app.xml|RAW", - "value": "7.6.2.1", - "value_start": 245, - "value_end": 252, - "variable": null, - "variable_start": -2, - "variable_end": -2, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 1.6042028126043455, - "valid": false - } - } - ] - }, { "api_validation": "NOT_AVAILABLE", "ml_validation": "NOT_AVAILABLE", @@ -12377,33 +10541,6 @@ } ] }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "VALIDATED_KEY", - "ml_probability": 1.0, - "rule": "JSON Web Token", - "severity": "medium", - "confidence": "moderate", - "line_data_list": [ - { - "line": "token in zip: eyJrIjoiMDAwMDAwNDAwMDAwODAwMDAwMDAwMDAwNDAwMDAwMDAwMDAwMDAyMSIsIm4iOiJ4eHh4IiwiaWQiOjQzMDh9Cg", - "line_num": 1, - "path": "tests/samples/test2.eml", - "info": "tests/samples/test2.eml|EML-DATA|ZIP|token.txt|RAW", - "value": "eyJrIjoiMDAwMDAwNDAwMDAwODAwMDAwMDAwMDAwNDAwMDAwMDAwMDAwMDAyMSIsIm4iOiJ4eHh4IiwiaWQiOjQzMDh9Cg", - "value_start": 14, - "value_end": 108, - "variable": null, - "variable_start": -2, - "variable_end": -2, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 4.006147345318248, - "valid": false - } - } - ] - }, { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", @@ -12458,33 +10595,6 @@ } ] }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.999, - "rule": "JSON Web Token", - "severity": "medium", - "confidence": "moderate", - "line_data_list": [ - { - "line": "token in text: eyJrIjoiMDAwMDAwNDAwMDAwODAwNDAwMDAwMDAwNDAwMDAwMDAwMDAwMDAyMSIsIm4iOiJ4eHh4IiwiaWQiOjQzMDh9Cg", - "line_num": 8, - "path": "tests/samples/test2.eml", - "info": "tests/samples/test2.eml|EML-TEXT", - "value": "eyJrIjoiMDAwMDAwNDAwMDAwODAwNDAwMDAwMDAwNDAwMDAwMDAwMDAwMDAyMSIsIm4iOiJ4eHh4IiwiaWQiOjQzMDh9Cg", - "value_start": 15, - "value_end": 109, - "variable": null, - "variable_start": -2, - "variable_end": -2, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 4.0296677144829305, - "valid": false - } - } - ] - }, { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", @@ -12782,6 +10892,60 @@ } ] }, + { + "api_validation": "NOT_AVAILABLE", + "ml_validation": "VALIDATED_KEY", + "ml_probability": 0.989, + "rule": "URL Credentials", + "severity": "high", + "confidence": "moderate", + "line_data_list": [ + { + "line": "email_as_login = \"smtps://example@gmail.com:FnD83JZs@smtp.gmail.com:465\";", + "line_num": 13, + "path": "tests/samples/url_cred.js", + "info": "tests/samples/url_cred.js|RAW", + "value": "FnD83JZs", + "value_start": 44, + "value_end": 52, + "variable": "smtps://", + "variable_start": 18, + "variable_end": 26, + "entropy_validation": { + "iterator": "BASE64_CHARS", + "entropy": 3.0, + "valid": false + } + } + ] + }, + { + "api_validation": "NOT_AVAILABLE", + "ml_validation": "NOT_AVAILABLE", + "ml_probability": null, + "rule": "UUID", + "severity": "info", + "confidence": "strong", + "line_data_list": [ + { + "line": "bace4d19-fa7e-beef-cafe-9129474bcd81 # tp", + "line_num": 1, + "path": "tests/samples/uuid", + "info": "tests/samples/uuid|RAW", + "value": "bace4d19-fa7e-beef-cafe-9129474bcd81", + "value_start": 0, + "value_end": 36, + "variable": null, + "variable_start": -2, + "variable_end": -2, + "entropy_validation": { + "iterator": "BASE36_CHARS", + "entropy": 3.2373263071270246, + "valid": true + } + } + ] + }, { "api_validation": "NOT_AVAILABLE", "ml_validation": "NOT_AVAILABLE", diff --git a/tests/data/doc.json b/tests/data/doc.json index a1c4ca0f3..01e58187e 100644 --- a/tests/data/doc.json +++ b/tests/data/doc.json @@ -438,6 +438,87 @@ } ] }, + { + "api_validation": "NOT_AVAILABLE", + "ml_validation": "NOT_AVAILABLE", + "ml_probability": null, + "rule": "JSON Web Token", + "severity": "medium", + "confidence": "strong", + "line_data_list": [ + { + "line": "eyJ0eXAiOiJKV1QiLCJhbGciOiJSUzI1NiIsIng1dCI6Imk2bEdrM0ZaenhSY1ViMkMzbkVRN3N5SEpsWSIsImtpZCI6Imk2bEdrM0ZaenhSY1ViMkMzbkVRN3N5SEpsWSJ9.eyJhdWQiOiJlZjFkYTlkNC1mZjc3LTRjM2UtYTAwNS04NDBjM2Y4MzA3NDUiLCJpc3MiOiJodHRwczovL3N0cy53aW5kb3dzLm5ldC9mYTE1ZDY5Mi1lOWM3LTQ0NjAtYTc0My0yOWYyOTUyMjIyOS8iLCJpYXQiOjE1MzcyMzMxMDYsIm5iZiI6MTUzNzIzMzEwNiwiZXhwIjoxNTM3MjM3MDA2LCJhY3IiOiIxIiwiYWlvIjoiQVhRQWkvOElBQUFBRm0rRS9RVEcrZ0ZuVnhMaldkdzhLKzYxQUdyU091TU1GNmViYU1qN1hPM0libUQzZkdtck95RCtOdlp5R24yVmFUL2tES1h3NE1JaHJnR1ZxNkJuOHdMWG9UMUxrSVorRnpRVmtKUFBMUU9WNEtjWHFTbENWUERTL0RpQ0RnRTIyMlRJbU12V05hRU1hVU9Uc0lHdlRRPT0iLCJhbXIiOlsid2lhIl0sImFwcGlkIjoiNzVkYmU3N2YtMTBhMy00ZTU5LTg1ZmQtOGMxMjc1NDRmMTdjIiwiYXBwaWRhY3IiOiIwIiwiZW1haWwiOiJBYmVMaUBtaWNyb3NvZnQuY29tIiwiZmFtaWx5X25hbWUiOiJMaW5jb2xuIiwiZ2l2ZW5fbmFtZSI6IkFiZSAoTVNGVCkiLCJpZHAiOiJodHRwczovL3N0cy53aW5kb3dzLm5ldC83MmY5ODhiZi04NmYxLTQxYWYtOTFhYi0yZDdjZDAxMjIyNDcvIiwiaXBhZGRyIjoiMjIyLjIyMi4yMjIuMjIiLCJuYW1lIjoiYWJlbGkiLCJvaWQiOiIwMjIyM2I2Yi1hYTFkLTQyZDQtOWVjMC0xYjJiYjkxOTQ0MzgiLCJyaCI6IkkiLCJzY3AiOiJ1c2VyX2ltcGVyc29uYXRpb24iLCJzdWIiOiJsM19yb0lTUVUyMjJiVUxTOXlpMmswWHBxcE9pTXo1SDNaQUNvMUdlWEEiLCJ0aWQiOiJmYTE1ZDY5Mi1lOWM3LTQ0NjAtYTc0My0yOWYyOTU2ZmQ0MjkiLCJ1bmlxdWVfbmFtZSI6ImFiZWxpQG1pY3Jvc29mdC5jb20iLCJ1dGkiOiJGVnNHeFlYSTMwLVR1aWt1dVVvRkFBIiwidmVyIjoiMS4wIn0.D3H6pMUtQnoJAGq6AHd", + "line_num": 1, + "path": "tests/samples/azure_access_token", + "info": "tests/samples/azure_access_token|RAW", + "value": "eyJ0eXAiOiJKV1QiLCJhbGciOiJSUzI1NiIsIng1dCI6Imk2bEdrM0ZaenhSY1ViMkMzbkVRN3N5SEpsWSIsImtpZCI6Imk2bEdrM0ZaenhSY1ViMkMzbkVRN3N5SEpsWSJ9.eyJhdWQiOiJlZjFkYTlkNC1mZjc3LTRjM2UtYTAwNS04NDBjM2Y4MzA3NDUiLCJpc3MiOiJodHRwczovL3N0cy53aW5kb3dzLm5ldC9mYTE1ZDY5Mi1lOWM3LTQ0NjAtYTc0My0yOWYyOTUyMjIyOS8iLCJpYXQiOjE1MzcyMzMxMDYsIm5iZiI6MTUzNzIzMzEwNiwiZXhwIjoxNTM3MjM3MDA2LCJhY3IiOiIxIiwiYWlvIjoiQVhRQWkvOElBQUFBRm0rRS9RVEcrZ0ZuVnhMaldkdzhLKzYxQUdyU091TU1GNmViYU1qN1hPM0libUQzZkdtck95RCtOdlp5R24yVmFUL2tES1h3NE1JaHJnR1ZxNkJuOHdMWG9UMUxrSVorRnpRVmtKUFBMUU9WNEtjWHFTbENWUERTL0RpQ0RnRTIyMlRJbU12V05hRU1hVU9Uc0lHdlRRPT0iLCJhbXIiOlsid2lhIl0sImFwcGlkIjoiNzVkYmU3N2YtMTBhMy00ZTU5LTg1ZmQtOGMxMjc1NDRmMTdjIiwiYXBwaWRhY3IiOiIwIiwiZW1haWwiOiJBYmVMaUBtaWNyb3NvZnQuY29tIiwiZmFtaWx5X25hbWUiOiJMaW5jb2xuIiwiZ2l2ZW5fbmFtZSI6IkFiZSAoTVNGVCkiLCJpZHAiOiJodHRwczovL3N0cy53aW5kb3dzLm5ldC83MmY5ODhiZi04NmYxLTQxYWYtOTFhYi0yZDdjZDAxMjIyNDcvIiwiaXBhZGRyIjoiMjIyLjIyMi4yMjIuMjIiLCJuYW1lIjoiYWJlbGkiLCJvaWQiOiIwMjIyM2I2Yi1hYTFkLTQyZDQtOWVjMC0xYjJiYjkxOTQ0MzgiLCJyaCI6IkkiLCJzY3AiOiJ1c2VyX2ltcGVyc29uYXRpb24iLCJzdWIiOiJsM19yb0lTUVUyMjJiVUxTOXlpMmswWHBxcE9pTXo1SDNaQUNvMUdlWEEiLCJ0aWQiOiJmYTE1ZDY5Mi1lOWM3LTQ0NjAtYTc0My0yOWYyOTU2ZmQ0MjkiLCJ1bmlxdWVfbmFtZSI6ImFiZWxpQG1pY3Jvc29mdC5jb20iLCJ1dGkiOiJGVnNHeFlYSTMwLVR1aWt1dVVvRkFBIiwidmVyIjoiMS4wIn0.D3H6pMUtQnoJAGq6AHd", + "value_start": 0, + "value_end": 1316, + "variable": null, + "variable_start": -2, + "variable_end": -2, + "entropy_validation": { + "iterator": "BASE64_CHARS", + "entropy": 5.615950458346115, + "valid": true + } + } + ] + }, + { + "api_validation": "NOT_AVAILABLE", + "ml_validation": "NOT_AVAILABLE", + "ml_probability": null, + "rule": "Azure Access Token", + "severity": "high", + "confidence": "strong", + "line_data_list": [ + { + "line": "eyJ0eXAiOiJKV1QiLCJhbGciOiJSUzI1NiIsImtpZCI6Imk2bEdrM0ZaenhSY1ViMkMzbkVRN3N5SEpsWSJ9.eyJhdWQiOiI2ZTc0MTcyYi1iZTU2LTQ4NDMtOWZmNC1lNjZhMzliYjEyZTMiLCJpc3MiOiJodHRwczovL2xvZ2luLm1pY3Jvc29mdG9ubGluZS5jb20vNzJmOTg4YmYtODZmMS00MWFmLTkxYWItMmQ3Y2QwMTFkYjQ3L3YyLjAiLCJpYXQiOjE1MzcyMzEwNDgsIm5iZiI6MTUzNzIzMTA0OCwiZXhwIjoxNTM3MjM0OTQ4LCJhaW8iOiJBWFFBaS84SUFBQUF0QWFaTG8zQ2hNaWY2S09udHRSQjdlQnE0L0RjY1F6amNKR3hQWXkvQzNqRGFOR3hYZDZ3TklJVkdSZ2hOUm53SjFsT2NBbk5aY2p2a295ckZ4Q3R0djMzMTQwUmlvT0ZKNGJDQ0dWdW9DYWcxdU9UVDIyMjIyZ0h3TFBZUS91Zjc5UVgrMEtJaWpkcm1wNjlSY3R6bVE9PSIsImF6cCI6IjZlNzQxNzJiLWJlNTYtNDg0My05ZmY0LWU2NmEzOWJiMTJlMyIsImF6cGFjciI6IjAiLCJuYW1lIjoiQWJlIExpbmNvbG4iLCJvaWQiOiI2OTAyMjJiZS1mZjFhLTRkNTYtYWJkMS03ZTRmN2QzOGU0NzQiLCJwcmVmZXJyZWRfdXNlcm5hbWUiOiJhYmVsaUBtaWNyb3NvZnQuY29tIiwicmgiOiJJIiwic2NwIjoiYWNjZXNzX2FzX3VzZXIiLCJzdWIiOiJIS1pwZmFIeVdhZGVPb3VZbGl0anJJLUtmZlRtMjIyWDVyclYzeERxZktRIiwidGlkIjoiNzJmOTg4YmYtODZmMS00MWFmLTkxYWItMmQ3Y2QwMTFkYjQ3IiwidXRpIjoiZnFpQnFYTFBqMGVRYTgyUy1JWUZBQSIsInZlciI6IjIuMCJ9.pj4N-w_3Us9DrBLfpCt", + "line_num": 2, + "path": "tests/samples/azure_access_token", + "info": "tests/samples/azure_access_token|RAW", + "value": "eyJ0eXAiOiJKV1QiLCJhbGciOiJSUzI1NiIsImtpZCI6Imk2bEdrM0ZaenhSY1ViMkMzbkVRN3N5SEpsWSJ9.eyJhdWQiOiI2ZTc0MTcyYi1iZTU2LTQ4NDMtOWZmNC1lNjZhMzliYjEyZTMiLCJpc3MiOiJodHRwczovL2xvZ2luLm1pY3Jvc29mdG9ubGluZS5jb20vNzJmOTg4YmYtODZmMS00MWFmLTkxYWItMmQ3Y2QwMTFkYjQ3L3YyLjAiLCJpYXQiOjE1MzcyMzEwNDgsIm5iZiI6MTUzNzIzMTA0OCwiZXhwIjoxNTM3MjM0OTQ4LCJhaW8iOiJBWFFBaS84SUFBQUF0QWFaTG8zQ2hNaWY2S09udHRSQjdlQnE0L0RjY1F6amNKR3hQWXkvQzNqRGFOR3hYZDZ3TklJVkdSZ2hOUm53SjFsT2NBbk5aY2p2a295ckZ4Q3R0djMzMTQwUmlvT0ZKNGJDQ0dWdW9DYWcxdU9UVDIyMjIyZ0h3TFBZUS91Zjc5UVgrMEtJaWpkcm1wNjlSY3R6bVE9PSIsImF6cCI6IjZlNzQxNzJiLWJlNTYtNDg0My05ZmY0LWU2NmEzOWJiMTJlMyIsImF6cGFjciI6IjAiLCJuYW1lIjoiQWJlIExpbmNvbG4iLCJvaWQiOiI2OTAyMjJiZS1mZjFhLTRkNTYtYWJkMS03ZTRmN2QzOGU0NzQiLCJwcmVmZXJyZWRfdXNlcm5hbWUiOiJhYmVsaUBtaWNyb3NvZnQuY29tIiwicmgiOiJJIiwic2NwIjoiYWNjZXNzX2FzX3VzZXIiLCJzdWIiOiJIS1pwZmFIeVdhZGVPb3VZbGl0anJJLUtmZlRtMjIyWDVyclYzeERxZktRIiwidGlkIjoiNzJmOTg4YmYtODZmMS00MWFmLTkxYWItMmQ3Y2QwMTFkYjQ3IiwidXRpIjoiZnFpQnFYTFBqMGVRYTgyUy1JWUZBQSIsInZlciI6IjIuMCJ9.pj4N-w_3Us9DrBLfpCt", + "value_start": 0, + "value_end": 1029, + "variable": null, + "variable_start": -2, + "variable_end": -2, + "entropy_validation": { + "iterator": "BASE64_CHARS", + "entropy": 5.6044494049575055, + "valid": true + } + } + ] + }, + { + "api_validation": "NOT_AVAILABLE", + "ml_validation": "NOT_AVAILABLE", + "ml_probability": null, + "rule": "JSON Web Token", + "severity": "medium", + "confidence": "strong", + "line_data_list": [ + { + "line": "eyJ0eXAiOiJKV1QiLCJhbGciOiJSUzI1NiIsImtpZCI6Imk2bEdrM0ZaenhSY1ViMkMzbkVRN3N5SEpsWSJ9.eyJhdWQiOiI2ZTc0MTcyYi1iZTU2LTQ4NDMtOWZmNC1lNjZhMzliYjEyZTMiLCJpc3MiOiJodHRwczovL2xvZ2luLm1pY3Jvc29mdG9ubGluZS5jb20vNzJmOTg4YmYtODZmMS00MWFmLTkxYWItMmQ3Y2QwMTFkYjQ3L3YyLjAiLCJpYXQiOjE1MzcyMzEwNDgsIm5iZiI6MTUzNzIzMTA0OCwiZXhwIjoxNTM3MjM0OTQ4LCJhaW8iOiJBWFFBaS84SUFBQUF0QWFaTG8zQ2hNaWY2S09udHRSQjdlQnE0L0RjY1F6amNKR3hQWXkvQzNqRGFOR3hYZDZ3TklJVkdSZ2hOUm53SjFsT2NBbk5aY2p2a295ckZ4Q3R0djMzMTQwUmlvT0ZKNGJDQ0dWdW9DYWcxdU9UVDIyMjIyZ0h3TFBZUS91Zjc5UVgrMEtJaWpkcm1wNjlSY3R6bVE9PSIsImF6cCI6IjZlNzQxNzJiLWJlNTYtNDg0My05ZmY0LWU2NmEzOWJiMTJlMyIsImF6cGFjciI6IjAiLCJuYW1lIjoiQWJlIExpbmNvbG4iLCJvaWQiOiI2OTAyMjJiZS1mZjFhLTRkNTYtYWJkMS03ZTRmN2QzOGU0NzQiLCJwcmVmZXJyZWRfdXNlcm5hbWUiOiJhYmVsaUBtaWNyb3NvZnQuY29tIiwicmgiOiJJIiwic2NwIjoiYWNjZXNzX2FzX3VzZXIiLCJzdWIiOiJIS1pwZmFIeVdhZGVPb3VZbGl0anJJLUtmZlRtMjIyWDVyclYzeERxZktRIiwidGlkIjoiNzJmOTg4YmYtODZmMS00MWFmLTkxYWItMmQ3Y2QwMTFkYjQ3IiwidXRpIjoiZnFpQnFYTFBqMGVRYTgyUy1JWUZBQSIsInZlciI6IjIuMCJ9.pj4N-w_3Us9DrBLfpCt", + "line_num": 2, + "path": "tests/samples/azure_access_token", + "info": "tests/samples/azure_access_token|RAW", + "value": "eyJ0eXAiOiJKV1QiLCJhbGciOiJSUzI1NiIsImtpZCI6Imk2bEdrM0ZaenhSY1ViMkMzbkVRN3N5SEpsWSJ9.eyJhdWQiOiI2ZTc0MTcyYi1iZTU2LTQ4NDMtOWZmNC1lNjZhMzliYjEyZTMiLCJpc3MiOiJodHRwczovL2xvZ2luLm1pY3Jvc29mdG9ubGluZS5jb20vNzJmOTg4YmYtODZmMS00MWFmLTkxYWItMmQ3Y2QwMTFkYjQ3L3YyLjAiLCJpYXQiOjE1MzcyMzEwNDgsIm5iZiI6MTUzNzIzMTA0OCwiZXhwIjoxNTM3MjM0OTQ4LCJhaW8iOiJBWFFBaS84SUFBQUF0QWFaTG8zQ2hNaWY2S09udHRSQjdlQnE0L0RjY1F6amNKR3hQWXkvQzNqRGFOR3hYZDZ3TklJVkdSZ2hOUm53SjFsT2NBbk5aY2p2a295ckZ4Q3R0djMzMTQwUmlvT0ZKNGJDQ0dWdW9DYWcxdU9UVDIyMjIyZ0h3TFBZUS91Zjc5UVgrMEtJaWpkcm1wNjlSY3R6bVE9PSIsImF6cCI6IjZlNzQxNzJiLWJlNTYtNDg0My05ZmY0LWU2NmEzOWJiMTJlMyIsImF6cGFjciI6IjAiLCJuYW1lIjoiQWJlIExpbmNvbG4iLCJvaWQiOiI2OTAyMjJiZS1mZjFhLTRkNTYtYWJkMS03ZTRmN2QzOGU0NzQiLCJwcmVmZXJyZWRfdXNlcm5hbWUiOiJhYmVsaUBtaWNyb3NvZnQuY29tIiwicmgiOiJJIiwic2NwIjoiYWNjZXNzX2FzX3VzZXIiLCJzdWIiOiJIS1pwZmFIeVdhZGVPb3VZbGl0anJJLUtmZlRtMjIyWDVyclYzeERxZktRIiwidGlkIjoiNzJmOTg4YmYtODZmMS00MWFmLTkxYWItMmQ3Y2QwMTFkYjQ3IiwidXRpIjoiZnFpQnFYTFBqMGVRYTgyUy1JWUZBQSIsInZlciI6IjIuMCJ9.pj4N-w_3Us9DrBLfpCt", + "value_start": 0, + "value_end": 1029, + "variable": null, + "variable_start": -2, + "variable_end": -2, + "entropy_validation": { + "iterator": "BASE64_CHARS", + "entropy": 5.6044494049575055, + "valid": true + } + } + ] + }, { "api_validation": "NOT_AVAILABLE", "ml_validation": "NOT_AVAILABLE", @@ -11439,6 +11520,33 @@ } ] }, + { + "api_validation": "NOT_AVAILABLE", + "ml_validation": "NOT_AVAILABLE", + "ml_probability": null, + "rule": "JSON Web Token", + "severity": "medium", + "confidence": "strong", + "line_data_list": [ + { + "line": "detected: eyJhbGciOiJSUzI1NiJ9Cg.eyJleHAiOjY1NTM2fQo.Ce7sh0ENPiBlE_dose0cBA", + "line_num": 1, + "path": "tests/samples/json_web_token", + "info": "tests/samples/json_web_token|RAW", + "value": "eyJhbGciOiJSUzI1NiJ9Cg.eyJleHAiOjY1NTM2fQo.Ce7sh0ENPiBlE_dose0cBA", + "value_start": 10, + "value_end": 75, + "variable": null, + "variable_start": -2, + "variable_end": -2, + "entropy_validation": { + "iterator": "BASE64_CHARS", + "entropy": 4.790963630103494, + "valid": true + } + } + ] + }, { "api_validation": "NOT_AVAILABLE", "ml_validation": "NOT_AVAILABLE", @@ -12956,6 +13064,33 @@ } ] }, + { + "api_validation": "NOT_AVAILABLE", + "ml_validation": "NOT_AVAILABLE", + "ml_probability": null, + "rule": "UUID", + "severity": "info", + "confidence": "strong", + "line_data_list": [ + { + "line": "bace4d19-fa7e-beef-cafe-9129474bcd81 # tp", + "line_num": 1, + "path": "tests/samples/uuid", + "info": "tests/samples/uuid|RAW", + "value": "bace4d19-fa7e-beef-cafe-9129474bcd81", + "value_start": 0, + "value_end": 36, + "variable": null, + "variable_start": -2, + "variable_end": -2, + "entropy_validation": { + "iterator": "BASE36_CHARS", + "entropy": 3.2373263071270246, + "valid": true + } + } + ] + }, { "api_validation": "NOT_AVAILABLE", "ml_validation": "NOT_AVAILABLE", diff --git a/tests/data/ml_threshold.json b/tests/data/ml_threshold.json index 9d9955e9b..2c5a32da5 100644 --- a/tests/data/ml_threshold.json +++ b/tests/data/ml_threshold.json @@ -336,7 +336,7 @@ "line_num": 9, "path": "tests/samples/auth_n.template", "info": "", - "value": "f8b7d9fdd5efc086722d2df3267d92f82edf1523225be33751597912f220b7c2", + "value": "eyJGRpVu1c2VzY2-823r_db32hbf4W1lbj", "value_start": 31, "value_end": 65, "variable": null, @@ -768,7 +768,7 @@ "ml_probability": 0.93, "rule": "JSON Web Token", "severity": "medium", - "confidence": "moderate", + "confidence": "strong", "line_data_list": [ { "line": "eyJ0eXAiOiJKV1QiLCJhbGciOiJSUzI1NiIsIng1dCI6Imk2bEdrM0ZaenhSY1ViMkMzbkVRN3N5SEpsWSIsImtpZCI6Imk2bEdrM0ZaenhSY1ViMkMzbkVRN3N5SEpsWSJ9.eyJhdWQiOiJlZjFkYTlkNC1mZjc3LTRjM2UtYTAwNS04NDBjM2Y4MzA3NDUiLCJpc3MiOiJodHRwczovL3N0cy53aW5kb3dzLm5ldC9mYTE1ZDY5Mi1lOWM3LTQ0NjAtYTc0My0yOWYyOTUyMjIyOS8iLCJpYXQiOjE1MzcyMzMxMDYsIm5iZiI6MTUzNzIzMzEwNiwiZXhwIjoxNTM3MjM3MDA2LCJhY3IiOiIxIiwiYWlvIjoiQVhRQWkvOElBQUFBRm0rRS9RVEcrZ0ZuVnhMaldkdzhLKzYxQUdyU091TU1GNmViYU1qN1hPM0libUQzZkdtck95RCtOdlp5R24yVmFUL2tES1h3NE1JaHJnR1ZxNkJuOHdMWG9UMUxrSVorRnpRVmtKUFBMUU9WNEtjWHFTbENWUERTL0RpQ0RnRTIyMlRJbU12V05hRU1hVU9Uc0lHdlRRPT0iLCJhbXIiOlsid2lhIl0sImFwcGlkIjoiNzVkYmU3N2YtMTBhMy00ZTU5LTg1ZmQtOGMxMjc1NDRmMTdjIiwiYXBwaWRhY3IiOiIwIiwiZW1haWwiOiJBYmVMaUBtaWNyb3NvZnQuY29tIiwiZmFtaWx5X25hbWUiOiJMaW5jb2xuIiwiZ2l2ZW5fbmFtZSI6IkFiZSAoTVNGVCkiLCJpZHAiOiJodHRwczovL3N0cy53aW5kb3dzLm5ldC83MmY5ODhiZi04NmYxLTQxYWYtOTFhYi0yZDdjZDAxMjIyNDcvIiwiaXBhZGRyIjoiMjIyLjIyMi4yMjIuMjIiLCJuYW1lIjoiYWJlbGkiLCJvaWQiOiIwMjIyM2I2Yi1hYTFkLTQyZDQtOWVjMC0xYjJiYjkxOTQ0MzgiLCJyaCI6IkkiLCJzY3AiOiJ1c2VyX2ltcGVyc29uYXRpb24iLCJzdWIiOiJsM19yb0lTUVUyMjJiVUxTOXlpMmswWHBxcE9pTXo1SDNaQUNvMUdlWEEiLCJ0aWQiOiJmYTE1ZDY5Mi1lOWM3LTQ0NjAtYTc0My0yOWYyOTU2ZmQ0MjkiLCJ1bmlxdWVfbmFtZSI6ImFiZWxpQG1pY3Jvc29mdC5jb20iLCJ1dGkiOiJGVnNHeFlYSTMwLVR1aWt1dVVvRkFBIiwidmVyIjoiMS4wIn0.D3H6pMUtQnoJAGq6AHd", @@ -789,6 +789,60 @@ } ] }, + { + "api_validation": "NOT_AVAILABLE", + "ml_validation": "NOT_AVAILABLE", + "ml_probability": null, + "rule": "Azure Access Token", + "severity": "high", + "confidence": "strong", + "line_data_list": [ + { + "line": "eyJ0eXAiOiJKV1QiLCJhbGciOiJSUzI1NiIsImtpZCI6Imk2bEdrM0ZaenhSY1ViMkMzbkVRN3N5SEpsWSJ9.eyJhdWQiOiI2ZTc0MTcyYi1iZTU2LTQ4NDMtOWZmNC1lNjZhMzliYjEyZTMiLCJpc3MiOiJodHRwczovL2xvZ2luLm1pY3Jvc29mdG9ubGluZS5jb20vNzJmOTg4YmYtODZmMS00MWFmLTkxYWItMmQ3Y2QwMTFkYjQ3L3YyLjAiLCJpYXQiOjE1MzcyMzEwNDgsIm5iZiI6MTUzNzIzMTA0OCwiZXhwIjoxNTM3MjM0OTQ4LCJhaW8iOiJBWFFBaS84SUFBQUF0QWFaTG8zQ2hNaWY2S09udHRSQjdlQnE0L0RjY1F6amNKR3hQWXkvQzNqRGFOR3hYZDZ3TklJVkdSZ2hOUm53SjFsT2NBbk5aY2p2a295ckZ4Q3R0djMzMTQwUmlvT0ZKNGJDQ0dWdW9DYWcxdU9UVDIyMjIyZ0h3TFBZUS91Zjc5UVgrMEtJaWpkcm1wNjlSY3R6bVE9PSIsImF6cCI6IjZlNzQxNzJiLWJlNTYtNDg0My05ZmY0LWU2NmEzOWJiMTJlMyIsImF6cGFjciI6IjAiLCJuYW1lIjoiQWJlIExpbmNvbG4iLCJvaWQiOiI2OTAyMjJiZS1mZjFhLTRkNTYtYWJkMS03ZTRmN2QzOGU0NzQiLCJwcmVmZXJyZWRfdXNlcm5hbWUiOiJhYmVsaUBtaWNyb3NvZnQuY29tIiwicmgiOiJJIiwic2NwIjoiYWNjZXNzX2FzX3VzZXIiLCJzdWIiOiJIS1pwZmFIeVdhZGVPb3VZbGl0anJJLUtmZlRtMjIyWDVyclYzeERxZktRIiwidGlkIjoiNzJmOTg4YmYtODZmMS00MWFmLTkxYWItMmQ3Y2QwMTFkYjQ3IiwidXRpIjoiZnFpQnFYTFBqMGVRYTgyUy1JWUZBQSIsInZlciI6IjIuMCJ9.pj4N-w_3Us9DrBLfpCt", + "line_num": 2, + "path": "tests/samples/azure_access_token", + "info": "", + "value": "eyJ0eXAiOiJKV1QiLCJhbGciOiJSUzI1NiIsImtpZCI6Imk2bEdrM0ZaenhSY1ViMkMzbkVRN3N5SEpsWSJ9.eyJhdWQiOiI2ZTc0MTcyYi1iZTU2LTQ4NDMtOWZmNC1lNjZhMzliYjEyZTMiLCJpc3MiOiJodHRwczovL2xvZ2luLm1pY3Jvc29mdG9ubGluZS5jb20vNzJmOTg4YmYtODZmMS00MWFmLTkxYWItMmQ3Y2QwMTFkYjQ3L3YyLjAiLCJpYXQiOjE1MzcyMzEwNDgsIm5iZiI6MTUzNzIzMTA0OCwiZXhwIjoxNTM3MjM0OTQ4LCJhaW8iOiJBWFFBaS84SUFBQUF0QWFaTG8zQ2hNaWY2S09udHRSQjdlQnE0L0RjY1F6amNKR3hQWXkvQzNqRGFOR3hYZDZ3TklJVkdSZ2hOUm53SjFsT2NBbk5aY2p2a295ckZ4Q3R0djMzMTQwUmlvT0ZKNGJDQ0dWdW9DYWcxdU9UVDIyMjIyZ0h3TFBZUS91Zjc5UVgrMEtJaWpkcm1wNjlSY3R6bVE9PSIsImF6cCI6IjZlNzQxNzJiLWJlNTYtNDg0My05ZmY0LWU2NmEzOWJiMTJlMyIsImF6cGFjciI6IjAiLCJuYW1lIjoiQWJlIExpbmNvbG4iLCJvaWQiOiI2OTAyMjJiZS1mZjFhLTRkNTYtYWJkMS03ZTRmN2QzOGU0NzQiLCJwcmVmZXJyZWRfdXNlcm5hbWUiOiJhYmVsaUBtaWNyb3NvZnQuY29tIiwicmgiOiJJIiwic2NwIjoiYWNjZXNzX2FzX3VzZXIiLCJzdWIiOiJIS1pwZmFIeVdhZGVPb3VZbGl0anJJLUtmZlRtMjIyWDVyclYzeERxZktRIiwidGlkIjoiNzJmOTg4YmYtODZmMS00MWFmLTkxYWItMmQ3Y2QwMTFkYjQ3IiwidXRpIjoiZnFpQnFYTFBqMGVRYTgyUy1JWUZBQSIsInZlciI6IjIuMCJ9.pj4N-w_3Us9DrBLfpCt", + "value_start": 0, + "value_end": 1029, + "variable": null, + "variable_start": -2, + "variable_end": -2, + "entropy_validation": { + "iterator": "BASE64_CHARS", + "entropy": 5.6044494049575055, + "valid": true + } + } + ] + }, + { + "api_validation": "NOT_AVAILABLE", + "ml_validation": "NOT_AVAILABLE", + "ml_probability": null, + "rule": "JSON Web Token", + "severity": "medium", + "confidence": "strong", + "line_data_list": [ + { + "line": "eyJ0eXAiOiJKV1QiLCJhbGciOiJSUzI1NiIsImtpZCI6Imk2bEdrM0ZaenhSY1ViMkMzbkVRN3N5SEpsWSJ9.eyJhdWQiOiI2ZTc0MTcyYi1iZTU2LTQ4NDMtOWZmNC1lNjZhMzliYjEyZTMiLCJpc3MiOiJodHRwczovL2xvZ2luLm1pY3Jvc29mdG9ubGluZS5jb20vNzJmOTg4YmYtODZmMS00MWFmLTkxYWItMmQ3Y2QwMTFkYjQ3L3YyLjAiLCJpYXQiOjE1MzcyMzEwNDgsIm5iZiI6MTUzNzIzMTA0OCwiZXhwIjoxNTM3MjM0OTQ4LCJhaW8iOiJBWFFBaS84SUFBQUF0QWFaTG8zQ2hNaWY2S09udHRSQjdlQnE0L0RjY1F6amNKR3hQWXkvQzNqRGFOR3hYZDZ3TklJVkdSZ2hOUm53SjFsT2NBbk5aY2p2a295ckZ4Q3R0djMzMTQwUmlvT0ZKNGJDQ0dWdW9DYWcxdU9UVDIyMjIyZ0h3TFBZUS91Zjc5UVgrMEtJaWpkcm1wNjlSY3R6bVE9PSIsImF6cCI6IjZlNzQxNzJiLWJlNTYtNDg0My05ZmY0LWU2NmEzOWJiMTJlMyIsImF6cGFjciI6IjAiLCJuYW1lIjoiQWJlIExpbmNvbG4iLCJvaWQiOiI2OTAyMjJiZS1mZjFhLTRkNTYtYWJkMS03ZTRmN2QzOGU0NzQiLCJwcmVmZXJyZWRfdXNlcm5hbWUiOiJhYmVsaUBtaWNyb3NvZnQuY29tIiwicmgiOiJJIiwic2NwIjoiYWNjZXNzX2FzX3VzZXIiLCJzdWIiOiJIS1pwZmFIeVdhZGVPb3VZbGl0anJJLUtmZlRtMjIyWDVyclYzeERxZktRIiwidGlkIjoiNzJmOTg4YmYtODZmMS00MWFmLTkxYWItMmQ3Y2QwMTFkYjQ3IiwidXRpIjoiZnFpQnFYTFBqMGVRYTgyUy1JWUZBQSIsInZlciI6IjIuMCJ9.pj4N-w_3Us9DrBLfpCt", + "line_num": 2, + "path": "tests/samples/azure_access_token", + "info": "", + "value": "eyJ0eXAiOiJKV1QiLCJhbGciOiJSUzI1NiIsImtpZCI6Imk2bEdrM0ZaenhSY1ViMkMzbkVRN3N5SEpsWSJ9.eyJhdWQiOiI2ZTc0MTcyYi1iZTU2LTQ4NDMtOWZmNC1lNjZhMzliYjEyZTMiLCJpc3MiOiJodHRwczovL2xvZ2luLm1pY3Jvc29mdG9ubGluZS5jb20vNzJmOTg4YmYtODZmMS00MWFmLTkxYWItMmQ3Y2QwMTFkYjQ3L3YyLjAiLCJpYXQiOjE1MzcyMzEwNDgsIm5iZiI6MTUzNzIzMTA0OCwiZXhwIjoxNTM3MjM0OTQ4LCJhaW8iOiJBWFFBaS84SUFBQUF0QWFaTG8zQ2hNaWY2S09udHRSQjdlQnE0L0RjY1F6amNKR3hQWXkvQzNqRGFOR3hYZDZ3TklJVkdSZ2hOUm53SjFsT2NBbk5aY2p2a295ckZ4Q3R0djMzMTQwUmlvT0ZKNGJDQ0dWdW9DYWcxdU9UVDIyMjIyZ0h3TFBZUS91Zjc5UVgrMEtJaWpkcm1wNjlSY3R6bVE9PSIsImF6cCI6IjZlNzQxNzJiLWJlNTYtNDg0My05ZmY0LWU2NmEzOWJiMTJlMyIsImF6cGFjciI6IjAiLCJuYW1lIjoiQWJlIExpbmNvbG4iLCJvaWQiOiI2OTAyMjJiZS1mZjFhLTRkNTYtYWJkMS03ZTRmN2QzOGU0NzQiLCJwcmVmZXJyZWRfdXNlcm5hbWUiOiJhYmVsaUBtaWNyb3NvZnQuY29tIiwicmgiOiJJIiwic2NwIjoiYWNjZXNzX2FzX3VzZXIiLCJzdWIiOiJIS1pwZmFIeVdhZGVPb3VZbGl0anJJLUtmZlRtMjIyWDVyclYzeERxZktRIiwidGlkIjoiNzJmOTg4YmYtODZmMS00MWFmLTkxYWItMmQ3Y2QwMTFkYjQ3IiwidXRpIjoiZnFpQnFYTFBqMGVRYTgyUy1JWUZBQSIsInZlciI6IjIuMCJ9.pj4N-w_3Us9DrBLfpCt", + "value_start": 0, + "value_end": 1029, + "variable": null, + "variable_start": -2, + "variable_end": -2, + "entropy_validation": { + "iterator": "BASE64_CHARS", + "entropy": 5.6044494049575055, + "valid": true + } + } + ] + }, { "api_validation": "NOT_AVAILABLE", "ml_validation": "NOT_AVAILABLE", @@ -2341,7 +2395,7 @@ "line_num": 50, "path": "tests/samples/doc_id_pair_passwd_pair", "info": "", - "value": "79edef7a351a4ad5f483a6a8ddf75b1f6d63ad08a774cee47e0f787e6c9bd790", + "value": "98.76.54.32", "value_start": 1, "value_end": 12, "variable": null, @@ -3248,26 +3302,26 @@ }, { "api_validation": "NOT_AVAILABLE", - "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.904, - "rule": "Password", - "severity": "medium", - "confidence": "moderate", + "ml_validation": "NOT_AVAILABLE", + "ml_probability": null, + "rule": "IPv4", + "severity": "info", + "confidence": "weak", "line_data_list": [ { "line": "ANYid:master pw:dipPr194Gg! ip:98.76.54.32", "line_num": 95, "path": "tests/samples/doc_id_pair_passwd_pair", "info": "", - "value": "51229c4f0ad32f657ee93a55ed8c6b09c2c8aa2951e9741c3838373a1b1d8105", - "value_start": 16, - "value_end": 27, - "variable": "pw", - "variable_start": 13, - "variable_end": 15, + "value": "98.76.54.32", + "value_start": 31, + "value_end": 42, + "variable": null, + "variable_start": -2, + "variable_end": -2, "entropy_validation": { "iterator": "BASE64_CHARS", - "entropy": 3.1449378351248165, + "entropy": 2.515950268099853, "valid": false } } @@ -3275,8 +3329,8 @@ }, { "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, + "ml_validation": "VALIDATED_KEY", + "ml_probability": 0.904, "rule": "IPv4", "severity": "info", "confidence": "weak", @@ -3313,7 +3367,7 @@ "line_num": 97, "path": "tests/samples/doc_id_pair_passwd_pair", "info": "", - "value": "79edef7a351a4ad5f483a6a8ddf75b1f6d63ad08a774cee47e0f787e6c9bd790", + "value": "98.76.54.32", "value_start": 7, "value_end": 18, "variable": null, @@ -3599,26 +3653,26 @@ }, { "api_validation": "NOT_AVAILABLE", - "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.435, - "rule": "Password", - "severity": "medium", - "confidence": "moderate", + "ml_validation": "NOT_AVAILABLE", + "ml_probability": null, + "rule": "IPv4", + "severity": "info", + "confidence": "weak", "line_data_list": [ { "line": "98.76.54.32(ID:master/PW:iPp10@GRq) # todo: move into other sample ?", "line_num": 11, "path": "tests/samples/doc_id_passwd_pair", "info": "", - "value": "01700ed04a09289c511fd137465ac0e5c5fd61665f3100fb17119dce90f20d1e", - "value_start": 25, - "value_end": 35, - "variable": "PW", - "variable_start": 22, - "variable_end": 24, + "value": "98.76.54.32", + "value_start": 0, + "value_end": 11, + "variable": null, + "variable_start": -2, + "variable_end": -2, "entropy_validation": { "iterator": "BASE64_CHARS", - "entropy": 2.65754247590989, + "entropy": 2.515950268099853, "valid": false } } @@ -3626,8 +3680,8 @@ }, { "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, + "ml_validation": "VALIDATED_KEY", + "ml_probability": 0.435, "rule": "IPv4", "severity": "info", "confidence": "weak", @@ -3707,26 +3761,26 @@ }, { "api_validation": "NOT_AVAILABLE", - "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.745, - "rule": "Password", - "severity": "medium", - "confidence": "moderate", + "ml_validation": "NOT_AVAILABLE", + "ml_probability": null, + "rule": "IPv4", + "severity": "info", + "confidence": "weak", "line_data_list": [ { "line": "98.76.54.32 id/pw:master/iPp19@GRq", "line_num": 20, "path": "tests/samples/doc_id_passwd_pair", "info": "", - "value": "35578a513fb58cf5c7124ce76f8caf0e2c416c69bdec652c2ad18f585edaffb7", - "value_start": 18, - "value_end": 34, - "variable": "pw", - "variable_start": 15, - "variable_end": 17, + "value": "98.76.54.32", + "value_start": 0, + "value_end": 11, + "variable": null, + "variable_start": -2, + "variable_end": -2, "entropy_validation": { "iterator": "BASE64_CHARS", - "entropy": 3.75, + "entropy": 2.515950268099853, "valid": false } } @@ -3734,8 +3788,8 @@ }, { "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, + "ml_validation": "VALIDATED_KEY", + "ml_probability": 0.745, "rule": "IPv4", "severity": "info", "confidence": "weak", @@ -4096,7 +4150,7 @@ "line_num": 14, "path": "tests/samples/doc_passwd_pair", "info": "", - "value": "79edef7a351a4ad5f483a6a8ddf75b1f6d63ad08a774cee47e0f787e6c9bd790", + "value": "98.76.54.32", "value_start": 7, "value_end": 18, "variable": null, @@ -5419,7 +5473,7 @@ "line_num": 1, "path": "tests/samples/doc_various", "info": "", - "value": "79edef7a351a4ad5f483a6a8ddf75b1f6d63ad08a774cee47e0f787e6c9bd790", + "value": "98.76.54.32", "value_start": 0, "value_end": 11, "variable": null, @@ -5446,7 +5500,7 @@ "line_num": 2, "path": "tests/samples/doc_various", "info": "", - "value": "79edef7a351a4ad5f483a6a8ddf75b1f6d63ad08a774cee47e0f787e6c9bd790", + "value": "98.76.54.32", "value_start": 0, "value_end": 11, "variable": null, @@ -5473,7 +5527,7 @@ "line_num": 3, "path": "tests/samples/doc_various", "info": "", - "value": "79edef7a351a4ad5f483a6a8ddf75b1f6d63ad08a774cee47e0f787e6c9bd790", + "value": "98.76.54.32", "value_start": 7, "value_end": 18, "variable": null, @@ -5581,7 +5635,7 @@ "line_num": 6, "path": "tests/samples/doc_various", "info": "", - "value": "79edef7a351a4ad5f483a6a8ddf75b1f6d63ad08a774cee47e0f787e6c9bd790", + "value": "98.76.54.32", "value_start": 17, "value_end": 28, "variable": null, @@ -5635,7 +5689,7 @@ "line_num": 7, "path": "tests/samples/doc_various", "info": "", - "value": "79edef7a351a4ad5f483a6a8ddf75b1f6d63ad08a774cee47e0f787e6c9bd790", + "value": "98.76.54.32", "value_start": 17, "value_end": 28, "variable": null, @@ -5689,7 +5743,7 @@ "line_num": 11, "path": "tests/samples/doc_various", "info": "", - "value": "79edef7a351a4ad5f483a6a8ddf75b1f6d63ad08a774cee47e0f787e6c9bd790", + "value": "98.76.54.32", "value_start": 29, "value_end": 40, "variable": null, @@ -5743,7 +5797,7 @@ "line_num": 14, "path": "tests/samples/doc_various", "info": "", - "value": "79edef7a351a4ad5f483a6a8ddf75b1f6d63ad08a774cee47e0f787e6c9bd790", + "value": "98.76.54.32", "value_start": 0, "value_end": 11, "variable": null, @@ -5770,7 +5824,7 @@ "line_num": 15, "path": "tests/samples/doc_various", "info": "", - "value": "79edef7a351a4ad5f483a6a8ddf75b1f6d63ad08a774cee47e0f787e6c9bd790", + "value": "98.76.54.32", "value_start": 13, "value_end": 24, "variable": null, @@ -5851,7 +5905,7 @@ "line_num": 17, "path": "tests/samples/doc_various", "info": "", - "value": "79edef7a351a4ad5f483a6a8ddf75b1f6d63ad08a774cee47e0f787e6c9bd790", + "value": "98.76.54.32", "value_start": 0, "value_end": 11, "variable": null, @@ -5905,7 +5959,7 @@ "line_num": 19, "path": "tests/samples/doc_various", "info": "", - "value": "79edef7a351a4ad5f483a6a8ddf75b1f6d63ad08a774cee47e0f787e6c9bd790", + "value": "98.76.54.32", "value_start": 0, "value_end": 11, "variable": null, @@ -6013,7 +6067,7 @@ "line_num": 22, "path": "tests/samples/doc_various", "info": "", - "value": "79edef7a351a4ad5f483a6a8ddf75b1f6d63ad08a774cee47e0f787e6c9bd790", + "value": "98.76.54.32", "value_start": 18, "value_end": 29, "variable": null, @@ -6067,7 +6121,7 @@ "line_num": 23, "path": "tests/samples/doc_various", "info": "", - "value": "79edef7a351a4ad5f483a6a8ddf75b1f6d63ad08a774cee47e0f787e6c9bd790", + "value": "98.76.54.32", "value_start": 13, "value_end": 24, "variable": null, @@ -6148,7 +6202,7 @@ "line_num": 25, "path": "tests/samples/doc_various", "info": "", - "value": "79edef7a351a4ad5f483a6a8ddf75b1f6d63ad08a774cee47e0f787e6c9bd790", + "value": "98.76.54.32", "value_start": 17, "value_end": 28, "variable": null, @@ -6202,7 +6256,7 @@ "line_num": 26, "path": "tests/samples/doc_various", "info": "", - "value": "79edef7a351a4ad5f483a6a8ddf75b1f6d63ad08a774cee47e0f787e6c9bd790", + "value": "98.76.54.32", "value_start": 17, "value_end": 28, "variable": null, @@ -6337,7 +6391,7 @@ "line_num": 34, "path": "tests/samples/doc_various", "info": "", - "value": "79edef7a351a4ad5f483a6a8ddf75b1f6d63ad08a774cee47e0f787e6c9bd790", + "value": "98.76.54.32", "value_start": 0, "value_end": 11, "variable": null, @@ -6391,7 +6445,7 @@ "line_num": 36, "path": "tests/samples/doc_various", "info": "", - "value": "79edef7a351a4ad5f483a6a8ddf75b1f6d63ad08a774cee47e0f787e6c9bd790", + "value": "98.76.54.32", "value_start": 0, "value_end": 11, "variable": null, @@ -6445,7 +6499,7 @@ "line_num": 38, "path": "tests/samples/doc_various", "info": "", - "value": "79edef7a351a4ad5f483a6a8ddf75b1f6d63ad08a774cee47e0f787e6c9bd790", + "value": "98.76.54.32", "value_start": 13, "value_end": 24, "variable": null, @@ -6526,7 +6580,7 @@ "line_num": 44, "path": "tests/samples/doc_various", "info": "", - "value": "79edef7a351a4ad5f483a6a8ddf75b1f6d63ad08a774cee47e0f787e6c9bd790", + "value": "98.76.54.32", "value_start": 13, "value_end": 24, "variable": null, @@ -6580,7 +6634,7 @@ "line_num": 47, "path": "tests/samples/doc_various", "info": "", - "value": "79edef7a351a4ad5f483a6a8ddf75b1f6d63ad08a774cee47e0f787e6c9bd790", + "value": "98.76.54.32", "value_start": 13, "value_end": 24, "variable": null, @@ -6634,7 +6688,7 @@ "line_num": 48, "path": "tests/samples/doc_various", "info": "", - "value": "79edef7a351a4ad5f483a6a8ddf75b1f6d63ad08a774cee47e0f787e6c9bd790", + "value": "98.76.54.32", "value_start": 0, "value_end": 11, "variable": null, @@ -6661,7 +6715,7 @@ "line_num": 49, "path": "tests/samples/doc_various", "info": "", - "value": "79edef7a351a4ad5f483a6a8ddf75b1f6d63ad08a774cee47e0f787e6c9bd790", + "value": "98.76.54.32", "value_start": 0, "value_end": 11, "variable": null, @@ -6688,7 +6742,7 @@ "line_num": 50, "path": "tests/samples/doc_various", "info": "", - "value": "79edef7a351a4ad5f483a6a8ddf75b1f6d63ad08a774cee47e0f787e6c9bd790", + "value": "98.76.54.32", "value_start": 5, "value_end": 16, "variable": null, @@ -6715,7 +6769,7 @@ "line_num": 51, "path": "tests/samples/doc_various", "info": "", - "value": "79edef7a351a4ad5f483a6a8ddf75b1f6d63ad08a774cee47e0f787e6c9bd790", + "value": "98.76.54.32", "value_start": 0, "value_end": 11, "variable": null, @@ -6796,7 +6850,7 @@ "line_num": 56, "path": "tests/samples/doc_various", "info": "", - "value": "79edef7a351a4ad5f483a6a8ddf75b1f6d63ad08a774cee47e0f787e6c9bd790", + "value": "98.76.54.32", "value_start": 13, "value_end": 24, "variable": null, @@ -6850,7 +6904,7 @@ "line_num": 59, "path": "tests/samples/doc_various", "info": "", - "value": "79edef7a351a4ad5f483a6a8ddf75b1f6d63ad08a774cee47e0f787e6c9bd790", + "value": "98.76.54.32", "value_start": 13, "value_end": 24, "variable": null, @@ -7012,7 +7066,7 @@ "line_num": 70, "path": "tests/samples/doc_various", "info": "", - "value": "79edef7a351a4ad5f483a6a8ddf75b1f6d63ad08a774cee47e0f787e6c9bd790", + "value": "98.76.54.32", "value_start": 0, "value_end": 11, "variable": null, @@ -7093,7 +7147,7 @@ "line_num": 73, "path": "tests/samples/doc_various", "info": "", - "value": "79edef7a351a4ad5f483a6a8ddf75b1f6d63ad08a774cee47e0f787e6c9bd790", + "value": "98.76.54.32", "value_start": 7, "value_end": 18, "variable": null, @@ -7147,7 +7201,7 @@ "line_num": 74, "path": "tests/samples/doc_various", "info": "", - "value": "79edef7a351a4ad5f483a6a8ddf75b1f6d63ad08a774cee47e0f787e6c9bd790", + "value": "98.76.54.32", "value_start": 7, "value_end": 18, "variable": null, @@ -7201,7 +7255,7 @@ "line_num": 75, "path": "tests/samples/doc_various", "info": "", - "value": "79edef7a351a4ad5f483a6a8ddf75b1f6d63ad08a774cee47e0f787e6c9bd790", + "value": "98.76.54.32", "value_start": 0, "value_end": 11, "variable": null, @@ -7282,7 +7336,7 @@ "line_num": 81, "path": "tests/samples/doc_various", "info": "", - "value": "79edef7a351a4ad5f483a6a8ddf75b1f6d63ad08a774cee47e0f787e6c9bd790", + "value": "98.76.54.32", "value_start": 0, "value_end": 11, "variable": null, @@ -7309,7 +7363,7 @@ "line_num": 82, "path": "tests/samples/doc_various", "info": "", - "value": "79edef7a351a4ad5f483a6a8ddf75b1f6d63ad08a774cee47e0f787e6c9bd790", + "value": "98.76.54.32", "value_start": 17, "value_end": 28, "variable": null, @@ -7390,7 +7444,7 @@ "line_num": 84, "path": "tests/samples/doc_various", "info": "", - "value": "79edef7a351a4ad5f483a6a8ddf75b1f6d63ad08a774cee47e0f787e6c9bd790", + "value": "98.76.54.32", "value_start": 13, "value_end": 24, "variable": null, @@ -7444,7 +7498,7 @@ "line_num": 85, "path": "tests/samples/doc_various", "info": "", - "value": "79edef7a351a4ad5f483a6a8ddf75b1f6d63ad08a774cee47e0f787e6c9bd790", + "value": "98.76.54.32", "value_start": 17, "value_end": 28, "variable": null, @@ -7498,7 +7552,7 @@ "line_num": 86, "path": "tests/samples/doc_various", "info": "", - "value": "79edef7a351a4ad5f483a6a8ddf75b1f6d63ad08a774cee47e0f787e6c9bd790", + "value": "98.76.54.32", "value_start": 13, "value_end": 24, "variable": null, @@ -7525,7 +7579,7 @@ "line_num": 87, "path": "tests/samples/doc_various", "info": "", - "value": "79edef7a351a4ad5f483a6a8ddf75b1f6d63ad08a774cee47e0f787e6c9bd790", + "value": "98.76.54.32", "value_start": 18, "value_end": 29, "variable": null, @@ -7579,7 +7633,7 @@ "line_num": 88, "path": "tests/samples/doc_various", "info": "", - "value": "79edef7a351a4ad5f483a6a8ddf75b1f6d63ad08a774cee47e0f787e6c9bd790", + "value": "98.76.54.32", "value_start": 17, "value_end": 28, "variable": null, @@ -7606,7 +7660,7 @@ "line_num": 89, "path": "tests/samples/doc_various", "info": "", - "value": "79edef7a351a4ad5f483a6a8ddf75b1f6d63ad08a774cee47e0f787e6c9bd790", + "value": "98.76.54.32", "value_start": 17, "value_end": 28, "variable": null, @@ -7633,7 +7687,7 @@ "line_num": 90, "path": "tests/samples/doc_various", "info": "", - "value": "79edef7a351a4ad5f483a6a8ddf75b1f6d63ad08a774cee47e0f787e6c9bd790", + "value": "98.76.54.32", "value_start": 17, "value_end": 28, "variable": null, @@ -7687,7 +7741,7 @@ "line_num": 91, "path": "tests/samples/doc_various", "info": "", - "value": "79edef7a351a4ad5f483a6a8ddf75b1f6d63ad08a774cee47e0f787e6c9bd790", + "value": "98.76.54.32", "value_start": 13, "value_end": 24, "variable": null, @@ -7714,7 +7768,7 @@ "line_num": 92, "path": "tests/samples/doc_various", "info": "", - "value": "79edef7a351a4ad5f483a6a8ddf75b1f6d63ad08a774cee47e0f787e6c9bd790", + "value": "98.76.54.32", "value_start": 17, "value_end": 28, "variable": null, @@ -7741,7 +7795,7 @@ "line_num": 93, "path": "tests/samples/doc_various", "info": "", - "value": "79edef7a351a4ad5f483a6a8ddf75b1f6d63ad08a774cee47e0f787e6c9bd790", + "value": "98.76.54.32", "value_start": 17, "value_end": 28, "variable": null, @@ -7795,7 +7849,7 @@ "line_num": 94, "path": "tests/samples/doc_various", "info": "", - "value": "79edef7a351a4ad5f483a6a8ddf75b1f6d63ad08a774cee47e0f787e6c9bd790", + "value": "98.76.54.32", "value_start": 13, "value_end": 24, "variable": null, @@ -7849,7 +7903,7 @@ "line_num": 96, "path": "tests/samples/doc_various", "info": "", - "value": "79edef7a351a4ad5f483a6a8ddf75b1f6d63ad08a774cee47e0f787e6c9bd790", + "value": "98.76.54.32", "value_start": 13, "value_end": 24, "variable": null, @@ -7903,7 +7957,7 @@ "line_num": 97, "path": "tests/samples/doc_various", "info": "", - "value": "79edef7a351a4ad5f483a6a8ddf75b1f6d63ad08a774cee47e0f787e6c9bd790", + "value": "98.76.54.32", "value_start": 13, "value_end": 24, "variable": null, @@ -7957,7 +8011,7 @@ "line_num": 99, "path": "tests/samples/doc_various", "info": "", - "value": "79edef7a351a4ad5f483a6a8ddf75b1f6d63ad08a774cee47e0f787e6c9bd790", + "value": "98.76.54.32", "value_start": 17, "value_end": 28, "variable": null, @@ -7984,7 +8038,7 @@ "line_num": 100, "path": "tests/samples/doc_various", "info": "", - "value": "79edef7a351a4ad5f483a6a8ddf75b1f6d63ad08a774cee47e0f787e6c9bd790", + "value": "98.76.54.32", "value_start": 17, "value_end": 28, "variable": null, @@ -8038,7 +8092,7 @@ "line_num": 101, "path": "tests/samples/doc_various", "info": "", - "value": "79edef7a351a4ad5f483a6a8ddf75b1f6d63ad08a774cee47e0f787e6c9bd790", + "value": "98.76.54.32", "value_start": 17, "value_end": 28, "variable": null, @@ -8119,7 +8173,7 @@ "line_num": 104, "path": "tests/samples/doc_various", "info": "", - "value": "79edef7a351a4ad5f483a6a8ddf75b1f6d63ad08a774cee47e0f787e6c9bd790", + "value": "98.76.54.32", "value_start": 17, "value_end": 28, "variable": null, @@ -8227,7 +8281,7 @@ "line_num": 107, "path": "tests/samples/doc_various", "info": "", - "value": "79edef7a351a4ad5f483a6a8ddf75b1f6d63ad08a774cee47e0f787e6c9bd790", + "value": "98.76.54.32", "value_start": 6, "value_end": 17, "variable": null, @@ -8254,7 +8308,7 @@ "line_num": 108, "path": "tests/samples/doc_various", "info": "", - "value": "79edef7a351a4ad5f483a6a8ddf75b1f6d63ad08a774cee47e0f787e6c9bd790", + "value": "98.76.54.32", "value_start": 7, "value_end": 18, "variable": null, @@ -9044,7 +9098,7 @@ "line_num": 1, "path": "tests/samples/grafana_provisioned_api_key", "info": "", - "value": "7d27d693f2a5bc3a082da6fa528493f249e0f00578615a7ab8acbf0f90c8e937", + "value": "eyJrIjoiMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMCIsIm4iOiJ4eHh4IiwiaWQiOjIwNDM2MH0=", "value_start": 11, "value_end": 107, "variable": null, @@ -9179,7 +9233,7 @@ "line_num": 13, "path": "tests/samples/ipv4", "info": "", - "value": "4b2228c26597aecab7d5894eb1ec83d915bc2e1a75d758b3b53471ce6aa2c91c", + "value": "100.64.0.0", "value_start": 0, "value_end": 10, "variable": null, @@ -9206,15 +9260,15 @@ "line_num": 14, "path": "tests/samples/ipv4", "info": "", - "value": "4b2228c26597aecab7d5894eb1ec83d915bc2e1a75d758b3b53471ce6aa2c91c", - "value_start": 0, - "value_end": 10, + "value": "100.127.255.255", + "value_start": 11, + "value_end": 26, "variable": null, "variable_start": -2, "variable_end": -2, "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 1.5253496664211537, + "iterator": "BASE36_CHARS", + "entropy": 2.008519976342584, "valid": false } } @@ -9233,15 +9287,15 @@ "line_num": 14, "path": "tests/samples/ipv4", "info": "", - "value": "7dcda6dac4bea638ef6f2c86184f4e7edf47d16bc9c6b312d87ed93a07f4008b", - "value_start": 11, - "value_end": 26, + "value": "100.64.0.0", + "value_start": 0, + "value_end": 10, "variable": null, "variable_start": -2, "variable_end": -2, "entropy_validation": { - "iterator": "BASE36_CHARS", - "entropy": 2.008519976342584, + "iterator": "BASE64_CHARS", + "entropy": 1.5253496664211537, "valid": false } } @@ -9260,7 +9314,7 @@ "line_num": 22, "path": "tests/samples/ipv4", "info": "", - "value": "37d39b8d08cb6f337be85850f685d08454d0185b7c07f5cc07fe399f87aed8b9", + "value": "192.0.0.255", "value_start": 10, "value_end": 21, "variable": null, @@ -9287,7 +9341,7 @@ "line_num": 25, "path": "tests/samples/ipv4", "info": "", - "value": "93765e9dc4a7c55cb33d612324027e75802c34b1afdea78aa084d3d626a73259", + "value": "192.88.99.0", "value_start": 0, "value_end": 11, "variable": null, @@ -9314,7 +9368,7 @@ "line_num": 26, "path": "tests/samples/ipv4", "info": "", - "value": "93765e9dc4a7c55cb33d612324027e75802c34b1afdea78aa084d3d626a73259", + "value": "192.88.99.0", "value_start": 0, "value_end": 11, "variable": null, @@ -9341,7 +9395,7 @@ "line_num": 26, "path": "tests/samples/ipv4", "info": "", - "value": "fa204d0bf8aa15ec16c72f41f2143528004e63728734475de6ec44df492f4fcc", + "value": "192.88.99.255", "value_start": 12, "value_end": 25, "variable": null, @@ -9368,7 +9422,7 @@ "line_num": 12, "path": "tests/samples/ipv6", "info": "", - "value": "9c186d7e1b6116581b7ec9388a9bc6eb292ba2bc7bf8b4c5f7f5c13e3ca3ea7e", + "value": "2004:5678::9324", "value_start": 0, "value_end": 15, "variable": null, @@ -9395,7 +9449,7 @@ "line_num": 13, "path": "tests/samples/ipv6", "info": "", - "value": "c6a5982a9eb3612a4d3fe10ab60ff0baa9d0acc926bce41e316601213a67e173", + "value": "2004::5678:9", "value_start": 0, "value_end": 12, "variable": null, @@ -9422,7 +9476,7 @@ "line_num": 14, "path": "tests/samples/ipv6", "info": "", - "value": "957d6a4bc388076cd945489926a1d701efaa378ef52d3b6c71f4e9d0b127300c", + "value": "2041:0000:140F::875B:131B", "value_start": 0, "value_end": 25, "variable": null, @@ -9449,7 +9503,7 @@ "line_num": 15, "path": "tests/samples/ipv6", "info": "", - "value": "cc37a3b3e3444f235804bc23cb60003657e237c5ddd86e3703e239387340a77f", + "value": "2041:0:140F::875B:131B", "value_start": 7010, "value_end": 7032, "variable": null, @@ -9476,7 +9530,7 @@ "line_num": 15, "path": "tests/samples/ipv6", "info": "", - "value": "cc37a3b3e3444f235804bc23cb60003657e237c5ddd86e3703e239387340a77f", + "value": "2041:0:140F::875B:131B", "value_start": 7989, "value_end": 8011, "variable": null, @@ -9631,14 +9685,14 @@ "ml_probability": 0.931, "rule": "JSON Web Token", "severity": "medium", - "confidence": "moderate", + "confidence": "strong", "line_data_list": [ { - "line": "$payload = 'eyJgsIZgeJhvNgFpSmlP.eyJcaaF9xCe7shE0ENPiBlEJOpS'", + "line": "detected: eyJhbGciOiJSUzI1NiJ9Cg.eyJleHAiOjY1NTM2fQo.Ce7sh0ENPiBlE_dose0cBA", "line_num": 1, - "path": "tests/samples/json_web_token.hs", + "path": "tests/samples/json_web_token", "info": "", - "value": "a39eeaaccad36b6726da66dd3c9e07320ba1b530a4c97b7c2bbe55d98fb11cf3", + "value": "eyJgsIZgeJhvNgFpSmlP.eyJcaaF9xCe7shE0ENPiBlEJOpS", "value_start": 12, "value_end": 60, "variable": null, @@ -9646,7 +9700,7 @@ "variable_end": -2, "entropy_validation": { "iterator": "BASE64_CHARS", - "entropy": 4.520488802699322, + "entropy": 4.790963630103494, "valid": true } } @@ -9769,11 +9823,11 @@ "confidence": "moderate", "line_data_list": [ { - "line": "secret_looks_like_linux_path__=\"VnpmUGWxhQW/9KQAwrL2ZYd/DJPNG1PQjEXAMPLE\"", + "line": "secret_looks_like_linux_path__=\"VnpmUGWxhQW/9KQAwrL2ZYd/DJPNG1PQjEXAMbLE\"", "line_num": 5, "path": "tests/samples/key.hs", "info": "", - "value": "042b9fe938bdb979ab754994cbc4e4a96d8fcd205cdef5fc5c7bdb689889ef53", + "value": "VnpmUGWxhQW/9KQAwrL2ZYd/DJPNG1PQjEXAMPLE", "value_start": 32, "value_end": 72, "variable": "secret_looks_like_linux_path__", @@ -9781,7 +9835,7 @@ "variable_end": 30, "entropy_validation": { "iterator": "BASE64_CHARS", - "entropy": 4.784183719779189, + "entropy": 4.8530559073332755, "valid": true } } @@ -11034,6 +11088,33 @@ } ] }, + { + "api_validation": "NOT_AVAILABLE", + "ml_validation": "VALIDATED_KEY", + "ml_probability": 0.192, + "rule": "Salt", + "severity": "medium", + "confidence": "moderate", + "line_data_list": [ + { + "line": "json_escaped = \"{\\\\\\\"salt8\\\\\\\":\\\\\\\"4b9a6d8b638eb0c6\\\\\\\"}\"", + "line_num": 5, + "path": "tests/samples/salt.py", + "info": "", + "value": "4b9a6d8b638eb0c6", + "value_start": 35, + "value_end": 51, + "variable": "salt8", + "variable_start": 21, + "variable_end": 26, + "entropy_validation": { + "iterator": "BASE36_CHARS", + "entropy": 3.2806390622295662, + "valid": true + } + } + ] + }, { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", @@ -11452,7 +11533,7 @@ "line_num": 17, "path": "tests/samples/test.eml", "info": "", - "value": "1c8ef02e0fe88c8749c7afb6f20cfd56d939d46bb648f1a68855b7a2dd803e28", + "value": "eyJUaGVyZSBpcyBub3QgdGhlIEpTT04geW91IGFyZSBsb29raW5nIGZvciJ9CjwvYm9keT4KPC9o", "value_start": 0, "value_end": 76, "variable": null, @@ -11506,7 +11587,7 @@ "line_num": 18, "path": "tests/samples/test2.eml", "info": "", - "value": "607f01c3981f1f136faaad65b3ac5d5b4a8429d5d2863e9593a6b49811ad0ddb", + "value": "eyJrIjoiMDAwMDAwNDAwMDAwODAwNDAwMDAwMDAwNDAwMDAwMDAwMDAwMDAyMSIsIm4iOiJ4eHh4IiwiaWQiOjQzMDh9Cg", "value_start": 15, "value_end": 109, "variable": null, @@ -11817,6 +11898,60 @@ } ] }, + { + "api_validation": "NOT_AVAILABLE", + "ml_validation": "VALIDATED_KEY", + "ml_probability": 0.989, + "rule": "URL Credentials", + "severity": "high", + "confidence": "moderate", + "line_data_list": [ + { + "line": "email_as_login = \"smtps://example@gmail.com:FnD83JZs@smtp.gmail.com:465\";", + "line_num": 13, + "path": "tests/samples/url_cred.js", + "info": "", + "value": "FnD83JZs", + "value_start": 44, + "value_end": 52, + "variable": "smtps://", + "variable_start": 18, + "variable_end": 26, + "entropy_validation": { + "iterator": "BASE64_CHARS", + "entropy": 3.0, + "valid": false + } + } + ] + }, + { + "api_validation": "NOT_AVAILABLE", + "ml_validation": "NOT_AVAILABLE", + "ml_probability": null, + "rule": "UUID", + "severity": "info", + "confidence": "strong", + "line_data_list": [ + { + "line": "bace4d19-fa7e-beef-cafe-9129474bcd81 # tp", + "line_num": 1, + "path": "tests/samples/uuid", + "info": "", + "value": "bace4d19-fa7e-beef-cafe-9129474bcd81", + "value_start": 0, + "value_end": 36, + "variable": null, + "variable_start": -2, + "variable_end": -2, + "entropy_validation": { + "iterator": "BASE36_CHARS", + "entropy": 3.2373263071270246, + "valid": true + } + } + ] + }, { "api_validation": "NOT_AVAILABLE", "ml_validation": "NOT_AVAILABLE", @@ -11898,4 +12033,4 @@ } ] } -] \ No newline at end of file +] diff --git a/tests/data/output.json b/tests/data/output.json index 2f3bee438..f6dcde344 100644 --- a/tests/data/output.json +++ b/tests/data/output.json @@ -299,7 +299,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.941, + "ml_probability": 0.963, "rule": "Auth", "severity": "medium", "confidence": "moderate", @@ -323,33 +323,6 @@ } ] }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.941, - "rule": "JSON Web Token", - "severity": "medium", - "confidence": "moderate", - "line_data_list": [ - { - "line": "curl -H \"Authorization: Bearer eyJGRpVu1c2VzY2-823r_db32hbf4W1lbj\" http://localhost:8080/.", - "line_num": 9, - "path": "tests/samples/auth_n.template", - "info": "", - "value": "eyJGRpVu1c2VzY2-823r_db32hbf4W1lbj", - "value_start": 31, - "value_end": 65, - "variable": null, - "variable_start": -2, - "variable_end": -2, - "entropy_validation": { - "iterator": "BASE36_CHARS", - "entropy": 3.2479906920322064, - "valid": true - } - } - ] - }, { "api_validation": "NOT_AVAILABLE", "ml_validation": "NOT_AVAILABLE", @@ -710,11 +683,11 @@ }, { "api_validation": "NOT_AVAILABLE", - "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.93, + "ml_validation": "NOT_AVAILABLE", + "ml_probability": null, "rule": "JSON Web Token", "severity": "medium", - "confidence": "moderate", + "confidence": "strong", "line_data_list": [ { "line": "eyJ0eXAiOiJKV1QiLCJhbGciOiJSUzI1NiIsIng1dCI6Imk2bEdrM0ZaenhSY1ViMkMzbkVRN3N5SEpsWSIsImtpZCI6Imk2bEdrM0ZaenhSY1ViMkMzbkVRN3N5SEpsWSJ9.eyJhdWQiOiJlZjFkYTlkNC1mZjc3LTRjM2UtYTAwNS04NDBjM2Y4MzA3NDUiLCJpc3MiOiJodHRwczovL3N0cy53aW5kb3dzLm5ldC9mYTE1ZDY5Mi1lOWM3LTQ0NjAtYTc0My0yOWYyOTUyMjIyOS8iLCJpYXQiOjE1MzcyMzMxMDYsIm5iZiI6MTUzNzIzMzEwNiwiZXhwIjoxNTM3MjM3MDA2LCJhY3IiOiIxIiwiYWlvIjoiQVhRQWkvOElBQUFBRm0rRS9RVEcrZ0ZuVnhMaldkdzhLKzYxQUdyU091TU1GNmViYU1qN1hPM0libUQzZkdtck95RCtOdlp5R24yVmFUL2tES1h3NE1JaHJnR1ZxNkJuOHdMWG9UMUxrSVorRnpRVmtKUFBMUU9WNEtjWHFTbENWUERTL0RpQ0RnRTIyMlRJbU12V05hRU1hVU9Uc0lHdlRRPT0iLCJhbXIiOlsid2lhIl0sImFwcGlkIjoiNzVkYmU3N2YtMTBhMy00ZTU5LTg1ZmQtOGMxMjc1NDRmMTdjIiwiYXBwaWRhY3IiOiIwIiwiZW1haWwiOiJBYmVMaUBtaWNyb3NvZnQuY29tIiwiZmFtaWx5X25hbWUiOiJMaW5jb2xuIiwiZ2l2ZW5fbmFtZSI6IkFiZSAoTVNGVCkiLCJpZHAiOiJodHRwczovL3N0cy53aW5kb3dzLm5ldC83MmY5ODhiZi04NmYxLTQxYWYtOTFhYi0yZDdjZDAxMjIyNDcvIiwiaXBhZGRyIjoiMjIyLjIyMi4yMjIuMjIiLCJuYW1lIjoiYWJlbGkiLCJvaWQiOiIwMjIyM2I2Yi1hYTFkLTQyZDQtOWVjMC0xYjJiYjkxOTQ0MzgiLCJyaCI6IkkiLCJzY3AiOiJ1c2VyX2ltcGVyc29uYXRpb24iLCJzdWIiOiJsM19yb0lTUVUyMjJiVUxTOXlpMmswWHBxcE9pTXo1SDNaQUNvMUdlWEEiLCJ0aWQiOiJmYTE1ZDY5Mi1lOWM3LTQ0NjAtYTc0My0yOWYyOTU2ZmQ0MjkiLCJ1bmlxdWVfbmFtZSI6ImFiZWxpQG1pY3Jvc29mdC5jb20iLCJ1dGkiOiJGVnNHeFlYSTMwLVR1aWt1dVVvRkFBIiwidmVyIjoiMS4wIn0.D3H6pMUtQnoJAGq6AHd", @@ -735,6 +708,60 @@ } ] }, + { + "api_validation": "NOT_AVAILABLE", + "ml_validation": "NOT_AVAILABLE", + "ml_probability": null, + "rule": "Azure Access Token", + "severity": "high", + "confidence": "strong", + "line_data_list": [ + { + "line": "eyJ0eXAiOiJKV1QiLCJhbGciOiJSUzI1NiIsImtpZCI6Imk2bEdrM0ZaenhSY1ViMkMzbkVRN3N5SEpsWSJ9.eyJhdWQiOiI2ZTc0MTcyYi1iZTU2LTQ4NDMtOWZmNC1lNjZhMzliYjEyZTMiLCJpc3MiOiJodHRwczovL2xvZ2luLm1pY3Jvc29mdG9ubGluZS5jb20vNzJmOTg4YmYtODZmMS00MWFmLTkxYWItMmQ3Y2QwMTFkYjQ3L3YyLjAiLCJpYXQiOjE1MzcyMzEwNDgsIm5iZiI6MTUzNzIzMTA0OCwiZXhwIjoxNTM3MjM0OTQ4LCJhaW8iOiJBWFFBaS84SUFBQUF0QWFaTG8zQ2hNaWY2S09udHRSQjdlQnE0L0RjY1F6amNKR3hQWXkvQzNqRGFOR3hYZDZ3TklJVkdSZ2hOUm53SjFsT2NBbk5aY2p2a295ckZ4Q3R0djMzMTQwUmlvT0ZKNGJDQ0dWdW9DYWcxdU9UVDIyMjIyZ0h3TFBZUS91Zjc5UVgrMEtJaWpkcm1wNjlSY3R6bVE9PSIsImF6cCI6IjZlNzQxNzJiLWJlNTYtNDg0My05ZmY0LWU2NmEzOWJiMTJlMyIsImF6cGFjciI6IjAiLCJuYW1lIjoiQWJlIExpbmNvbG4iLCJvaWQiOiI2OTAyMjJiZS1mZjFhLTRkNTYtYWJkMS03ZTRmN2QzOGU0NzQiLCJwcmVmZXJyZWRfdXNlcm5hbWUiOiJhYmVsaUBtaWNyb3NvZnQuY29tIiwicmgiOiJJIiwic2NwIjoiYWNjZXNzX2FzX3VzZXIiLCJzdWIiOiJIS1pwZmFIeVdhZGVPb3VZbGl0anJJLUtmZlRtMjIyWDVyclYzeERxZktRIiwidGlkIjoiNzJmOTg4YmYtODZmMS00MWFmLTkxYWItMmQ3Y2QwMTFkYjQ3IiwidXRpIjoiZnFpQnFYTFBqMGVRYTgyUy1JWUZBQSIsInZlciI6IjIuMCJ9.pj4N-w_3Us9DrBLfpCt", + "line_num": 2, + "path": "tests/samples/azure_access_token", + "info": "", + "value": "eyJ0eXAiOiJKV1QiLCJhbGciOiJSUzI1NiIsImtpZCI6Imk2bEdrM0ZaenhSY1ViMkMzbkVRN3N5SEpsWSJ9.eyJhdWQiOiI2ZTc0MTcyYi1iZTU2LTQ4NDMtOWZmNC1lNjZhMzliYjEyZTMiLCJpc3MiOiJodHRwczovL2xvZ2luLm1pY3Jvc29mdG9ubGluZS5jb20vNzJmOTg4YmYtODZmMS00MWFmLTkxYWItMmQ3Y2QwMTFkYjQ3L3YyLjAiLCJpYXQiOjE1MzcyMzEwNDgsIm5iZiI6MTUzNzIzMTA0OCwiZXhwIjoxNTM3MjM0OTQ4LCJhaW8iOiJBWFFBaS84SUFBQUF0QWFaTG8zQ2hNaWY2S09udHRSQjdlQnE0L0RjY1F6amNKR3hQWXkvQzNqRGFOR3hYZDZ3TklJVkdSZ2hOUm53SjFsT2NBbk5aY2p2a295ckZ4Q3R0djMzMTQwUmlvT0ZKNGJDQ0dWdW9DYWcxdU9UVDIyMjIyZ0h3TFBZUS91Zjc5UVgrMEtJaWpkcm1wNjlSY3R6bVE9PSIsImF6cCI6IjZlNzQxNzJiLWJlNTYtNDg0My05ZmY0LWU2NmEzOWJiMTJlMyIsImF6cGFjciI6IjAiLCJuYW1lIjoiQWJlIExpbmNvbG4iLCJvaWQiOiI2OTAyMjJiZS1mZjFhLTRkNTYtYWJkMS03ZTRmN2QzOGU0NzQiLCJwcmVmZXJyZWRfdXNlcm5hbWUiOiJhYmVsaUBtaWNyb3NvZnQuY29tIiwicmgiOiJJIiwic2NwIjoiYWNjZXNzX2FzX3VzZXIiLCJzdWIiOiJIS1pwZmFIeVdhZGVPb3VZbGl0anJJLUtmZlRtMjIyWDVyclYzeERxZktRIiwidGlkIjoiNzJmOTg4YmYtODZmMS00MWFmLTkxYWItMmQ3Y2QwMTFkYjQ3IiwidXRpIjoiZnFpQnFYTFBqMGVRYTgyUy1JWUZBQSIsInZlciI6IjIuMCJ9.pj4N-w_3Us9DrBLfpCt", + "value_start": 0, + "value_end": 1029, + "variable": null, + "variable_start": -2, + "variable_end": -2, + "entropy_validation": { + "iterator": "BASE64_CHARS", + "entropy": 5.6044494049575055, + "valid": true + } + } + ] + }, + { + "api_validation": "NOT_AVAILABLE", + "ml_validation": "NOT_AVAILABLE", + "ml_probability": null, + "rule": "JSON Web Token", + "severity": "medium", + "confidence": "strong", + "line_data_list": [ + { + "line": "eyJ0eXAiOiJKV1QiLCJhbGciOiJSUzI1NiIsImtpZCI6Imk2bEdrM0ZaenhSY1ViMkMzbkVRN3N5SEpsWSJ9.eyJhdWQiOiI2ZTc0MTcyYi1iZTU2LTQ4NDMtOWZmNC1lNjZhMzliYjEyZTMiLCJpc3MiOiJodHRwczovL2xvZ2luLm1pY3Jvc29mdG9ubGluZS5jb20vNzJmOTg4YmYtODZmMS00MWFmLTkxYWItMmQ3Y2QwMTFkYjQ3L3YyLjAiLCJpYXQiOjE1MzcyMzEwNDgsIm5iZiI6MTUzNzIzMTA0OCwiZXhwIjoxNTM3MjM0OTQ4LCJhaW8iOiJBWFFBaS84SUFBQUF0QWFaTG8zQ2hNaWY2S09udHRSQjdlQnE0L0RjY1F6amNKR3hQWXkvQzNqRGFOR3hYZDZ3TklJVkdSZ2hOUm53SjFsT2NBbk5aY2p2a295ckZ4Q3R0djMzMTQwUmlvT0ZKNGJDQ0dWdW9DYWcxdU9UVDIyMjIyZ0h3TFBZUS91Zjc5UVgrMEtJaWpkcm1wNjlSY3R6bVE9PSIsImF6cCI6IjZlNzQxNzJiLWJlNTYtNDg0My05ZmY0LWU2NmEzOWJiMTJlMyIsImF6cGFjciI6IjAiLCJuYW1lIjoiQWJlIExpbmNvbG4iLCJvaWQiOiI2OTAyMjJiZS1mZjFhLTRkNTYtYWJkMS03ZTRmN2QzOGU0NzQiLCJwcmVmZXJyZWRfdXNlcm5hbWUiOiJhYmVsaUBtaWNyb3NvZnQuY29tIiwicmgiOiJJIiwic2NwIjoiYWNjZXNzX2FzX3VzZXIiLCJzdWIiOiJIS1pwZmFIeVdhZGVPb3VZbGl0anJJLUtmZlRtMjIyWDVyclYzeERxZktRIiwidGlkIjoiNzJmOTg4YmYtODZmMS00MWFmLTkxYWItMmQ3Y2QwMTFkYjQ3IiwidXRpIjoiZnFpQnFYTFBqMGVRYTgyUy1JWUZBQSIsInZlciI6IjIuMCJ9.pj4N-w_3Us9DrBLfpCt", + "line_num": 2, + "path": "tests/samples/azure_access_token", + "info": "", + "value": "eyJ0eXAiOiJKV1QiLCJhbGciOiJSUzI1NiIsImtpZCI6Imk2bEdrM0ZaenhSY1ViMkMzbkVRN3N5SEpsWSJ9.eyJhdWQiOiI2ZTc0MTcyYi1iZTU2LTQ4NDMtOWZmNC1lNjZhMzliYjEyZTMiLCJpc3MiOiJodHRwczovL2xvZ2luLm1pY3Jvc29mdG9ubGluZS5jb20vNzJmOTg4YmYtODZmMS00MWFmLTkxYWItMmQ3Y2QwMTFkYjQ3L3YyLjAiLCJpYXQiOjE1MzcyMzEwNDgsIm5iZiI6MTUzNzIzMTA0OCwiZXhwIjoxNTM3MjM0OTQ4LCJhaW8iOiJBWFFBaS84SUFBQUF0QWFaTG8zQ2hNaWY2S09udHRSQjdlQnE0L0RjY1F6amNKR3hQWXkvQzNqRGFOR3hYZDZ3TklJVkdSZ2hOUm53SjFsT2NBbk5aY2p2a295ckZ4Q3R0djMzMTQwUmlvT0ZKNGJDQ0dWdW9DYWcxdU9UVDIyMjIyZ0h3TFBZUS91Zjc5UVgrMEtJaWpkcm1wNjlSY3R6bVE9PSIsImF6cCI6IjZlNzQxNzJiLWJlNTYtNDg0My05ZmY0LWU2NmEzOWJiMTJlMyIsImF6cGFjciI6IjAiLCJuYW1lIjoiQWJlIExpbmNvbG4iLCJvaWQiOiI2OTAyMjJiZS1mZjFhLTRkNTYtYWJkMS03ZTRmN2QzOGU0NzQiLCJwcmVmZXJyZWRfdXNlcm5hbWUiOiJhYmVsaUBtaWNyb3NvZnQuY29tIiwicmgiOiJJIiwic2NwIjoiYWNjZXNzX2FzX3VzZXIiLCJzdWIiOiJIS1pwZmFIeVdhZGVPb3VZbGl0anJJLUtmZlRtMjIyWDVyclYzeERxZktRIiwidGlkIjoiNzJmOTg4YmYtODZmMS00MWFmLTkxYWItMmQ3Y2QwMTFkYjQ3IiwidXRpIjoiZnFpQnFYTFBqMGVRYTgyUy1JWUZBQSIsInZlciI6IjIuMCJ9.pj4N-w_3Us9DrBLfpCt", + "value_start": 0, + "value_end": 1029, + "variable": null, + "variable_start": -2, + "variable_end": -2, + "entropy_validation": { + "iterator": "BASE64_CHARS", + "entropy": 5.6044494049575055, + "valid": true + } + } + ] + }, { "api_validation": "NOT_AVAILABLE", "ml_validation": "NOT_AVAILABLE", @@ -2166,33 +2193,6 @@ } ] }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "IPv4", - "severity": "info", - "confidence": "weak", - "line_data_list": [ - { - "line": "(98.76.54.32)ID:master PW:dipPr149Gg!", - "line_num": 50, - "path": "tests/samples/doc_id_pair_passwd_pair", - "info": "", - "value": "98.76.54.32", - "value_start": 1, - "value_end": 12, - "variable": null, - "variable_start": -2, - "variable_end": -2, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 2.515950268099853, - "valid": false - } - } - ] - }, { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", @@ -2976,33 +2976,6 @@ } ] }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "IPv4", - "severity": "info", - "confidence": "weak", - "line_data_list": [ - { - "line": "ANYid:master pw:dipPr194Gg! ip:98.76.54.32", - "line_num": 95, - "path": "tests/samples/doc_id_pair_passwd_pair", - "info": "", - "value": "98.76.54.32", - "value_start": 31, - "value_end": 42, - "variable": null, - "variable_start": -2, - "variable_end": -2, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 2.515950268099853, - "valid": false - } - } - ] - }, { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", @@ -3030,33 +3003,6 @@ } ] }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "IPv4", - "severity": "info", - "confidence": "weak", - "line_data_list": [ - { - "line": "master@98.76.54.32 mailto:{1} (password-dipPr196Gg!) # skip", - "line_num": 97, - "path": "tests/samples/doc_id_pair_passwd_pair", - "info": "", - "value": "98.76.54.32", - "value_start": 7, - "value_end": 18, - "variable": null, - "variable_start": -2, - "variable_end": -2, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 2.515950268099853, - "valid": false - } - } - ] - }, { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", @@ -3138,60 +3084,6 @@ } ] }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "IPv4", - "severity": "info", - "confidence": "weak", - "line_data_list": [ - { - "line": "98.76.54.32(ID:master/PW:iPp10@GRq) # todo: move into other sample ?", - "line_num": 11, - "path": "tests/samples/doc_id_passwd_pair", - "info": "", - "value": "98.76.54.32", - "value_start": 0, - "value_end": 11, - "variable": null, - "variable_start": -2, - "variable_end": -2, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 2.515950268099853, - "valid": false - } - } - ] - }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "IPv4", - "severity": "info", - "confidence": "weak", - "line_data_list": [ - { - "line": "98.76.54.32 id/pw:master/iPp19@GRq", - "line_num": 20, - "path": "tests/samples/doc_id_passwd_pair", - "info": "", - "value": "98.76.54.32", - "value_start": 0, - "value_end": 11, - "variable": null, - "variable_start": -2, - "variable_end": -2, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 2.515950268099853, - "valid": false - } - } - ] - }, { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", @@ -3435,33 +3327,6 @@ } ] }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "IPv4", - "severity": "info", - "confidence": "weak", - "line_data_list": [ - { - "line": "master@98.76.54.32 password:Prl23Db#@", - "line_num": 14, - "path": "tests/samples/doc_passwd_pair", - "info": "", - "value": "98.76.54.32", - "value_start": 7, - "value_end": 18, - "variable": null, - "variable_start": -2, - "variable_end": -2, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 2.515950268099853, - "valid": false - } - } - ] - }, { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", @@ -4652,26 +4517,26 @@ }, { "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "IPv4", - "severity": "info", - "confidence": "weak", + "ml_validation": "VALIDATED_KEY", + "ml_probability": 0.956, + "rule": "Password", + "severity": "medium", + "confidence": "moderate", "line_data_list": [ { - "line": "98.76.54.32 (master/IhqSb1Gg)", - "line_num": 1, + "line": "master@98.76.54.32(pw:IhqSb1Gg)", + "line_num": 3, "path": "tests/samples/doc_various", "info": "", - "value": "98.76.54.32", - "value_start": 0, - "value_end": 11, - "variable": null, - "variable_start": -2, - "variable_end": -2, + "value": "IhqSb1Gg)", + "value_start": 22, + "value_end": 31, + "variable": "pw", + "variable_start": 19, + "variable_end": 21, "entropy_validation": { "iterator": "BASE64_CHARS", - "entropy": 2.515950268099853, + "entropy": 2.8177111123931664, "valid": false } } @@ -4679,96 +4544,15 @@ }, { "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "IPv4", - "severity": "info", - "confidence": "weak", + "ml_validation": "VALIDATED_KEY", + "ml_probability": 0.896, + "rule": "Password", + "severity": "medium", + "confidence": "moderate", "line_data_list": [ { - "line": "98.76.54.32(master/IhqSb1Gg)", - "line_num": 2, - "path": "tests/samples/doc_various", - "info": "", - "value": "98.76.54.32", - "value_start": 0, - "value_end": 11, - "variable": null, - "variable_start": -2, - "variable_end": -2, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 2.515950268099853, - "valid": false - } - } - ] - }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "IPv4", - "severity": "info", - "confidence": "weak", - "line_data_list": [ - { - "line": "master@98.76.54.32(pw:IhqSb1Gg)", - "line_num": 3, - "path": "tests/samples/doc_various", - "info": "", - "value": "98.76.54.32", - "value_start": 7, - "value_end": 18, - "variable": null, - "variable_start": -2, - "variable_end": -2, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 2.515950268099853, - "valid": false - } - } - ] - }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.956, - "rule": "Password", - "severity": "medium", - "confidence": "moderate", - "line_data_list": [ - { - "line": "master@98.76.54.32(pw:IhqSb1Gg)", - "line_num": 3, - "path": "tests/samples/doc_various", - "info": "", - "value": "IhqSb1Gg)", - "value_start": 22, - "value_end": 31, - "variable": "pw", - "variable_start": 19, - "variable_end": 21, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 2.8177111123931664, - "valid": false - } - } - ] - }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.896, - "rule": "Password", - "severity": "medium", - "confidence": "moderate", - "line_data_list": [ - { - "line": "ID:gildong.hong@example.com mailto:{1} PW:IhqSb1Gg", - "line_num": 4, + "line": "ID:gildong.hong@example.com mailto:{1} PW:IhqSb1Gg", + "line_num": 4, "path": "tests/samples/doc_various", "info": "", "value": "IhqSb1Gg", @@ -4812,33 +4596,6 @@ } ] }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "IPv4", - "severity": "info", - "confidence": "weak", - "line_data_list": [ - { - "line": "ssh gildong.hong@98.76.54.32 mailto:{1} (PW:IhqSb1Gg)", - "line_num": 6, - "path": "tests/samples/doc_various", - "info": "", - "value": "98.76.54.32", - "value_start": 17, - "value_end": 28, - "variable": null, - "variable_start": -2, - "variable_end": -2, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 2.515950268099853, - "valid": false - } - } - ] - }, { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", @@ -4866,33 +4623,6 @@ } ] }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "IPv4", - "severity": "info", - "confidence": "weak", - "line_data_list": [ - { - "line": "ssh gildong.hong@98.76.54.32 mailto:{1} password:IhqSb1Gg", - "line_num": 7, - "path": "tests/samples/doc_various", - "info": "", - "value": "98.76.54.32", - "value_start": 17, - "value_end": 28, - "variable": null, - "variable_start": -2, - "variable_end": -2, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 2.515950268099853, - "valid": false - } - } - ] - }, { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", @@ -4920,33 +4650,6 @@ } ] }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "IPv4", - "severity": "info", - "confidence": "weak", - "line_data_list": [ - { - "line": "ssh -P IhqSb1Gg gildong.hong@98.76.54.32 mailto:{1} (password:IhqSb1Gg)", - "line_num": 11, - "path": "tests/samples/doc_various", - "info": "", - "value": "98.76.54.32", - "value_start": 29, - "value_end": 40, - "variable": null, - "variable_start": -2, - "variable_end": -2, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 2.515950268099853, - "valid": false - } - } - ] - }, { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", @@ -4974,60 +4677,6 @@ } ] }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "IPv4", - "severity": "info", - "confidence": "weak", - "line_data_list": [ - { - "line": "98.76.54.32 xxxx (master/IhqSb1Gg)", - "line_num": 14, - "path": "tests/samples/doc_various", - "info": "", - "value": "98.76.54.32", - "value_start": 0, - "value_end": 11, - "variable": null, - "variable_start": -2, - "variable_end": -2, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 2.515950268099853, - "valid": false - } - } - ] - }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "IPv4", - "severity": "info", - "confidence": "weak", - "line_data_list": [ - { - "line": "gildong.hong@98.76.54.32 pwd:IhqSb1Gg", - "line_num": 15, - "path": "tests/samples/doc_various", - "info": "", - "value": "98.76.54.32", - "value_start": 13, - "value_end": 24, - "variable": null, - "variable_start": -2, - "variable_end": -2, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 2.515950268099853, - "valid": false - } - } - ] - }, { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", @@ -5082,33 +4731,6 @@ } ] }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "IPv4", - "severity": "info", - "confidence": "weak", - "line_data_list": [ - { - "line": "98.76.54.32(pw:IhqSb1Gg)", - "line_num": 17, - "path": "tests/samples/doc_various", - "info": "", - "value": "98.76.54.32", - "value_start": 0, - "value_end": 11, - "variable": null, - "variable_start": -2, - "variable_end": -2, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 2.515950268099853, - "valid": false - } - } - ] - }, { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", @@ -5136,33 +4758,6 @@ } ] }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "IPv4", - "severity": "info", - "confidence": "weak", - "line_data_list": [ - { - "line": "98.76.54.32/pw:IhqSb1Gg", - "line_num": 19, - "path": "tests/samples/doc_various", - "info": "", - "value": "98.76.54.32", - "value_start": 0, - "value_end": 11, - "variable": null, - "variable_start": -2, - "variable_end": -2, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 2.515950268099853, - "valid": false - } - } - ] - }, { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", @@ -5246,50 +4841,23 @@ }, { "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "IPv4", - "severity": "info", - "confidence": "weak", + "ml_validation": "VALIDATED_KEY", + "ml_probability": 0.982, + "rule": "Password", + "severity": "medium", + "confidence": "moderate", "line_data_list": [ { "line": "sftp gildong.hong@98.76.54.32 mailto:{1} (pw:IhqSb1Gg)", "line_num": 22, "path": "tests/samples/doc_various", "info": "", - "value": "98.76.54.32", - "value_start": 18, - "value_end": 29, - "variable": null, - "variable_start": -2, - "variable_end": -2, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 2.515950268099853, - "valid": false - } - } - ] - }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.982, - "rule": "Password", - "severity": "medium", - "confidence": "moderate", - "line_data_list": [ - { - "line": "sftp gildong.hong@98.76.54.32 mailto:{1} (pw:IhqSb1Gg)", - "line_num": 22, - "path": "tests/samples/doc_various", - "info": "", - "value": "IhqSb1Gg)", - "value_start": 45, - "value_end": 54, - "variable": "pw", - "variable_start": 42, - "variable_end": 44, + "value": "IhqSb1Gg)", + "value_start": 45, + "value_end": 54, + "variable": "pw", + "variable_start": 42, + "variable_end": 44, "entropy_validation": { "iterator": "BASE64_CHARS", "entropy": 2.8177111123931664, @@ -5298,33 +4866,6 @@ } ] }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "IPv4", - "severity": "info", - "confidence": "weak", - "line_data_list": [ - { - "line": "gildong.hong@98.76.54.32 mailto:{1} (pw:IhqSb1Gg)", - "line_num": 23, - "path": "tests/samples/doc_various", - "info": "", - "value": "98.76.54.32", - "value_start": 13, - "value_end": 24, - "variable": null, - "variable_start": -2, - "variable_end": -2, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 2.515950268099853, - "valid": false - } - } - ] - }, { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", @@ -5379,33 +4920,6 @@ } ] }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "IPv4", - "severity": "info", - "confidence": "weak", - "line_data_list": [ - { - "line": "ssh gildong.hong@98.76.54.32 mailto:{1} (password:IhqSb1Gg)", - "line_num": 25, - "path": "tests/samples/doc_various", - "info": "", - "value": "98.76.54.32", - "value_start": 17, - "value_end": 28, - "variable": null, - "variable_start": -2, - "variable_end": -2, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 2.515950268099853, - "valid": false - } - } - ] - }, { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", @@ -5433,33 +4947,6 @@ } ] }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "IPv4", - "severity": "info", - "confidence": "weak", - "line_data_list": [ - { - "line": "ssh gildong.hong@98.76.54.32 mailto:{1} master/IhqSb1Gg", - "line_num": 26, - "path": "tests/samples/doc_various", - "info": "", - "value": "98.76.54.32", - "value_start": 17, - "value_end": 28, - "variable": null, - "variable_start": -2, - "variable_end": -2, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 2.515950268099853, - "valid": false - } - } - ] - }, { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", @@ -5568,33 +5055,6 @@ } ] }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "IPv4", - "severity": "info", - "confidence": "weak", - "line_data_list": [ - { - "line": "98.76.54.32 ANY_PW:IhqSb1Gg", - "line_num": 34, - "path": "tests/samples/doc_various", - "info": "", - "value": "98.76.54.32", - "value_start": 0, - "value_end": 11, - "variable": null, - "variable_start": -2, - "variable_end": -2, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 2.515950268099853, - "valid": false - } - } - ] - }, { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", @@ -5622,33 +5082,6 @@ } ] }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "IPv4", - "severity": "info", - "confidence": "weak", - "line_data_list": [ - { - "line": "98.76.54.32(ID/PW:IhqSb1Gg)", - "line_num": 36, - "path": "tests/samples/doc_various", - "info": "", - "value": "98.76.54.32", - "value_start": 0, - "value_end": 11, - "variable": null, - "variable_start": -2, - "variable_end": -2, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 2.515950268099853, - "valid": false - } - } - ] - }, { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", @@ -5676,33 +5109,6 @@ } ] }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "IPv4", - "severity": "info", - "confidence": "weak", - "line_data_list": [ - { - "line": "gildong.hong@98.76.54.32 (pwd:IhqSb1Gg)", - "line_num": 38, - "path": "tests/samples/doc_various", - "info": "", - "value": "98.76.54.32", - "value_start": 13, - "value_end": 24, - "variable": null, - "variable_start": -2, - "variable_end": -2, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 2.515950268099853, - "valid": false - } - } - ] - }, { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", @@ -5757,33 +5163,6 @@ } ] }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "IPv4", - "severity": "info", - "confidence": "weak", - "line_data_list": [ - { - "line": "gildong.hong@98.76.54.32(master/IhqSb1Gg)", - "line_num": 44, - "path": "tests/samples/doc_various", - "info": "", - "value": "98.76.54.32", - "value_start": 13, - "value_end": 24, - "variable": null, - "variable_start": -2, - "variable_end": -2, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 2.515950268099853, - "valid": false - } - } - ] - }, { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", @@ -5813,26 +5192,26 @@ }, { "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "IPv4", - "severity": "info", - "confidence": "weak", + "ml_validation": "VALIDATED_KEY", + "ml_probability": 0.896, + "rule": "Password", + "severity": "medium", + "confidence": "moderate", "line_data_list": [ { - "line": "gildong.hong@98.76.54.32,pw:IhqSb1Gg", - "line_num": 47, + "line": "98.76.54.32:xxxx(PW:IhqSb1Gg)", + "line_num": 51, "path": "tests/samples/doc_various", "info": "", - "value": "98.76.54.32", - "value_start": 13, - "value_end": 24, - "variable": null, - "variable_start": -2, - "variable_end": -2, + "value": "IhqSb1Gg)", + "value_start": 20, + "value_end": 29, + "variable": "PW", + "variable_start": 17, + "variable_end": 19, "entropy_validation": { "iterator": "BASE64_CHARS", - "entropy": 2.515950268099853, + "entropy": 2.8177111123931664, "valid": false } } @@ -5840,26 +5219,26 @@ }, { "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "IPv4", - "severity": "info", - "confidence": "weak", + "ml_validation": "VALIDATED_KEY", + "ml_probability": 0.949, + "rule": "Password", + "severity": "medium", + "confidence": "moderate", "line_data_list": [ { - "line": "98.76.54.32(master/IhqSb1Gg,master/IhqSb1Gg)", - "line_num": 48, + "line": "gildong.hong@98.76.54.32 PW:IhqSb1Gg", + "line_num": 56, "path": "tests/samples/doc_various", "info": "", - "value": "98.76.54.32", - "value_start": 0, - "value_end": 11, - "variable": null, - "variable_start": -2, - "variable_end": -2, + "value": "IhqSb1Gg", + "value_start": 28, + "value_end": 36, + "variable": "PW", + "variable_start": 25, + "variable_end": 27, "entropy_validation": { "iterator": "BASE64_CHARS", - "entropy": 2.515950268099853, + "entropy": 3.0, "valid": false } } @@ -5867,204 +5246,15 @@ }, { "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "IPv4", - "severity": "info", - "confidence": "weak", + "ml_validation": "VALIDATED_KEY", + "ml_probability": 0.999, + "rule": "Password", + "severity": "medium", + "confidence": "moderate", "line_data_list": [ { - "line": "98.76.54.32(master/IhqSb1Gg master/IhqSb1Gg)", - "line_num": 49, - "path": "tests/samples/doc_various", - "info": "", - "value": "98.76.54.32", - "value_start": 0, - "value_end": 11, - "variable": null, - "variable_start": -2, - "variable_end": -2, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 2.515950268099853, - "valid": false - } - } - ] - }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "IPv4", - "severity": "info", - "confidence": "weak", - "line_data_list": [ - { - "line": "xxxx:98.76.54.32(master/IhqSb1Gg)", - "line_num": 50, - "path": "tests/samples/doc_various", - "info": "", - "value": "98.76.54.32", - "value_start": 5, - "value_end": 16, - "variable": null, - "variable_start": -2, - "variable_end": -2, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 2.515950268099853, - "valid": false - } - } - ] - }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "IPv4", - "severity": "info", - "confidence": "weak", - "line_data_list": [ - { - "line": "98.76.54.32:xxxx(PW:IhqSb1Gg)", - "line_num": 51, - "path": "tests/samples/doc_various", - "info": "", - "value": "98.76.54.32", - "value_start": 0, - "value_end": 11, - "variable": null, - "variable_start": -2, - "variable_end": -2, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 2.515950268099853, - "valid": false - } - } - ] - }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.896, - "rule": "Password", - "severity": "medium", - "confidence": "moderate", - "line_data_list": [ - { - "line": "98.76.54.32:xxxx(PW:IhqSb1Gg)", - "line_num": 51, - "path": "tests/samples/doc_various", - "info": "", - "value": "IhqSb1Gg)", - "value_start": 20, - "value_end": 29, - "variable": "PW", - "variable_start": 17, - "variable_end": 19, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 2.8177111123931664, - "valid": false - } - } - ] - }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "IPv4", - "severity": "info", - "confidence": "weak", - "line_data_list": [ - { - "line": "gildong.hong@98.76.54.32 PW:IhqSb1Gg", - "line_num": 56, - "path": "tests/samples/doc_various", - "info": "", - "value": "98.76.54.32", - "value_start": 13, - "value_end": 24, - "variable": null, - "variable_start": -2, - "variable_end": -2, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 2.515950268099853, - "valid": false - } - } - ] - }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.949, - "rule": "Password", - "severity": "medium", - "confidence": "moderate", - "line_data_list": [ - { - "line": "gildong.hong@98.76.54.32 PW:IhqSb1Gg", - "line_num": 56, - "path": "tests/samples/doc_various", - "info": "", - "value": "IhqSb1Gg", - "value_start": 28, - "value_end": 36, - "variable": "PW", - "variable_start": 25, - "variable_end": 27, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 3.0, - "valid": false - } - } - ] - }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "IPv4", - "severity": "info", - "confidence": "weak", - "line_data_list": [ - { - "line": "http|https://98.76.54.32/xxxx(master/IhqSb1Gg)", - "line_num": 59, - "path": "tests/samples/doc_various", - "info": "", - "value": "98.76.54.32", - "value_start": 13, - "value_end": 24, - "variable": null, - "variable_start": -2, - "variable_end": -2, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 2.515950268099853, - "valid": false - } - } - ] - }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.999, - "rule": "Password", - "severity": "medium", - "confidence": "moderate", - "line_data_list": [ - { - "line": "ANY_user:xxxx ANY_pwd:IhqSb1Gg", - "line_num": 61, + "line": "ANY_user:xxxx ANY_pwd:IhqSb1Gg", + "line_num": 61, "path": "tests/samples/doc_various", "info": "", "value": "IhqSb1Gg", @@ -6189,60 +5379,6 @@ } ] }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "IPv4", - "severity": "info", - "confidence": "weak", - "line_data_list": [ - { - "line": "98.76.54.32(ID/PW:master/IhqSb1Gg)", - "line_num": 70, - "path": "tests/samples/doc_various", - "info": "", - "value": "98.76.54.32", - "value_start": 0, - "value_end": 11, - "variable": null, - "variable_start": -2, - "variable_end": -2, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 2.515950268099853, - "valid": false - } - } - ] - }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "IPv4", - "severity": "info", - "confidence": "weak", - "line_data_list": [ - { - "line": "master@98.76.54.32(pw:IhqSb1Gg)", - "line_num": 73, - "path": "tests/samples/doc_various", - "info": "", - "value": "98.76.54.32", - "value_start": 7, - "value_end": 18, - "variable": null, - "variable_start": -2, - "variable_end": -2, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 2.515950268099853, - "valid": false - } - } - ] - }, { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", @@ -6270,60 +5406,6 @@ } ] }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "IPv4", - "severity": "info", - "confidence": "weak", - "line_data_list": [ - { - "line": "master@98.76.54.32,PW:IhqSb1Gg", - "line_num": 74, - "path": "tests/samples/doc_various", - "info": "", - "value": "98.76.54.32", - "value_start": 7, - "value_end": 18, - "variable": null, - "variable_start": -2, - "variable_end": -2, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 2.515950268099853, - "valid": false - } - } - ] - }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "IPv4", - "severity": "info", - "confidence": "weak", - "line_data_list": [ - { - "line": "98.76.54.32 pw:IhqSb1Gg", - "line_num": 75, - "path": "tests/samples/doc_various", - "info": "", - "value": "98.76.54.32", - "value_start": 0, - "value_end": 11, - "variable": null, - "variable_start": -2, - "variable_end": -2, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 2.515950268099853, - "valid": false - } - } - ] - }, { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", @@ -6378,60 +5460,6 @@ } ] }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "IPv4", - "severity": "info", - "confidence": "weak", - "line_data_list": [ - { - "line": "98.76.54.32-->master/IhqSb1Gg", - "line_num": 81, - "path": "tests/samples/doc_various", - "info": "", - "value": "98.76.54.32", - "value_start": 0, - "value_end": 11, - "variable": null, - "variable_start": -2, - "variable_end": -2, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 2.515950268099853, - "valid": false - } - } - ] - }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "IPv4", - "severity": "info", - "confidence": "weak", - "line_data_list": [ - { - "line": "scp gildong.hong@98.76.54.32 mailto:{1} pw:IhqSb1Gg", - "line_num": 82, - "path": "tests/samples/doc_various", - "info": "", - "value": "98.76.54.32", - "value_start": 17, - "value_end": 28, - "variable": null, - "variable_start": -2, - "variable_end": -2, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 2.515950268099853, - "valid": false - } - } - ] - }, { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", @@ -6488,26 +5516,26 @@ }, { "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "IPv4", - "severity": "info", - "confidence": "weak", + "ml_validation": "VALIDATED_KEY", + "ml_probability": 0.943, + "rule": "Password", + "severity": "medium", + "confidence": "moderate", "line_data_list": [ { "line": "gildong.hong@98.76.54.32 mailto:{1} pw:IhqSb1Gg", "line_num": 84, "path": "tests/samples/doc_various", "info": "", - "value": "98.76.54.32", - "value_start": 13, - "value_end": 24, - "variable": null, - "variable_start": -2, - "variable_end": -2, + "value": "IhqSb1Gg", + "value_start": 39, + "value_end": 47, + "variable": "pw", + "variable_start": 36, + "variable_end": 38, "entropy_validation": { "iterator": "BASE64_CHARS", - "entropy": 2.515950268099853, + "entropy": 3.0, "valid": false } } @@ -6516,22 +5544,22 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.943, + "ml_probability": 0.996, "rule": "Password", "severity": "medium", "confidence": "moderate", "line_data_list": [ { - "line": "gildong.hong@98.76.54.32 mailto:{1} pw:IhqSb1Gg", - "line_num": 84, + "line": "ssh gildong.hong@98.76.54.32 mailto:{1},pw:IhqSb1Gg", + "line_num": 85, "path": "tests/samples/doc_various", "info": "", "value": "IhqSb1Gg", - "value_start": 39, - "value_end": 47, + "value_start": 43, + "value_end": 51, "variable": "pw", - "variable_start": 36, - "variable_end": 38, + "variable_start": 40, + "variable_end": 42, "entropy_validation": { "iterator": "BASE64_CHARS", "entropy": 3.0, @@ -6542,26 +5570,26 @@ }, { "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "IPv4", - "severity": "info", - "confidence": "weak", + "ml_validation": "VALIDATED_KEY", + "ml_probability": 0.995, + "rule": "Password", + "severity": "medium", + "confidence": "moderate", "line_data_list": [ { - "line": "ssh gildong.hong@98.76.54.32 mailto:{1},pw:IhqSb1Gg", - "line_num": 85, + "line": "(ssh gildong.hong@98.76.54.32 mailto{1}) pwd:IhqSb1Gg", + "line_num": 87, "path": "tests/samples/doc_various", "info": "", - "value": "98.76.54.32", - "value_start": 17, - "value_end": 28, - "variable": null, - "variable_start": -2, - "variable_end": -2, + "value": "IhqSb1Gg", + "value_start": 45, + "value_end": 53, + "variable": "pwd", + "variable_start": 41, + "variable_end": 44, "entropy_validation": { "iterator": "BASE64_CHARS", - "entropy": 2.515950268099853, + "entropy": 3.0, "valid": false } } @@ -6576,16 +5604,16 @@ "confidence": "moderate", "line_data_list": [ { - "line": "ssh gildong.hong@98.76.54.32 mailto:{1},pw:IhqSb1Gg", - "line_num": 85, + "line": "ssh gildong.hong@98.76.54.32 mailto:{1}, pw:IhqSb1Gg", + "line_num": 90, "path": "tests/samples/doc_various", "info": "", "value": "IhqSb1Gg", - "value_start": 43, - "value_end": 51, + "value_start": 44, + "value_end": 52, "variable": "pw", - "variable_start": 40, - "variable_end": 42, + "variable_start": 41, + "variable_end": 43, "entropy_validation": { "iterator": "BASE64_CHARS", "entropy": 3.0, @@ -6596,26 +5624,26 @@ }, { "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "IPv4", - "severity": "info", - "confidence": "weak", + "ml_validation": "VALIDATED_KEY", + "ml_probability": 0.998, + "rule": "Password", + "severity": "medium", + "confidence": "moderate", "line_data_list": [ { - "line": "gildong.hong@98.76.54.32 mailto:{1} (master/IhqSb1Gg)", - "line_num": 86, + "line": "ssh gildong.hong@98.76.54.32 mailto:{1} (pwd:IhqSb1Gg)", + "line_num": 93, "path": "tests/samples/doc_various", "info": "", - "value": "98.76.54.32", - "value_start": 13, - "value_end": 24, - "variable": null, - "variable_start": -2, - "variable_end": -2, + "value": "IhqSb1Gg)", + "value_start": 45, + "value_end": 54, + "variable": "pwd", + "variable_start": 41, + "variable_end": 44, "entropy_validation": { "iterator": "BASE64_CHARS", - "entropy": 2.515950268099853, + "entropy": 2.8177111123931664, "valid": false } } @@ -6623,26 +5651,26 @@ }, { "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "IPv4", - "severity": "info", - "confidence": "weak", + "ml_validation": "VALIDATED_KEY", + "ml_probability": 0.997, + "rule": "Password", + "severity": "medium", + "confidence": "moderate", "line_data_list": [ { - "line": "(ssh gildong.hong@98.76.54.32 mailto{1}) pwd:IhqSb1Gg", - "line_num": 87, + "line": "gildong.hong@98.76.54.32 mailto:{1} (password:IhqSb1Gg)", + "line_num": 94, "path": "tests/samples/doc_various", "info": "", - "value": "98.76.54.32", - "value_start": 18, - "value_end": 29, - "variable": null, - "variable_start": -2, - "variable_end": -2, + "value": "IhqSb1Gg)", + "value_start": 46, + "value_end": 55, + "variable": "password", + "variable_start": 37, + "variable_end": 45, "entropy_validation": { "iterator": "BASE64_CHARS", - "entropy": 2.515950268099853, + "entropy": 2.8177111123931664, "valid": false } } @@ -6657,15 +5685,15 @@ "confidence": "moderate", "line_data_list": [ { - "line": "(ssh gildong.hong@98.76.54.32 mailto{1}) pwd:IhqSb1Gg", - "line_num": 87, + "line": "gildong.hong@98.76.54.32 mailto:{1} Password:IhqSb1Gg", + "line_num": 96, "path": "tests/samples/doc_various", "info": "", "value": "IhqSb1Gg", "value_start": 45, "value_end": 53, - "variable": "pwd", - "variable_start": 41, + "variable": "Password", + "variable_start": 36, "variable_end": 44, "entropy_validation": { "iterator": "BASE64_CHARS", @@ -6677,26 +5705,26 @@ }, { "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "IPv4", - "severity": "info", - "confidence": "weak", + "ml_validation": "VALIDATED_KEY", + "ml_probability": 0.979, + "rule": "Password", + "severity": "medium", + "confidence": "moderate", "line_data_list": [ { - "line": "ssh gildong.hong@98.76.54.32 (master/IhqSb1Gg)", - "line_num": 88, + "line": "gildong.hong@98.76.54.32 mailto:{1} (pass:IhqSb1Gg)", + "line_num": 97, "path": "tests/samples/doc_various", "info": "", - "value": "98.76.54.32", - "value_start": 17, - "value_end": 28, - "variable": null, - "variable_start": -2, - "variable_end": -2, + "value": "IhqSb1Gg)", + "value_start": 42, + "value_end": 51, + "variable": "pass", + "variable_start": 37, + "variable_end": 41, "entropy_validation": { "iterator": "BASE64_CHARS", - "entropy": 2.515950268099853, + "entropy": 2.8177111123931664, "valid": false } } @@ -6704,26 +5732,26 @@ }, { "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "IPv4", - "severity": "info", - "confidence": "weak", + "ml_validation": "VALIDATED_KEY", + "ml_probability": 0.994, + "rule": "Password", + "severity": "medium", + "confidence": "moderate", "line_data_list": [ { - "line": "ssh gildong.hong@98.76.54.32 mailto:[1} \uacc4\uc815master/IhqSb1Gg", - "line_num": 89, + "line": "ssh gildong.hong@98.76.54.32 mailto:{1} pw:IhqSb1Gg", + "line_num": 100, "path": "tests/samples/doc_various", "info": "", - "value": "98.76.54.32", - "value_start": 17, - "value_end": 28, - "variable": null, - "variable_start": -2, - "variable_end": -2, + "value": "IhqSb1Gg", + "value_start": 43, + "value_end": 51, + "variable": "pw", + "variable_start": 40, + "variable_end": 42, "entropy_validation": { "iterator": "BASE64_CHARS", - "entropy": 2.515950268099853, + "entropy": 3.0, "valid": false } } @@ -6731,26 +5759,26 @@ }, { "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "IPv4", - "severity": "info", - "confidence": "weak", + "ml_validation": "VALIDATED_KEY", + "ml_probability": 0.999, + "rule": "Password", + "severity": "medium", + "confidence": "moderate", "line_data_list": [ { - "line": "ssh gildong.hong@98.76.54.32 mailto:{1}, pw:IhqSb1Gg", - "line_num": 90, + "line": "ssh gildong.hong@98.76.54.32 mailto:{1} pass:IhqSb1Gg", + "line_num": 101, "path": "tests/samples/doc_various", "info": "", - "value": "98.76.54.32", - "value_start": 17, - "value_end": 28, - "variable": null, - "variable_start": -2, - "variable_end": -2, + "value": "IhqSb1Gg", + "value_start": 45, + "value_end": 53, + "variable": "pass", + "variable_start": 40, + "variable_end": 44, "entropy_validation": { "iterator": "BASE64_CHARS", - "entropy": 2.515950268099853, + "entropy": 3.0, "valid": false } } @@ -6759,21 +5787,21 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.996, + "ml_probability": 0.999, "rule": "Password", "severity": "medium", "confidence": "moderate", "line_data_list": [ { - "line": "ssh gildong.hong@98.76.54.32 mailto:{1}, pw:IhqSb1Gg", - "line_num": 90, + "line": "id:gildong.hong@xxx.com mailto:{1}/password:IhqSb1Gg", + "line_num": 102, "path": "tests/samples/doc_various", "info": "", "value": "IhqSb1Gg", "value_start": 44, "value_end": 52, - "variable": "pw", - "variable_start": 41, + "variable": "password", + "variable_start": 35, "variable_end": 43, "entropy_validation": { "iterator": "BASE64_CHARS", @@ -6783,214 +5811,25 @@ } ] }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "IPv4", - "severity": "info", - "confidence": "weak", - "line_data_list": [ - { - "line": "gildong.hong@98.76.54.32 mailto:{1} (\ube44\ubc88 IhqSb1Gg)", - "line_num": 91, - "path": "tests/samples/doc_various", - "info": "", - "value": "98.76.54.32", - "value_start": 13, - "value_end": 24, - "variable": null, - "variable_start": -2, - "variable_end": -2, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 2.515950268099853, - "valid": false - } - } - ] - }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "IPv4", - "severity": "info", - "confidence": "weak", - "line_data_list": [ - { - "line": "ssh gildong.hong@98.76.54.32 mailto:{1} password IhqSb1Gg", - "line_num": 92, - "path": "tests/samples/doc_various", - "info": "", - "value": "98.76.54.32", - "value_start": 17, - "value_end": 28, - "variable": null, - "variable_start": -2, - "variable_end": -2, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 2.515950268099853, - "valid": false - } - } - ] - }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "IPv4", - "severity": "info", - "confidence": "weak", - "line_data_list": [ - { - "line": "ssh gildong.hong@98.76.54.32 mailto:{1} (pwd:IhqSb1Gg)", - "line_num": 93, - "path": "tests/samples/doc_various", - "info": "", - "value": "98.76.54.32", - "value_start": 17, - "value_end": 28, - "variable": null, - "variable_start": -2, - "variable_end": -2, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 2.515950268099853, - "valid": false - } - } - ] - }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.998, - "rule": "Password", - "severity": "medium", - "confidence": "moderate", - "line_data_list": [ - { - "line": "ssh gildong.hong@98.76.54.32 mailto:{1} (pwd:IhqSb1Gg)", - "line_num": 93, - "path": "tests/samples/doc_various", - "info": "", - "value": "IhqSb1Gg)", - "value_start": 45, - "value_end": 54, - "variable": "pwd", - "variable_start": 41, - "variable_end": 44, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 2.8177111123931664, - "valid": false - } - } - ] - }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "IPv4", - "severity": "info", - "confidence": "weak", - "line_data_list": [ - { - "line": "gildong.hong@98.76.54.32 mailto:{1} (password:IhqSb1Gg)", - "line_num": 94, - "path": "tests/samples/doc_various", - "info": "", - "value": "98.76.54.32", - "value_start": 13, - "value_end": 24, - "variable": null, - "variable_start": -2, - "variable_end": -2, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 2.515950268099853, - "valid": false - } - } - ] - }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.997, - "rule": "Password", - "severity": "medium", - "confidence": "moderate", - "line_data_list": [ - { - "line": "gildong.hong@98.76.54.32 mailto:{1} (password:IhqSb1Gg)", - "line_num": 94, - "path": "tests/samples/doc_various", - "info": "", - "value": "IhqSb1Gg)", - "value_start": 46, - "value_end": 55, - "variable": "password", - "variable_start": 37, - "variable_end": 45, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 2.8177111123931664, - "valid": false - } - } - ] - }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "IPv4", - "severity": "info", - "confidence": "weak", - "line_data_list": [ - { - "line": "gildong.hong@98.76.54.32 mailto:{1} Password:IhqSb1Gg", - "line_num": 96, - "path": "tests/samples/doc_various", - "info": "", - "value": "98.76.54.32", - "value_start": 13, - "value_end": 24, - "variable": null, - "variable_start": -2, - "variable_end": -2, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 2.515950268099853, - "valid": false - } - } - ] - }, { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.995, + "ml_probability": 1.0, "rule": "Password", "severity": "medium", "confidence": "moderate", "line_data_list": [ { - "line": "gildong.hong@98.76.54.32 mailto:{1} Password:IhqSb1Gg", - "line_num": 96, + "line": "ssh gildong.hong@98.76.54.32 mailto:{1}/password:IhqSb1Gg", + "line_num": 104, "path": "tests/samples/doc_various", "info": "", "value": "IhqSb1Gg", - "value_start": 45, - "value_end": 53, - "variable": "Password", - "variable_start": 36, - "variable_end": 44, + "value_start": 49, + "value_end": 57, + "variable": "password", + "variable_start": 40, + "variable_end": 48, "entropy_validation": { "iterator": "BASE64_CHARS", "entropy": 3.0, @@ -6999,287 +5838,17 @@ } ] }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "IPv4", - "severity": "info", - "confidence": "weak", - "line_data_list": [ - { - "line": "gildong.hong@98.76.54.32 mailto:{1} (pass:IhqSb1Gg)", - "line_num": 97, - "path": "tests/samples/doc_various", - "info": "", - "value": "98.76.54.32", - "value_start": 13, - "value_end": 24, - "variable": null, - "variable_start": -2, - "variable_end": -2, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 2.515950268099853, - "valid": false - } - } - ] - }, { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.979, + "ml_probability": 0.934, "rule": "Password", "severity": "medium", "confidence": "moderate", "line_data_list": [ { - "line": "gildong.hong@98.76.54.32 mailto:{1} (pass:IhqSb1Gg)", - "line_num": 97, - "path": "tests/samples/doc_various", - "info": "", - "value": "IhqSb1Gg)", - "value_start": 42, - "value_end": 51, - "variable": "pass", - "variable_start": 37, - "variable_end": 41, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 2.8177111123931664, - "valid": false - } - } - ] - }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "IPv4", - "severity": "info", - "confidence": "weak", - "line_data_list": [ - { - "line": "ssh gildong.hong@98.76.54.32 mailto:{1} (xxxx//IhqSb1Gg)", - "line_num": 99, - "path": "tests/samples/doc_various", - "info": "", - "value": "98.76.54.32", - "value_start": 17, - "value_end": 28, - "variable": null, - "variable_start": -2, - "variable_end": -2, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 2.515950268099853, - "valid": false - } - } - ] - }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "IPv4", - "severity": "info", - "confidence": "weak", - "line_data_list": [ - { - "line": "ssh gildong.hong@98.76.54.32 mailto:{1} pw:IhqSb1Gg", - "line_num": 100, - "path": "tests/samples/doc_various", - "info": "", - "value": "98.76.54.32", - "value_start": 17, - "value_end": 28, - "variable": null, - "variable_start": -2, - "variable_end": -2, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 2.515950268099853, - "valid": false - } - } - ] - }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.994, - "rule": "Password", - "severity": "medium", - "confidence": "moderate", - "line_data_list": [ - { - "line": "ssh gildong.hong@98.76.54.32 mailto:{1} pw:IhqSb1Gg", - "line_num": 100, - "path": "tests/samples/doc_various", - "info": "", - "value": "IhqSb1Gg", - "value_start": 43, - "value_end": 51, - "variable": "pw", - "variable_start": 40, - "variable_end": 42, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 3.0, - "valid": false - } - } - ] - }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "IPv4", - "severity": "info", - "confidence": "weak", - "line_data_list": [ - { - "line": "ssh gildong.hong@98.76.54.32 mailto:{1} pass:IhqSb1Gg", - "line_num": 101, - "path": "tests/samples/doc_various", - "info": "", - "value": "98.76.54.32", - "value_start": 17, - "value_end": 28, - "variable": null, - "variable_start": -2, - "variable_end": -2, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 2.515950268099853, - "valid": false - } - } - ] - }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.999, - "rule": "Password", - "severity": "medium", - "confidence": "moderate", - "line_data_list": [ - { - "line": "ssh gildong.hong@98.76.54.32 mailto:{1} pass:IhqSb1Gg", - "line_num": 101, - "path": "tests/samples/doc_various", - "info": "", - "value": "IhqSb1Gg", - "value_start": 45, - "value_end": 53, - "variable": "pass", - "variable_start": 40, - "variable_end": 44, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 3.0, - "valid": false - } - } - ] - }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.999, - "rule": "Password", - "severity": "medium", - "confidence": "moderate", - "line_data_list": [ - { - "line": "id:gildong.hong@xxx.com mailto:{1}/password:IhqSb1Gg", - "line_num": 102, - "path": "tests/samples/doc_various", - "info": "", - "value": "IhqSb1Gg", - "value_start": 44, - "value_end": 52, - "variable": "password", - "variable_start": 35, - "variable_end": 43, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 3.0, - "valid": false - } - } - ] - }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "IPv4", - "severity": "info", - "confidence": "weak", - "line_data_list": [ - { - "line": "ssh gildong.hong@98.76.54.32 mailto:{1}/password:IhqSb1Gg", - "line_num": 104, - "path": "tests/samples/doc_various", - "info": "", - "value": "98.76.54.32", - "value_start": 17, - "value_end": 28, - "variable": null, - "variable_start": -2, - "variable_end": -2, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 2.515950268099853, - "valid": false - } - } - ] - }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "VALIDATED_KEY", - "ml_probability": 1.0, - "rule": "Password", - "severity": "medium", - "confidence": "moderate", - "line_data_list": [ - { - "line": "ssh gildong.hong@98.76.54.32 mailto:{1}/password:IhqSb1Gg", - "line_num": 104, - "path": "tests/samples/doc_various", - "info": "", - "value": "IhqSb1Gg", - "value_start": 49, - "value_end": 57, - "variable": "password", - "variable_start": 40, - "variable_end": 48, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 3.0, - "valid": false - } - } - ] - }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.934, - "rule": "Password", - "severity": "medium", - "confidence": "moderate", - "line_data_list": [ - { - "line": "-ANYID:gildong.hong@example.com mailto:{1} -pw:IhqSb1Gg", - "line_num": 105, + "line": "-ANYID:gildong.hong@example.com mailto:{1} -pw:IhqSb1Gg", + "line_num": 105, "path": "tests/samples/doc_various", "info": "", "value": "IhqSb1Gg", @@ -7290,413 +5859,8 @@ "variable_end": 46, "entropy_validation": { "iterator": "BASE64_CHARS", - "entropy": 3.0, - "valid": false - } - } - ] - }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.97, - "rule": "Password", - "severity": "medium", - "confidence": "moderate", - "line_data_list": [ - { - "line": "ID:gildong.hong@xxxx.net mailto:{1} pw:IhqSb1Gg", - "line_num": 106, - "path": "tests/samples/doc_various", - "info": "", - "value": "IhqSb1Gg", - "value_start": 39, - "value_end": 47, - "variable": "pw", - "variable_start": 36, - "variable_end": 38, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 3.0, - "valid": false - } - } - ] - }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "IPv4", - "severity": "info", - "confidence": "weak", - "line_data_list": [ - { - "line": "ID/PW 98.76.54.32:xxx master/IhqSb1Gg", - "line_num": 107, - "path": "tests/samples/doc_various", - "info": "", - "value": "98.76.54.32", - "value_start": 6, - "value_end": 17, - "variable": null, - "variable_start": -2, - "variable_end": -2, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 2.515950268099853, - "valid": false - } - } - ] - }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "IPv4", - "severity": "info", - "confidence": "weak", - "line_data_list": [ - { - "line": "http://98.76.54.32:xxx(pw:IhqSb1Gg)", - "line_num": 108, - "path": "tests/samples/doc_various", - "info": "", - "value": "98.76.54.32", - "value_start": 7, - "value_end": 18, - "variable": null, - "variable_start": -2, - "variable_end": -2, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 2.515950268099853, - "valid": false - } - } - ] - }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.978, - "rule": "Password", - "severity": "medium", - "confidence": "moderate", - "line_data_list": [ - { - "line": "http://98.76.54.32:xxx(pw:IhqSb1Gg)", - "line_num": 108, - "path": "tests/samples/doc_various", - "info": "", - "value": "IhqSb1Gg)", - "value_start": 26, - "value_end": 35, - "variable": "pw", - "variable_start": 23, - "variable_end": 25, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 2.8177111123931664, - "valid": false - } - } - ] - }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "Dropbox API secret (long term)", - "severity": "high", - "confidence": "weak", - "line_data_list": [ - { - "line": "var g = '7rBynGo0b1cAAAAAAAAAAc72L3T6rQK51mB5a06ijnwRG91deTxvSqdZNAlxq8pZ'", - "line_num": 1, - "path": "tests/samples/dropbox_api_secret_long_term", - "info": "", - "value": "7rBynGo0b1cAAAAAAAAAAc72L3T6rQK51mB5a06ijnwRG91deTxvSqdZNAlxq8pZ", - "value_start": 9, - "value_end": 73, - "variable": null, - "variable_start": -2, - "variable_end": -2, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 4.89361507332541, - "valid": true - } - } - ] - }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "Dropbox App secret", - "severity": "info", - "confidence": "weak", - "line_data_list": [ - { - "line": "var app_unique_val_s = 'wpv1jq9xwanbn3n';", - "line_num": 1, - "path": "tests/samples/dropbox_app_secret", - "info": "", - "value": "wpv1jq9xwanbn3n", - "value_start": 24, - "value_end": 39, - "variable": null, - "variable_start": -2, - "variable_end": -2, - "entropy_validation": { - "iterator": "BASE36_CHARS", - "entropy": 3.4565647621309536, - "valid": true - } - } - ] - }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "Dropbox OAuth2 API Access Token", - "severity": "high", - "confidence": "moderate", - "line_data_list": [ - { - "line": "var dropbox = 'sl.BdmpmC82mhhySscKk2oQGyE5l--8LdAQftLTXVGQhP39Z8FtAK1BhePhyevurA-Elt7ToIr6OpwzKAYE7RBqpu6VVyQU5WlCTL_Q7N4gElXahaWou6aPpOIwgGCIOq9aeC3YFoc';", - "line_num": 1, - "path": "tests/samples/dropbox_oauth_token", - "info": "", - "value": "sl.BdmpmC82mhhySscKk2oQGyE5l--8LdAQftLTXVGQhP39Z8FtAK1BhePhyevurA-Elt7ToIr6OpwzKAYE7RBqpu6VVyQU5WlCTL_Q7N4gElXahaWou6aPpOIwgGCIOq9aeC3YFoc", - "value_start": 15, - "value_end": 153, - "variable": null, - "variable_start": -2, - "variable_end": -2, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 5.395844179446957, - "valid": true - } - } - ] - }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "Dynatrace API Token", - "severity": "high", - "confidence": "moderate", - "line_data_list": [ - { - "line": "dt0c01.ST2EY72KQINMH574WMNVI7YN.G3DFPBEJYMODIDAEX454M7YWBUVEFOWKPRVMWFASS64NFH52PX6BNDVFFM572RZM", - "line_num": 1, - "path": "tests/samples/dynatrace_api.hs", - "info": "", - "value": "dt0c01.ST2EY72KQINMH574WMNVI7YN.G3DFPBEJYMODIDAEX454M7YWBUVEFOWKPRVMWFASS64NFH52PX6BNDVFFM572RZM", - "value_start": 0, - "value_end": 96, - "variable": null, - "variable_start": -2, - "variable_end": -2, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 4.808191506786782, - "valid": true - } - } - ] - }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "Facebook Access Token", - "severity": "high", - "confidence": "moderate", - "line_data_list": [ - { - "line": "GI_REO_GI_FACEBOOK_TOKEN = \"EAACEb00Kse0BAlGy7KeQ5YnaCEd09Eose0cBAlGy7KeQ5Yna9CoDsup39tiYdoQ4jH9Coup39tiYdWoQ4jHFZD\"", - "line_num": 1, - "path": "tests/samples/facebook_key", - "info": "", - "value": "EAACEb00Kse0BAlGy7KeQ5YnaCEd09Eose0cBAlGy7KeQ5Yna9CoDsup39tiYdoQ4jH9Coup39tiYdWoQ4jHFZD", - "value_start": 28, - "value_end": 115, - "variable": null, - "variable_start": -2, - "variable_end": -2, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 4.936120692057916, - "valid": true - } - } - ] - }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.999, - "rule": "Token", - "severity": "medium", - "confidence": "moderate", - "line_data_list": [ - { - "line": "GI_REO_GI_FACEBOOK_TOKEN = \"EAACEb00Kse0BAlGy7KeQ5YnaCEd09Eose0cBAlGy7KeQ5Yna9CoDsup39tiYdoQ4jH9Coup39tiYdWoQ4jHFZD\"", - "line_num": 1, - "path": "tests/samples/facebook_key", - "info": "", - "value": "EAACEb00Kse0BAlGy7KeQ5YnaCEd09Eose0cBAlGy7KeQ5Yna9CoDsup39tiYdoQ4jH9Coup39tiYdWoQ4jHFZD", - "value_start": 28, - "value_end": 115, - "variable": "GI_REO_GI_FACEBOOK_TOKEN", - "variable_start": 0, - "variable_end": 24, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 4.936120692057916, - "valid": true - } - } - ] - }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "Facebook App Token", - "severity": "high", - "confidence": "moderate", - "line_data_list": [ - { - "line": "1527194624358273|qbBf2-fdB9zZpqLA0_2nNzZDw2M", - "line_num": 2, - "path": "tests/samples/facebook_key", - "info": "", - "value": "1527194624358273|qbBf2-fdB9zZpqLA0_2nNzZDw2M", - "value_start": 0, - "value_end": 44, - "variable": null, - "variable_start": -2, - "variable_end": -2, - "entropy_validation": { - "iterator": "BASE36_CHARS", - "entropy": 3.2089099270924217, - "valid": true - } - } - ] - }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "Firebase Domain", - "severity": "info", - "confidence": "moderate", - "line_data_list": [ - { - "line": "test-app-domain-42.firebaseapp.com", - "line_num": 1, - "path": "tests/samples/firebase_domain", - "info": "", - "value": "test-app-domain-42.firebaseapp.com", - "value_start": 0, - "value_end": 34, - "variable": null, - "variable_start": -2, - "variable_end": -2, - "entropy_validation": { - "iterator": "BASE36_CHARS", - "entropy": 3.4347510262969525, - "valid": true - } - } - ] - }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "Firebase Domain", - "severity": "info", - "confidence": "moderate", - "line_data_list": [ - { - "line": "test2.io.firebaseio.com", - "line_num": 2, - "path": "tests/samples/firebase_domain", - "info": "", - "value": "test2.io.firebaseio.com", - "value_start": 0, - "value_end": 23, - "variable": null, - "variable_start": -2, - "variable_end": -2, - "entropy_validation": { - "iterator": "BASE36_CHARS", - "entropy": 3.1394163745499943, - "valid": true - } - } - ] - }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "Github Classic Token", - "severity": "high", - "confidence": "strong", - "line_data_list": [ - { - "line": "ghp_00000000000000000000000000000004WZ4EQ # classic", - "line_num": 1, - "path": "tests/samples/github_classic_token", - "info": "", - "value": "ghp_00000000000000000000000000000004WZ4EQ", - "value_start": 0, - "value_end": 41, - "variable": null, - "variable_start": -2, - "variable_end": -2, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 1.4322437698226884, - "valid": false - } - } - ] - }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "Github Fine-granted Token", - "severity": "high", - "confidence": "strong", - "line_data_list": [ - { - "line": "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", - "line_num": 2, - "path": "tests/samples/github_fine_granted_token", - "info": "", - "value": "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", - "value_start": 0, - "value_end": 93, - "variable": null, - "variable_start": -2, - "variable_end": -2, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 5.255374790203285, - "valid": true + "entropy": 3.0, + "valid": false } } ] @@ -7704,26 +5868,26 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 1.0, - "rule": "Github Old Token", - "severity": "high", + "ml_probability": 0.97, + "rule": "Password", + "severity": "medium", "confidence": "moderate", "line_data_list": [ { - "line": "GITHUB_ACCESS_TOKEN = \"lbyxnhqjfnzhlpnvcvaxjumgvdlnktgugdofmzyy\"", - "line_num": 1, - "path": "tests/samples/github_key.groovy", + "line": "ID:gildong.hong@xxxx.net mailto:{1} pw:IhqSb1Gg", + "line_num": 106, + "path": "tests/samples/doc_various", "info": "", - "value": "lbyxnhqjfnzhlpnvcvaxjumgvdlnktgugdofmzyy", - "value_start": 23, - "value_end": 63, - "variable": null, - "variable_start": -2, - "variable_end": -2, + "value": "IhqSb1Gg", + "value_start": 39, + "value_end": 47, + "variable": "pw", + "variable_start": 36, + "variable_end": 38, "entropy_validation": { - "iterator": "BASE36_CHARS", - "entropy": 4.246439344671015, - "valid": true + "iterator": "BASE64_CHARS", + "entropy": 3.0, + "valid": false } } ] @@ -7731,26 +5895,26 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 1.0, - "rule": "Token", + "ml_probability": 0.978, + "rule": "Password", "severity": "medium", "confidence": "moderate", "line_data_list": [ { - "line": "GITHUB_ACCESS_TOKEN = \"lbyxnhqjfnzhlpnvcvaxjumgvdlnktgugdofmzyy\"", - "line_num": 1, - "path": "tests/samples/github_key.groovy", + "line": "http://98.76.54.32:xxx(pw:IhqSb1Gg)", + "line_num": 108, + "path": "tests/samples/doc_various", "info": "", - "value": "lbyxnhqjfnzhlpnvcvaxjumgvdlnktgugdofmzyy", - "value_start": 23, - "value_end": 63, - "variable": "GITHUB_ACCESS_TOKEN", - "variable_start": 0, - "variable_end": 19, + "value": "IhqSb1Gg)", + "value_start": 26, + "value_end": 35, + "variable": "pw", + "variable_start": 23, + "variable_end": 25, "entropy_validation": { - "iterator": "BASE36_CHARS", - "entropy": 4.246439344671015, - "valid": true + "iterator": "BASE64_CHARS", + "entropy": 2.8177111123931664, + "valid": false } } ] @@ -7759,24 +5923,24 @@ "api_validation": "NOT_AVAILABLE", "ml_validation": "NOT_AVAILABLE", "ml_probability": null, - "rule": "Gitlab Incoming Email Token", - "severity": "info", + "rule": "Dropbox API secret (long term)", + "severity": "high", "confidence": "weak", "line_data_list": [ { - "line": "var email_t = '7e4v6v5j2nepcc8f5zvatgl9g';", + "line": "var g = '7rBynGo0b1cAAAAAAAAAAc72L3T6rQK51mB5a06ijnwRG91deTxvSqdZNAlxq8pZ'", "line_num": 1, - "path": "tests/samples/gitlab_email_token", + "path": "tests/samples/dropbox_api_secret_long_term", "info": "", - "value": "7e4v6v5j2nepcc8f5zvatgl9g", - "value_start": 15, - "value_end": 40, + "value": "7rBynGo0b1cAAAAAAAAAAc72L3T6rQK51mB5a06ijnwRG91deTxvSqdZNAlxq8pZ", + "value_start": 9, + "value_end": 73, "variable": null, "variable_start": -2, "variable_end": -2, "entropy_validation": { - "iterator": "BASE36_CHARS", - "entropy": 4.133660689688185, + "iterator": "BASE64_CHARS", + "entropy": 4.89361507332541, "valid": true } } @@ -7786,25 +5950,25 @@ "api_validation": "NOT_AVAILABLE", "ml_validation": "NOT_AVAILABLE", "ml_probability": null, - "rule": "Gitlab Feed Token", + "rule": "Dropbox App secret", "severity": "info", "confidence": "weak", "line_data_list": [ { - "line": "feed_n = 'o9aEaH32LN618KhF7e_L'", + "line": "var app_unique_val_s = 'wpv1jq9xwanbn3n';", "line_num": 1, - "path": "tests/samples/gitlab_feed_token", + "path": "tests/samples/dropbox_app_secret", "info": "", - "value": "o9aEaH32LN618KhF7e_L", - "value_start": 10, - "value_end": 30, + "value": "wpv1jq9xwanbn3n", + "value_start": 24, + "value_end": 39, "variable": null, "variable_start": -2, "variable_end": -2, "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 3.9058316901429944, - "valid": false + "iterator": "BASE36_CHARS", + "entropy": 3.4565647621309536, + "valid": true } } ] @@ -7813,25 +5977,25 @@ "api_validation": "NOT_AVAILABLE", "ml_validation": "NOT_AVAILABLE", "ml_probability": null, - "rule": "Gitlab PAT", + "rule": "Dropbox OAuth2 API Access Token", "severity": "high", - "confidence": "strong", + "confidence": "moderate", "line_data_list": [ { - "line": "var pat = 'glpat-a6N2pFAr2L2A6iRsA_mw';", + "line": "var dropbox = 'sl.BdmpmC82mhhySscKk2oQGyE5l--8LdAQftLTXVGQhP39Z8FtAK1BhePhyevurA-Elt7ToIr6OpwzKAYE7RBqpu6VVyQU5WlCTL_Q7N4gElXahaWou6aPpOIwgGCIOq9aeC3YFoc';", "line_num": 1, - "path": "tests/samples/gitlab_pat_api", + "path": "tests/samples/dropbox_oauth_token", "info": "", - "value": "glpat-a6N2pFAr2L2A6iRsA_mw", - "value_start": 11, - "value_end": 37, + "value": "sl.BdmpmC82mhhySscKk2oQGyE5l--8LdAQftLTXVGQhP39Z8FtAK1BhePhyevurA-Elt7ToIr6OpwzKAYE7RBqpu6VVyQU5WlCTL_Q7N4gElXahaWou6aPpOIwgGCIOq9aeC3YFoc", + "value_start": 15, + "value_end": 153, "variable": null, "variable_start": -2, "variable_end": -2, "entropy_validation": { "iterator": "BASE64_CHARS", - "entropy": 3.7423376242715105, - "valid": false + "entropy": 5.395844179446957, + "valid": true } } ] @@ -7840,24 +6004,24 @@ "api_validation": "NOT_AVAILABLE", "ml_validation": "NOT_AVAILABLE", "ml_probability": null, - "rule": "Gitlab Pipeline Trigger Token", + "rule": "Dynatrace API Token", "severity": "high", - "confidence": "strong", + "confidence": "moderate", "line_data_list": [ { - "line": "trigger = 'glptt-33276248c9748113e978392e5c074b7f974f8683';", + "line": "dt0c01.ST2EY72KQINMH574WMNVI7YN.G3DFPBEJYMODIDAEX454M7YWBUVEFOWKPRVMWFASS64NFH52PX6BNDVFFM572RZM", "line_num": 1, - "path": "tests/samples/gitlab_pipeline_trigger_token", + "path": "tests/samples/dynatrace_api.hs", "info": "", - "value": "glptt-33276248c9748113e978392e5c074b7f974f8683", - "value_start": 11, - "value_end": 57, + "value": "dt0c01.ST2EY72KQINMH574WMNVI7YN.G3DFPBEJYMODIDAEX454M7YWBUVEFOWKPRVMWFASS64NFH52PX6BNDVFFM572RZM", + "value_start": 0, + "value_end": 96, "variable": null, "variable_start": -2, "variable_end": -2, "entropy_validation": { - "iterator": "BASE36_CHARS", - "entropy": 3.8494857514609038, + "iterator": "BASE64_CHARS", + "entropy": 4.808191506786782, "valid": true } } @@ -7867,24 +6031,24 @@ "api_validation": "NOT_AVAILABLE", "ml_validation": "NOT_AVAILABLE", "ml_probability": null, - "rule": "Gitlab Registration Runner Token", + "rule": "Facebook Access Token", "severity": "high", - "confidence": "strong", + "confidence": "moderate", "line_data_list": [ { - "line": "gitlab_runner = 'GR1348941jG6xeSsmN8DFVKoyBYu2';", + "line": "GI_REO_GI_FACEBOOK_TOKEN = \"EAACEb00Kse0BAlGy7KeQ5YnaCEd09Eose0cBAlGy7KeQ5Yna9CoDsup39tiYdoQ4jH9Coup39tiYdWoQ4jHFZD\"", "line_num": 1, - "path": "tests/samples/gitlab_registration_runner", + "path": "tests/samples/facebook_key", "info": "", - "value": "GR1348941jG6xeSsmN8DFVKoyBYu2", - "value_start": 17, - "value_end": 46, + "value": "EAACEb00Kse0BAlGy7KeQ5YnaCEd09Eose0cBAlGy7KeQ5Yna9CoDsup39tiYdoQ4jH9Coup39tiYdWoQ4jHFZD", + "value_start": 28, + "value_end": 115, "variable": null, "variable_start": -2, "variable_end": -2, "entropy_validation": { "iterator": "BASE64_CHARS", - "entropy": 4.582118926162056, + "entropy": 4.936120692057916, "valid": true } } @@ -7892,27 +6056,27 @@ }, { "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "Gitlab Registration Runner Token 2023", - "severity": "high", - "confidence": "strong", + "ml_validation": "VALIDATED_KEY", + "ml_probability": 0.999, + "rule": "Token", + "severity": "medium", + "confidence": "moderate", "line_data_list": [ { - "line": "return \"glrt-2CR8_eVxiio-1QmzPZwa\"", + "line": "GI_REO_GI_FACEBOOK_TOKEN = \"EAACEb00Kse0BAlGy7KeQ5YnaCEd09Eose0cBAlGy7KeQ5Yna9CoDsup39tiYdoQ4jH9Coup39tiYdWoQ4jHFZD\"", "line_num": 1, - "path": "tests/samples/gitlab_registration_runner_2023", + "path": "tests/samples/facebook_key", "info": "", - "value": "glrt-2CR8_eVxiio-1QmzPZwa", - "value_start": 8, - "value_end": 33, - "variable": null, - "variable_start": -2, - "variable_end": -2, + "value": "EAACEb00Kse0BAlGy7KeQ5YnaCEd09Eose0cBAlGy7KeQ5Yna9CoDsup39tiYdoQ4jH9Coup39tiYdWoQ4jHFZD", + "value_start": 28, + "value_end": 115, + "variable": "GI_REO_GI_FACEBOOK_TOKEN", + "variable_start": 0, + "variable_end": 24, "entropy_validation": { "iterator": "BASE64_CHARS", - "entropy": 4.006593447001756, - "valid": false + "entropy": 4.936120692057916, + "valid": true } } ] @@ -7921,24 +6085,24 @@ "api_validation": "NOT_AVAILABLE", "ml_validation": "NOT_AVAILABLE", "ml_probability": null, - "rule": "Google API Key", + "rule": "Facebook App Token", "severity": "high", "confidence": "moderate", "line_data_list": [ { - "line": "AIzaGiReoG-CrackleCrackle12315618_12315", - "line_num": 1, - "path": "tests/samples/google_api_key.toml", + "line": "1527194624358273|qbBf2-fdB9zZpqLA0_2nNzZDw2M", + "line_num": 2, + "path": "tests/samples/facebook_key", "info": "", - "value": "AIzaGiReoG-CrackleCrackle12315618_12315", + "value": "1527194624358273|qbBf2-fdB9zZpqLA0_2nNzZDw2M", "value_start": 0, - "value_end": 39, + "value_end": 44, "variable": null, "variable_start": -2, "variable_end": -2, "entropy_validation": { "iterator": "BASE36_CHARS", - "entropy": 3.165196181720608, + "entropy": 3.2089099270924217, "valid": true } } @@ -7948,43 +6112,26 @@ "api_validation": "NOT_AVAILABLE", "ml_validation": "NOT_AVAILABLE", "ml_probability": null, - "rule": "Google Multi", - "severity": "high", + "rule": "Firebase Domain", + "severity": "info", "confidence": "moderate", "line_data_list": [ { - "line": "194206074328-qp89pdv6fi35vsi71258g1eh31q6h7c3.apps.googleusercontent.com", - "line_num": 2, - "path": "tests/samples/google_multi", + "line": "test-app-domain-42.firebaseapp.com", + "line_num": 1, + "path": "tests/samples/firebase_domain", "info": "", - "value": "194206074328-qp89pdv6fi35vsi71258g1eh31q6h7c3.apps.googleusercontent.com", + "value": "test-app-domain-42.firebaseapp.com", "value_start": 0, - "value_end": 72, + "value_end": 34, "variable": null, "variable_start": -2, "variable_end": -2, "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 4.531537327540733, + "iterator": "BASE36_CHARS", + "entropy": 3.4347510262969525, "valid": true } - }, - { - "line": "4L2QMyTm6Rr0o46ytGiReoG1", - "line_num": 4, - "path": "tests/samples/google_multi", - "info": "", - "value": "4L2QMyTm6Rr0o46ytGiReoG1", - "value_start": 0, - "value_end": 24, - "variable": null, - "variable_start": -2, - "variable_end": -2, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 4.084962500721157, - "valid": false - } } ] }, @@ -7992,41 +6139,51 @@ "api_validation": "NOT_AVAILABLE", "ml_validation": "NOT_AVAILABLE", "ml_probability": null, - "rule": "Google Multi", - "severity": "high", + "rule": "Firebase Domain", + "severity": "info", "confidence": "moderate", "line_data_list": [ { - "line": "\"id\":\"194206074328-qdv6fi3eh31q6h7c35vsi7p89p1258g1.apps.googleusercontent.com\",\"CEKPET\":\"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\",", - "line_num": 18, - "path": "tests/samples/google_multi", + "line": "test2.io.firebaseio.com", + "line_num": 2, + "path": "tests/samples/firebase_domain", "info": "", - "value": "194206074328-qdv6fi3eh31q6h7c35vsi7p89p1258g1.apps.googleusercontent.com", - "value_start": 6, - "value_end": 78, + "value": "test2.io.firebaseio.com", + "value_start": 0, + "value_end": 23, "variable": null, "variable_start": -2, "variable_end": -2, "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 4.531537327540733, + "iterator": "BASE36_CHARS", + "entropy": 3.1394163745499943, "valid": true } - }, + } + ] + }, + { + "api_validation": "NOT_AVAILABLE", + "ml_validation": "NOT_AVAILABLE", + "ml_probability": null, + "rule": "Github Classic Token", + "severity": "high", + "confidence": "strong", + "line_data_list": [ { - "line": "\"id\":\"194206074328-qdv6fi3eh31q6h7c35vsi7p89p1258g1.apps.googleusercontent.com\",\"CEKPET\":\"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\",", - "line_num": 18, - "path": "tests/samples/google_multi", + "line": "ghp_00000000000000000000000000000004WZ4EQ # classic", + "line_num": 1, + "path": "tests/samples/github_classic_token", "info": "", - "value": "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", - "value_start": 90, - "value_end": 125, + "value": "ghp_00000000000000000000000000000004WZ4EQ", + "value_start": 0, + "value_end": 41, "variable": null, "variable_start": -2, "variable_end": -2, "entropy_validation": { "iterator": "BASE64_CHARS", - "entropy": 4.436181130262395, + "entropy": 1.4322437698226884, "valid": false } } @@ -8036,51 +6193,78 @@ "api_validation": "NOT_AVAILABLE", "ml_validation": "NOT_AVAILABLE", "ml_probability": null, - "rule": "Google OAuth Secret", + "rule": "Github Fine-granted Token", "severity": "high", "confidence": "strong", "line_data_list": [ { - "line": "\"id\":\"194206074328-qdv6fi3eh31q6h7c35vsi7p89p1258g1.apps.googleusercontent.com\",\"CEKPET\":\"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\",", - "line_num": 18, - "path": "tests/samples/google_multi", + "line": "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", + "line_num": 2, + "path": "tests/samples/github_fine_granted_token", "info": "", - "value": "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", - "value_start": 90, - "value_end": 125, + "value": "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", + "value_start": 0, + "value_end": 93, "variable": null, "variable_start": -2, "variable_end": -2, "entropy_validation": { "iterator": "BASE64_CHARS", - "entropy": 4.436181130262395, - "valid": false + "entropy": 5.255374790203285, + "valid": true } } ] }, { "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "Google OAuth Access Token", + "ml_validation": "VALIDATED_KEY", + "ml_probability": 1.0, + "rule": "Github Old Token", "severity": "high", "confidence": "moderate", "line_data_list": [ { - "line": "google_oauth_key = \"ya29.gi_reo_gi_crackle_ln22\"", + "line": "GITHUB_ACCESS_TOKEN = \"lbyxnhqjfnzhlpnvcvaxjumgvdlnktgugdofmzyy\"", "line_num": 1, - "path": "tests/samples/google_oauth_key", + "path": "tests/samples/github_key.groovy", "info": "", - "value": "ya29.gi_reo_gi_crackle_ln22", - "value_start": 20, - "value_end": 47, + "value": "lbyxnhqjfnzhlpnvcvaxjumgvdlnktgugdofmzyy", + "value_start": 23, + "value_end": 63, "variable": null, "variable_start": -2, "variable_end": -2, "entropy_validation": { "iterator": "BASE36_CHARS", - "entropy": 3.1797273164975133, + "entropy": 4.246439344671015, + "valid": true + } + } + ] + }, + { + "api_validation": "NOT_AVAILABLE", + "ml_validation": "VALIDATED_KEY", + "ml_probability": 1.0, + "rule": "Token", + "severity": "medium", + "confidence": "moderate", + "line_data_list": [ + { + "line": "GITHUB_ACCESS_TOKEN = \"lbyxnhqjfnzhlpnvcvaxjumgvdlnktgugdofmzyy\"", + "line_num": 1, + "path": "tests/samples/github_key.groovy", + "info": "", + "value": "lbyxnhqjfnzhlpnvcvaxjumgvdlnktgugdofmzyy", + "value_start": 23, + "value_end": 63, + "variable": "GITHUB_ACCESS_TOKEN", + "variable_start": 0, + "variable_end": 19, + "entropy_validation": { + "iterator": "BASE36_CHARS", + "entropy": 4.246439344671015, "valid": true } } @@ -8090,24 +6274,24 @@ "api_validation": "NOT_AVAILABLE", "ml_validation": "NOT_AVAILABLE", "ml_probability": null, - "rule": "Grafana Access Policy Token", - "severity": "high", - "confidence": "strong", + "rule": "Gitlab Incoming Email Token", + "severity": "info", + "confidence": "weak", "line_data_list": [ { - "line": "grafana_policy = 'glc_eyJvIjoiMjA0NjMwIiwibiI6InRlc3QtdG9rZW4iLCJrIjoidklnbjJ2WHc5MTVXOWtNOWxsNHcyZHEyIiwibSI6eyJyIjoicHJvZC0wIn19'", + "line": "var email_t = '7e4v6v5j2nepcc8f5zvatgl9g';", "line_num": 1, - "path": "tests/samples/grafana_access_policy_token", + "path": "tests/samples/gitlab_email_token", "info": "", - "value": "glc_eyJvIjoiMjA0NjMwIiwibiI6InRlc3QtdG9rZW4iLCJrIjoidklnbjJ2WHc5MTVXOWtNOWxsNHcyZHEyIiwibSI6eyJyIjoicHJvZC0wIn19", - "value_start": 18, - "value_end": 130, + "value": "7e4v6v5j2nepcc8f5zvatgl9g", + "value_start": 15, + "value_end": 40, "variable": null, "variable_start": -2, "variable_end": -2, "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 5.097632476604024, + "iterator": "BASE36_CHARS", + "entropy": 4.133660689688185, "valid": true } } @@ -8117,24 +6301,24 @@ "api_validation": "NOT_AVAILABLE", "ml_validation": "NOT_AVAILABLE", "ml_probability": null, - "rule": "Grafana Provisioned API Key", - "severity": "high", - "confidence": "strong", + "rule": "Gitlab Feed Token", + "severity": "info", + "confidence": "weak", "line_data_list": [ { - "line": "grafana = 'eyJrIjoiMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMCIsIm4iOiJ4eHh4IiwiaWQiOjIwNDM2MH0='", + "line": "feed_n = 'o9aEaH32LN618KhF7e_L'", "line_num": 1, - "path": "tests/samples/grafana_provisioned_api_key", + "path": "tests/samples/gitlab_feed_token", "info": "", - "value": "eyJrIjoiMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMCIsIm4iOiJ4eHh4IiwiaWQiOjIwNDM2MH0=", - "value_start": 11, - "value_end": 107, + "value": "o9aEaH32LN618KhF7e_L", + "value_start": 10, + "value_end": 30, "variable": null, "variable_start": -2, "variable_end": -2, "entropy_validation": { "iterator": "BASE64_CHARS", - "entropy": 3.8153130511409934, + "entropy": 3.9058316901429944, "valid": false } } @@ -8142,26 +6326,26 @@ }, { "api_validation": "NOT_AVAILABLE", - "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.985, - "rule": "JSON Web Token", - "severity": "medium", - "confidence": "moderate", + "ml_validation": "NOT_AVAILABLE", + "ml_probability": null, + "rule": "Gitlab PAT", + "severity": "high", + "confidence": "strong", "line_data_list": [ { - "line": "grafana = 'eyJrIjoiMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMCIsIm4iOiJ4eHh4IiwiaWQiOjIwNDM2MH0='", + "line": "var pat = 'glpat-a6N2pFAr2L2A6iRsA_mw';", "line_num": 1, - "path": "tests/samples/grafana_provisioned_api_key", + "path": "tests/samples/gitlab_pat_api", "info": "", - "value": "eyJrIjoiMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMCIsIm4iOiJ4eHh4IiwiaWQiOjIwNDM2MH0=", + "value": "glpat-a6N2pFAr2L2A6iRsA_mw", "value_start": 11, - "value_end": 107, + "value_end": 37, "variable": null, "variable_start": -2, "variable_end": -2, "entropy_validation": { "iterator": "BASE64_CHARS", - "entropy": 3.8153130511409934, + "entropy": 3.7423376242715105, "valid": false } } @@ -8171,24 +6355,24 @@ "api_validation": "NOT_AVAILABLE", "ml_validation": "NOT_AVAILABLE", "ml_probability": null, - "rule": "Grafana Service Account Token", + "rule": "Gitlab Pipeline Trigger Token", "severity": "high", "confidence": "strong", "line_data_list": [ { - "line": "glsa_ThisI5NtTheTok3nYou8reLo0k1ngF0r_0a2a3df7", + "line": "trigger = 'glptt-33276248c9748113e978392e5c074b7f974f8683';", "line_num": 1, - "path": "tests/samples/grafana_service_accounts", + "path": "tests/samples/gitlab_pipeline_trigger_token", "info": "", - "value": "glsa_ThisI5NtTheTok3nYou8reLo0k1ngF0r_0a2a3df7", - "value_start": 0, - "value_end": 46, + "value": "glptt-33276248c9748113e978392e5c074b7f974f8683", + "value_start": 11, + "value_end": 57, "variable": null, "variable_start": -2, "variable_end": -2, "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 4.52211252299684, + "iterator": "BASE36_CHARS", + "entropy": 3.8494857514609038, "valid": true } } @@ -8198,24 +6382,24 @@ "api_validation": "NOT_AVAILABLE", "ml_validation": "NOT_AVAILABLE", "ml_probability": null, - "rule": "Hashicorp Terraform Token", + "rule": "Gitlab Registration Runner Token", "severity": "high", "confidence": "strong", "line_data_list": [ { - "line": "Z28P3STmkBQi1Y.atlasv1.YE7RBqu6VVyQIOq9a1eC3YFU5Elt7ToIr6OwzKAWlCTQ7N4gElXaWou6aPpOIwGCoc0", + "line": "gitlab_runner = 'GR1348941jG6xeSsmN8DFVKoyBYu2';", "line_num": 1, - "path": "tests/samples/hashicorp_terraform", + "path": "tests/samples/gitlab_registration_runner", "info": "", - "value": "Z28P3STmkBQi1Y.atlasv1.YE7RBqu6VVyQIOq9a1eC3YFU5Elt7ToIr6OwzKAWlCTQ7N4gElXaWou6aPpOIwGCoc0", - "value_start": 0, - "value_end": 90, + "value": "GR1348941jG6xeSsmN8DFVKoyBYu2", + "value_start": 17, + "value_end": 46, "variable": null, "variable_start": -2, "variable_end": -2, "entropy_validation": { "iterator": "BASE64_CHARS", - "entropy": 5.348551883097512, + "entropy": 4.582118926162056, "valid": true } } @@ -8225,25 +6409,25 @@ "api_validation": "NOT_AVAILABLE", "ml_validation": "NOT_AVAILABLE", "ml_probability": null, - "rule": "Heroku API Key", + "rule": "Gitlab Registration Runner Token 2023", "severity": "high", - "confidence": "moderate", + "confidence": "strong", "line_data_list": [ { - "line": "HerOkUa04b8c1d-A147-b252-3b6a8f9c2b16", + "line": "return \"glrt-2CR8_eVxiio-1QmzPZwa\"", "line_num": 1, - "path": "tests/samples/heroku_api.toml", + "path": "tests/samples/gitlab_registration_runner_2023", "info": "", - "value": "HerOkUa04b8c1d-A147-b252-3b6a8f9c2b16", - "value_start": 0, - "value_end": 37, + "value": "glrt-2CR8_eVxiio-1QmzPZwa", + "value_start": 8, + "value_end": 33, "variable": null, "variable_start": -2, "variable_end": -2, "entropy_validation": { - "iterator": "BASE36_CHARS", - "entropy": 3.48037367471734, - "valid": true + "iterator": "BASE64_CHARS", + "entropy": 4.006593447001756, + "valid": false } } ] @@ -8252,24 +6436,24 @@ "api_validation": "NOT_AVAILABLE", "ml_validation": "NOT_AVAILABLE", "ml_probability": null, - "rule": "Instagram Access Token", + "rule": "Google API Key", "severity": "high", - "confidence": "strong", + "confidence": "moderate", "line_data_list": [ { - "line": "IGQVJWS3pUNmZA2MUJDVlRwLW9ac1lrU05nZAmpzWHQtWHFJSEFRMF9tWVpRdEd70HQ5Wk8wSnY0R0VEQnVQdUU0MnpxNWxocUYyNmZAXSTUtVVNrMmh1ZAHZAQQno2ZA0VHR0lLWkk1N1R5RDFvM0dmVEpIYQZGZX", + "line": "AIzaGiReoG-CrackleCrackle12315618_12315", "line_num": 1, - "path": "tests/samples/instagram_access_token", + "path": "tests/samples/google_api_key.toml", "info": "", - "value": "IGQVJWS3pUNmZA2MUJDVlRwLW9ac1lrU05nZAmpzWHQtWHFJSEFRMF9tWVpRdEd70HQ5Wk8wSnY0R0VEQnVQdUU0MnpxNWxocUYyNmZAXSTUtVVNrMmh1ZAHZAQQno2ZA0VHR0lLWkk1N1R5RDFvM0dmVEpIYQZGZX", + "value": "AIzaGiReoG-CrackleCrackle12315618_12315", "value_start": 0, - "value_end": 162, + "value_end": 39, "variable": null, "variable_start": -2, "variable_end": -2, "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 5.231644837540696, + "iterator": "BASE36_CHARS", + "entropy": 3.165196181720608, "valid": true } } @@ -8279,51 +6463,41 @@ "api_validation": "NOT_AVAILABLE", "ml_validation": "NOT_AVAILABLE", "ml_probability": null, - "rule": "IPv4", - "severity": "info", - "confidence": "weak", + "rule": "Google Multi", + "severity": "high", + "confidence": "moderate", "line_data_list": [ { - "line": "100.64.0.0/10", - "line_num": 13, - "path": "tests/samples/ipv4", + "line": "194206074328-qp89pdv6fi35vsi71258g1eh31q6h7c3.apps.googleusercontent.com", + "line_num": 2, + "path": "tests/samples/google_multi", "info": "", - "value": "100.64.0.0", + "value": "194206074328-qp89pdv6fi35vsi71258g1eh31q6h7c3.apps.googleusercontent.com", "value_start": 0, - "value_end": 10, + "value_end": 72, "variable": null, "variable_start": -2, "variable_end": -2, "entropy_validation": { "iterator": "BASE64_CHARS", - "entropy": 1.5253496664211537, - "valid": false + "entropy": 4.531537327540733, + "valid": true } - } - ] - }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "IPv4", - "severity": "info", - "confidence": "weak", - "line_data_list": [ + }, { - "line": "100.64.0.0\u2013100.127.255.255", - "line_num": 14, - "path": "tests/samples/ipv4", + "line": "4L2QMyTm6Rr0o46ytGiReoG1", + "line_num": 4, + "path": "tests/samples/google_multi", "info": "", - "value": "100.127.255.255", - "value_start": 11, - "value_end": 26, + "value": "4L2QMyTm6Rr0o46ytGiReoG1", + "value_start": 0, + "value_end": 24, "variable": null, "variable_start": -2, "variable_end": -2, "entropy_validation": { - "iterator": "BASE36_CHARS", - "entropy": 2.008519976342584, + "iterator": "BASE64_CHARS", + "entropy": 4.084962500721157, "valid": false } } @@ -8333,51 +6507,41 @@ "api_validation": "NOT_AVAILABLE", "ml_validation": "NOT_AVAILABLE", "ml_probability": null, - "rule": "IPv4", - "severity": "info", - "confidence": "weak", + "rule": "Google Multi", + "severity": "high", + "confidence": "moderate", "line_data_list": [ { - "line": "100.64.0.0\u2013100.127.255.255", - "line_num": 14, - "path": "tests/samples/ipv4", + "line": "\"id\":\"194206074328-qdv6fi3eh31q6h7c35vsi7p89p1258g1.apps.googleusercontent.com\",\"CEKPET\":\"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\",", + "line_num": 18, + "path": "tests/samples/google_multi", "info": "", - "value": "100.64.0.0", - "value_start": 0, - "value_end": 10, + "value": "194206074328-qdv6fi3eh31q6h7c35vsi7p89p1258g1.apps.googleusercontent.com", + "value_start": 6, + "value_end": 78, "variable": null, "variable_start": -2, "variable_end": -2, "entropy_validation": { "iterator": "BASE64_CHARS", - "entropy": 1.5253496664211537, - "valid": false + "entropy": 4.531537327540733, + "valid": true } - } - ] - }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "IPv4", - "severity": "info", - "confidence": "weak", - "line_data_list": [ + }, { - "line": "192.0.0.0\u2013192.0.0.255", - "line_num": 22, - "path": "tests/samples/ipv4", + "line": "\"id\":\"194206074328-qdv6fi3eh31q6h7c35vsi7p89p1258g1.apps.googleusercontent.com\",\"CEKPET\":\"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\",", + "line_num": 18, + "path": "tests/samples/google_multi", "info": "", - "value": "192.0.0.255", - "value_start": 10, - "value_end": 21, + "value": "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", + "value_start": 90, + "value_end": 125, "variable": null, "variable_start": -2, "variable_end": -2, "entropy_validation": { "iterator": "BASE64_CHARS", - "entropy": 1.9704957226453073, + "entropy": 4.436181130262395, "valid": false } } @@ -8387,24 +6551,24 @@ "api_validation": "NOT_AVAILABLE", "ml_validation": "NOT_AVAILABLE", "ml_probability": null, - "rule": "IPv4", - "severity": "info", - "confidence": "weak", + "rule": "Google OAuth Secret", + "severity": "high", + "confidence": "strong", "line_data_list": [ { - "line": "192.88.99.0/24", - "line_num": 25, - "path": "tests/samples/ipv4", + "line": "\"id\":\"194206074328-qdv6fi3eh31q6h7c35vsi7p89p1258g1.apps.googleusercontent.com\",\"CEKPET\":\"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\",", + "line_num": 18, + "path": "tests/samples/google_multi", "info": "", - "value": "192.88.99.0", - "value_start": 0, - "value_end": 11, + "value": "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", + "value_start": 90, + "value_end": 125, "variable": null, "variable_start": -2, "variable_end": -2, "entropy_validation": { - "iterator": "BASE36_CHARS", - "entropy": 1.9018695860849921, + "iterator": "BASE64_CHARS", + "entropy": 4.436181130262395, "valid": false } } @@ -8414,25 +6578,25 @@ "api_validation": "NOT_AVAILABLE", "ml_validation": "NOT_AVAILABLE", "ml_probability": null, - "rule": "IPv4", - "severity": "info", - "confidence": "weak", + "rule": "Google OAuth Access Token", + "severity": "high", + "confidence": "moderate", "line_data_list": [ { - "line": "192.88.99.0\u2013192.88.99.255", - "line_num": 26, - "path": "tests/samples/ipv4", + "line": "google_oauth_key = \"ya29.gi_reo_gi_crackle_ln22\"", + "line_num": 1, + "path": "tests/samples/google_oauth_key", "info": "", - "value": "192.88.99.0", - "value_start": 0, - "value_end": 11, + "value": "ya29.gi_reo_gi_crackle_ln22", + "value_start": 20, + "value_end": 47, "variable": null, "variable_start": -2, "variable_end": -2, "entropy_validation": { "iterator": "BASE36_CHARS", - "entropy": 1.9018695860849921, - "valid": false + "entropy": 3.1797273164975133, + "valid": true } } ] @@ -8441,25 +6605,25 @@ "api_validation": "NOT_AVAILABLE", "ml_validation": "NOT_AVAILABLE", "ml_probability": null, - "rule": "IPv4", - "severity": "info", - "confidence": "weak", + "rule": "Grafana Access Policy Token", + "severity": "high", + "confidence": "strong", "line_data_list": [ { - "line": "192.88.99.0\u2013192.88.99.255", - "line_num": 26, - "path": "tests/samples/ipv4", + "line": "grafana_policy = 'glc_eyJvIjoiMjA0NjMwIiwibiI6InRlc3QtdG9rZW4iLCJrIjoidklnbjJ2WHc5MTVXOWtNOWxsNHcyZHEyIiwibSI6eyJyIjoicHJvZC0wIn19'", + "line_num": 1, + "path": "tests/samples/grafana_access_policy_token", "info": "", - "value": "192.88.99.255", - "value_start": 12, - "value_end": 25, + "value": "glc_eyJvIjoiMjA0NjMwIiwibiI6InRlc3QtdG9rZW4iLCJrIjoidklnbjJ2WHc5MTVXOWtNOWxsNHcyZHEyIiwibSI6eyJyIjoicHJvZC0wIn19", + "value_start": 18, + "value_end": 130, "variable": null, "variable_start": -2, "variable_end": -2, "entropy_validation": { "iterator": "BASE64_CHARS", - "entropy": 2.019193052249804, - "valid": false + "entropy": 5.097632476604024, + "valid": true } } ] @@ -8468,24 +6632,24 @@ "api_validation": "NOT_AVAILABLE", "ml_validation": "NOT_AVAILABLE", "ml_probability": null, - "rule": "IPv6", - "severity": "info", + "rule": "Grafana Provisioned API Key", + "severity": "high", "confidence": "strong", "line_data_list": [ { - "line": "2004:5678::9324", - "line_num": 12, - "path": "tests/samples/ipv6", + "line": "grafana = 'eyJrIjoiMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMCIsIm4iOiJ4eHh4IiwiaWQiOjIwNDM2MH0='", + "line_num": 1, + "path": "tests/samples/grafana_provisioned_api_key", "info": "", - "value": "2004:5678::9324", - "value_start": 0, - "value_end": 15, + "value": "eyJrIjoiMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMCIsIm4iOiJ4eHh4IiwiaWQiOjIwNDM2MH0=", + "value_start": 11, + "value_end": 107, "variable": null, "variable_start": -2, "variable_end": -2, "entropy_validation": { - "iterator": "BASE36_CHARS", - "entropy": 2.725512476486815, + "iterator": "BASE64_CHARS", + "entropy": 3.8153130511409934, "valid": false } } @@ -8495,25 +6659,25 @@ "api_validation": "NOT_AVAILABLE", "ml_validation": "NOT_AVAILABLE", "ml_probability": null, - "rule": "IPv6", - "severity": "info", + "rule": "Grafana Service Account Token", + "severity": "high", "confidence": "strong", "line_data_list": [ { - "line": "2004::5678:9", - "line_num": 13, - "path": "tests/samples/ipv6", + "line": "glsa_ThisI5NtTheTok3nYou8reLo0k1ngF0r_0a2a3df7", + "line_num": 1, + "path": "tests/samples/grafana_service_accounts", "info": "", - "value": "2004::5678:9", + "value": "glsa_ThisI5NtTheTok3nYou8reLo0k1ngF0r_0a2a3df7", "value_start": 0, - "value_end": 12, + "value_end": 46, "variable": null, "variable_start": -2, "variable_end": -2, "entropy_validation": { - "iterator": "BASE36_CHARS", - "entropy": 2.5220552088742005, - "valid": false + "iterator": "BASE64_CHARS", + "entropy": 4.52211252299684, + "valid": true } } ] @@ -8522,25 +6686,25 @@ "api_validation": "NOT_AVAILABLE", "ml_validation": "NOT_AVAILABLE", "ml_probability": null, - "rule": "IPv6", - "severity": "info", + "rule": "Hashicorp Terraform Token", + "severity": "high", "confidence": "strong", "line_data_list": [ { - "line": "2041:0000:140F::875B:131B", - "line_num": 14, - "path": "tests/samples/ipv6", + "line": "Z28P3STmkBQi1Y.atlasv1.YE7RBqu6VVyQIOq9a1eC3YFU5Elt7ToIr6OwzKAWlCTQ7N4gElXaWou6aPpOIwGCoc0", + "line_num": 1, + "path": "tests/samples/hashicorp_terraform", "info": "", - "value": "2041:0000:140F::875B:131B", + "value": "Z28P3STmkBQi1Y.atlasv1.YE7RBqu6VVyQIOq9a1eC3YFU5Elt7ToIr6OwzKAWlCTQ7N4gElXaWou6aPpOIwGCoc0", "value_start": 0, - "value_end": 25, + "value_end": 90, "variable": null, "variable_start": -2, "variable_end": -2, "entropy_validation": { - "iterator": "HEX_CHARS", - "entropy": 2.6146939516467023, - "valid": false + "iterator": "BASE64_CHARS", + "entropy": 5.348551883097512, + "valid": true } } ] @@ -8549,25 +6713,25 @@ "api_validation": "NOT_AVAILABLE", "ml_validation": "NOT_AVAILABLE", "ml_probability": null, - "rule": "IPv6", - "severity": "info", - "confidence": "strong", + "rule": "Heroku API Key", + "severity": "high", + "confidence": "moderate", "line_data_list": [ { - "line": "LONG LINE TEST: 2041:0:140F::875B:131B 2041:0:140F::875B:131B ;", - "line_num": 15, - "path": "tests/samples/ipv6", + "line": "HerOkUa04b8c1d-A147-b252-3b6a8f9c2b16", + "line_num": 1, + "path": "tests/samples/heroku_api.toml", "info": "", - "value": "2041:0:140F::875B:131B", - "value_start": 7010, - "value_end": 7032, + "value": "HerOkUa04b8c1d-A147-b252-3b6a8f9c2b16", + "value_start": 0, + "value_end": 37, "variable": null, "variable_start": -2, "variable_end": -2, "entropy_validation": { - "iterator": "HEX_CHARS", - "entropy": 2.684338637030481, - "valid": false + "iterator": "BASE36_CHARS", + "entropy": 3.48037367471734, + "valid": true } } ] @@ -8576,25 +6740,25 @@ "api_validation": "NOT_AVAILABLE", "ml_validation": "NOT_AVAILABLE", "ml_probability": null, - "rule": "IPv6", - "severity": "info", + "rule": "Instagram Access Token", + "severity": "high", "confidence": "strong", "line_data_list": [ { - "line": "LONG LINE TEST: 2041:0:140F::875B:131B 2041:0:140F::875B:131B ;", - "line_num": 15, - "path": "tests/samples/ipv6", + "line": "IGQVJWS3pUNmZA2MUJDVlRwLW9ac1lrU05nZAmpzWHQtWHFJSEFRMF9tWVpRdEd70HQ5Wk8wSnY0R0VEQnVQdUU0MnpxNWxocUYyNmZAXSTUtVVNrMmh1ZAHZAQQno2ZA0VHR0lLWkk1N1R5RDFvM0dmVEpIYQZGZX", + "line_num": 1, + "path": "tests/samples/instagram_access_token", "info": "", - "value": "2041:0:140F::875B:131B", - "value_start": 7989, - "value_end": 8011, + "value": "IGQVJWS3pUNmZA2MUJDVlRwLW9ac1lrU05nZAmpzWHQtWHFJSEFRMF9tWVpRdEd70HQ5Wk8wSnY0R0VEQnVQdUU0MnpxNWxocUYyNmZAXSTUtVVNrMmh1ZAHZAQQno2ZA0VHR0lLWkk1N1R5RDFvM0dmVEpIYQZGZX", + "value_start": 0, + "value_end": 162, "variable": null, "variable_start": -2, "variable_end": -2, "entropy_validation": { - "iterator": "HEX_CHARS", - "entropy": 2.684338637030481, - "valid": false + "iterator": "BASE64_CHARS", + "entropy": 5.231644837540696, + "valid": true } } ] @@ -8736,26 +6900,26 @@ }, { "api_validation": "NOT_AVAILABLE", - "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.931, + "ml_validation": "NOT_AVAILABLE", + "ml_probability": null, "rule": "JSON Web Token", "severity": "medium", - "confidence": "moderate", + "confidence": "strong", "line_data_list": [ { - "line": "$payload = 'eyJgsIZgeJhvNgFpSmlP.eyJcaaF9xCe7shE0ENPiBlEJOpS'", + "line": "detected: eyJhbGciOiJSUzI1NiJ9Cg.eyJleHAiOjY1NTM2fQo.Ce7sh0ENPiBlE_dose0cBA", "line_num": 1, - "path": "tests/samples/json_web_token.hs", + "path": "tests/samples/json_web_token", "info": "", - "value": "eyJgsIZgeJhvNgFpSmlP.eyJcaaF9xCe7shE0ENPiBlEJOpS", - "value_start": 12, - "value_end": 60, + "value": "eyJhbGciOiJSUzI1NiJ9Cg.eyJleHAiOjY1NTM2fQo.Ce7sh0ENPiBlE_dose0cBA", + "value_start": 10, + "value_end": 75, "variable": null, "variable_start": -2, "variable_end": -2, "entropy_validation": { "iterator": "BASE64_CHARS", - "entropy": 4.520488802699322, + "entropy": 4.790963630103494, "valid": true } } @@ -8845,17 +7009,17 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.986, + "ml_probability": 0.999, "rule": "Secret", "severity": "medium", "confidence": "moderate", "line_data_list": [ { - "line": "secret_looks_like_linux_path__=\"VnpmUGWxhQW/9KQAwrL2ZYd/DJPNG1PQjEXAMPLE\"", + "line": "secret_looks_like_linux_path__=\"VnpmUGWxhQW/9KQAwrL2ZYd/DJPNG1PQjEXAMbLE\"", "line_num": 5, "path": "tests/samples/key.hs", "info": "", - "value": "VnpmUGWxhQW/9KQAwrL2ZYd/DJPNG1PQjEXAMPLE", + "value": "VnpmUGWxhQW/9KQAwrL2ZYd/DJPNG1PQjEXAMbLE", "value_start": 32, "value_end": 72, "variable": "secret_looks_like_linux_path__", @@ -8863,7 +7027,7 @@ "variable_end": 30, "entropy_validation": { "iterator": "BASE64_CHARS", - "entropy": 4.784183719779189, + "entropy": 4.8530559073332755, "valid": true } } @@ -9981,6 +8145,33 @@ } ] }, + { + "api_validation": "NOT_AVAILABLE", + "ml_validation": "VALIDATED_KEY", + "ml_probability": 0.665, + "rule": "Salt", + "severity": "medium", + "confidence": "moderate", + "line_data_list": [ + { + "line": "json_escaped = \"{\\\\\\\"salt8\\\\\\\":\\\\\\\"4b9a6d8b638eb0c6\\\\\\\"}\"", + "line_num": 5, + "path": "tests/samples/salt.py", + "info": "", + "value": "4b9a6d8b638eb0c6", + "value_start": 35, + "value_end": 51, + "variable": "salt8", + "variable_start": 21, + "variable_end": 26, + "entropy_validation": { + "iterator": "BASE36_CHARS", + "entropy": 3.2806390622295662, + "valid": true + } + } + ] + }, { "api_validation": "NOT_AVAILABLE", "ml_validation": "NOT_AVAILABLE", @@ -10386,33 +8577,6 @@ } ] }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.999, - "rule": "JSON Web Token", - "severity": "medium", - "confidence": "moderate", - "line_data_list": [ - { - "line": "token in text: eyJrIjoiMDAwMDAwNDAwMDAwODAwNDAwMDAwMDAwNDAwMDAwMDAwMDAwMDAyMSIsIm4iOiJ4eHh4IiwiaWQiOjQzMDh9Cg", - "line_num": 18, - "path": "tests/samples/test2.eml", - "info": "", - "value": "eyJrIjoiMDAwMDAwNDAwMDAwODAwNDAwMDAwMDAwNDAwMDAwMDAwMDAwMDAyMSIsIm4iOiJ4eHh4IiwiaWQiOjQzMDh9Cg", - "value_start": 15, - "value_end": 109, - "variable": null, - "variable_start": -2, - "variable_end": -2, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 4.0296677144829305, - "valid": false - } - } - ] - }, { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", @@ -10710,6 +8874,60 @@ } ] }, + { + "api_validation": "NOT_AVAILABLE", + "ml_validation": "VALIDATED_KEY", + "ml_probability": 0.989, + "rule": "URL Credentials", + "severity": "high", + "confidence": "moderate", + "line_data_list": [ + { + "line": "email_as_login = \"smtps://example@gmail.com:FnD83JZs@smtp.gmail.com:465\";", + "line_num": 13, + "path": "tests/samples/url_cred.js", + "info": "", + "value": "FnD83JZs", + "value_start": 44, + "value_end": 52, + "variable": "smtps://", + "variable_start": 18, + "variable_end": 26, + "entropy_validation": { + "iterator": "BASE64_CHARS", + "entropy": 3.0, + "valid": false + } + } + ] + }, + { + "api_validation": "NOT_AVAILABLE", + "ml_validation": "NOT_AVAILABLE", + "ml_probability": null, + "rule": "UUID", + "severity": "info", + "confidence": "strong", + "line_data_list": [ + { + "line": "bace4d19-fa7e-beef-cafe-9129474bcd81 # tp", + "line_num": 1, + "path": "tests/samples/uuid", + "info": "", + "value": "bace4d19-fa7e-beef-cafe-9129474bcd81", + "value_start": 0, + "value_end": 36, + "variable": null, + "variable_start": -2, + "variable_end": -2, + "entropy_validation": { + "iterator": "BASE36_CHARS", + "entropy": 3.2373263071270246, + "valid": true + } + } + ] + }, { "api_validation": "NOT_AVAILABLE", "ml_validation": "NOT_AVAILABLE", diff --git a/tests/filters/test_line_specific_key_check.py b/tests/filters/test_line_specific_key_check.py index ca123d839..c55a3cb3f 100644 --- a/tests/filters/test_line_specific_key_check.py +++ b/tests/filters/test_line_specific_key_check.py @@ -18,9 +18,11 @@ def test_line_specific_key_check_p(self, file_path: pytest.fixture, line: str) - @pytest.mark.parametrize("line", [ '"AwsAccessKey": enc("AKIAGIREOGIAWSKEY123"),', - '"AwsAccessKey": "AKIAGIREXAMPLEKEY123"', + '"AwsAccessKey as example": "AKIAGIREXAMPLEKEY123"', ]) def test_line_specific_key_check_n(self, file_path: pytest.fixture, line: str) -> None: cred_candidate = get_line_data(file_path, line=line, pattern=LINE_VALUE_PATTERN) + # LINE_VALUE_PATTERN does not detect a value position + cred_candidate.value_start = line.find("AKIA") target = AnalysisTarget(line_pos=0, lines=[line], line_nums=[1], descriptor=DUMMY_DESCRIPTOR) assert LineSpecificKeyCheck().run(cred_candidate, target) is True diff --git a/tests/filters/test_value_azure_token_check.py b/tests/filters/test_value_azure_token_check.py new file mode 100644 index 000000000..fab078169 --- /dev/null +++ b/tests/filters/test_value_azure_token_check.py @@ -0,0 +1,26 @@ +import unittest + +from credsweeper.filters import ValueAzureTokenCheck +from tests.filters.conftest import LINE_VALUE_PATTERN, DUMMY_ANALYSIS_TARGET +from tests.test_utils.dummy_line_data import get_line_data + + +class TestValueAzureTokenCheck(unittest.TestCase): + + def test_value_AzureToken_check_p(self): + self.assertTrue(ValueAzureTokenCheck().run(get_line_data(line=""), DUMMY_ANALYSIS_TARGET)) + self.assertTrue(ValueAzureTokenCheck().run(get_line_data(line="eyJungle", pattern=LINE_VALUE_PATTERN), + DUMMY_ANALYSIS_TARGET)) + self.assertTrue(ValueAzureTokenCheck().run( + get_line_data(line="eyJhbGciOjEsInR5cCI6Miwia2lkIjozfQo", pattern=LINE_VALUE_PATTERN), + DUMMY_ANALYSIS_TARGET)) + self.assertTrue(ValueAzureTokenCheck().run( + get_line_data(line="eyJhbGciOjEsInR5cCI6Miwia2lkIjozfQo.eyJhbGciOjEsInR5cCI6Miwia2lkIjozfQo" + ".eyJhbGciOjEsInR5cCI6Miwia2lkIjozfQo", + pattern=LINE_VALUE_PATTERN), DUMMY_ANALYSIS_TARGET)) + + def test_value_AzureToken_check_n(self): + self.assertFalse(ValueAzureTokenCheck().run( + get_line_data(line="eyJhbGciOjEsInR5cCI6Miwia2lkIjozfQo.eyJpc3MiOjEsImV4cCI6MiwiaWF0IjozfQo" + ".1234567890qwertyuiopasdfghjklzxc", + pattern=LINE_VALUE_PATTERN), DUMMY_ANALYSIS_TARGET)) diff --git a/tests/filters/test_value_base64_part_check.py b/tests/filters/test_value_base64_part_check.py new file mode 100644 index 000000000..790ca9008 --- /dev/null +++ b/tests/filters/test_value_base64_part_check.py @@ -0,0 +1,33 @@ +import re +import unittest + +from credsweeper.credentials import LineData +from credsweeper.filters import ValueBase64PartCheck +from tests.filters.conftest import DUMMY_ANALYSIS_TARGET + + +class TestValueBase64PartCheck(unittest.TestCase): + EAA_PATTERN = re.compile(r"(?P\bEAA[0-9A-Za-z]{32})") + + def test_value_check_n(self) -> None: + line_data = LineData(config=None, + path="dummy", + file_type="", + line="qcE81rS+FJHuvg39lz4T/EAACEb00Kse0BAlGy7KeQ5YnaCEd09Eo" + "se0cBAlGy7KeQ5Yna9CoDsup39tiYdoQ4jH9Coup39tiYdWoQ4jHFZD", + info="", + line_num=1, + line_pos=0, + pattern=TestValueBase64PartCheck.EAA_PATTERN) + self.assertTrue(ValueBase64PartCheck().run(line_data, DUMMY_ANALYSIS_TARGET)) + + def test_value_check_p(self) -> None: + line_data = LineData(config=None, + path="dummy", + file_type="", + line="http://meta.test/api/EAACRvAWiwzR8rcXFsLiUH13ybj0tdEa?x=login", + info="", + line_num=1, + line_pos=0, + pattern=TestValueBase64PartCheck.EAA_PATTERN) + self.assertFalse(ValueBase64PartCheck().run(line_data, DUMMY_ANALYSIS_TARGET)) diff --git a/tests/filters/test_value_json_web_token_check.py b/tests/filters/test_value_json_web_token_check.py index fc6f1d2d2..4cb701956 100644 --- a/tests/filters/test_value_json_web_token_check.py +++ b/tests/filters/test_value_json_web_token_check.py @@ -1,28 +1,43 @@ -import base64 - -import pytest +import unittest from credsweeper.filters import ValueJsonWebTokenCheck from tests.filters.conftest import LINE_VALUE_PATTERN, DUMMY_ANALYSIS_TARGET from tests.test_utils.dummy_line_data import get_line_data -class TestValueJsonWebTokenCheck: +class TestValueJsonWebTokenCheck(unittest.TestCase): - @pytest.mark.parametrize("line", ["12345:asbdsa:28yd"]) - def test_value_jwt_check_p(self, file_path: pytest.fixture, line: str) -> None: - encoded_line = base64.b64encode(line.encode('ascii')).decode('ascii') - jwt_like_line = base64.b64encode('{"typ":"JWT", "dummy": false}'.encode('ascii')).decode('ascii') - jwt_line_data = get_line_data(file_path, line=f"{jwt_like_line}.{encoded_line}", pattern=LINE_VALUE_PATTERN) - assert ValueJsonWebTokenCheck().run(jwt_line_data, DUMMY_ANALYSIS_TARGET) is False - # partially line - jwt_line_data = get_line_data(file_path, line=f"{jwt_like_line}.AnyTailOfString", pattern=LINE_VALUE_PATTERN) - assert ValueJsonWebTokenCheck().run(jwt_line_data, DUMMY_ANALYSIS_TARGET) is False + def test_value_jwt_check_p(self): + self.assertTrue(ValueJsonWebTokenCheck().run(get_line_data(line="", pattern=LINE_VALUE_PATTERN), + DUMMY_ANALYSIS_TARGET)) + self.assertTrue(ValueJsonWebTokenCheck().run(get_line_data(line="eyJungle", pattern=LINE_VALUE_PATTERN), + DUMMY_ANALYSIS_TARGET)) + self.assertTrue(ValueJsonWebTokenCheck().run( + get_line_data(line="1234567890qwertyuiopasdfghjklzxc", pattern=LINE_VALUE_PATTERN), DUMMY_ANALYSIS_TARGET)) + self.assertTrue(ValueJsonWebTokenCheck().run( + get_line_data(line="eyJhbGciOiJSUzI1NiJ9Cg.eyJleHAiOjY1NTM2fQo.eyJleHAiOjY1NTM2fQo", + pattern=LINE_VALUE_PATTERN), DUMMY_ANALYSIS_TARGET)) + self.assertTrue(ValueJsonWebTokenCheck().run( + get_line_data(line="eyJhbGciOiJSUzI1NiJ9Cg.eyJleHAiOjY1NTM2fQo.AAAAAAAAAAAAAAAAAAAAAAA", + pattern=LINE_VALUE_PATTERN), DUMMY_ANALYSIS_TARGET)) + self.assertTrue(ValueJsonWebTokenCheck().run( + get_line_data(line="eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.x3.GFsFyGiCUIP5VHI9CEJL9thWsGjSZf1fJfarNk-LGTM", + pattern=LINE_VALUE_PATTERN), DUMMY_ANALYSIS_TARGET)) - @pytest.mark.parametrize("line", ["1234f:asbdsa:28yd"]) - def test_value_jwt_check_n(self, file_path: pytest.fixture, line: str) -> None: - encoded_line = base64.b64encode(line.encode('ascii')).decode('ascii') - jwt_line_data = get_line_data(file_path, line=f"eyJungle.{encoded_line}", pattern=LINE_VALUE_PATTERN) - assert ValueJsonWebTokenCheck().run(jwt_line_data, DUMMY_ANALYSIS_TARGET) is True - jwt_line_data = get_line_data(file_path, line="eyJungle", pattern=LINE_VALUE_PATTERN) - assert ValueJsonWebTokenCheck().run(jwt_line_data, DUMMY_ANALYSIS_TARGET) is True + def test_value_jwt_check_n(self): + self.assertFalse(ValueJsonWebTokenCheck().run( + get_line_data(line="eyJhbGciOiJQUzM4NCJ9.eyJkdW1teSI6bnVsbH0.eyJpc3MiOiJqb2UifQ." \ + "_VP9ZxcPkOptWScOUMXriLH31bTcrg0YhlYL-A7TTHLX7LTDKjggtVq3Nmdl4GIS" \ + "gJdM7GHHZOJHckUjgD-T3X6oHQanKqjpWjU-GxcnOkM86e0joZgJUL7CpHUt7e3W" \ + "MhbUrOCyCFRrxOXtuTvGr2m_LdS7I5OyZ4xEP4JRcsOgOnGq-MEWmLqrRvc4zy5m" \ + "pM6tJwJXI8fr1tF4pcAZxXR17ITCrocVSRC6NuWOVzh_XyyEVRUfqlDbJnU2Z_I0" \ + "dfEQIcC6K5hAgQGSZQC_pQDA51RUoUHa9KfNskerI681fJ8mbjIlbf68CFdXZnjE" \ + "zobUhMn5Z544PF9DjW1BVtsQgXtHlSDFxl6MIMVdvM8oLRbrjlf6BYCRnCxuTA_y" \ + "Ui1o9ndy7ckISHQVhuYFKu78l7nqC4heghK_Gw4h7EB7s8eEuUC-D6JjVtX10IyS" \ + "vCRkRo7f8dWQTjFLs7mlPowjRz0cP5J-MmCoegKHYagOHZ_ArXOR91_u8jMdwmOf", + pattern=LINE_VALUE_PATTERN), DUMMY_ANALYSIS_TARGET)) + self.assertFalse(ValueJsonWebTokenCheck().run( + get_line_data(line="eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9." \ + "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9." \ + ".e30.GFsFyGiCUIP5VHI9CEJL9thWsGjSZf1fJfarNk-LGTM", + pattern=LINE_VALUE_PATTERN), DUMMY_ANALYSIS_TARGET)) diff --git a/tests/filters/test_value_length_check.py b/tests/filters/test_value_length_check.py deleted file mode 100644 index 69bd5f809..000000000 --- a/tests/filters/test_value_length_check.py +++ /dev/null @@ -1,19 +0,0 @@ -import pytest - -from credsweeper.config import Config -from credsweeper.filters import ValueLengthCheck -from tests.filters.conftest import LINE_VALUE_PATTERN, DUMMY_ANALYSIS_TARGET -from tests.test_utils.dummy_line_data import get_line_data - - -class TestValueLengthCheck: - - def test_value_length_check_p(self, file_path: pytest.fixture, config: Config, - success_line: pytest.fixture) -> None: - line_data = get_line_data(file_path, line=success_line, pattern=LINE_VALUE_PATTERN) - assert ValueLengthCheck(config).run(line_data, DUMMY_ANALYSIS_TARGET) is False - - @pytest.mark.parametrize("line", ["Cra"]) - def test_value_length_check_n(self, file_path: pytest.fixture, config: Config, line: str) -> None: - line_data = get_line_data(file_path, line=line, pattern=LINE_VALUE_PATTERN) - assert ValueLengthCheck(config).run(line_data, DUMMY_ANALYSIS_TARGET) is True diff --git a/tests/filters/test_value_useless_word_check.py b/tests/filters/test_value_useless_word_check.py index 2cbdcf192..911f3781f 100644 --- a/tests/filters/test_value_useless_word_check.py +++ b/tests/filters/test_value_useless_word_check.py @@ -11,7 +11,7 @@ def test_value_useless_word_check_p(self, file_path: pytest.fixture, success_lin line_data = get_line_data(file_path=file_path, line=success_line, pattern=LINE_VALUE_PATTERN) assert ValueUselessWordCheck().run(line_data, DUMMY_ANALYSIS_TARGET) is False - @pytest.mark.parametrize("line", ["{0x943058439}", "0x%", "->gi_reo_gi", "xxxxxGIREOGI", " GIREOGI"]) + @pytest.mark.parametrize("line", ["{0x943058439}", "0x%", "->gi_reo_gi", "GIREOGIEXAMPLE"]) def test_value_useless_word_check_n(self, file_path: pytest.fixture, line: str) -> None: line_data = get_line_data(file_path=file_path, line=line, pattern=LINE_VALUE_PATTERN) assert ValueUselessWordCheck().run(line_data, DUMMY_ANALYSIS_TARGET) is True diff --git a/tests/ml_model/test_ml_validator.py b/tests/ml_model/test_ml_validator.py index 9322eae63..3e61d2e1a 100644 --- a/tests/ml_model/test_ml_validator.py +++ b/tests/ml_model/test_ml_validator.py @@ -10,7 +10,7 @@ from credsweeper.credentials import Candidate, CandidateKey from credsweeper.ml_model import MlValidator from credsweeper.utils import Util -from tests import AZ_STRING, NEGLIGIBLE_ML_THRESHOLD +from tests import NEGLIGIBLE_ML_THRESHOLD class TestMlValidator(unittest.TestCase): @@ -48,22 +48,48 @@ def validate(_candidate: Candidate) -> Tuple[bool, float]: candidate.line_data_list[0].value = "Ahga%$FiQ@Ei8" decision, probability = validate(candidate) - self.assertAlmostEqual(probability, 0.9997520446777344, delta=NEGLIGIBLE_ML_THRESHOLD) + self.assertAlmostEqual(0.9997520446777344, probability, delta=NEGLIGIBLE_ML_THRESHOLD) candidate.line_data_list[0].path = "sample.py" candidate.line_data_list[0].file_type = ".yaml" decision, probability = validate(candidate) - self.assertAlmostEqual(probability, 0.9994515776634216, delta=NEGLIGIBLE_ML_THRESHOLD) + self.assertAlmostEqual(0.9994515776634216, probability, delta=NEGLIGIBLE_ML_THRESHOLD) candidate.line_data_list[0].path = "test.zip" candidate.line_data_list[0].file_type = ".zip" decision, probability = validate(candidate) - self.assertAlmostEqual(probability, 0.9994281530380249, delta=NEGLIGIBLE_ML_THRESHOLD) + self.assertAlmostEqual(0.9994281530380249, probability, delta=NEGLIGIBLE_ML_THRESHOLD) candidate.line_data_list[0].path = "other.txt" candidate.line_data_list[0].file_type = ".txt" decision, probability = validate(candidate) - self.assertAlmostEqual(probability, 0.9980608820915222, delta=NEGLIGIBLE_ML_THRESHOLD) + self.assertAlmostEqual(0.9980608820915222, probability, delta=NEGLIGIBLE_ML_THRESHOLD) + + def test_ml_validator_auxiliary_p(self): + candidate = Candidate.get_dummy_candidate(self.config, "secret", "", "") + candidate.rule_name = "Secret" + candidate.line_data_list[0].line = "secret=bace4d19-dead-beef-cafe-9129474bcd81" + candidate.line_data_list[0].variable = "secret" + candidate.line_data_list[0].value_start = 7 + candidate.line_data_list[0].value_end = 43 + candidate.line_data_list[0].value = "bace4d19-dead-beef-cafe-9129474bcd81" + # auxiliary candidate for a pattern rule - without variable + aux_candidate = copy.deepcopy(candidate) + aux_candidate.line_data_list[0].variable = None + + # todo: the scores are low for current ML model - will be changed after train + + candidate_key = CandidateKey(candidate.line_data_list[0]) + sample_as_batch = [(candidate_key, [candidate])] + is_cred_batch, probability_batch = self.ml_validator.validate_groups(sample_as_batch, 2) + self.assertAlmostEqual(0.16333681344985962, probability_batch[0], delta=NEGLIGIBLE_ML_THRESHOLD) + + # auxiliary rule in train does not increase ML probability yet - will be used after next train + + aux_candidate.rule_name = "UUID" + sample_as_batch = [(candidate_key, [candidate, aux_candidate])] + is_cred_batch, probability_batch = self.ml_validator.validate_groups(sample_as_batch, 2) + self.assertAlmostEqual(0.16333681344985962, probability_batch[0], delta=NEGLIGIBLE_ML_THRESHOLD) def test_extract_features_p(self): candidate1 = Candidate.get_dummy_candidate(self.config, "main.py", ".py", "info") @@ -74,10 +100,10 @@ def test_extract_features_p(self): candidate1.line_data_list[0].value = "123" candidate1.rule_name = "Password" features1 = self.ml_validator.extract_features([candidate1]) - self.assertEqual(18, np.count_nonzero(features1)) + self.assertAlmostEqual(18, np.count_nonzero(features1), delta=NEGLIGIBLE_ML_THRESHOLD) candidate2 = copy.deepcopy(candidate1) features2 = self.ml_validator.extract_features([candidate1, candidate2]) - self.assertEqual(18, np.count_nonzero(features2)) + self.assertAlmostEqual(18, np.count_nonzero(features2), delta=NEGLIGIBLE_ML_THRESHOLD) candidate2.rule_name = "Secret" features3 = self.ml_validator.extract_features([candidate1, candidate2]) - self.assertEqual(19, np.count_nonzero(features3)) + self.assertAlmostEqual(19, np.count_nonzero(features3), delta=NEGLIGIBLE_ML_THRESHOLD) diff --git a/tests/rules/test_jwt.py b/tests/rules/test_jwt.py index 10ec210ac..8ac1c4ec1 100644 --- a/tests/rules/test_jwt.py +++ b/tests/rules/test_jwt.py @@ -8,11 +8,11 @@ class TestJwt(BaseTestRule): @pytest.fixture(params=[[ - "jwt: 'eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiIxMjM0NTY3ODkwIiwibmFtZSI6IkpvaG4gRG9lIiwiaWF0IjoxN", - "TE2MjM5MDIyLCJ0ZXN0IjoiSSBuZWVkIHJlYWxseSByZWFsbHkgbG9uZyBsb25nIGxvbmcgbG9uZyBsb25nIGxvbmcgbG9uZyBsb25nI", - "GxvbmcgbG9uZyBsb25nIGxvbmcgbG9uZyBsb25nIGxvbmcgbG9uZyBsb25nIGxvbmcgbG9uZyBsb25nIGxvbmcgbG9uZyBsb25nIGxvb", - "mcgbG9uZyBsb25nIGxvbmcgbG9uZyBsb25nIGxvbmcgbG9uZyBsb25nIGxvbmcgbG9uZyBsb25nIGxvbmcgbG9uZyBsb25nIGxvbmcgb", - "G9uZyBsb25nIGxvbmcgbG9uZyBsb25nIGxvbmcgbG9uZyBsb25nIGxvbmcgbG9uZyBsb25nIGxvbmcgbG9uZyBsb25nIGxvbmcgbG9uZ", + "jwt: 'eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiIxMjM0NTY3ODkwIiwibmFtZSI6IkpvaG4gRG9lIiwiaWF0IjoxN" + "TE2MjM5MDIyLCJ0ZXN0IjoiSSBuZWVkIHJlYWxseSByZWFsbHkgbG9uZyBsb25nIGxvbmcgbG9uZyBsb25nIGxvbmcgbG9uZyBsb25nI" + "GxvbmcgbG9uZyBsb25nIGxvbmcgbG9uZyBsb25nIGxvbmcgbG9uZyBsb25nIGxvbmcgbG9uZyBsb25nIGxvbmcgbG9uZyBsb25nIGxvb" + "mcgbG9uZyBsb25nIGxvbmcgbG9uZyBsb25nIGxvbmcgbG9uZyBsb25nIGxvbmcgbG9uZyBsb25nIGxvbmcgbG9uZyBsb25nIGxvbmcgb" + "G9uZyBsb25nIGxvbmcgbG9uZyBsb25nIGxvbmcgbG9uZyBsb25nIGxvbmcgbG9uZyBsb25nIGxvbmcgbG9uZyBsb25nIGxvbmcgbG9uZ" "yBsb25nIGxvbmcgbG9uZyBsb25nIGxvbmcgbG9uZyBqd3QgdG9rZW4ifQ.4pWgA4mthx4FPPh1AZQY0luTKTQ7VOj6PGwwiANvtqg'" ]]) def lines(self, request) -> List[str]: diff --git a/tests/samples/aws_client_id b/tests/samples/aws_client_id index 3685378f5..b5a83275c 100644 --- a/tests/samples/aws_client_id +++ b/tests/samples/aws_client_id @@ -1,2 +1,4 @@ The items are AKIAGIREOGIAWSKEY123,AKIAGIREOGIAWSKEY45X the coma is necessary there ^ bariers thesting !!! +must be filtered: AKIAGIREOGIAEXAMPLE7 +filtered too: AKIALGSBKLIKEAREAL12 --access-key diff --git a/tests/samples/azure_access_token b/tests/samples/azure_access_token index 4d5d0c454..d85be0700 100644 --- a/tests/samples/azure_access_token +++ b/tests/samples/azure_access_token @@ -1 +1,3 @@ -eyJ0eXAiOiJKV1QiLCJhbGciOiJSUzI1NiIsIng1dCI6Imk2bEdrM0ZaenhSY1ViMkMzbkVRN3N5SEpsWSIsImtpZCI6Imk2bEdrM0ZaenhSY1ViMkMzbkVRN3N5SEpsWSJ9.eyJhdWQiOiJlZjFkYTlkNC1mZjc3LTRjM2UtYTAwNS04NDBjM2Y4MzA3NDUiLCJpc3MiOiJodHRwczovL3N0cy53aW5kb3dzLm5ldC9mYTE1ZDY5Mi1lOWM3LTQ0NjAtYTc0My0yOWYyOTUyMjIyOS8iLCJpYXQiOjE1MzcyMzMxMDYsIm5iZiI6MTUzNzIzMzEwNiwiZXhwIjoxNTM3MjM3MDA2LCJhY3IiOiIxIiwiYWlvIjoiQVhRQWkvOElBQUFBRm0rRS9RVEcrZ0ZuVnhMaldkdzhLKzYxQUdyU091TU1GNmViYU1qN1hPM0libUQzZkdtck95RCtOdlp5R24yVmFUL2tES1h3NE1JaHJnR1ZxNkJuOHdMWG9UMUxrSVorRnpRVmtKUFBMUU9WNEtjWHFTbENWUERTL0RpQ0RnRTIyMlRJbU12V05hRU1hVU9Uc0lHdlRRPT0iLCJhbXIiOlsid2lhIl0sImFwcGlkIjoiNzVkYmU3N2YtMTBhMy00ZTU5LTg1ZmQtOGMxMjc1NDRmMTdjIiwiYXBwaWRhY3IiOiIwIiwiZW1haWwiOiJBYmVMaUBtaWNyb3NvZnQuY29tIiwiZmFtaWx5X25hbWUiOiJMaW5jb2xuIiwiZ2l2ZW5fbmFtZSI6IkFiZSAoTVNGVCkiLCJpZHAiOiJodHRwczovL3N0cy53aW5kb3dzLm5ldC83MmY5ODhiZi04NmYxLTQxYWYtOTFhYi0yZDdjZDAxMjIyNDcvIiwiaXBhZGRyIjoiMjIyLjIyMi4yMjIuMjIiLCJuYW1lIjoiYWJlbGkiLCJvaWQiOiIwMjIyM2I2Yi1hYTFkLTQyZDQtOWVjMC0xYjJiYjkxOTQ0MzgiLCJyaCI6IkkiLCJzY3AiOiJ1c2VyX2ltcGVyc29uYXRpb24iLCJzdWIiOiJsM19yb0lTUVUyMjJiVUxTOXlpMmswWHBxcE9pTXo1SDNaQUNvMUdlWEEiLCJ0aWQiOiJmYTE1ZDY5Mi1lOWM3LTQ0NjAtYTc0My0yOWYyOTU2ZmQ0MjkiLCJ1bmlxdWVfbmFtZSI6ImFiZWxpQG1pY3Jvc29mdC5jb20iLCJ1dGkiOiJGVnNHeFlYSTMwLVR1aWt1dVVvRkFBIiwidmVyIjoiMS4wIn0.D3H6pMUtQnoJAGq6AHd \ No newline at end of file +eyJ0eXAiOiJKV1QiLCJhbGciOiJSUzI1NiIsIng1dCI6Imk2bEdrM0ZaenhSY1ViMkMzbkVRN3N5SEpsWSIsImtpZCI6Imk2bEdrM0ZaenhSY1ViMkMzbkVRN3N5SEpsWSJ9.eyJhdWQiOiJlZjFkYTlkNC1mZjc3LTRjM2UtYTAwNS04NDBjM2Y4MzA3NDUiLCJpc3MiOiJodHRwczovL3N0cy53aW5kb3dzLm5ldC9mYTE1ZDY5Mi1lOWM3LTQ0NjAtYTc0My0yOWYyOTUyMjIyOS8iLCJpYXQiOjE1MzcyMzMxMDYsIm5iZiI6MTUzNzIzMzEwNiwiZXhwIjoxNTM3MjM3MDA2LCJhY3IiOiIxIiwiYWlvIjoiQVhRQWkvOElBQUFBRm0rRS9RVEcrZ0ZuVnhMaldkdzhLKzYxQUdyU091TU1GNmViYU1qN1hPM0libUQzZkdtck95RCtOdlp5R24yVmFUL2tES1h3NE1JaHJnR1ZxNkJuOHdMWG9UMUxrSVorRnpRVmtKUFBMUU9WNEtjWHFTbENWUERTL0RpQ0RnRTIyMlRJbU12V05hRU1hVU9Uc0lHdlRRPT0iLCJhbXIiOlsid2lhIl0sImFwcGlkIjoiNzVkYmU3N2YtMTBhMy00ZTU5LTg1ZmQtOGMxMjc1NDRmMTdjIiwiYXBwaWRhY3IiOiIwIiwiZW1haWwiOiJBYmVMaUBtaWNyb3NvZnQuY29tIiwiZmFtaWx5X25hbWUiOiJMaW5jb2xuIiwiZ2l2ZW5fbmFtZSI6IkFiZSAoTVNGVCkiLCJpZHAiOiJodHRwczovL3N0cy53aW5kb3dzLm5ldC83MmY5ODhiZi04NmYxLTQxYWYtOTFhYi0yZDdjZDAxMjIyNDcvIiwiaXBhZGRyIjoiMjIyLjIyMi4yMjIuMjIiLCJuYW1lIjoiYWJlbGkiLCJvaWQiOiIwMjIyM2I2Yi1hYTFkLTQyZDQtOWVjMC0xYjJiYjkxOTQ0MzgiLCJyaCI6IkkiLCJzY3AiOiJ1c2VyX2ltcGVyc29uYXRpb24iLCJzdWIiOiJsM19yb0lTUVUyMjJiVUxTOXlpMmswWHBxcE9pTXo1SDNaQUNvMUdlWEEiLCJ0aWQiOiJmYTE1ZDY5Mi1lOWM3LTQ0NjAtYTc0My0yOWYyOTU2ZmQ0MjkiLCJ1bmlxdWVfbmFtZSI6ImFiZWxpQG1pY3Jvc29mdC5jb20iLCJ1dGkiOiJGVnNHeFlYSTMwLVR1aWt1dVVvRkFBIiwidmVyIjoiMS4wIn0.D3H6pMUtQnoJAGq6AHd +eyJ0eXAiOiJKV1QiLCJhbGciOiJSUzI1NiIsImtpZCI6Imk2bEdrM0ZaenhSY1ViMkMzbkVRN3N5SEpsWSJ9.eyJhdWQiOiI2ZTc0MTcyYi1iZTU2LTQ4NDMtOWZmNC1lNjZhMzliYjEyZTMiLCJpc3MiOiJodHRwczovL2xvZ2luLm1pY3Jvc29mdG9ubGluZS5jb20vNzJmOTg4YmYtODZmMS00MWFmLTkxYWItMmQ3Y2QwMTFkYjQ3L3YyLjAiLCJpYXQiOjE1MzcyMzEwNDgsIm5iZiI6MTUzNzIzMTA0OCwiZXhwIjoxNTM3MjM0OTQ4LCJhaW8iOiJBWFFBaS84SUFBQUF0QWFaTG8zQ2hNaWY2S09udHRSQjdlQnE0L0RjY1F6amNKR3hQWXkvQzNqRGFOR3hYZDZ3TklJVkdSZ2hOUm53SjFsT2NBbk5aY2p2a295ckZ4Q3R0djMzMTQwUmlvT0ZKNGJDQ0dWdW9DYWcxdU9UVDIyMjIyZ0h3TFBZUS91Zjc5UVgrMEtJaWpkcm1wNjlSY3R6bVE9PSIsImF6cCI6IjZlNzQxNzJiLWJlNTYtNDg0My05ZmY0LWU2NmEzOWJiMTJlMyIsImF6cGFjciI6IjAiLCJuYW1lIjoiQWJlIExpbmNvbG4iLCJvaWQiOiI2OTAyMjJiZS1mZjFhLTRkNTYtYWJkMS03ZTRmN2QzOGU0NzQiLCJwcmVmZXJyZWRfdXNlcm5hbWUiOiJhYmVsaUBtaWNyb3NvZnQuY29tIiwicmgiOiJJIiwic2NwIjoiYWNjZXNzX2FzX3VzZXIiLCJzdWIiOiJIS1pwZmFIeVdhZGVPb3VZbGl0anJJLUtmZlRtMjIyWDVyclYzeERxZktRIiwidGlkIjoiNzJmOTg4YmYtODZmMS00MWFmLTkxYWItMmQ3Y2QwMTFkYjQ3IiwidXRpIjoiZnFpQnFYTFBqMGVRYTgyUy1JWUZBQSIsInZlciI6IjIuMCJ9.pj4N-w_3Us9DrBLfpCt +^^^ examples from https://learn.microsoft.com/en-us/entra/identity-platform/access-tokens \ No newline at end of file diff --git a/tests/samples/ipv4 b/tests/samples/ipv4 deleted file mode 100644 index d1da7b509..000000000 --- a/tests/samples/ipv4 +++ /dev/null @@ -1,42 +0,0 @@ -# wrong values: -299.199.99.0 -321.500.312.32 -99.199.299.0 -version 8.8.8.8 -long line check rfc 1.2.3.4 OVERSIZE; - -# list from https://en.wikipedia.org/wiki/Reserved_IP_addresses -0.0.0.0/8 -0.0.0.0–0.255.255.255 -10.0.0.0/8 -10.0.0.0–10.255.255.255 -100.64.0.0/10 -100.64.0.0–100.127.255.255 -127.0.0.0/8 -127.0.0.0–127.255.255.255 -169.254.0.0/16 -169.254.0.0–169.254.255.255 -172.16.0.0/12 -172.16.0.0–172.31.255.255 -192.0.0.0/24 -192.0.0.0–192.0.0.255 -192.0.2.0/24 -192.0.2.0–192.0.2.255 -192.88.99.0/24 -192.88.99.0–192.88.99.255 -192.168.0.0/16 -192.168.0.0–192.168.255.255 -198.18.0.0/15 -198.18.0.0–198.19.255.255 -198.51.100.0/24 -198.51.100.0–198.51.100.255 -203.0.113.0/24 -203.0.113.0–203.0.113.255 -224.0.0.0/4 -224.0.0.0–239.255.255.255 -233.252.0.0/24 -233.252.0.0-233.252.0.255 -240.0.0.0/4 -240.0.0.0–255.255.255.254 -255.255.255.255/32 -255.255.255.255 diff --git a/tests/samples/ipv6 b/tests/samples/ipv6 deleted file mode 100644 index e807faa9e..000000000 --- a/tests/samples/ipv6 +++ /dev/null @@ -1,15 +0,0 @@ -# not an ipv6 -abba:03911 -1234::5678::9 - -# loopback -0000:0000:0000:0000:0000:0000:0000:0001 -::1 # fill -2001:db8:85a3:8d3:1319:8a2e:370:7348 private -fe80::1ff:fe23:4567:890a # link_local - -# dummy but valid -2004:5678::9324 -2004::5678:9 -2041:0000:140F::875B:131B -LONG LINE TEST: 2041:0:140F::875B:131B 2041:0:140F::875B:131B ; diff --git a/tests/samples/json_web_token b/tests/samples/json_web_token new file mode 100644 index 000000000..45e244772 --- /dev/null +++ b/tests/samples/json_web_token @@ -0,0 +1,2 @@ +detected: eyJhbGciOiJSUzI1NiJ9Cg.eyJleHAiOjY1NTM2fQo.Ce7sh0ENPiBlE_dose0cBA +not detected: eyJhbGciOiJSUzI1NiIsInR5cCI6IkpXVCJ9.NiIsInR5cCI6IkpXV.NiIsInR5cCI6IkpXV diff --git a/tests/samples/json_web_token.hs b/tests/samples/json_web_token.hs deleted file mode 100644 index 76f919b6c..000000000 --- a/tests/samples/json_web_token.hs +++ /dev/null @@ -1 +0,0 @@ -$payload = 'eyJgsIZgeJhvNgFpSmlP.eyJcaaF9xCe7shE0ENPiBlEJOpS' diff --git a/tests/samples/key.hs b/tests/samples/key.hs index 5a13fab50..3d197e623 100644 --- a/tests/samples/key.hs +++ b/tests/samples/key.hs @@ -2,6 +2,6 @@ prKeyValid=LS0tLS1CRUdJTiBQUklWQVRFIENDcUdTTTQ5QXdFSEJHMHdhd0lCQVFRZ0ViVnpmUGWxh secret_looks_like_linux_path_1="/VnpmUGWxhQW9KQAwrL2ZYdDJPNG1PQjYxMXNPaF" secret_looks_like_linux_path_2="VnpmUGWxhQW/9KQAwrL2ZYd/DJPNG1PQjYxMXNPF" secret_looks_like_linux_path_3="VnpmUGWxhQW/9KQAwrL2ZYdDJPNG1PQjYxMXNPF=" -secret_looks_like_linux_path__="VnpmUGWxhQW/9KQAwrL2ZYd/DJPNG1PQjEXAMPLE" +secret_looks_like_linux_path__="VnpmUGWxhQW/9KQAwrL2ZYd/DJPNG1PQjEXAMbLE" "https://example.com/api/js?key=dhd0lCQVFRZ0ViVnpmUGWxhQW9KQWwrLzZYdDJPNG1PQjYxMXNPaFJB&bug=true" diff --git a/tests/samples/salt.py b/tests/samples/salt.py index 4140c4e5a..60b2fcd07 100644 --- a/tests/samples/salt.py +++ b/tests/samples/salt.py @@ -2,3 +2,4 @@ salt2 = r"""\0x12\0x3s""" salt3 = u"\u0020827634876" salt4 = {"salt5": "my124%#$@s\x04clt\0"} +json_escaped = "{\\\"salt8\\\":\\\"4b9a6d8b638eb0c6\\\"}" diff --git a/tests/samples/test.html b/tests/samples/test.html index d07d0420f..65964bccb 100644 --- a/tests/samples/test.html +++ b/tests/samples/test.html @@ -114,7 +114,7 @@ nested table - + xml capability diff --git a/tests/samples/url_cred.js b/tests/samples/url_cred.js index 4b21bc971..d555f7233 100644 --- a/tests/samples/url_cred.js +++ b/tests/samples/url_cred.js @@ -9,3 +9,5 @@ url = "https://secure.com/83675/39084?Credential=546DFS64N90P3AW7DX%2Fkeep%26cut /* partially line to sanitize url-like items 39084?Credential=546DFS64N90P3AW7DX&key=3487263-2384579834-234732875-345&hasToBefound=2 */ + +email_as_login = "smtps://example@gmail.com:FnD83JZs@smtp.gmail.com:465"; diff --git a/tests/samples/uuid b/tests/samples/uuid new file mode 100644 index 000000000..0ce05f451 --- /dev/null +++ b/tests/samples/uuid @@ -0,0 +1,2 @@ +bace4d19-fa7e-beef-cafe-9129474bcd81 # tp +12345678-1234-1234-1234-1234567890ab # fp diff --git a/tests/test_main.py b/tests/test_main.py index d796e471a..dea02e9ac 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -490,7 +490,7 @@ def test_pdf_p(self) -> None: # may be tested with # https://www.dcc.edu/documents/administration/offices/information-technology/password-examples.pdf content_provider: AbstractProvider = FilesProvider([SAMPLES_PATH / "sample.pdf"]) - cred_sweeper = CredSweeper(depth=33) + cred_sweeper = CredSweeper(depth=7) cred_sweeper.run(content_provider=content_provider) found_credentials = cred_sweeper.credential_manager.get_credentials() self.assertSetEqual({"AWS Client ID", "Password", "Github Classic Token", "Key"}, @@ -755,7 +755,7 @@ def prepare(report: List[Dict[str, Any]]): tmp_file = Path(tmp_dir) / cfg["json_filename"] # apply the current path to keep equivalence in path os.chdir(TESTS_PATH.parent) - content_provider: AbstractProvider = FilesProvider(["tests/samples"]) + content_provider: AbstractProvider = FilesProvider([Path("tests") / "samples"]) # replace output report file to place in tmp_dir cfg["json_filename"] = str(tmp_file) cred_sweeper = CredSweeper(**cfg) @@ -792,6 +792,7 @@ def test_param_n(self) -> None: ("pager.rs", b"token: impl AsRef,"), # ("pager.rs", b" let tokens = quote::quote! {"), # ("pager.rs", b" let cert_chain = x509_rx"), # + ("my.kt", b'val password: String? = null'), # ] content_provider: AbstractProvider = FilesProvider([(file_name, io.BytesIO(data_line)) for file_name, data_line in items]) @@ -825,7 +826,7 @@ def test_param_p(self) -> None: ("accept.py", b"password='Ahga%$FiQ@Ei8'", "password", "Ahga%$FiQ@Ei8"), # ("test.template", b" NAMED_API_KEY=qii7t1m6423127xto389xc914l34451qz5135865564sg ", "NAMED_API_KEY", "qii7t1m6423127xto389xc914l34451qz5135865564sg"), # - ("my.kt", b'val password: String? = "Ahga%$FiQ@Ei8"', "password", "Ahga%$FiQ@Ei8"), # + ("my.kt", b'val password: String = "Ahga%$FiQ@Ei8"', "password", "Ahga%$FiQ@Ei8"), # ] for file_name, data_line, variable, value in items: content_provider: AbstractProvider = FilesProvider([