Skip to content

Commit

Permalink
Merge branch 'main' into update/gcpapplicationdefaultcreds-detector
Browse files Browse the repository at this point in the history
  • Loading branch information
kashifkhan0771 authored Oct 28, 2024
2 parents 8088855 + 03d999c commit 61fa101
Show file tree
Hide file tree
Showing 14 changed files with 211 additions and 474 deletions.
23 changes: 23 additions & 0 deletions CONTRIBUTING.md
Original file line number Diff line number Diff line change
Expand Up @@ -11,3 +11,26 @@ Contributors need to [sign our CLA](https://cla-assistant.io/trufflesecurity/tru
## Adding new secret detectors

We have published some [documentation and tooling to get started on adding new secret detectors](hack/docs/Adding_Detectors_external.md). Let's improve detection together!

## Logging in TruffleHog

**Use fields over format strings**. For structured logging, fields allow us to better filter and search through logs than embedding data in the message.

**Differentiate logs coming from dependencies**. This can be done with a `"dep"` field that gets passed to the library. Sometimes it’s not possible to do this.

Limit log levels to _**info**_ (indicate normal or expected operation) and _**error**_ (functionality is impeded and should be checked by an engineer)

**Choose an appropriate verbosity level**
```
0. — logs we always want to see
1. — logs we could possibly want to turn off
2. — logs that are useful for debugging
3. — frequently called logs that may produce a lot of output
4. — extremely verbose logs or logs containing sensitive information
5. — ultimate verbosity
```
Example: `Logger().V(2).Info("skipping file: extension is ignored", "ext", mimeExt)`

**Either log an error or return it**. Doing one or the other will help defer logging for when there is more context for it and prevent duplicate “bubbling up” logs.

**Log contextual information**. Every log emitted should contain this context via fields to easily filter and search.
10 changes: 5 additions & 5 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ replace github.com/coinbase/waas-client-library-go => github.com/trufflesecurity
replace github.com/STARRY-S/zip => github.com/STARRY-S/zip v0.1.0

require (
cloud.google.com/go/secretmanager v1.14.1
cloud.google.com/go/secretmanager v1.14.2
cloud.google.com/go/storage v1.45.0
github.com/Azure/go-autorest/autorest/azure/auth v0.5.13
github.com/AzureAD/microsoft-authentication-library-for-go v1.2.2
Expand All @@ -27,7 +27,7 @@ require (
github.com/bill-rich/go-syslog v0.0.0-20220413021637-49edb52a574c
github.com/bitfinexcom/bitfinex-api-go v0.0.0-20210608095005-9e0b26f200fb
github.com/bradleyfalzon/ghinstallation/v2 v2.11.0
github.com/brianvoe/gofakeit/v7 v7.0.4
github.com/brianvoe/gofakeit/v7 v7.1.1
github.com/charmbracelet/bubbles v0.18.0
github.com/charmbracelet/bubbletea v0.27.0
github.com/charmbracelet/glamour v0.7.0
Expand Down Expand Up @@ -81,7 +81,7 @@ require (
github.com/prometheus/client_golang v1.20.5
github.com/rabbitmq/amqp091-go v1.10.0
github.com/sassoftware/go-rpmutils v0.4.0
github.com/schollz/progressbar/v3 v3.16.1
github.com/schollz/progressbar/v3 v3.17.0
github.com/sendgrid/sendgrid-go v3.16.0+incompatible
github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3
github.com/shuheiktgw/go-travis v0.3.1
Expand All @@ -108,7 +108,7 @@ require (
golang.org/x/oauth2 v0.23.0
golang.org/x/sync v0.8.0
golang.org/x/text v0.19.0
google.golang.org/api v0.202.0
google.golang.org/api v0.203.0
google.golang.org/protobuf v1.35.1
gopkg.in/h2non/gock.v1 v1.1.2
gopkg.in/yaml.v2 v2.4.0
Expand All @@ -121,7 +121,7 @@ require (
require (
cel.dev/expr v0.16.1 // indirect
cloud.google.com/go v0.116.0 // indirect
cloud.google.com/go/auth v0.9.8 // indirect
cloud.google.com/go/auth v0.9.9 // indirect
cloud.google.com/go/auth/oauth2adapt v0.2.4 // indirect
cloud.google.com/go/compute/metadata v0.5.2 // indirect
cloud.google.com/go/iam v1.2.1 // indirect
Expand Down
10 changes: 10 additions & 0 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,8 @@ cloud.google.com/go/auth v0.9.5 h1:4CTn43Eynw40aFVr3GpPqsQponx2jv0BQpjvajsbbzw=
cloud.google.com/go/auth v0.9.5/go.mod h1:Xo0n7n66eHyOWWCnitop6870Ilwo3PiZyodVkkH1xWM=
cloud.google.com/go/auth v0.9.8 h1:+CSJ0Gw9iVeSENVCKJoLHhdUykDgXSc4Qn+gu2BRtR8=
cloud.google.com/go/auth v0.9.8/go.mod h1:xxA5AqpDrvS+Gkmo9RqrGGRh6WSNKKOXhY3zNOr38tI=
cloud.google.com/go/auth v0.9.9 h1:BmtbpNQozo8ZwW2t7QJjnrQtdganSdmqeIBxHxNkEZQ=
cloud.google.com/go/auth v0.9.9/go.mod h1:xxA5AqpDrvS+Gkmo9RqrGGRh6WSNKKOXhY3zNOr38tI=
cloud.google.com/go/auth/oauth2adapt v0.2.4 h1:0GWE/FUsXhf6C+jAkWgYm7X9tK8cuEIfy19DBn6B6bY=
cloud.google.com/go/auth/oauth2adapt v0.2.4/go.mod h1:jC/jOpwFP6JBxhB3P5Rr0a9HLMC/Pe3eaL4NmdvqPtc=
cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o=
Expand All @@ -38,6 +40,8 @@ cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2k
cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw=
cloud.google.com/go/secretmanager v1.14.1 h1:xlWSIg8rtBn5qCr2f3XtQP19+5COyf/ll49SEvi/0vM=
cloud.google.com/go/secretmanager v1.14.1/go.mod h1:L+gO+u2JA9CCyXpSR8gDH0o8EV7i/f0jdBOrUXcIV0U=
cloud.google.com/go/secretmanager v1.14.2 h1:2XscWCfy//l/qF96YE18/oUaNJynAx749Jg3u0CjQr8=
cloud.google.com/go/secretmanager v1.14.2/go.mod h1:Q18wAPMM6RXLC/zVpWTlqq2IBSbbm7pKBlM3lCKsmjw=
cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw=
cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos=
cloud.google.com/go/storage v1.44.0 h1:abBzXf4UJKMmQ04xxJf9dYM/fNl24KHoTuBjyJDX2AI=
Expand Down Expand Up @@ -172,6 +176,8 @@ github.com/bradleyfalzon/ghinstallation/v2 v2.11.0 h1:R9d0v+iobRHSaE4wKUnXFiZp53
github.com/bradleyfalzon/ghinstallation/v2 v2.11.0/go.mod h1:0LWKQwOHewXO/1acI6TtyE0Xc4ObDb2rFN7eHBAG71M=
github.com/brianvoe/gofakeit/v7 v7.0.4 h1:Mkxwz9jYg8Ad8NvT9HA27pCMZGFQo08MK6jD0QTKEww=
github.com/brianvoe/gofakeit/v7 v7.0.4/go.mod h1:QXuPeBw164PJCzCUZVmgpgHJ3Llj49jSLVkKPMtxtxA=
github.com/brianvoe/gofakeit/v7 v7.1.1 h1:/DEG+f/mFtqqNjhZ0AXA0aDzrnfE85AcAKVE+mMdxAQ=
github.com/brianvoe/gofakeit/v7 v7.1.1/go.mod h1:QXuPeBw164PJCzCUZVmgpgHJ3Llj49jSLVkKPMtxtxA=
github.com/bwesterb/go-ristretto v1.2.3/go.mod h1:fUIoIZaG73pV5biE2Blr2xEzDoMj7NFEuV9ekS419A0=
github.com/cenkalti/backoff/v4 v4.2.1 h1:y4OZtCnogmCPw98Zjyt5a6+QwPLGkiQsYW5oUqylYbM=
github.com/cenkalti/backoff/v4 v4.2.1/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE=
Expand Down Expand Up @@ -707,6 +713,8 @@ github.com/sassoftware/go-rpmutils v0.4.0 h1:ojND82NYBxgwrV+mX1CWsd5QJvvEZTKddtC
github.com/sassoftware/go-rpmutils v0.4.0/go.mod h1:3goNWi7PGAT3/dlql2lv3+MSN5jNYPjT5mVcQcIsYzI=
github.com/schollz/progressbar/v3 v3.16.1 h1:RnF1neWZFzLCoGx8yp1yF7SDl4AzNDI5y4I0aUJRrZQ=
github.com/schollz/progressbar/v3 v3.16.1/go.mod h1:I2ILR76gz5VXqYMIY/LdLecvMHDPVcQm3W/MSKi1TME=
github.com/schollz/progressbar/v3 v3.17.0 h1:Fv+vG6O6jnJwdjCelvfyYO7sF2jaUGQVmdH4CxcZdsQ=
github.com/schollz/progressbar/v3 v3.17.0/go.mod h1:5H4fLgifX+KeQCsEJnZTOepgZLe1jFF1lpPXb68IJTA=
github.com/sendgrid/rest v2.6.9+incompatible h1:1EyIcsNdn9KIisLW50MKwmSRSK+ekueiEMJ7NEoxJo0=
github.com/sendgrid/rest v2.6.9+incompatible/go.mod h1:kXX7q3jZtJXK5c5qK83bSGMdV6tsOE70KbHoqJls4lE=
github.com/sendgrid/sendgrid-go v3.16.0+incompatible h1:i8eE6IMkiCy7vusSdacHHSBUpXyTcTXy/Rl9N9aZ/Qw=
Expand Down Expand Up @@ -1135,6 +1143,8 @@ google.golang.org/api v0.201.0 h1:+7AD9JNM3tREtawRMu8sOjSbb8VYcYXJG/2eEOmfDu0=
google.golang.org/api v0.201.0/go.mod h1:HVY0FCHVs89xIW9fzf/pBvOEm+OolHa86G/txFezyq4=
google.golang.org/api v0.202.0 h1:y1iuVHMqokQbimW79ZqPZWo4CiyFu6HcCYHwSNyzlfo=
google.golang.org/api v0.202.0/go.mod h1:3Jjeq7M/SFblTNCp7ES2xhq+WvGL0KeXI0joHQBfwTQ=
google.golang.org/api v0.203.0 h1:SrEeuwU3S11Wlscsn+LA1kb/Y5xT8uggJSkIhD08NAU=
google.golang.org/api v0.203.0/go.mod h1:BuOVyCSYEPwJb3npWvDnNmFI92f3GeRnHNkETneT3SI=
google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
Expand Down
4 changes: 2 additions & 2 deletions pkg/detectors/aeroworkflow/aeroworkflow.go
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@ var (
defaultClient = common.SaneHttpClient()

// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"aeroworkflow"}) + `([a-zA-Z0-9^!?#:*;]{20})\b`)
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"aeroworkflow"}) + `\b([a-zA-Z0-9^!?#:*;]{20})`)
idPat = regexp.MustCompile(detectors.PrefixRegex([]string{"aeroworkflow"}) + `\b([0-9]{1,})\b`)
)

Expand Down Expand Up @@ -85,7 +85,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
}

func verifyAeroworkflow(ctx context.Context, client *http.Client, resMatch, resIdMatch string) (bool, error) {
req, err := http.NewRequestWithContext(ctx, http.MethodGet, aeroworkflowURL+"/api/"+resIdMatch+"/v1/AeroAppointments", nil)
req, err := http.NewRequestWithContext(ctx, http.MethodGet, aeroworkflowURL+"/api/"+resIdMatch+"/me", nil)
if err != nil {
return false, err
}
Expand Down
80 changes: 52 additions & 28 deletions pkg/detectors/alegra/alegra.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,8 @@ package alegra

import (
"context"
"fmt"
"io"
"net/http"
"strings"

Expand All @@ -13,18 +15,17 @@ import (
)

type Scanner struct {
detectors.DefaultMultiPartCredentialProvider
client *http.Client
}

// Ensure the Scanner satisfies the interface at compile time.
var _ detectors.Detector = (*Scanner)(nil)

var (
client = common.SaneHttpClient()

defaultClient = common.SaneHttpClient()
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"alegra"}) + `\b([a-z0-9-]{20})\b`)
idPat = regexp.MustCompile(detectors.PrefixRegex([]string{"alegra"}) + `\b([a-zA-Z0-9\.\-\@]{25,30})\b`)
idPat = regexp.MustCompile(detectors.PrefixRegex([]string{"alegra"}) + common.EmailPattern)
)

// Keywords are used for efficiently pre-filtering chunks.
Expand All @@ -37,41 +38,38 @@ func (s Scanner) Keywords() []string {
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
dataStr := string(data)

matches := keyPat.FindAllStringSubmatch(dataStr, -1)
keyMatches := keyPat.FindAllStringSubmatch(dataStr, -1)
idMatches := idPat.FindAllStringSubmatch(dataStr, -1)

for _, match := range matches {
if len(match) != 2 {
continue
}
tokenPatMatch := strings.TrimSpace(match[1])
uniqueTokens := make(map[string]struct{})
uniqueIDs := make(map[string]struct{})

for _, idMatch := range idMatches {
if len(idMatch) != 2 {
continue
}
for _, match := range keyMatches {
uniqueTokens[match[1]] = struct{}{}
}

userPatMatch := strings.TrimSpace(idMatch[1])
for _, match := range idMatches {
id := match[0][strings.LastIndex(match[0], " ")+1:]
uniqueIDs[id] = struct{}{}
}

for token := range uniqueTokens {
for id := range uniqueIDs {
s1 := detectors.Result{
DetectorType: detectorspb.DetectorType_Alegra,
Raw: []byte(tokenPatMatch),
RawV2: []byte(tokenPatMatch + userPatMatch),
Raw: []byte(token),
RawV2: []byte(token + ":" + id),
}

if verify {
req, err := http.NewRequestWithContext(ctx, "GET", "https://api.alegra.com/api/v1/users", nil)
if err != nil {
continue
}
req.SetBasicAuth(userPatMatch, tokenPatMatch)
res, err := client.Do(req)
if err == nil {
defer res.Body.Close()
if res.StatusCode >= 200 && res.StatusCode < 300 {
s1.Verified = true
}
client := s.client
if client == nil {
client = defaultClient
}

isVerified, verificationErr := verifyCredentials(ctx, client, id, token)
s1.Verified = isVerified
s1.SetVerificationError(verificationErr, token)
}

results = append(results, s1)
Expand All @@ -81,6 +79,32 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
return results, nil
}

func verifyCredentials(ctx context.Context, client *http.Client, username, token string) (bool, error) {
req, err := http.NewRequestWithContext(ctx, http.MethodGet, "https://api.alegra.com/api/v1/users/self", nil)
if err != nil {
return false, nil
}
req.SetBasicAuth(username, token)

res, err := client.Do(req)
if err != nil {
return false, err
}
defer func() {
_, _ = io.Copy(io.Discard, res.Body)
_ = res.Body.Close()
}()

switch res.StatusCode {
case http.StatusOK:
return true, nil
case http.StatusUnauthorized:
return false, nil
default:
return false, fmt.Errorf("unexpected HTTP response status %d", res.StatusCode)
}
}

func (s Scanner) Type() detectorspb.DetectorType {
return detectorspb.DetectorType_Alegra
}
Expand Down
1 change: 1 addition & 0 deletions pkg/detectors/alegra/alegra_integration_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -96,6 +96,7 @@ func TestAlegra_FromChunk(t *testing.T) {
t.Fatalf("no raw secret present: \n %+v", got[i])
}
got[i].Raw = nil
got[i].RawV2 = nil
}
if diff := pretty.Compare(got, tt.want); diff != "" {
t.Errorf("Alegra.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
Expand Down
22 changes: 9 additions & 13 deletions pkg/detectors/alegra/alegra_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ import (
var (
validPattern = "wdvn-usa87a-fxp9ioas/[email protected]"
validSpecialCharPattern = "wdvn-usa87a-fxp9ioas / [email protected]"
invalidPattern = "wdvn-usa87a-fxp9ioas/testUser$1005@example.com"
invalidPattern = "wdvn-usa87a-fxp9ioasQQsstestUsQQ@example"
)

func TestAlegra_Pattern(t *testing.T) {
Expand All @@ -28,30 +28,30 @@ func TestAlegra_Pattern(t *testing.T) {
}{
{
name: "valid pattern",
input: fmt.Sprintf("alegra: '%s'", validPattern),
want: []string{"wdvn-usa87a-fxp9ioastestUser[email protected]"},
input: fmt.Sprintf("alegra: %s", validPattern),
want: []string{"wdvn-usa87a-fxp9ioas:wdvn-usa87a-fxp9ioas/testUser[email protected]"},
},
{
name: "valid pattern - with special characters",
input: fmt.Sprintf("alegra: '%s'", validSpecialCharPattern),
want: []string{"wdvn-usa87a-fxp9ioastest[email protected]"},
input: fmt.Sprintf("alegra: %s", validSpecialCharPattern),
want: []string{"wdvn-usa87a-fxp9ioas:test[email protected]"},
},
{
name: "valid pattern - key out of prefix range",
input: fmt.Sprintf("alegra keyword is not close to the real key and id = '%s'", validPattern),
input: fmt.Sprintf("alegra keyword is not close to the real key and id = %s", validPattern),
want: nil,
},
{
name: "invalid pattern",
input: fmt.Sprintf("alegra: '%s'", invalidPattern),
input: fmt.Sprintf("alegra: %s", invalidPattern),
want: nil,
},
}

for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
if len(matchedDetectors) == 0 {
if len(matchedDetectors) == 0 && test.want != nil {
t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
return
}
Expand All @@ -63,11 +63,7 @@ func TestAlegra_Pattern(t *testing.T) {
}

if len(results) != len(test.want) {
if len(results) == 0 {
t.Errorf("did not receive result")
} else {
t.Errorf("expected %d results, only received %d", len(test.want), len(results))
}
t.Errorf("expected %d results, got %d", len(test.want), len(results))
return
}

Expand Down
Loading

0 comments on commit 61fa101

Please sign in to comment.