diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 16fe1035b2f9..786b4a806c76 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -11,3 +11,26 @@ Contributors need to [sign our CLA](https://cla-assistant.io/trufflesecurity/tru ## Adding new secret detectors We have published some [documentation and tooling to get started on adding new secret detectors](hack/docs/Adding_Detectors_external.md). Let's improve detection together! + +## Logging in TruffleHog + +**Use fields over format strings**. For structured logging, fields allow us to better filter and search through logs than embedding data in the message. + +**Differentiate logs coming from dependencies**. This can be done with a `"dep"` field that gets passed to the library. Sometimes it’s not possible to do this. + +Limit log levels to _**info**_ (indicate normal or expected operation) and _**error**_ (functionality is impeded and should be checked by an engineer) + +**Choose an appropriate verbosity level** +``` +0. — logs we always want to see +1. — logs we could possibly want to turn off +2. — logs that are useful for debugging +3. — frequently called logs that may produce a lot of output +4. — extremely verbose logs or logs containing sensitive information +5. — ultimate verbosity +``` +Example: `Logger().V(2).Info("skipping file: extension is ignored", "ext", mimeExt)` + +**Either log an error or return it**. Doing one or the other will help defer logging for when there is more context for it and prevent duplicate “bubbling up” logs. + +**Log contextual information**. Every log emitted should contain this context via fields to easily filter and search. diff --git a/go.mod b/go.mod index e29107df6a1f..45b70a6a50ad 100644 --- a/go.mod +++ b/go.mod @@ -14,7 +14,7 @@ replace github.com/coinbase/waas-client-library-go => github.com/trufflesecurity replace github.com/STARRY-S/zip => github.com/STARRY-S/zip v0.1.0 require ( - cloud.google.com/go/secretmanager v1.14.1 + cloud.google.com/go/secretmanager v1.14.2 cloud.google.com/go/storage v1.45.0 github.com/Azure/go-autorest/autorest/azure/auth v0.5.13 github.com/AzureAD/microsoft-authentication-library-for-go v1.2.2 @@ -27,7 +27,7 @@ require ( github.com/bill-rich/go-syslog v0.0.0-20220413021637-49edb52a574c github.com/bitfinexcom/bitfinex-api-go v0.0.0-20210608095005-9e0b26f200fb github.com/bradleyfalzon/ghinstallation/v2 v2.11.0 - github.com/brianvoe/gofakeit/v7 v7.0.4 + github.com/brianvoe/gofakeit/v7 v7.1.1 github.com/charmbracelet/bubbles v0.18.0 github.com/charmbracelet/bubbletea v0.27.0 github.com/charmbracelet/glamour v0.7.0 @@ -81,7 +81,7 @@ require ( github.com/prometheus/client_golang v1.20.5 github.com/rabbitmq/amqp091-go v1.10.0 github.com/sassoftware/go-rpmutils v0.4.0 - github.com/schollz/progressbar/v3 v3.16.1 + github.com/schollz/progressbar/v3 v3.17.0 github.com/sendgrid/sendgrid-go v3.16.0+incompatible github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3 github.com/shuheiktgw/go-travis v0.3.1 @@ -108,7 +108,7 @@ require ( golang.org/x/oauth2 v0.23.0 golang.org/x/sync v0.8.0 golang.org/x/text v0.19.0 - google.golang.org/api v0.202.0 + google.golang.org/api v0.203.0 google.golang.org/protobuf v1.35.1 gopkg.in/h2non/gock.v1 v1.1.2 gopkg.in/yaml.v2 v2.4.0 @@ -121,7 +121,7 @@ require ( require ( cel.dev/expr v0.16.1 // indirect cloud.google.com/go v0.116.0 // indirect - cloud.google.com/go/auth v0.9.8 // indirect + cloud.google.com/go/auth v0.9.9 // indirect cloud.google.com/go/auth/oauth2adapt v0.2.4 // indirect cloud.google.com/go/compute/metadata v0.5.2 // indirect cloud.google.com/go/iam v1.2.1 // indirect diff --git a/go.sum b/go.sum index e3f027f88c28..a8859b417cc0 100644 --- a/go.sum +++ b/go.sum @@ -17,6 +17,8 @@ cloud.google.com/go/auth v0.9.5 h1:4CTn43Eynw40aFVr3GpPqsQponx2jv0BQpjvajsbbzw= cloud.google.com/go/auth v0.9.5/go.mod h1:Xo0n7n66eHyOWWCnitop6870Ilwo3PiZyodVkkH1xWM= cloud.google.com/go/auth v0.9.8 h1:+CSJ0Gw9iVeSENVCKJoLHhdUykDgXSc4Qn+gu2BRtR8= cloud.google.com/go/auth v0.9.8/go.mod h1:xxA5AqpDrvS+Gkmo9RqrGGRh6WSNKKOXhY3zNOr38tI= +cloud.google.com/go/auth v0.9.9 h1:BmtbpNQozo8ZwW2t7QJjnrQtdganSdmqeIBxHxNkEZQ= +cloud.google.com/go/auth v0.9.9/go.mod h1:xxA5AqpDrvS+Gkmo9RqrGGRh6WSNKKOXhY3zNOr38tI= cloud.google.com/go/auth/oauth2adapt v0.2.4 h1:0GWE/FUsXhf6C+jAkWgYm7X9tK8cuEIfy19DBn6B6bY= cloud.google.com/go/auth/oauth2adapt v0.2.4/go.mod h1:jC/jOpwFP6JBxhB3P5Rr0a9HLMC/Pe3eaL4NmdvqPtc= cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= @@ -38,6 +40,8 @@ cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2k cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= cloud.google.com/go/secretmanager v1.14.1 h1:xlWSIg8rtBn5qCr2f3XtQP19+5COyf/ll49SEvi/0vM= cloud.google.com/go/secretmanager v1.14.1/go.mod h1:L+gO+u2JA9CCyXpSR8gDH0o8EV7i/f0jdBOrUXcIV0U= +cloud.google.com/go/secretmanager v1.14.2 h1:2XscWCfy//l/qF96YE18/oUaNJynAx749Jg3u0CjQr8= +cloud.google.com/go/secretmanager v1.14.2/go.mod h1:Q18wAPMM6RXLC/zVpWTlqq2IBSbbm7pKBlM3lCKsmjw= cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos= cloud.google.com/go/storage v1.44.0 h1:abBzXf4UJKMmQ04xxJf9dYM/fNl24KHoTuBjyJDX2AI= @@ -172,6 +176,8 @@ github.com/bradleyfalzon/ghinstallation/v2 v2.11.0 h1:R9d0v+iobRHSaE4wKUnXFiZp53 github.com/bradleyfalzon/ghinstallation/v2 v2.11.0/go.mod h1:0LWKQwOHewXO/1acI6TtyE0Xc4ObDb2rFN7eHBAG71M= github.com/brianvoe/gofakeit/v7 v7.0.4 h1:Mkxwz9jYg8Ad8NvT9HA27pCMZGFQo08MK6jD0QTKEww= github.com/brianvoe/gofakeit/v7 v7.0.4/go.mod h1:QXuPeBw164PJCzCUZVmgpgHJ3Llj49jSLVkKPMtxtxA= +github.com/brianvoe/gofakeit/v7 v7.1.1 h1:/DEG+f/mFtqqNjhZ0AXA0aDzrnfE85AcAKVE+mMdxAQ= +github.com/brianvoe/gofakeit/v7 v7.1.1/go.mod h1:QXuPeBw164PJCzCUZVmgpgHJ3Llj49jSLVkKPMtxtxA= github.com/bwesterb/go-ristretto v1.2.3/go.mod h1:fUIoIZaG73pV5biE2Blr2xEzDoMj7NFEuV9ekS419A0= github.com/cenkalti/backoff/v4 v4.2.1 h1:y4OZtCnogmCPw98Zjyt5a6+QwPLGkiQsYW5oUqylYbM= github.com/cenkalti/backoff/v4 v4.2.1/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= @@ -707,6 +713,8 @@ github.com/sassoftware/go-rpmutils v0.4.0 h1:ojND82NYBxgwrV+mX1CWsd5QJvvEZTKddtC github.com/sassoftware/go-rpmutils v0.4.0/go.mod h1:3goNWi7PGAT3/dlql2lv3+MSN5jNYPjT5mVcQcIsYzI= github.com/schollz/progressbar/v3 v3.16.1 h1:RnF1neWZFzLCoGx8yp1yF7SDl4AzNDI5y4I0aUJRrZQ= github.com/schollz/progressbar/v3 v3.16.1/go.mod h1:I2ILR76gz5VXqYMIY/LdLecvMHDPVcQm3W/MSKi1TME= +github.com/schollz/progressbar/v3 v3.17.0 h1:Fv+vG6O6jnJwdjCelvfyYO7sF2jaUGQVmdH4CxcZdsQ= +github.com/schollz/progressbar/v3 v3.17.0/go.mod h1:5H4fLgifX+KeQCsEJnZTOepgZLe1jFF1lpPXb68IJTA= github.com/sendgrid/rest v2.6.9+incompatible h1:1EyIcsNdn9KIisLW50MKwmSRSK+ekueiEMJ7NEoxJo0= github.com/sendgrid/rest v2.6.9+incompatible/go.mod h1:kXX7q3jZtJXK5c5qK83bSGMdV6tsOE70KbHoqJls4lE= github.com/sendgrid/sendgrid-go v3.16.0+incompatible h1:i8eE6IMkiCy7vusSdacHHSBUpXyTcTXy/Rl9N9aZ/Qw= @@ -1135,6 +1143,8 @@ google.golang.org/api v0.201.0 h1:+7AD9JNM3tREtawRMu8sOjSbb8VYcYXJG/2eEOmfDu0= google.golang.org/api v0.201.0/go.mod h1:HVY0FCHVs89xIW9fzf/pBvOEm+OolHa86G/txFezyq4= google.golang.org/api v0.202.0 h1:y1iuVHMqokQbimW79ZqPZWo4CiyFu6HcCYHwSNyzlfo= google.golang.org/api v0.202.0/go.mod h1:3Jjeq7M/SFblTNCp7ES2xhq+WvGL0KeXI0joHQBfwTQ= +google.golang.org/api v0.203.0 h1:SrEeuwU3S11Wlscsn+LA1kb/Y5xT8uggJSkIhD08NAU= +google.golang.org/api v0.203.0/go.mod h1:BuOVyCSYEPwJb3npWvDnNmFI92f3GeRnHNkETneT3SI= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= diff --git a/pkg/detectors/aeroworkflow/aeroworkflow.go b/pkg/detectors/aeroworkflow/aeroworkflow.go index 4ec0022f0876..d5ce4b33f533 100644 --- a/pkg/detectors/aeroworkflow/aeroworkflow.go +++ b/pkg/detectors/aeroworkflow/aeroworkflow.go @@ -27,7 +27,7 @@ var ( defaultClient = common.SaneHttpClient() // Make sure that your group is surrounded in boundary characters such as below to reduce false positives. - keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"aeroworkflow"}) + `([a-zA-Z0-9^!?#:*;]{20})\b`) + keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"aeroworkflow"}) + `\b([a-zA-Z0-9^!?#:*;]{20})`) idPat = regexp.MustCompile(detectors.PrefixRegex([]string{"aeroworkflow"}) + `\b([0-9]{1,})\b`) ) @@ -85,7 +85,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result } func verifyAeroworkflow(ctx context.Context, client *http.Client, resMatch, resIdMatch string) (bool, error) { - req, err := http.NewRequestWithContext(ctx, http.MethodGet, aeroworkflowURL+"/api/"+resIdMatch+"/v1/AeroAppointments", nil) + req, err := http.NewRequestWithContext(ctx, http.MethodGet, aeroworkflowURL+"/api/"+resIdMatch+"/me", nil) if err != nil { return false, err } diff --git a/pkg/detectors/alegra/alegra.go b/pkg/detectors/alegra/alegra.go index 1fcbfcb2bdce..856d77237681 100644 --- a/pkg/detectors/alegra/alegra.go +++ b/pkg/detectors/alegra/alegra.go @@ -2,6 +2,8 @@ package alegra import ( "context" + "fmt" + "io" "net/http" "strings" @@ -13,18 +15,17 @@ import ( ) type Scanner struct { - detectors.DefaultMultiPartCredentialProvider + client *http.Client } // Ensure the Scanner satisfies the interface at compile time. var _ detectors.Detector = (*Scanner)(nil) var ( - client = common.SaneHttpClient() - + defaultClient = common.SaneHttpClient() // Make sure that your group is surrounded in boundary characters such as below to reduce false positives. keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"alegra"}) + `\b([a-z0-9-]{20})\b`) - idPat = regexp.MustCompile(detectors.PrefixRegex([]string{"alegra"}) + `\b([a-zA-Z0-9\.\-\@]{25,30})\b`) + idPat = regexp.MustCompile(detectors.PrefixRegex([]string{"alegra"}) + common.EmailPattern) ) // Keywords are used for efficiently pre-filtering chunks. @@ -37,41 +38,38 @@ func (s Scanner) Keywords() []string { func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) { dataStr := string(data) - matches := keyPat.FindAllStringSubmatch(dataStr, -1) + keyMatches := keyPat.FindAllStringSubmatch(dataStr, -1) idMatches := idPat.FindAllStringSubmatch(dataStr, -1) - for _, match := range matches { - if len(match) != 2 { - continue - } - tokenPatMatch := strings.TrimSpace(match[1]) + uniqueTokens := make(map[string]struct{}) + uniqueIDs := make(map[string]struct{}) - for _, idMatch := range idMatches { - if len(idMatch) != 2 { - continue - } + for _, match := range keyMatches { + uniqueTokens[match[1]] = struct{}{} + } - userPatMatch := strings.TrimSpace(idMatch[1]) + for _, match := range idMatches { + id := match[0][strings.LastIndex(match[0], " ")+1:] + uniqueIDs[id] = struct{}{} + } + for token := range uniqueTokens { + for id := range uniqueIDs { s1 := detectors.Result{ DetectorType: detectorspb.DetectorType_Alegra, - Raw: []byte(tokenPatMatch), - RawV2: []byte(tokenPatMatch + userPatMatch), + Raw: []byte(token), + RawV2: []byte(token + ":" + id), } if verify { - req, err := http.NewRequestWithContext(ctx, "GET", "https://api.alegra.com/api/v1/users", nil) - if err != nil { - continue - } - req.SetBasicAuth(userPatMatch, tokenPatMatch) - res, err := client.Do(req) - if err == nil { - defer res.Body.Close() - if res.StatusCode >= 200 && res.StatusCode < 300 { - s1.Verified = true - } + client := s.client + if client == nil { + client = defaultClient } + + isVerified, verificationErr := verifyCredentials(ctx, client, id, token) + s1.Verified = isVerified + s1.SetVerificationError(verificationErr, token) } results = append(results, s1) @@ -81,6 +79,32 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result return results, nil } +func verifyCredentials(ctx context.Context, client *http.Client, username, token string) (bool, error) { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, "https://api.alegra.com/api/v1/users/self", nil) + if err != nil { + return false, nil + } + req.SetBasicAuth(username, token) + + res, err := client.Do(req) + if err != nil { + return false, err + } + defer func() { + _, _ = io.Copy(io.Discard, res.Body) + _ = res.Body.Close() + }() + + switch res.StatusCode { + case http.StatusOK: + return true, nil + case http.StatusUnauthorized: + return false, nil + default: + return false, fmt.Errorf("unexpected HTTP response status %d", res.StatusCode) + } +} + func (s Scanner) Type() detectorspb.DetectorType { return detectorspb.DetectorType_Alegra } diff --git a/pkg/detectors/alegra/alegra_integration_test.go b/pkg/detectors/alegra/alegra_integration_test.go index 19913d9c8aeb..35961518bcb9 100644 --- a/pkg/detectors/alegra/alegra_integration_test.go +++ b/pkg/detectors/alegra/alegra_integration_test.go @@ -96,6 +96,7 @@ func TestAlegra_FromChunk(t *testing.T) { t.Fatalf("no raw secret present: \n %+v", got[i]) } got[i].Raw = nil + got[i].RawV2 = nil } if diff := pretty.Compare(got, tt.want); diff != "" { t.Errorf("Alegra.FromData() %s diff: (-got +want)\n%s", tt.name, diff) diff --git a/pkg/detectors/alegra/alegra_test.go b/pkg/detectors/alegra/alegra_test.go index d2eb6e3469a6..5594c9ba4c1f 100644 --- a/pkg/detectors/alegra/alegra_test.go +++ b/pkg/detectors/alegra/alegra_test.go @@ -14,7 +14,7 @@ import ( var ( validPattern = "wdvn-usa87a-fxp9ioas/testUser.1005@example.com" validSpecialCharPattern = "wdvn-usa87a-fxp9ioas / test-User.1005@example.com" - invalidPattern = "wdvn-usa87a-fxp9ioas/testUser$1005@example.com" + invalidPattern = "wdvn-usa87a-fxp9ioasQQsstestUsQQ@example" ) func TestAlegra_Pattern(t *testing.T) { @@ -28,22 +28,22 @@ func TestAlegra_Pattern(t *testing.T) { }{ { name: "valid pattern", - input: fmt.Sprintf("alegra: '%s'", validPattern), - want: []string{"wdvn-usa87a-fxp9ioastestUser.1005@example.com"}, + input: fmt.Sprintf("alegra: %s", validPattern), + want: []string{"wdvn-usa87a-fxp9ioas:wdvn-usa87a-fxp9ioas/testUser.1005@example.com"}, }, { name: "valid pattern - with special characters", - input: fmt.Sprintf("alegra: '%s'", validSpecialCharPattern), - want: []string{"wdvn-usa87a-fxp9ioastest-User.1005@example.com"}, + input: fmt.Sprintf("alegra: %s", validSpecialCharPattern), + want: []string{"wdvn-usa87a-fxp9ioas:test-User.1005@example.com"}, }, { name: "valid pattern - key out of prefix range", - input: fmt.Sprintf("alegra keyword is not close to the real key and id = '%s'", validPattern), + input: fmt.Sprintf("alegra keyword is not close to the real key and id = %s", validPattern), want: nil, }, { name: "invalid pattern", - input: fmt.Sprintf("alegra: '%s'", invalidPattern), + input: fmt.Sprintf("alegra: %s", invalidPattern), want: nil, }, } @@ -51,7 +51,7 @@ func TestAlegra_Pattern(t *testing.T) { for _, test := range tests { t.Run(test.name, func(t *testing.T) { matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input)) - if len(matchedDetectors) == 0 { + if len(matchedDetectors) == 0 && test.want != nil { t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input) return } @@ -63,11 +63,7 @@ func TestAlegra_Pattern(t *testing.T) { } if len(results) != len(test.want) { - if len(results) == 0 { - t.Errorf("did not receive result") - } else { - t.Errorf("expected %d results, only received %d", len(test.want), len(results)) - } + t.Errorf("expected %d results, got %d", len(test.want), len(results)) return } diff --git a/pkg/detectors/algoliaadminkey/algoliaadminkey.go b/pkg/detectors/algoliaadminkey/algoliaadminkey.go index c9436ee11ff0..a72a5ddeecb7 100644 --- a/pkg/detectors/algoliaadminkey/algoliaadminkey.go +++ b/pkg/detectors/algoliaadminkey/algoliaadminkey.go @@ -2,6 +2,8 @@ package algoliaadminkey import ( "context" + "fmt" + "encoding/json" regexp "github.com/wasilibs/go-re2" "net/http" "strings" @@ -22,14 +24,14 @@ var ( client = common.SaneHttpClient() // Make sure that your group is surrounded in boundary characters such as below to reduce false positives. - keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"algolia"}) + `\b([a-zA-Z0-9]{32})\b`) - idPat = regexp.MustCompile(detectors.PrefixRegex([]string{"algolia"}) + `\b([A-Z0-9]{10})\b`) + keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"algolia", "docsearch", "apiKey"}) + `\b([a-zA-Z0-9]{32})\b`) + idPat = regexp.MustCompile(detectors.PrefixRegex([]string{"algolia", "docsearch", "appId"}) + `\b([A-Z0-9]{10})\b`) ) // Keywords are used for efficiently pre-filtering chunks. // Use identifiers in the secret preferably, or the provider name. func (s Scanner) Keywords() []string { - return []string{"algolia"} + return []string{"algolia", "docsearch"} } // FromData will find and optionally verify AlgoliaAdminKey secrets in a given set of bytes. @@ -57,19 +59,16 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result } if verify { - req, err := http.NewRequestWithContext(ctx, "GET", "https://"+resIdMatch+"-dsn.algolia.net/1/keys", nil) - if err != nil { - continue - } - req.Header.Add("X-Algolia-Application-Id", resIdMatch) - req.Header.Add("X-Algolia-API-Key", resMatch) - res, err := client.Do(req) - if err == nil { - defer res.Body.Close() - if res.StatusCode >= 200 && res.StatusCode < 300 { - s1.Verified = true - } + // Verify if the key is a valid Algolia Admin Key. + isVerified, verificationErr := verifyAlgoliaKey(ctx, resIdMatch, resMatch) + + // Verify if the key has sensitive permissions, even if it's not an Admin Key. + if !isVerified { + isVerified, verificationErr = verifyAlgoliaKeyACL(ctx, resIdMatch, resMatch) } + + s1.SetVerificationError(verificationErr, resMatch) + s1.Verified = isVerified } results = append(results, s1) @@ -78,6 +77,68 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result return results, nil } +func verifyAlgoliaKey(ctx context.Context, appId, apiKey string) (bool, error) { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, "https://"+appId+"-dsn.algolia.net/1/keys", nil) + if err != nil { + return false, err + } + + req.Header.Add("X-Algolia-Application-Id", appId) + req.Header.Add("X-Algolia-API-Key", apiKey) + + res, err := client.Do(req) + if err != nil { + return false, err + } + defer res.Body.Close() + + if res.StatusCode == 403 { + return false, nil + } else if res.StatusCode < 200 || res.StatusCode > 299 { + return false, fmt.Errorf("unexpected HTTP response status %d", res.StatusCode) + } + + return true, nil +} + +func verifyAlgoliaKeyACL(ctx context.Context, appId, apiKey string) (bool, error) { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, "https://"+appId+".algolia.net/1/keys/"+apiKey, nil) + if err != nil { + return false, err + } + + req.Header.Add("X-Algolia-Application-Id", appId) + req.Header.Add("X-Algolia-API-Key", apiKey) + + res, err := client.Do(req) + if err != nil { + return false, err + } + defer res.Body.Close() + + if res.StatusCode == 403 { + return false, nil + } else if res.StatusCode < 200 || res.StatusCode > 299 { + return false, fmt.Errorf("unexpected HTTP response status %d", res.StatusCode) + } + + var jsonResponse struct { + ACL []string `json:"acl"` + } + + if err := json.NewDecoder(res.Body).Decode(&jsonResponse); err != nil { + return false, err + } + + for _, acl := range jsonResponse.ACL { + if acl != "search" && acl != "listIndexes" && acl != "settings" { + return true, nil // Other permissions are sensitive. + } + } + + return false, nil +} + func (s Scanner) Type() detectorspb.DetectorType { return detectorspb.DetectorType_AlgoliaAdminKey } diff --git a/pkg/detectors/algoliaadminkey/algoliaadminkey_integration_test.go b/pkg/detectors/algoliaadminkey/algoliaadminkey_integration_test.go index 3a6985580f32..a5fe493f280b 100644 --- a/pkg/detectors/algoliaadminkey/algoliaadminkey_integration_test.go +++ b/pkg/detectors/algoliaadminkey/algoliaadminkey_integration_test.go @@ -51,6 +51,7 @@ func TestAlgoliaAdminKey_FromChunk(t *testing.T) { { DetectorType: detectorspb.DetectorType_AlgoliaAdminKey, Verified: true, + RawV2: []byte(fmt.Sprintf("%s%s", secret, id)), }, }, wantErr: false, @@ -67,6 +68,7 @@ func TestAlgoliaAdminKey_FromChunk(t *testing.T) { { DetectorType: detectorspb.DetectorType_AlgoliaAdminKey, Verified: false, + RawV2: []byte(fmt.Sprintf("%s%s", inactiveSecret, id)), }, }, wantErr: false, diff --git a/pkg/detectors/bulksms/bulksms.go b/pkg/detectors/bulksms/bulksms.go index b543fd5a7fed..0a80a97bb426 100644 --- a/pkg/detectors/bulksms/bulksms.go +++ b/pkg/detectors/bulksms/bulksms.go @@ -2,18 +2,17 @@ package bulksms import ( "context" - b64 "encoding/base64" - "fmt" - regexp "github.com/wasilibs/go-re2" + "io" "net/http" - "strings" + + regexp "github.com/wasilibs/go-re2" "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" ) -type Scanner struct{ +type Scanner struct { detectors.DefaultMultiPartCredentialProvider } @@ -24,12 +23,11 @@ var ( client = common.SaneHttpClient() // Make sure that your group is surrounded in boundary characters such as below to reduce false positives - keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"bulksms"}) + `\b([a-fA-Z0-9*]{29})\b`) + keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"bulksms"}) + `\b([a-zA-Z0-9!@#$%^&*()]{29})\b`) idPat = regexp.MustCompile(detectors.PrefixRegex([]string{"bulksms"}) + `\b([A-F0-9-]{37})\b`) ) // Keywords are used for efficiently pre-filtering chunks. -// Use identifiers in the secret preferably, or the provider name. func (s Scanner) Keywords() []string { return []string{"bulksms"} } @@ -38,46 +36,51 @@ func (s Scanner) Keywords() []string { func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) { dataStr := string(data) - matches := keyPat.FindAllStringSubmatch(dataStr, -1) - idMatches := idPat.FindAllStringSubmatch(dataStr, -1) + var uniqueIds = make(map[string]struct{}) + var uniqueKeys = make(map[string]struct{}) - for _, match := range matches { - if len(match) != 2 { - continue - } - resMatch := strings.TrimSpace(match[1]) - for _, idmatch := range idMatches { - if len(match) != 2 { - continue - } - resIdMatch := strings.TrimSpace(idmatch[1]) + for _, match := range idPat.FindAllStringSubmatch(dataStr, -1) { + uniqueIds[match[1]] = struct{}{} + } + + for _, match := range keyPat.FindAllStringSubmatch(dataStr, -1) { + uniqueKeys[match[1]] = struct{}{} + } + for id := range uniqueIds { + for key := range uniqueKeys { s1 := detectors.Result{ DetectorType: detectorspb.DetectorType_Bulksms, - Raw: []byte(resMatch), - RawV2: []byte(resMatch + resIdMatch), + Raw: []byte(key), + RawV2: []byte(key + id), } if verify { - data := fmt.Sprintf("%s:%s", resIdMatch, resMatch) - sEnc := b64.StdEncoding.EncodeToString([]byte(data)) req, err := http.NewRequestWithContext(ctx, "GET", "https://api.bulksms.com/v1/messages", nil) if err != nil { continue } - req.Header.Add("Authorization", fmt.Sprintf("Basic %s", sEnc)) + req.SetBasicAuth(id, key) res, err := client.Do(req) if err == nil { - defer res.Body.Close() - if res.StatusCode >= 200 && res.StatusCode < 300 { + defer func() { + _, _ = io.Copy(io.Discard, res.Body) + _ = res.Body.Close() + }() + + if res.StatusCode == http.StatusOK { s1.Verified = true + results = append(results, s1) + // move to next id, by skipping remaining key's + break } + } else { + s1.SetVerificationError(err, key) } } results = append(results, s1) } - } return results, nil diff --git a/pkg/detectors/bulksms/bulksms_test.go b/pkg/detectors/bulksms/bulksms_test.go index b087cacc6ded..aed029be3775 100644 --- a/pkg/detectors/bulksms/bulksms_test.go +++ b/pkg/detectors/bulksms/bulksms_test.go @@ -96,6 +96,7 @@ func TestBulksms_FromChunk(t *testing.T) { t.Fatalf("no raw secret present: \n %+v", got[i]) } got[i].Raw = nil + got[i].RawV2 = nil } if diff := pretty.Compare(got, tt.want); diff != "" { t.Errorf("Bulksms.FromData() %s diff: (-got +want)\n%s", tt.name, diff) diff --git a/pkg/log/level.go b/pkg/log/level.go index cc8f8d4336d8..99dae3cba653 100644 --- a/pkg/log/level.go +++ b/pkg/log/level.go @@ -2,10 +2,8 @@ package log import ( "sort" - "sync" "github.com/go-logr/logr" - "github.com/go-logr/zapr" "go.uber.org/zap" "go.uber.org/zap/zapcore" ) @@ -14,15 +12,6 @@ import ( var ( // Global, default log level control. globalLogLevel levelSetter = zap.NewAtomicLevel() - - // Map of name -> level control for independently setting log levels. A new - // control is registered via WithNamedLevel. This map is never cleaned up - // and new entries will overwrite previous values. Currently, this is - // acceptable behavior because WithNamedLevel is used sparingly. - globalControls map[string]levelSetter = make(map[string]levelSetter, 16) - // globalControls is protected (both read and write) by a mutex to make it - // thread safe. Access is low frequency, so performance is not a concern. - globalControlsLock sync.Mutex ) type levelSetter interface { @@ -45,65 +34,6 @@ func SetLevelForControl(control levelSetter, level int8) { control.SetLevel(zapcore.Level(-level)) } -// SetLevelFor sets the log level for a given named control. -func SetLevelFor(name string, level int8) { - globalControlsLock.Lock() - defer globalControlsLock.Unlock() - if control, ok := globalControls[name]; ok { - SetLevelForControl(control, level) - return - } - // Create a new control so registering a control with the same name will - // inherit the existing level. - globalControls[name] = newAtomicLevelAt(level) -} - -// AddLeveler adds a log level control to a logr.Logger. -func AddLeveler(l logr.Logger, control levelSetter) (logr.Logger, error) { - zapLogger, err := getZapLogger(l) - if err != nil { - return l, err - } - - zapLogger = zapLogger.WithOptions(zap.WrapCore(func(core zapcore.Core) zapcore.Core { - return NewLevelCore(core, control) - })) - return zapr.NewLogger(zapLogger), nil -} - -// WithNamedLevel creates a child logger with a new name and independent log -// level control (see SetLevelFor). NOTE: if name already exists, the existing -// controller will be used, otherwise a new controller is created with level -// matching the parent's log level. -func WithNamedLevel(logger logr.Logger, name string) logr.Logger { - logger = logger.WithName(name) - - globalControlsLock.Lock() - defer globalControlsLock.Unlock() - - var leveler levelSetter - if currentControl, ok := globalControls[name]; ok { - leveler = currentControl - } else { - leveler = newAtomicLevelAt(findLevel(logger)) - globalControls[name] = leveler - } - newLogger, err := AddLeveler(logger, leveler) - if err != nil { - return logger - } - return newLogger -} - -// newAtomicLevelAt is a helper function to create a zap.AtomicLevel -// initialized with a level. We cannot use zap.NewAtomicLevelAt here because of -// a quirk with logr levels (see SetLevelForControl). -func newAtomicLevelAt(level int8) zap.AtomicLevel { - control := zap.NewAtomicLevel() - SetLevelForControl(control, level) - return control -} - // findLevel probes a logr.Logger to figure out what level it is at via binary // search. We only search [0, 128), so worst case is ~7 checks. func findLevel(logger logr.Logger) int8 { diff --git a/pkg/log/log_test.go b/pkg/log/log_test.go index 65593b7890c4..b158ce2d975d 100644 --- a/pkg/log/log_test.go +++ b/pkg/log/log_test.go @@ -3,16 +3,12 @@ package log import ( "bytes" "encoding/json" - "fmt" "io" - "sort" "strings" - "sync" "testing" "time" "github.com/getsentry/sentry-go" - "github.com/go-logr/logr" "github.com/stretchr/testify/assert" "go.uber.org/zap" ) @@ -217,290 +213,6 @@ func TestWithLeveler(t *testing.T) { assert.Contains(t, buf2.String(), "line 3") } -func TestWithNamedLevelMoreVerbose(t *testing.T) { - var buf bytes.Buffer - globalControls = make(map[string]levelSetter, 16) - - l1 := zap.NewAtomicLevel() - logger, flush := New( - "service-name", - WithConsoleSink(&buf, WithLeveler(l1)), - ) - - childLogger := WithNamedLevel(logger, "child") - - SetLevelForControl(l1, 1) - SetLevelFor("child", 2) - - logger.V(0).Info("line 1") - logger.V(1).Info("line 2") - logger.V(2).Info("line 3") - childLogger.V(0).Info("line A") - childLogger.V(1).Info("line B") - childLogger.V(2).Info("line C") - assert.Nil(t, flush()) - - // output should contain up to verbosity 1 - assert.Equal(t, []string{ - "info-0\tservice-name\tline 1", - "info-1\tservice-name\tline 2", - "info-0\tservice-name.child\tline A", - "info-1\tservice-name.child\tline B", - }, splitLines(buf.String())) -} - -func TestWithNamedLevelLessVerbose(t *testing.T) { - var buf bytes.Buffer - globalControls = make(map[string]levelSetter, 16) - - l1 := zap.NewAtomicLevel() - logger, flush := New( - "service-name", - WithConsoleSink(&buf, WithLeveler(l1)), - ) - - childLogger := WithNamedLevel(logger, "child") - - SetLevelForControl(l1, 1) - SetLevelFor("child", 0) - - logger.V(0).Info("line 1") - logger.V(1).Info("line 2") - logger.V(2).Info("line 3") - childLogger.V(0).Info("line A") - childLogger.V(1).Info("line B") - childLogger.V(2).Info("line C") - assert.Nil(t, flush()) - - // output should contain up to verbosity 1 for parent - // and verbosity 0 for child - assert.Equal(t, []string{ - "info-0\tservice-name\tline 1", - "info-1\tservice-name\tline 2", - "info-0\tservice-name.child\tline A", - }, splitLines(buf.String())) -} - -func TestNestedWithNamedLevel(t *testing.T) { - var buf bytes.Buffer - globalControls = make(map[string]levelSetter, 16) - - grandParent, flush := New("grandParent", WithConsoleSink(&buf, WithLevel(1))) - parent := WithNamedLevel(grandParent, "parent") - child := WithNamedLevel(parent, "child") - - SetLevelFor("parent", 0) - SetLevelFor("child", 2) - - grandParent.V(0).Info("line 1") - parent.V(0).Info("line 2") - child.V(0).Info("line 3") - - grandParent.V(1).Info("line 4") - parent.V(1).Info("line 5") - child.V(1).Info("line 6") - - grandParent.V(2).Info("line 7") - parent.V(2).Info("line 8") - child.V(2).Info("line 9") - - assert.Nil(t, flush()) - - lines := splitLines(buf.String()) - assert.Equal(t, 4, len(lines)) - - assert.Equal(t, `info-0 grandParent line 1`, lines[0]) - assert.Equal(t, `info-0 grandParent.parent line 2`, lines[1]) - assert.Equal(t, `info-0 grandParent.parent.child line 3`, lines[2]) - assert.Equal(t, `info-1 grandParent line 4`, lines[3]) -} - -func TestSiblingsWithNamedLevel(t *testing.T) { - var buf bytes.Buffer - globalControls = make(map[string]levelSetter, 16) - - parent, flush := New("parent", WithConsoleSink(&buf, WithLevel(1))) - alice := WithNamedLevel(parent, "alice") - bob := WithNamedLevel(parent, "bob") - - SetLevelFor("alice", 0) - SetLevelFor("bob", 2) - - parent.V(0).Info("line 1") - alice.V(0).Info("line 2") - bob.V(0).Info("line 3") - - parent.V(1).Info("line 4") - alice.V(1).Info("line 5") - bob.V(1).Info("line 6") - - parent.V(2).Info("line 7") - alice.V(2).Info("line 8") - bob.V(2).Info("line 9") - - assert.Nil(t, flush()) - lines := splitLines(buf.String()) - assert.Equal(t, 5, len(lines)) - - assert.Equal(t, `info-0 parent line 1`, lines[0]) - assert.Equal(t, `info-0 parent.alice line 2`, lines[1]) - assert.Equal(t, `info-0 parent.bob line 3`, lines[2]) - assert.Equal(t, `info-1 parent line 4`, lines[3]) - assert.Equal(t, `info-1 parent.bob line 6`, lines[4]) -} - -func TestWithNamedLevelConcurrency(t *testing.T) { - var buf bytes.Buffer - globalControls = make(map[string]levelSetter, 16) - - parent, flush := New("parent", WithConsoleSink(&buf)) - - alice := WithNamedLevel(parent, "alice") - bob := WithNamedLevel(parent, "bob") - - var wg sync.WaitGroup - f := func(logger logr.Logger) { - defer wg.Done() - for i := 0; i < 100_000; i++ { - logger.Info(fmt.Sprintf("%06d", i)) - } - } - wg.Add(3) - go f(parent) - go f(alice) - go f(bob) - wg.Wait() - - assert.Nil(t, flush()) - logLines := splitLines(buf.String()) - assert.Equal(t, 300_000, len(logLines)) - sort.Slice(logLines, func(i, j int) bool { - return logLines[i] < logLines[j] - }) - - for i := 0; i < 100_000; i++ { - assert.Equal(t, fmt.Sprintf("info-0\tparent\t%06d", i), logLines[i]) - assert.Equal(t, fmt.Sprintf("info-0\tparent.alice\t%06d", i), logLines[i+100_000]) - assert.Equal(t, fmt.Sprintf("info-0\tparent.bob\t%06d", i), logLines[i+200_000]) - } -} - -func TestWithNamedLevelInheritance(t *testing.T) { - t.Run("child inherits parent level", func(t *testing.T) { - var buf bytes.Buffer - globalControls = make(map[string]levelSetter, 16) - - parent, flush := New("parent", WithConsoleSink(&buf, WithLevel(2))) - parent = parent.WithValues("key", "value") - // child will inherit parent's log level 2 - child := WithNamedLevel(parent, "child") - - parent.V(2).Info("yay") - child.V(2).Info("yay again") - assert.Nil(t, flush()) - - logLines := splitLines(buf.String()) - assert.Equal(t, []string{ - `info-2 parent yay {"key": "value"}`, - `info-2 parent.child yay again {"key": "value"}`, - }, logLines) - }) - - t.Run("child inherits existing named level", func(t *testing.T) { - var buf bytes.Buffer - globalControls = make(map[string]levelSetter, 16) - - parent, flush := New("parent", WithConsoleSink(&buf, WithLevel(2))) - parent = parent.WithValues("key", "value") - SetLevelFor("child", 0) - // child will inherit existing named level 0 - child := WithNamedLevel(parent, "child") - - parent.V(2).Info("yay") - child.V(2).Info("yay again") - assert.Nil(t, flush()) - - logLines := splitLines(buf.String()) - assert.Equal(t, []string{`info-2 parent yay {"key": "value"}`}, logLines) - }) -} - -func TestExistingChildLevel(t *testing.T) { - var buf bytes.Buffer - globalControls = make(map[string]levelSetter, 16) - - parent, flush := New("parent", WithConsoleSink(&buf, WithLevel(2))) - - SetLevelFor("child", 2) - // child should start with a level of 2 due to SetLevelFor above - child := WithNamedLevel(parent, "child") - - parent.V(2).Info("yay") - child.V(2).Info("yay again") - assert.Nil(t, flush()) - - assert.Contains(t, buf.String(), "info-2\tparent\tyay") - assert.Contains(t, buf.String(), "info-2\tparent.child\tyay again") -} - -func TestSinkWithNamedLevel(t *testing.T) { - var buf1, buf2 bytes.Buffer - globalControls = make(map[string]levelSetter, 16) - - parent, flush := New( - "parent", - WithConsoleSink(&buf1, WithLevel(0)), - WithConsoleSink(&buf2, WithLevel(2)), - ) - child := WithNamedLevel(parent, "child") - - for level := 0; level < 3; level++ { - SetLevelFor("child", int8(level)) - child.Info("") - child.V(1).Info("") - child.V(2).Info("") - } - assert.Nil(t, flush()) - - // buf1 should get only level 0 logs - assert.Equal(t, []string{ - "info-0\tparent.child", - "info-0\tparent.child", - "info-0\tparent.child", - }, splitLines(buf1.String())) - - assert.Equal(t, []string{ - // child level 0 - "info-0\tparent.child", - // child level 1 - "info-0\tparent.child", - "info-1\tparent.child", - // child level 2 - "info-0\tparent.child", - "info-1\tparent.child", - "info-2\tparent.child", - }, splitLines(buf2.String())) -} - -func TestAddLeveler(t *testing.T) { - l1, l2 := zap.NewAtomicLevel(), zap.NewAtomicLevel() - logger, _ := New("parent", WithConsoleSink(io.Discard, WithLeveler(l1))) - - t.Run("child level more verbose", func(t *testing.T) { - l1.SetLevel(0) - l2.SetLevel(1) - _, err := AddLeveler(logger, l2) - assert.Nil(t, err) - }) - - t.Run("child level less verbose", func(t *testing.T) { - l1.SetLevel(1) - l2.SetLevel(0) - _, err := AddLeveler(logger, l2) - assert.Nil(t, err) - }) -} - func splitLines(s string) []string { lines := strings.Split(strings.TrimSpace(s), "\n") logLines := make([]string, len(lines)) @@ -521,28 +233,3 @@ func TestFindLevel(t *testing.T) { assert.Equal(t, i8, findLevel(logger)) } } - -func TestOverwriteWithNamedLevel(t *testing.T) { - var buf bytes.Buffer - globalControls = make(map[string]levelSetter, 16) - - parent, flush := New( - "parent", - WithConsoleSink(&buf, WithLevel(2)), - ) - SetLevelFor("child", 0) - child1 := WithNamedLevel(parent, "child") - child2 := WithNamedLevel(parent, "child") - SetLevelFor("child", 2) - - child1.V(2).Info("") - child2.V(2).Info("") - - assert.Nil(t, flush()) - - // buf1 should get only level 0 logs - assert.Equal(t, []string{ - "info-2\tparent.child", - "info-2\tparent.child", - }, splitLines(buf.String())) -} diff --git a/pkg/sources/s3/s3.go b/pkg/sources/s3/s3.go index f5df0fd3a0d1..8c3aabca85b3 100644 --- a/pkg/sources/s3/s3.go +++ b/pkg/sources/s3/s3.go @@ -131,7 +131,6 @@ func (s *Source) setMaxObjectSize(maxObjectSize int64) { func (s *Source) newClient(region, roleArn string) (*s3.S3, error) { cfg := aws.NewConfig() cfg.CredentialsChainVerboseErrors = aws.Bool(true) - cfg.LogLevel = aws.LogLevel(aws.LogDebugWithRequestErrors) cfg.Region = aws.String(region) switch cred := s.conn.GetCredential().(type) {