From e2fecb1a31e67901ae0cd31fbc0bc4583967cd37 Mon Sep 17 00:00:00 2001 From: Zachary Rice Date: Mon, 25 Sep 2023 11:16:37 -0500 Subject: [PATCH] rm brightlocal, hotwire, update dynadot test --- pkg/detectors/brightlocal/brightlocal.go | 92 -------------- pkg/detectors/brightlocal/brightlocal_test.go | 116 ------------------ pkg/detectors/dynadot/dynadot_test.go | 2 +- pkg/detectors/hotwire/hotwire.go | 85 ------------- pkg/detectors/hotwire/hotwire_test.go | 116 ------------------ 5 files changed, 1 insertion(+), 410 deletions(-) delete mode 100644 pkg/detectors/brightlocal/brightlocal.go delete mode 100644 pkg/detectors/brightlocal/brightlocal_test.go delete mode 100644 pkg/detectors/hotwire/hotwire.go delete mode 100644 pkg/detectors/hotwire/hotwire_test.go diff --git a/pkg/detectors/brightlocal/brightlocal.go b/pkg/detectors/brightlocal/brightlocal.go deleted file mode 100644 index a4d3ac54b91a..000000000000 --- a/pkg/detectors/brightlocal/brightlocal.go +++ /dev/null @@ -1,92 +0,0 @@ -package brightlocal - -import ( - "context" - "fmt" - "net/http" - "regexp" - "strings" - "time" - - "github.com/trufflesecurity/trufflehog/v3/pkg/common" - "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" - "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" -) - -// Ensure the Scanner satisfies the interface at compile time -type Scanner struct { - client *http.Client -} - -var _ detectors.Detector = (*Scanner)(nil) - -var ( - defaultClient = common.SaneHttpClientTimeOut(5 * time.Second) - - //Make sure that your group is surrounded in boundry characters such as below to reduce false positives - keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"brightlocal"}) + `\b([a-f0-9]{40})\b`) -) - -// Keywords are used for efficiently pre-filtering chunks. -// Use identifiers in the secret preferably, or the provider name. -func (s Scanner) Keywords() []string { - return []string{"brightlocal"} -} - -// FromData will find and optionally verify Brightlocal secrets in a given set of bytes. -func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) { - dataStr := string(data) - - matches := keyPat.FindAllStringSubmatch(dataStr, -1) - - for _, match := range matches { - if len(match) != 2 { - continue - } - resMatch := strings.TrimSpace(match[1]) - - s1 := detectors.Result{ - DetectorType: detectorspb.DetectorType_Brightlocal, - Raw: []byte(resMatch), - } - - if verify { - client := s.client - if client == nil { - client = defaultClient - } - payload := strings.NewReader(fmt.Sprintf("api-key=%s", resMatch)) - req, err := http.NewRequestWithContext(ctx, "POST", "https://tools.brightlocal.com/seo-tools/api/v4/batch", payload) - if err != nil { - s1.VerificationError = err - } else { - req.Header.Add("Content-Type", "application/x-www-form-urlencoded") - res, err := client.Do(req) - if err != nil { - s1.VerificationError = err - } else { - defer res.Body.Close() - if res.StatusCode >= 200 && res.StatusCode < 300 { - s1.Verified = true - } else if res.StatusCode != 401 { - s1.VerificationError = fmt.Errorf("unexpected HTTP response status %d", res.StatusCode) - } - - //This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key - - } - } - } - if !s1.Verified && detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) { - continue - } - - results = append(results, s1) - } - - return detectors.CleanResults(results), nil -} - -func (s Scanner) Type() detectorspb.DetectorType { - return detectorspb.DetectorType_Brightlocal -} diff --git a/pkg/detectors/brightlocal/brightlocal_test.go b/pkg/detectors/brightlocal/brightlocal_test.go deleted file mode 100644 index e868b783e21e..000000000000 --- a/pkg/detectors/brightlocal/brightlocal_test.go +++ /dev/null @@ -1,116 +0,0 @@ -package brightlocal - -import ( - "context" - "fmt" - "testing" - "time" - - "github.com/kylelemons/godebug/pretty" - "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" - - "github.com/trufflesecurity/trufflehog/v3/pkg/common" - "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" -) - -func TestBrightlocal_FromChunk(t *testing.T) { - ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) - defer cancel() - testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors4") - if err != nil { - t.Fatalf("could not get test secrets from GCP: %s", err) - } - secret := testSecrets.MustGetField("BRIGHTLOCAL") - inactiveSecret := testSecrets.MustGetField("BRIGHTLOCAL_INACTIVE") - - type args struct { - ctx context.Context - data []byte - verify bool - } - tests := []struct { - name string - s Scanner - args args - want []detectors.Result - wantErr bool - }{ - { - name: "found, verified", - s: Scanner{}, - args: args{ - ctx: context.Background(), - data: []byte(fmt.Sprintf("You can find a brightlocal secret %s within", secret)), - verify: true, - }, - want: []detectors.Result{ - { - DetectorType: detectorspb.DetectorType_Brightlocal, - Verified: true, - }, - }, - wantErr: false, - }, - { - name: "found, unverified", - s: Scanner{}, - args: args{ - ctx: context.Background(), - data: []byte(fmt.Sprintf("You can find a brightlocal secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation - verify: true, - }, - want: []detectors.Result{ - { - DetectorType: detectorspb.DetectorType_Brightlocal, - Verified: false, - }, - }, - wantErr: false, - }, - { - name: "not found", - s: Scanner{}, - args: args{ - ctx: context.Background(), - data: []byte("You cannot find the secret within"), - verify: true, - }, - want: nil, - wantErr: false, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - s := Scanner{} - got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data) - if (err != nil) != tt.wantErr { - t.Errorf("Brightlocal.FromData() error = %v, wantErr %v", err, tt.wantErr) - return - } - for i := range got { - if len(got[i].Raw) == 0 { - t.Fatalf("no raw secret present: \n %+v", got[i]) - } - got[i].Raw = nil - } - if diff := pretty.Compare(got, tt.want); diff != "" { - t.Errorf("Brightlocal.FromData() %s diff: (-got +want)\n%s", tt.name, diff) - } - }) - } -} - -func BenchmarkFromData(benchmark *testing.B) { - ctx := context.Background() - s := Scanner{} - for name, data := range detectors.MustGetBenchmarkData() { - benchmark.Run(name, func(b *testing.B) { - for n := 0; n < b.N; n++ { - _, err := s.FromData(ctx, false, data) - if err != nil { - b.Fatal(err) - } - } - }) - } -} diff --git a/pkg/detectors/dynadot/dynadot_test.go b/pkg/detectors/dynadot/dynadot_test.go index 1ba857bf6263..be2fb1531eb1 100644 --- a/pkg/detectors/dynadot/dynadot_test.go +++ b/pkg/detectors/dynadot/dynadot_test.go @@ -16,7 +16,7 @@ import ( func TestDynadot_FromChunk(t *testing.T) { ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) defer cancel() - testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors4") + testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors5") if err != nil { t.Fatalf("could not get test secrets from GCP: %s", err) } diff --git a/pkg/detectors/hotwire/hotwire.go b/pkg/detectors/hotwire/hotwire.go deleted file mode 100644 index 4036fb766d40..000000000000 --- a/pkg/detectors/hotwire/hotwire.go +++ /dev/null @@ -1,85 +0,0 @@ -package hotwire - -import ( - "context" - "fmt" - "net/http" - "regexp" - "strings" - "time" - - "github.com/trufflesecurity/trufflehog/v3/pkg/common" - "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" - "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" -) - -type Scanner struct { - client *http.Client -} - -// Ensure the Scanner satisfies the interface at compile time -var _ detectors.Detector = (*Scanner)(nil) - -var ( - defaultClient = common.SaneHttpClientTimeOut(5 * time.Second) - - // Make sure that your group is surrounded in boundary characters such as below to reduce false positives - keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"hotwire"}) + `\b([a-z0-9]{24})\b`) -) - -func (s Scanner) Keywords() []string { - return []string{"hotwire"} -} - -func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) { - dataStr := string(data) - - matches := keyPat.FindAllStringSubmatch(dataStr, -1) - - for _, match := range matches { - if len(match) != 2 { - continue - } - resMatch := strings.TrimSpace(match[1]) - - s1 := detectors.Result{ - DetectorType: detectorspb.DetectorType_Hotwire, - Raw: []byte(resMatch), - } - - if verify { - client := s.client - if client == nil { - client = defaultClient - } - req, err := http.NewRequestWithContext(ctx, "GET", "http://api.hotwire.com/v1/search/car?apikey="+resMatch+"&dest=LAX&startdate=04/23/2022&enddate=04/25/2022&pickuptime=10:00&dropofftime=13:30", nil) - if err != nil { - s1.VerificationError = err - } else { - res, err := client.Do(req) - if err != nil { - s1.VerificationError = err - } else { - defer res.Body.Close() - if res.StatusCode >= 200 && res.StatusCode < 300 { - s1.Verified = true - } else if res.StatusCode != 401 { - s1.VerificationError = fmt.Errorf("unexpected HTTP response status %d", res.StatusCode) - } - } - } - } - - if !s1.Verified && detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) { - continue - } - - results = append(results, s1) - } - - return detectors.CleanResults(results), nil -} - -func (s Scanner) Type() detectorspb.DetectorType { - return detectorspb.DetectorType_Hotwire -} diff --git a/pkg/detectors/hotwire/hotwire_test.go b/pkg/detectors/hotwire/hotwire_test.go deleted file mode 100644 index 3f0bda1fb2e1..000000000000 --- a/pkg/detectors/hotwire/hotwire_test.go +++ /dev/null @@ -1,116 +0,0 @@ -package hotwire - -import ( - "context" - "fmt" - "testing" - "time" - - "github.com/kylelemons/godebug/pretty" - "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" - - "github.com/trufflesecurity/trufflehog/v3/pkg/common" - "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" -) - -func TestHotwire_FromChunk(t *testing.T) { - ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) - defer cancel() - testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors4") - if err != nil { - t.Fatalf("could not get test secrets from GCP: %s", err) - } - secret := testSecrets.MustGetField("HOTWIRE") - inactiveSecret := testSecrets.MustGetField("HOTWIRE_INACTIVE") - - type args struct { - ctx context.Context - data []byte - verify bool - } - tests := []struct { - name string - s Scanner - args args - want []detectors.Result - wantErr bool - }{ - { - name: "found, verified", - s: Scanner{}, - args: args{ - ctx: context.Background(), - data: []byte(fmt.Sprintf("You can find a hotwire secret %s within", secret)), - verify: true, - }, - want: []detectors.Result{ - { - DetectorType: detectorspb.DetectorType_Hotwire, - Verified: true, - }, - }, - wantErr: false, - }, - { - name: "found, unverified", - s: Scanner{}, - args: args{ - ctx: context.Background(), - data: []byte(fmt.Sprintf("You can find a hotwire secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation - verify: true, - }, - want: []detectors.Result{ - { - DetectorType: detectorspb.DetectorType_Hotwire, - Verified: false, - }, - }, - wantErr: false, - }, - { - name: "not found", - s: Scanner{}, - args: args{ - ctx: context.Background(), - data: []byte("You cannot find the secret within"), - verify: true, - }, - want: nil, - wantErr: false, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - s := Scanner{} - got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data) - if (err != nil) != tt.wantErr { - t.Errorf("Hotwire.FromData() error = %v, wantErr %v", err, tt.wantErr) - return - } - for i := range got { - if len(got[i].Raw) == 0 { - t.Fatalf("no raw secret present: \n %+v", got[i]) - } - got[i].Raw = nil - } - if diff := pretty.Compare(got, tt.want); diff != "" { - t.Errorf("Hotwire.FromData() %s diff: (-got +want)\n%s", tt.name, diff) - } - }) - } -} - -func BenchmarkFromData(benchmark *testing.B) { - ctx := context.Background() - s := Scanner{} - for name, data := range detectors.MustGetBenchmarkData() { - benchmark.Run(name, func(b *testing.B) { - for n := 0; n < b.N; n++ { - _, err := s.FromData(ctx, false, data) - if err != nil { - b.Fatal(err) - } - } - }) - } -}