From 51b7fcc5d64f226430cad3f8ab9b413aa9f2f9a5 Mon Sep 17 00:00:00 2001 From: Corben Leo <19563282+lc@users.noreply.github.com> Date: Wed, 25 Oct 2023 17:45:40 -0500 Subject: [PATCH] Detector-Competition-Fix: Fix/Remove BlaBlaBus, API retired (#1996) * Detector-Competition-Fix: Fix/Remove BlaBlaBus, API retired * Detector-Competition-Fix: Depreciate Blabus proto --- pkg/detectors/blablabus/blablabus.go | 79 -------------- pkg/detectors/blablabus/blablabus_test.go | 120 ---------------------- pkg/engine/defaults.go | 2 - proto/detectors.proto | 2 +- 4 files changed, 1 insertion(+), 202 deletions(-) delete mode 100644 pkg/detectors/blablabus/blablabus.go delete mode 100644 pkg/detectors/blablabus/blablabus_test.go diff --git a/pkg/detectors/blablabus/blablabus.go b/pkg/detectors/blablabus/blablabus.go deleted file mode 100644 index 9bca40925805..000000000000 --- a/pkg/detectors/blablabus/blablabus.go +++ /dev/null @@ -1,79 +0,0 @@ -package blablabus - -import ( - "context" - "fmt" - "net/http" - "regexp" - "strings" - - "github.com/trufflesecurity/trufflehog/v3/pkg/common" - "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" - "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" -) - -type Scanner struct{} - -// Ensure the Scanner satisfies the interface at compile time. -var _ detectors.Detector = (*Scanner)(nil) - -var ( - client = common.SaneHttpClient() - - // Make sure that your group is surrounded in boundary characters such as below to reduce false positives. - keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"blablabus"}) + `\b([0-9A-Za-z]{22})\b`) -) - -// Keywords are used for efficiently pre-filtering chunks. -// Use identifiers in the secret preferably, or the provider name. -func (s Scanner) Keywords() []string { - return []string{"blablabus"} -} - -// FromData will find and optionally verify Blablabus secrets in a given set of bytes. -func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) { - dataStr := string(data) - - matches := keyPat.FindAllStringSubmatch(dataStr, -1) - - for _, match := range matches { - if len(match) != 2 { - continue - } - resMatch := strings.TrimSpace(match[1]) - - s1 := detectors.Result{ - DetectorType: detectorspb.DetectorType_Blablabus, - Raw: []byte(resMatch), - } - - if verify { - req, err := http.NewRequestWithContext(ctx, "GET", "https://api.idbus.com/v3/stops", nil) - if err != nil { - continue - } - req.Header.Add("Content-Type", "application/json") - req.Header.Add("Authorization", fmt.Sprintf("Token %s", resMatch)) - res, err := client.Do(req) - if err == nil { - defer res.Body.Close() - if res.StatusCode >= 200 && res.StatusCode < 300 { - s1.Verified = true - } else { - // This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key. - if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) { - continue - } - } - } - } - - results = append(results, s1) - } - - return results, nil -} - -func (s Scanner) Type() detectorspb.DetectorType { - return detectorspb.DetectorType_Blablabus -} diff --git a/pkg/detectors/blablabus/blablabus_test.go b/pkg/detectors/blablabus/blablabus_test.go deleted file mode 100644 index f2d987b572a0..000000000000 --- a/pkg/detectors/blablabus/blablabus_test.go +++ /dev/null @@ -1,120 +0,0 @@ -//go:build detectors -// +build detectors - -package blablabus - -import ( - "context" - "fmt" - "testing" - "time" - - "github.com/kylelemons/godebug/pretty" - "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" - - "github.com/trufflesecurity/trufflehog/v3/pkg/common" - "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" -) - -func TestBlablabus_FromChunk(t *testing.T) { - ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) - defer cancel() - testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1") - if err != nil { - t.Fatalf("could not get test secrets from GCP: %s", err) - } - secret := testSecrets.MustGetField("BLABLABUS") - inactiveSecret := testSecrets.MustGetField("BLABLABUS_INACTIVE") - - type args struct { - ctx context.Context - data []byte - verify bool - } - tests := []struct { - name string - s Scanner - args args - want []detectors.Result - wantErr bool - }{ - { - name: "found, verified", - s: Scanner{}, - args: args{ - ctx: context.Background(), - data: []byte(fmt.Sprintf("You can find a blablabus secret %s within", secret)), - verify: true, - }, - want: []detectors.Result{ - { - DetectorType: detectorspb.DetectorType_Blablabus, - Verified: true, - }, - }, - wantErr: false, - }, - { - name: "found, unverified", - s: Scanner{}, - args: args{ - ctx: context.Background(), - data: []byte(fmt.Sprintf("You can find a blablabus secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation - verify: true, - }, - want: []detectors.Result{ - { - DetectorType: detectorspb.DetectorType_Blablabus, - Verified: false, - }, - }, - wantErr: false, - }, - { - name: "not found", - s: Scanner{}, - args: args{ - ctx: context.Background(), - data: []byte("You cannot find the secret within"), - verify: true, - }, - want: nil, - wantErr: false, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - s := Scanner{} - got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data) - if (err != nil) != tt.wantErr { - t.Errorf("Blablabus.FromData() error = %v, wantErr %v", err, tt.wantErr) - return - } - for i := range got { - if len(got[i].Raw) == 0 { - t.Fatalf("no raw secret present: \n %+v", got[i]) - } - got[i].Raw = nil - } - if diff := pretty.Compare(got, tt.want); diff != "" { - t.Errorf("Blablabus.FromData() %s diff: (-got +want)\n%s", tt.name, diff) - } - }) - } -} - -func BenchmarkFromData(benchmark *testing.B) { - ctx := context.Background() - s := Scanner{} - for name, data := range detectors.MustGetBenchmarkData() { - benchmark.Run(name, func(b *testing.B) { - b.ResetTimer() - for n := 0; n < b.N; n++ { - _, err := s.FromData(ctx, false, data) - if err != nil { - b.Fatal(err) - } - } - }) - } -} diff --git a/pkg/engine/defaults.go b/pkg/engine/defaults.go index 7fa9225a900b..9bd6c64e066a 100644 --- a/pkg/engine/defaults.go +++ b/pkg/engine/defaults.go @@ -86,7 +86,6 @@ import ( "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/bitfinex" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/bitlyaccesstoken" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/bitmex" - "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/blablabus" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/blazemeter" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/blitapp" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/blocknative" @@ -1064,7 +1063,6 @@ func DefaultDetectors() []detectors.Detector { deepgram.Scanner{}, brandfetch.Scanner{}, typeform.Scanner{}, - blablabus.Scanner{}, fxmarket.Scanner{}, ipapi.Scanner{}, clearbit.Scanner{}, diff --git a/proto/detectors.proto b/proto/detectors.proto index 3610a893d6e2..fcad85b5d833 100644 --- a/proto/detectors.proto +++ b/proto/detectors.proto @@ -361,7 +361,7 @@ enum DetectorType { Imagga = 349; SMSApi = 350; Distribusion = 351; - Blablabus = 352; + Blablabus = 352 [deprecated = true]; WordsApi = 353; Currencylayer = 354; Html2Pdf = 355;