Skip to content

Commit

Permalink
Detectors Updates 1 for Tristate Verification (#2187)
Browse files Browse the repository at this point in the history 
* updating alibaba

* updating agora

* updating aeroworkflow

* updating aha

* updating artifactory

* updating abbysale

* updating abstract

* updating abuseipdb

* updating accuweather

* updating adafruitio

* updating adzuna

* cleanup on abuseipdb

* cleanup on aha

* cleanup on abuseipdb

* cleanup on aeroworkflow

* cleanup on adzuna

* cleanup on accuweather

* cleanup/refactor

* update token pattern to be explicitly 73char (old) or 64char (new)

* comment to clarify 403 on Aha

* mocking out verified case for aha + adding inactive account test

* using contact response instead of gock

* update 403 to be determinate
  • Loading branch information
@0x1
0x1 authored Jan 30, 2024
1 parent 2320324 commit 7ece4c3
Show file tree
Hide file tree
Showing 22 changed files with 1,247 additions and 355 deletions.
72 changes: 50 additions & 22 deletions pkg/detectors/abbysale/abbysale.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,22 +2,28 @@ package abbysale

import (
"context"
regexp "github.com/wasilibs/go-re2"
"fmt"
"net/http"
"strings"

regexp "github.com/wasilibs/go-re2"

"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
)

type Scanner struct{}
type Scanner struct {
client *http.Client
}

// Ensure the Scanner satisfies the interface at compile time.
var _ detectors.Detector = (*Scanner)(nil)
const abbysaleURL = "https://api.abyssale.com"

var (
client = common.SaneHttpClient()
// Ensure the Scanner satisfies the interface at compile time.
_ detectors.Detector = (*Scanner)(nil)

defaultClient = common.SaneHttpClient()

// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"abbysale"}) + `\b([a-z0-9A-Z]{40})\b`)
Expand All @@ -29,6 +35,13 @@ func (s Scanner) Keywords() []string {
return []string{"abbysale"}
}

func (s Scanner) getClient() *http.Client {
if s.client != nil {
return s.client
}
return defaultClient
}

// FromData will find and optionally verify Abbysale secrets in a given set of bytes.
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
dataStr := string(data)
Expand All @@ -47,23 +60,15 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
}

if verify {
req, err := http.NewRequestWithContext(ctx, "GET", "https://api.abyssale.com/ready", nil)
if err != nil {
continue
}
req.Header.Add("x-api-key", resMatch)
res, err := client.Do(req)
if err == nil {
defer res.Body.Close()
if res.StatusCode >= 200 && res.StatusCode < 300 {
s1.Verified = true
} else {
// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
continue
}
}
}
client := s.getClient()
isVerified, verificationErr := verifyAbbysale(ctx, client, resMatch)
s1.Verified = isVerified
s1.SetVerificationError(verificationErr, resMatch)
}

// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
if !s1.Verified && detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
continue
}

results = append(results, s1)
Expand All @@ -72,6 +77,29 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
return results, nil
}

func verifyAbbysale(ctx context.Context, client *http.Client, resMatch string) (bool, error) {
// https://developers.abyssale.com/rest-api/authentication
req, err := http.NewRequestWithContext(ctx, http.MethodGet, abbysaleURL+"/ready", nil)
if err != nil {
return false, err
}
req.Header.Add("x-api-key", resMatch)
res, err := client.Do(req)
if err != nil {
return false, err
}
defer res.Body.Close()

switch res.StatusCode {
case http.StatusOK:
return true, nil
case http.StatusForbidden:
return false, nil
default:
return false, fmt.Errorf("unexpected HTTP response status %d", res.StatusCode)
}
}

func (s Scanner) Type() detectorspb.DetectorType {
return detectorspb.DetectorType_Abbysale
}
62 changes: 55 additions & 7 deletions pkg/detectors/abbysale/abbysale_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,8 @@ import (
"testing"
"time"

"github.com/kylelemons/godebug/pretty"
"github.com/google/go-cmp/cmp"
"github.com/google/go-cmp/cmp/cmpopts"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"

"github.com/trufflesecurity/trufflehog/v3/pkg/common"
Expand Down Expand Up @@ -43,7 +44,7 @@ func TestAbbysale_FromChunk(t *testing.T) {
s: Scanner{},
args: args{
ctx: context.Background(),
data: []byte(fmt.Sprintf("You can find a abbysale secret %s within", secret)),
data: []byte(fmt.Sprintf("You can find a abbysale secret %s within but verified", secret)),
verify: true,
},
want: []detectors.Result{
Expand All @@ -54,12 +55,48 @@ func TestAbbysale_FromChunk(t *testing.T) {
},
wantErr: false,
},
{
name: "found, real secrets, verification error due to timeout",
s: Scanner{client: common.SaneHttpClientTimeOut(1 * time.Microsecond)},
args: args{
ctx: context.Background(),
data: []byte(fmt.Sprintf("You can find a abbysale secret %s within but verified", secret)),
verify: true,
},
want: func() []detectors.Result {
r := detectors.Result{
DetectorType: detectorspb.DetectorType_Abbysale,
Verified: false,
}
r.SetVerificationError(context.DeadlineExceeded)
return []detectors.Result{r}
}(),
wantErr: false,
},
{
name: "found, real secrets, verification error due to unexpected api surface",
s: Scanner{client: common.ConstantResponseHttpClient(500, "{}")},
args: args{
ctx: context.Background(),
data: []byte(fmt.Sprintf("You can find a abbysale secret %s within but verified", secret)),
verify: true,
},
want: func() []detectors.Result {
r := detectors.Result{
DetectorType: detectorspb.DetectorType_Abbysale,
Verified: false,
}
r.SetVerificationError(fmt.Errorf("unexpected HTTP response status 500"))
return []detectors.Result{r}
}(),
wantErr: false,
},
{
name: "found, unverified",
s: Scanner{},
args: args{
ctx: context.Background(),
data: []byte(fmt.Sprintf("You can find a abbysale secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
data: []byte(fmt.Sprintf("You can find a abbysale secret %s within but verified", inactiveSecret)), // the secret would satisfy the regex but not pass validation
verify: true,
},
want: []detectors.Result{
Expand All @@ -84,8 +121,7 @@ func TestAbbysale_FromChunk(t *testing.T) {
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
s := Scanner{}
got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
if (err != nil) != tt.wantErr {
t.Errorf("Abbysale.FromData() error = %v, wantErr %v", err, tt.wantErr)
return
Expand All @@ -94,9 +130,21 @@ func TestAbbysale_FromChunk(t *testing.T) {
if len(got[i].Raw) == 0 {
t.Fatalf("no raw secret present: \n %+v", got[i])
}
got[i].Raw = nil
gotErr := ""
if got[i].VerificationError() != nil {
gotErr = got[i].VerificationError().Error()
}
wantErr := ""
if tt.want[i].VerificationError() != nil {
wantErr = tt.want[i].VerificationError().Error()
}

if gotErr != wantErr {
t.Fatalf("wantVerificationError = %v, verification error = %v", tt.want[i].VerificationError(), got[i].VerificationError())
}
}
if diff := pretty.Compare(got, tt.want); diff != "" {
ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "RawV2", "verificationError")
if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {
t.Errorf("Abbysale.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
}
})
Expand Down
69 changes: 48 additions & 21 deletions pkg/detectors/abstract/abstract.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,13 +12,17 @@ import (
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
)

type Scanner struct{}
type Scanner struct {
client *http.Client
}

// Ensure the Scanner satisfies the interface at compile time.
var _ detectors.Detector = (*Scanner)(nil)
const abstractURL = "https://exchange-rates.abstractapi.com"

var (
client = common.SaneHttpClient()
// Ensure the Scanner satisfies the interface at compile time.
_ detectors.Detector = (*Scanner)(nil)

defaultClient = common.SaneHttpClient()

// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"abstract"}) + `\b([0-9a-z]{32})\b`)
Expand All @@ -30,6 +34,13 @@ func (s Scanner) Keywords() []string {
return []string{"abstract"}
}

func (s Scanner) getClient() *http.Client {
if s.client != nil {
return s.client
}
return defaultClient
}

// FromData will find and optionally verify Abstract secrets in a given set of bytes.
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
dataStr := string(data)
Expand All @@ -48,23 +59,15 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
}

if verify {
req, err := http.NewRequestWithContext(ctx, "GET", fmt.Sprintf("https://exchange-rates.abstractapi.com/v1/live/?api_key=%s&base=USD", resMatch), nil)
if err != nil {
continue
}
req.Header.Add("Content-Type", "application/json")
res, err := client.Do(req)
if err == nil {
defer res.Body.Close()
if res.StatusCode >= 200 && res.StatusCode < 300 {
s1.Verified = true
} else {
// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
continue
}
}
}
client := s.getClient()
isVerified, verificationErr := verifyAbstract(ctx, client, resMatch)
s1.Verified = isVerified
s1.SetVerificationError(verificationErr, resMatch)
}

// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
if !s1.Verified && detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
continue
}

results = append(results, s1)
Expand All @@ -73,6 +76,30 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
return results, nil
}

func verifyAbstract(ctx context.Context, client *http.Client, resMatch string) (bool, error) {
// https://docs.abstractapi.com/exchange-rates#response-and-error-codes
req, err := http.NewRequestWithContext(ctx, http.MethodGet, abstractURL+fmt.Sprintf("/v1/live/?api_key=%s&base=USD", resMatch), nil)
if err != nil {
return false, err
}
req.Header.Add("Content-Type", "application/json")
res, err := client.Do(req)
if err != nil {
return false, err
}
defer res.Body.Close()

// https://docs.abstractapi.com/exchange-rates#response-and-error-codes
switch res.StatusCode {
case http.StatusOK:
return true, nil
case http.StatusUnauthorized:
return false, nil
default:
return false, fmt.Errorf("unexpected HTTP response status %d", res.StatusCode)
}
}

func (s Scanner) Type() detectorspb.DetectorType {
return detectorspb.DetectorType_Abstract
}
Loading

0 comments on commit 7ece4c3

Please sign in to comment.