Merge branch 'main' of github.com:trufflesecurity/trufflehog into fum…

…blehool/add-replicate-detector

Dockerfile

@@ -9,7 +9,7 @@ RUN  --mount=type=cache,target=/go/pkg/mod \
      GOOS=${TARGETOS} GOARCH=${TARGETARCH} go build -o trufflehog .
 
 FROM alpine:3.18
-RUN apk add --no-cache bash git openssh-client ca-certificates rpm2cpio \
+RUN apk add --no-cache bash git openssh-client ca-certificates rpm2cpio binutils cpio \
     && rm -rf /var/cache/apk/* && update-ca-certificates
 COPY --from=builder /build/trufflehog /usr/bin/trufflehog
 COPY entrypoint.sh /etc/entrypoint.sh

Dockerfile.goreleaser

@@ -1,6 +1,6 @@
 FROM alpine:3.18
 
-RUN apk add --no-cache bash git openssh-client ca-certificates rpm2cpio \
+RUN apk add --no-cache bash git openssh-client ca-certificates rpm2cpio binutils cpio \
     && rm -rf /var/cache/apk/* && update-ca-certificates
 WORKDIR /usr/bin/
 COPY trufflehog .

hack/snifftest/main.go

@@ -204,7 +204,7 @@ func main() {
 					})
 
 				logger.Info("scanning repo", "repo", r)
-				err = s.ScanRepo(ctx, repo, path, git.NewScanOptions(), chunksChan)
+				err = s.ScanRepo(ctx, repo, path, git.NewScanOptions(), sources.ChanReporter{Ch: chunksChan})
 				if err != nil {
 					logFatal(err, "error scanning repo")
 				}

pkg/cache/memory/memory.go

@@ -94,7 +94,7 @@ func (c *Cache) Values() []string {
 	return res
 }
 
-// Contents returns all key-value pairs in the cache encodes as a string.
+// Contents returns a comma-separated string containing all keys in the cache.
 func (c *Cache) Contents() string {
 	items := c.c.Items()
 	res := make([]string, 0, len(items))

pkg/detectors/appfollow/appfollow.go

@@ -2,8 +2,6 @@ package appfollow
 
 import (
 	"context"
-	b64 "encoding/base64"
-	"fmt"
 	"net/http"
 	"regexp"
 	"strings"
@@ -22,7 +20,7 @@ var (
 	client = common.SaneHttpClient()
 
 	// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
-	keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"appfollow"}) + `\b([0-9A-Za-z]{20})\b`)
+	keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"appfollow"}) + `\b(eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9\.[0-9A-Za-z]{74}\.[0-9A-Z-a-z\-_]{43})\b`)
 )
 
 // Keywords are used for efficiently pre-filtering chunks.
@@ -49,13 +47,11 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 		}
 
 		if verify {
-			data := fmt.Sprintf("%s:", resMatch)
-			sEnc := b64.StdEncoding.EncodeToString([]byte(data))
-			req, err := http.NewRequestWithContext(ctx, "GET", "https://api.appfollow.io/test", nil)
+			req, err := http.NewRequestWithContext(ctx, "GET", "https://api.appfollow.io/api/v2/account/users", nil)
 			if err != nil {
 				continue
 			}
-			req.Header.Add("Authorization", fmt.Sprintf("Basic %s", sEnc))
+			req.Header.Add("X-AppFollow-API-Token", resMatch)
 			res, err := client.Do(req)
 			if err == nil {
 				defer res.Body.Close()

pkg/detectors/aws/aws.go

@@ -22,6 +22,22 @@ type scanner struct {
 	skipIDs            map[string]struct{}
 }
 
+// resourceTypes derived from: https://docs.aws.amazon.com/IAM/latest/UserGuide/reference_identifiers.html#identifiers-unique-ids
+var resourceTypes = map[string]string{
+	"ABIA": "AWS STS service bearer token",
+	"ACCA": "Context-specific credential",
+	"AGPA": "User group",
+	"AIDA": "IAM user",
+	"AIPA": "Amazon EC2 instance profile",
+	"AKIA": "Access key",
+	"ANPA": "Managed policy",
+	"ANVA": "Version in a managed policy",
+	"APKA": "Public key",
+	"AROA": "Role",
+	"ASCA": "Certificate",
+	"ASIA": "Temporary (AWS STS) access key IDs",
+}
+
 func New(opts ...func(*scanner)) *scanner {
 	scanner := &scanner{
 		skipIDs: map[string]struct{}{},
@@ -53,7 +69,7 @@ var (
 
 	// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
 	// Key types are from this list https://docs.aws.amazon.com/IAM/latest/UserGuide/reference_identifiers.html#identifiers-unique-ids
-	idPat     = regexp.MustCompile(`\b((?:AKIA|ABIA|ACCA|ASIA)[0-9A-Z]{16})\b`)
+	idPat     = regexp.MustCompile(`\b((AKIA|ABIA|ACCA)[0-9A-Z]{16})\b`)
 	secretPat = regexp.MustCompile(`[^A-Za-z0-9+\/]{0,1}([A-Za-z0-9+\/]{40})[^A-Za-z0-9+\/]{0,1}`)
 	// Hashes, like those for git, do technically match the secret pattern.
 	// But they are extremely unlikely to be generated as an actual AWS secret.
@@ -93,7 +109,7 @@ func (s scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 	secretMatches := secretPat.FindAllStringSubmatch(dataStr, -1)
 
 	for _, idMatch := range idMatches {
-		if len(idMatch) != 2 {
+		if len(idMatch) != 3 {
 			continue
 		}
 		resIDMatch := strings.TrimSpace(idMatch[1])
@@ -115,12 +131,19 @@ func (s scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 				Raw:          []byte(resIDMatch),
 				Redacted:     resIDMatch,
 				RawV2:        []byte(resIDMatch + resSecretMatch),
+				ExtraData: map[string]string{
+					"resource_type": resourceTypes[idMatch[2]],
+				},
 			}
 
 			if verify {
 				verified, extraData, verificationErr := s.verifyMatch(ctx, resIDMatch, resSecretMatch, true)
 				s1.Verified = verified
-				s1.ExtraData = extraData
+				//Append the extraData to the existing ExtraData map.
+				// This will overwrite with the new verified values.
+				for k, v := range extraData {
+					s1.ExtraData[k] = v
+				}
 				s1.VerificationError = verificationErr
 			}
 

pkg/detectors/aws/aws_test.go

@@ -61,9 +61,11 @@ func TestAWS_FromChunk(t *testing.T) {
 					Verified:     true,
 					Redacted:     "AKIASP2TPHJSQH3FJRUX",
 					ExtraData: map[string]string{
-						"account": "171436882533",
-						"arn":     "arn:aws:iam::171436882533:user/canarytokens.com@@4dxkh0pdeop3bzu9zx5wob793",
-						"user_id": "AIDASP2TPHJSUFRSTTZX4",
+						"resource_type":  "Access key",
+						"rotation_guide": "https://howtorotate.com/docs/tutorials/aws/",
+						"account":        "171436882533",
+						"arn":            "arn:aws:iam::171436882533:user/canarytokens.com@@4dxkh0pdeop3bzu9zx5wob793",
+						"user_id":        "AIDASP2TPHJSUFRSTTZX4",
 					},
 				},
 			},
@@ -82,7 +84,7 @@ func TestAWS_FromChunk(t *testing.T) {
 					DetectorType: detectorspb.DetectorType_AWS,
 					Verified:     false,
 					Redacted:     "AKIASP2TPHJSQH3FJRUX",
-					ExtraData:    nil,
+					ExtraData:    map[string]string{"resource_type": "Access key"},
 				},
 			},
 			wantErr: false,
@@ -111,15 +113,20 @@ func TestAWS_FromChunk(t *testing.T) {
 					DetectorType: detectorspb.DetectorType_AWS,
 					Verified:     false,
 					Redacted:     "AKIASP2TPHJSQH3FJXYZ",
+					ExtraData: map[string]string{
+						"resource_type": "Access key",
+					},
 				},
 				{
 					DetectorType: detectorspb.DetectorType_AWS,
 					Verified:     true,
 					Redacted:     "AKIASP2TPHJSQH3FJRUX",
 					ExtraData: map[string]string{
-						"account": "171436882533",
-						"arn":     "arn:aws:iam::171436882533:user/canarytokens.com@@4dxkh0pdeop3bzu9zx5wob793",
-						"user_id": "AIDASP2TPHJSUFRSTTZX4",
+						"resource_type":  "Access key",
+						"rotation_guide": "https://howtorotate.com/docs/tutorials/aws/",
+						"account":        "171436882533",
+						"arn":            "arn:aws:iam::171436882533:user/canarytokens.com@@4dxkh0pdeop3bzu9zx5wob793",
+						"user_id":        "AIDASP2TPHJSUFRSTTZX4",
 					},
 				},
 			},
@@ -150,9 +157,11 @@ func TestAWS_FromChunk(t *testing.T) {
 					Verified:     true,
 					Redacted:     "AKIASP2TPHJSQH3FJRUX",
 					ExtraData: map[string]string{
-						"account": "171436882533",
-						"arn":     "arn:aws:iam::171436882533:user/canarytokens.com@@4dxkh0pdeop3bzu9zx5wob793",
-						"user_id": "AIDASP2TPHJSUFRSTTZX4",
+						"resource_type":  "Access key",
+						"rotation_guide": "https://howtorotate.com/docs/tutorials/aws/",
+						"account":        "171436882533",
+						"arn":            "arn:aws:iam::171436882533:user/canarytokens.com@@4dxkh0pdeop3bzu9zx5wob793",
+						"user_id":        "AIDASP2TPHJSUFRSTTZX4",
 					},
 				},
 				{
@@ -176,6 +185,9 @@ func TestAWS_FromChunk(t *testing.T) {
 					DetectorType: detectorspb.DetectorType_AWS,
 					Verified:     false,
 					Redacted:     "AKIASP2TPHJSQH3FJRUX",
+					ExtraData: map[string]string{
+						"resource_type": "Access key",
+					},
 				},
 			},
 			wantErr: false,
@@ -207,6 +219,9 @@ func TestAWS_FromChunk(t *testing.T) {
 					DetectorType: detectorspb.DetectorType_AWS,
 					Verified:     false,
 					Redacted:     "AKIASP2TPHJSQH3FJRUX",
+					ExtraData: map[string]string{
+						"resource_type": "Access key",
+					},
 				},
 			},
 			wantErr:               false,
@@ -225,6 +240,9 @@ func TestAWS_FromChunk(t *testing.T) {
 					DetectorType: detectorspb.DetectorType_AWS,
 					Verified:     false,
 					Redacted:     "AKIASP2TPHJSQH3FJRUX",
+					ExtraData: map[string]string{
+						"resource_type": "Access key",
+					},
 				},
 			},
 			wantErr:               false,
@@ -243,6 +261,9 @@ func TestAWS_FromChunk(t *testing.T) {
 					DetectorType: detectorspb.DetectorType_AWS,
 					Verified:     false,
 					Redacted:     "AKIASP2TPHJSQH3FJRUX",
+					ExtraData: map[string]string{
+						"resource_type": "Access key",
+					},
 				},
 			},
 			wantErr:               false,
@@ -262,9 +283,11 @@ func TestAWS_FromChunk(t *testing.T) {
 					Verified:     true,
 					Redacted:     "AKIASP2TPHJSQH3FJRUX",
 					ExtraData: map[string]string{
-						"account": "171436882533",
-						"arn":     "arn:aws:iam::171436882533:user/canarytokens.com@@4dxkh0pdeop3bzu9zx5wob793",
-						"user_id": "AIDASP2TPHJSUFRSTTZX4",
+						"resource_type":  "Access key",
+						"rotation_guide": "https://howtorotate.com/docs/tutorials/aws/",
+						"account":        "171436882533",
+						"arn":            "arn:aws:iam::171436882533:user/canarytokens.com@@4dxkh0pdeop3bzu9zx5wob793",
+						"user_id":        "AIDASP2TPHJSUFRSTTZX4",
 					},
 				},
 			},