Improve private key detector (#1760)

* Surface extra data and check private keys directly against gitlab and github

* fix encrpypted private key test

* implement feedback

* mod tidy

* fix change

* Set timeout for SSH connections

go.mod

@@ -11,6 +11,7 @@ require (
 	cloud.google.com/go/secretmanager v1.11.1
 	cloud.google.com/go/storage v1.31.0
 	github.com/Azure/go-autorest/autorest/azure/auth v0.5.11
+	github.com/AzureAD/microsoft-authentication-library-for-go v0.5.1
 	github.com/BobuSumisu/aho-corasick v1.0.3
 	github.com/TheZeroSlave/zapsentry v1.17.0
 	github.com/aws/aws-sdk-go v1.44.83
@@ -203,7 +204,7 @@ require (
 	github.com/minio/asm2plan9s v0.0.0-20200509001527-cdd76441f9d8 // indirect
 	github.com/minio/c2goasm v0.0.0-20190812172519-36a3d3bbc4f3 // indirect
 	github.com/mitchellh/go-homedir v1.1.0 // indirect
-	github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe // indirect
+	github.com/montanaflynn/stats v0.6.6 // indirect
 	github.com/mtibben/percent v0.2.1 // indirect
 	github.com/muesli/ansi v0.0.0-20211031195517-c9f0611b6c70 // indirect
 	github.com/muesli/cancelreader v0.2.2 // indirect

go.sum

@@ -304,6 +304,7 @@ github.com/goccy/go-json v0.10.0 h1:mXKd9Qw4NuzShiRlOXKews24ufknHO7gx30lsDyokKA=
 github.com/goccy/go-json v0.10.0/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I=
 github.com/godbus/dbus v0.0.0-20190726142602-4481cbc300e2 h1:ZpnhV/YsD2/4cESfV5+Hoeu/iUR3ruzNvZ+yQfO03a0=
 github.com/godbus/dbus v0.0.0-20190726142602-4481cbc300e2/go.mod h1:bBOAhwG1umN6/6ZUMtDFBMQR8jRg9O75tm9K00oMsK4=
+github.com/golang-jwt/jwt v3.2.1+incompatible/go.mod h1:8pz2t5EyA70fFQQSrl6XZXzqecmYZeUEB8OUGHkxJ+I=
 github.com/golang-jwt/jwt v3.2.2+incompatible h1:IfV12K8xAKAnZqdXVzCZ+TOjboZ2keLg81eXfW3O+oY=
 github.com/golang-jwt/jwt v3.2.2+incompatible/go.mod h1:8pz2t5EyA70fFQQSrl6XZXzqecmYZeUEB8OUGHkxJ+I=
 github.com/golang-jwt/jwt/v4 v4.0.0/go.mod h1:/xlHOz8bRuivTWchD4jCa+NbatV+wEUSzwAxVc6locg=
@@ -382,6 +383,7 @@ github.com/google/pprof v0.0.0-20211214055906-6f57359322fd/go.mod h1:KgnwoLYCZ8I
 github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
 github.com/google/s2a-go v0.1.4 h1:1kZ/sQM3srePvKs3tXAvQzo66XfcReoqFpIpIccE7Oc=
 github.com/google/s2a-go v0.1.4/go.mod h1:Ej+mSEMGRnqRzjc7VtF+jdBwYG5fuJfiZ8ELkjEwM0A=
+github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I=
 github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
@@ -518,8 +520,9 @@ github.com/minio/c2goasm v0.0.0-20190812172519-36a3d3bbc4f3/go.mod h1:RagcQ7I8Ie
 github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y=
 github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
 github.com/modocache/gover v0.0.0-20171022184752-b58185e213c5/go.mod h1:caMODM3PzxT8aQXRPkAt8xlV/e7d7w8GM5g0fa5F0D8=
-github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe h1:iruDEfMl2E6fbMZ9s0scYfZQ84/6SPL6zC8ACM2oIL0=
 github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc=
+github.com/montanaflynn/stats v0.6.6 h1:Duep6KMIDpY4Yo11iFsvyqJDyfzLF9+sndUKT+v64GQ=
+github.com/montanaflynn/stats v0.6.6/go.mod h1:etXPPgVO6n31NxCd9KQUMvCM+ve0ruNzt6R8Bnaayow=
 github.com/mtibben/percent v0.2.1 h1:5gssi8Nqo8QU/r2pynCm+hBQHpkB/uNK7BJCFogWdzs=
 github.com/mtibben/percent v0.2.1/go.mod h1:KG9uO+SZkUp+VkRHsCdYQV3XSZrrSpR3O9ibNBTZrns=
 github.com/muesli/ansi v0.0.0-20211031195517-c9f0611b6c70 h1:kMlmsLSbjkikxQJ1IPwaM+7LJ9ltFu/fi8CRzvSnQmA=
@@ -565,6 +568,7 @@ github.com/pingcap/errors v0.11.4/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTw
 github.com/pjbgf/sha1cd v0.3.0 h1:4D5XXmUUBUl/xQ6IjCkEAbqXskkq/4O7LmGn0AqMDs4=
 github.com/pjbgf/sha1cd v0.3.0/go.mod h1:nZ1rrWOcGJ5uZgEEVL1VUM9iRQiZvWdbZjkKyFzPPsI=
 github.com/pkg/browser v0.0.0-20180916011732-0a3d74bf9ce4/go.mod h1:4OwLy04Bl9Ef3GJJCoec+30X3LQs/0/m4HFRt/2LUSA=
+github.com/pkg/browser v0.0.0-20210115035449-ce105d075bb4/go.mod h1:N6UoU20jOqggOuDwUaBQpluzLNDqif3kq9z2wpdYEfQ=
 github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8 h1:KoWmjvw+nsYOo29YJK9vDA65RGE3NrOnUtO7a+RF9HU=
 github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8/go.mod h1:HKlIX3XHQyzLZPlr7++PzdhaXEj94dEiJgZDTsxEqUI=
 github.com/pkg/diff v0.0.0-20200914180035-5b29258ca4f7 h1:+/+DxvQaYifJ+grD4klzrS5y+KJXldn/2YTl5JG+vZ8=

pkg/detectors/privatekey/cracker.go

@@ -9,7 +9,7 @@ import (
 	"golang.org/x/crypto/ssh"
 )
 
-//go:embed "rockyou-15.txt"
+//go:embed "list.txt"
 var rawCrackList []byte
 var passphrases [][]byte
 
@@ -21,17 +21,17 @@ var (
 	ErrUncrackable = errors.New("unable to crack encryption")
 )
 
-func crack(in []byte) (interface{}, error) {
+func crack(in []byte) (interface{}, string, error) {
 	for _, passphrase := range passphrases {
 		parsed, err := ssh.ParseRawPrivateKeyWithPassphrase(in, passphrase)
 		if err != nil {
 			if errors.Is(err, x509.IncorrectPasswordError) {
 				continue
 			} else {
-				return nil, err
+				return nil, "", err
 			}
 		}
-		return parsed, nil
+		return parsed, string(passphrase), nil
 	}
-	return nil, ErrUncrackable
+	return nil, "", ErrUncrackable
 }

pkg/detectors/privatekey/cracker_test.go

@@ -56,23 +56,29 @@ Hx7UPVlTK8dyvk1Z+Yw0nrfNClI=
 
 func Test_crack(t *testing.T) {
 	tests := []struct {
-		name    string
-		in      []byte
-		wantErr bool
+		name             string
+		in               []byte
+		wantedPassphrase string
+		wantErr          bool
 	}{
 		{
-			name:    "crackable",
-			wantErr: false,
-			in:      testEncryptedKey,
+			name:             "crackable",
+			wantErr:          false,
+			in:               testEncryptedKey,
+			wantedPassphrase: string(testEncryptedKeyCorrectPassword),
 		},
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			_, err := crack(tt.in)
+			_, passphrase, err := crack(tt.in)
 			if (err != nil) != tt.wantErr {
 				t.Errorf("crack() error = %v, wantErr %v", err, tt.wantErr)
 				return
 			}
+
+			if passphrase != string(tt.wantedPassphrase) {
+				t.Errorf("crack() passphrase = %v, want %v", passphrase, string(testEncryptedKeyCorrectPassword))
+			}
 		})
 	}
 }

pkg/detectors/privatekey/fingerprint.go

@@ -5,10 +5,12 @@ import (
 	"crypto/ed25519"
 	"crypto/rsa"
 	"crypto/sha1"
+	"crypto/sha256"
 	"crypto/x509"
+	"encoding/base64"
 	"encoding/hex"
 	"errors"
-	"strings"
+	"fmt"
 
 	"golang.org/x/crypto/ssh"
 )
@@ -18,18 +20,8 @@ var (
 	ErrEncryptedKey = errors.New("key is encrypted")
 )
 
-func FingerprintPEMKey(in []byte) (string, error) {
-	parsedKey, err := ssh.ParseRawPrivateKey(in)
-	if err != nil && strings.Contains(err.Error(), "private key is passphrase protected") {
-		parsedKey, err = crack(in)
-		if err != nil {
-			return "", err
-		}
-	} else if err != nil {
-		return "", err
-	}
-
-	var pubKey interface{}
+func FingerprintPEMKey(parsedKey any) (string, error) {
+	var pubKey any
 	switch privateKey := parsedKey.(type) {
 	case *rsa.PrivateKey:
 		pubKey = &privateKey.PublicKey
@@ -44,6 +36,10 @@ func FingerprintPEMKey(in []byte) (string, error) {
 		return "", ErrNotSupported
 	}
 
+	return fingerprintPublicKey(pubKey)
+}
+
+func fingerprintPublicKey(pubKey any) (string, error) {
 	publickeyBytes, err := x509.MarshalPKIXPublicKey(pubKey)
 	if err != nil {
 		return "", err
@@ -53,3 +49,8 @@ func FingerprintPEMKey(in []byte) (string, error) {
 	publicKeyFingerprintInHex := hex.EncodeToString(publicKeyFingerprint[:])
 	return publicKeyFingerprintInHex, nil
 }
+
+func fingerprintSSHPublicKey(pubKey ssh.PublicKey) string {
+	publicKeyFingerprint := sha256.Sum256(pubKey.Marshal())
+	return fmt.Sprintf("SHA256:%s", base64.RawStdEncoding.EncodeToString(publicKeyFingerprint[:]))
+}

pkg/detectors/privatekey/privatekey.go

@@ -6,11 +6,14 @@ import (
 	"fmt"
 	"net/http"
 	"regexp"
+	"strings"
 	"time"
 
+	"github.com/AzureAD/microsoft-authentication-library-for-go/apps/errors"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+	"golang.org/x/crypto/ssh"
 )
 
 type Scanner struct {
@@ -53,44 +56,99 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) ([]dete
 			Redacted:     token[0:64],
 		}
 
-		fingerprint, err := FingerprintPEMKey([]byte(token))
+		secret.ExtraData = make(map[string]string)
+
+		var passphrase string
+		parsedKey, err := ssh.ParseRawPrivateKey([]byte(token))
+		if err != nil && strings.Contains(err.Error(), "private key is passphrase protected") {
+			secret.ExtraData["encrypted"] = "true"
+			parsedKey, passphrase, err = crack([]byte(token))
+			if err != nil {
+				secret.VerificationError = err
+				continue
+			}
+			if passphrase != "" {
+				secret.ExtraData["cracked_encryption_passphrase"] = "true"
+			}
+		} else if err != nil {
+			// couldn't parse key, probably invalid
+			continue
+		}
+
+		fingerprint, err := FingerprintPEMKey(parsedKey)
 		if err != nil {
 			continue
 		}
 
 		if verify {
+			verificationErrors := []string{}
 			data, err := lookupFingerprint(fingerprint, s.IncludeExpired)
 			if err == nil {
-				secret.StructuredData = data
 				if data != nil {
 					secret.Verified = true
+					secret.ExtraData["certificate_urls"] = strings.Join(data.CertificateURLs, ", ")
 				}
+			} else {
+				verificationErrors = append(verificationErrors, err.Error())
+			}
+
+			user, err := verifyGitHubUser(parsedKey)
+			if err != nil && !errors.Is(err, errPermissionDenied) {
+				verificationErrors = append(verificationErrors, err.Error())
+			}
+			if user != nil {
+				secret.Verified = true
+				secret.ExtraData["github_user"] = *user
+			}
+
+			user, err = verifyGitLabUser(parsedKey)
+			if err != nil && !errors.Is(err, errPermissionDenied) {
+				verificationErrors = append(verificationErrors, err.Error())
+			}
+			if user != nil {
+				secret.Verified = true
+				secret.ExtraData["gitlab_user"] = *user
+			}
+
+			if !secret.Verified && len(verificationErrors) > 0 {
+				secret.VerificationError = fmt.Errorf("verification failures: %s", strings.Join(verificationErrors, ", "))
 			}
 		}
 
+		if len(secret.ExtraData) == 0 {
+			secret.ExtraData = nil
+		}
+
 		results = append(results, secret)
 	}
 
 	return results, nil
 }
 
-func lookupFingerprint(publicKeyFingerprintInHex string, includeExpired bool) (data *detectorspb.StructuredData, err error) {
+type result struct {
+	CertificateURLs []string
+	GitHubUsername  string
+}
+
+func lookupFingerprint(publicKeyFingerprintInHex string, includeExpired bool) (*result, error) {
 	req, err := http.NewRequest("GET", fmt.Sprintf("https://keychecker.trufflesecurity.com/fingerprint/%s", publicKeyFingerprintInHex), nil)
 	if err != nil {
-		return
+		return nil, err
 	}
 	res, err := client.Do(req)
 	if err != nil {
-		return
+		return nil, err
 	}
 	defer res.Body.Close()
 
 	results := DriftwoodResult{}
 	err = json.NewDecoder(res.Body).Decode(&results)
 	if err != nil {
-		return
+		return nil, err
 	}
 
+	var data *result
+
 	seen := map[string]struct{}{}
 	for _, r := range results.CertificateResults {
 		if _, ok := seen[r.CertificateFingerprint]; ok {
@@ -100,36 +158,13 @@ func lookupFingerprint(publicKeyFingerprintInHex string, includeExpired bool) (d
 			continue
 		}
 		if data == nil {
-			data = &detectorspb.StructuredData{}
-		}
-		if data.TlsPrivateKey == nil {
-			data.TlsPrivateKey = make([]*detectorspb.TlsPrivateKey, 0)
+			data = &result{}
 		}
-		data.TlsPrivateKey = append(data.TlsPrivateKey, &detectorspb.TlsPrivateKey{
-			CertificateFingerprint: r.CertificateFingerprint,
-			ExpirationTimestamp:    r.ExpirationTimestamp.Unix(),
-			VerificationUrl:        fmt.Sprintf("https://crt.sh/?q=%s", r.CertificateFingerprint),
-		})
+		data.CertificateURLs = append(data.CertificateURLs, fmt.Sprintf("https://crt.sh/?q=%s", r.CertificateFingerprint))
 		seen[r.CertificateFingerprint] = struct{}{}
 	}
 
-	for _, r := range results.GitHubSSHResults {
-		if _, ok := seen[r.Username]; ok {
-			continue
-		}
-		if data == nil {
-			data = &detectorspb.StructuredData{}
-		}
-		if data.GithubSshKey == nil {
-			data.GithubSshKey = make([]*detectorspb.GitHubSSHKey, 0)
-		}
-		data.GithubSshKey = append(data.GithubSshKey, &detectorspb.GitHubSSHKey{
-			User: r.Username,
-		})
-		seen[r.Username] = struct{}{}
-	}
-
-	return
+	return data, nil
 }
 
 type DriftwoodResult struct {