diff --git a/pkg/detectors/aws/aws.go b/pkg/detectors/aws/aws.go index 0047f761176d..05b6675fb11d 100644 --- a/pkg/detectors/aws/aws.go +++ b/pkg/detectors/aws/aws.go @@ -53,7 +53,7 @@ var ( // Make sure that your group is surrounded in boundary characters such as below to reduce false positives. // Key types are from this list https://docs.aws.amazon.com/IAM/latest/UserGuide/reference_identifiers.html#identifiers-unique-ids - idPat = regexp.MustCompile(`\b((?:AKIA|ABIA|ACCA|ASIA)[0-9A-Z]{16})\b`) + idPat = regexp.MustCompile(`\b((?:AKIA|ABIA|ACCA)[0-9A-Z]{16})\b`) secretPat = regexp.MustCompile(`[^A-Za-z0-9+\/]{0,1}([A-Za-z0-9+\/]{40})[^A-Za-z0-9+\/]{0,1}`) // Hashes, like those for git, do technically match the secret pattern. // But they are extremely unlikely to be generated as an actual AWS secret. diff --git a/pkg/detectors/awssessionkeys/awssessionkey.go b/pkg/detectors/awssessionkeys/awssessionkey.go new file mode 100644 index 000000000000..12fbd688930e --- /dev/null +++ b/pkg/detectors/awssessionkeys/awssessionkey.go @@ -0,0 +1,332 @@ +package awssessionkey + +import ( + "context" + "crypto/hmac" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "fmt" + "net/http" + "regexp" + "strings" + "time" + + "github.com/trufflesecurity/trufflehog/v3/pkg/common" + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" + "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" +) + +type scanner struct { + verificationClient *http.Client + skipIDs map[string]struct{} +} + +func New(opts ...func(*scanner)) *scanner { + scanner := &scanner{ + skipIDs: map[string]struct{}{}, + } + for _, opt := range opts { + + opt(scanner) + } + + return scanner +} + +func WithSkipIDs(skipIDs []string) func(*scanner) { + return func(s *scanner) { + ids := map[string]struct{}{} + for _, id := range skipIDs { + ids[id] = struct{}{} + } + + s.skipIDs = ids + } +} + +// Ensure the scanner satisfies the interface at compile time. +var _ detectors.Detector = (*scanner)(nil) + +var ( + defaultVerificationClient = common.SaneHttpClient() + + // Make sure that your group is surrounded in boundary characters such as below to reduce false positives. + // Key types are from this list https://docs.aws.amazon.com/IAM/latest/UserGuide/reference_identifiers.html#identifiers-unique-ids + idPat = regexp.MustCompile(`\b((?:AKIA|ABIA|ACCA|ASIA)[0-9A-Z]{16})\b`) + secretPat = regexp.MustCompile(`\b[^A-Za-z0-9+\/]{0,1}([A-Za-z0-9+\/]{40})[^A-Za-z0-9+\/]{0,1}\b`) + sessionPat = regexp.MustCompile(`\b[^A-Za-z0-9+\/]{0,1}([A-Za-z0-9+=\/]{41,1000})[^A-Za-z0-9+=\/]{0,1}\b`) + // Hashes, like those for git, do technically match the secret pattern. + // But they are extremely unlikely to be generated as an actual AWS secret. + // So when we find them, if they're not verified, we should ignore the result. + falsePositiveSecretCheck = regexp.MustCompile(`[a-f0-9]{40}`) +) + +// Keywords are used for efficiently pre-filtering chunks. +// Use identifiers in the secret preferably, or the provider name. +func (s scanner) Keywords() []string { + return []string{ + "ASIA", + } +} + +func GetHash(input string) string { + data := []byte(input) + hasher := sha256.New() + hasher.Write(data) + return hex.EncodeToString(hasher.Sum(nil)) +} + +func GetHMAC(key []byte, data []byte) []byte { + hasher := hmac.New(sha256.New, key) + hasher.Write(data) + return hasher.Sum(nil) +} + +func checkSessionToken(sessionToken string, secret string) bool { + if !strings.Contains(sessionToken, "YXdz") || strings.Contains(sessionToken, secret){ + // Handle error if the sessionToken is not a valid base64 string + return false + } + return true +} + + +// FromData will find and optionally verify AWS secrets in a given set of bytes. +func (s scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) { + dataStr := string(data) + + idMatches := idPat.FindAllStringSubmatch(dataStr, -1) + secretMatches := secretPat.FindAllStringSubmatch(dataStr, -1) + sessionMatches := sessionPat.FindAllStringSubmatch(dataStr, -1) + + for _, idMatch := range idMatches { + if len(idMatch) != 2 { + continue + } + resIDMatch := strings.TrimSpace(idMatch[1]) + + if s.skipIDs != nil { + if _, ok := s.skipIDs[resIDMatch]; ok { + continue + } + } + + for _, secretMatch := range secretMatches { + if len(secretMatch) != 2 { + continue + } + resSecretMatch := strings.TrimSpace(secretMatch[1]) + + for _, sessionMatch := range sessionMatches { + if len(sessionMatch) != 2 { + continue + } + resSessionMatch := strings.TrimSpace(sessionMatch[1]) + if !checkSessionToken(resSessionMatch, resSecretMatch){ + continue + } + s1 := detectors.Result{ + DetectorType: detectorspb.DetectorType_AWSSessionKey, + Raw: []byte(resIDMatch), + Redacted: resIDMatch, + RawV2: []byte(resIDMatch + resSecretMatch + resSessionMatch), + } + + if verify { + verified, extraData, verificationErr := s.verifyMatch(ctx, resIDMatch, resSecretMatch, resSessionMatch, true) + s1.Verified = verified + s1.ExtraData = extraData + s1.VerificationError = verificationErr + } + + if !s1.Verified { + // Unverified results that contain common test words are probably not secrets + if detectors.IsKnownFalsePositive(resSecretMatch, detectors.DefaultFalsePositives, true) { + continue + } + // Unverified results that look like hashes are probably not secrets + if falsePositiveSecretCheck.MatchString(resSecretMatch) { + continue + } + } + + results = append(results, s1) + // If we've found a verified match with this ID, we don't need to look for any more. So move on to the next ID. + if s1.Verified { + break + } + } + } + } + return awsCustomCleanResults(results), nil +} + +func (s scanner) verifyMatch(ctx context.Context, resIDMatch, resSecretMatch string, resSessionMatch string, retryOn403 bool) (bool, map[string]string, error) { + + // REQUEST VALUES. + method := "GET" + service := "sts" + host := "sts.amazonaws.com" + region := "us-east-1" + endpoint := "https://sts.amazonaws.com" + now := time.Now().UTC() + datestamp := now.Format("20060102") + amzDate := now.Format("20060102T150405Z") + + req, err := http.NewRequestWithContext(ctx, method, endpoint, nil) + if err != nil { + return false, nil, err + } + req.Header.Set("Accept", "application/json") + + canonicalURI := "/" + canonicalHeaders := "host:" + host + "\n" + "x-amz-date:" + amzDate + "\n" + "x-amz-security-token:" + resSessionMatch + "\n" + signedHeaders := "host;x-amz-date;x-amz-security-token" + algorithm := "AWS4-HMAC-SHA256" + credentialScope := fmt.Sprintf("%s/%s/%s/aws4_request", datestamp, region, service) + + params := req.URL.Query() + params.Add("Action", "GetCallerIdentity") + params.Add("Version", "2011-06-15") + canonicalQuerystring := params.Encode() + payloadHash := GetHash("") // empty payload + canonicalRequest := method + "\n" + canonicalURI + "\n" + canonicalQuerystring + "\n" + canonicalHeaders + "\n" + signedHeaders + "\n" + payloadHash + + stringToSign := algorithm + "\n" + amzDate + "\n" + credentialScope + "\n" + GetHash(canonicalRequest) + + hash := GetHMAC([]byte(fmt.Sprintf("AWS4%s", resSecretMatch)), []byte(datestamp)) + hash = GetHMAC(hash, []byte(region)) + hash = GetHMAC(hash, []byte(service)) + hash = GetHMAC(hash, []byte("aws4_request")) + + signature2 := GetHMAC(hash, []byte(stringToSign)) // Get Signature HMAC SHA256 + signature := hex.EncodeToString(signature2) + + authorizationHeader := fmt.Sprintf("%s Credential=%s/%s, SignedHeaders=%s, Signature=%s", + algorithm, resIDMatch, credentialScope, signedHeaders, signature) + + req.Header.Add("Authorization", authorizationHeader) + req.Header.Add("x-amz-date", amzDate) + req.Header.Add("x-amz-security-token", resSessionMatch) + + // Rest of your code... + + req.URL.RawQuery = params.Encode() + + client := s.verificationClient + if client == nil { + client = defaultVerificationClient + } + + extraData := map[string]string{ + "rotation_guide": "https://howtorotate.com/docs/tutorials/aws/", + } + + res, err := client.Do(req) + if err == nil { + defer res.Body.Close() + if res.StatusCode >= 200 && res.StatusCode < 300 { + identityInfo := identityRes{} + err := json.NewDecoder(res.Body).Decode(&identityInfo) + if err == nil { + extraData["account"] = identityInfo.GetCallerIdentityResponse.GetCallerIdentityResult.Account + extraData["user_id"] = identityInfo.GetCallerIdentityResponse.GetCallerIdentityResult.UserID + extraData["arn"] = identityInfo.GetCallerIdentityResponse.GetCallerIdentityResult.Arn + return true, extraData, nil + } else { + return false, nil, err + } + } else if res.StatusCode == 403 { + // Experimentation has indicated that if you make two GetCallerIdentity requests within five seconds that + // share a key ID but are signed with different secrets the second one will be rejected with a 403 that + // carries a SignatureDoesNotMatch code in its body. This happens even if the second ID-secret pair is + // valid. Since this is exactly our access pattern, we need to work around it. + // + // Fortunately, experimentation has also revealed a workaround: simply resubmit the second request. The + // response to the resubmission will be as expected. But there's a caveat: You can't have closed the body of + // the response to the original second request, or read to its end, or the resubmission will also yield a + // SignatureDoesNotMatch. For this reason, we have to re-request all 403s. We can't re-request only + // SignatureDoesNotMatch responses, because we can only tell whether a given 403 is a SignatureDoesNotMatch + // after decoding its response body, which requires reading the entire response body, which disables the + // workaround. + // + // We are clearly deep in the guts of AWS implementation details here, so this all might change with no + // notice. If you're here because something in this detector broke, you have my condolences. + if retryOn403 { + return s.verifyMatch(ctx, resIDMatch, resSecretMatch, resSessionMatch, false) + } + var body awsErrorResponseBody + err = json.NewDecoder(res.Body).Decode(&body) + if err == nil { + // All instances of the code I've seen in the wild are PascalCased but this check is + // case-insensitive out of an abundance of caution + if strings.EqualFold(body.Error.Code, "InvalidClientTokenId") { + return false, nil, nil + } else { + return false, nil, fmt.Errorf("request to %v returned status %d with an unexpected reason (%s: %s)", res.Request.URL, res.StatusCode, body.Error.Code, body.Error.Message) + } + } else { + return false, nil, fmt.Errorf("couldn't parse the sts response body (%v)", err) + } + } else { + return false, nil, fmt.Errorf("request to %v returned unexpected status %d", res.Request.URL, res.StatusCode) + } + } else { + return false, nil, err + } +} + +func awsCustomCleanResults(results []detectors.Result) []detectors.Result { + if len(results) == 0 { + return results + } + + // For every ID, we want at most one result, preferably verified. + idResults := map[string]detectors.Result{} + for _, result := range results { + // Always accept the verified result as the result for the given ID. + if result.Verified { + idResults[result.Redacted] = result + continue + } + + // Only include an unverified result if we don't already have a result for a given ID. + if _, exist := idResults[result.Redacted]; !exist { + idResults[result.Redacted] = result + } + } + + out := []detectors.Result{} + for _, r := range idResults { + out = append(out, r) + } + return out +} + +type awsError struct { + Code string `json:"Code"` + Message string `json:"Message"` +} + +type awsErrorResponseBody struct { + Error awsError `json:"Error"` +} + +type identityRes struct { + GetCallerIdentityResponse struct { + GetCallerIdentityResult struct { + Account string `json:"Account"` + Arn string `json:"Arn"` + UserID string `json:"UserId"` + } `json:"GetCallerIdentityResult"` + ResponseMetadata struct { + RequestID string `json:"RequestId"` + } `json:"ResponseMetadata"` + } `json:"GetCallerIdentityResponse"` +} + +func (s scanner) Type() detectorspb.DetectorType { + return detectorspb.DetectorType_AWSSessionKey +} + diff --git a/pkg/engine/defaults.go b/pkg/engine/defaults.go index 52ed3fc74e62..9e7788f4cf77 100644 --- a/pkg/engine/defaults.go +++ b/pkg/engine/defaults.go @@ -69,6 +69,7 @@ import ( "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/avazapersonalaccesstoken" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/aviationstack" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/aws" + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/awssessionkeys" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/axonaut" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/aylien" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/ayrshare" @@ -774,6 +775,7 @@ func DefaultDetectors() []detectors.Detector { &linearapi.Scanner{}, &alibaba.Scanner{}, aws.New(), + awssessionkey.New(), &azure.Scanner{}, &slack.Scanner{}, // has 4 secret types &gitlab.Scanner{}, diff --git a/pkg/pb/detectorspb/detectors.pb.go b/pkg/pb/detectorspb/detectors.pb.go index 23cdb2317e0b..d9658b2663f0 100644 --- a/pkg/pb/detectorspb/detectors.pb.go +++ b/pkg/pb/detectorspb/detectors.pb.go @@ -1013,6 +1013,7 @@ const ( DetectorType_IPInfo DetectorType = 939 DetectorType_Ip2location DetectorType = 940 DetectorType_Instamojo DetectorType = 941 + DetectorType_AWSSessionKey DetectorType = 942 ) // Enum value maps for DetectorType. @@ -1956,6 +1957,7 @@ var ( 939: "IPInfo", 940: "Ip2location", 941: "Instamojo", + 942: "AWSSessionKey", } DetectorType_value = map[string]int32{ "Alibaba": 0, @@ -2896,6 +2898,7 @@ var ( "IPInfo": 939, "Ip2location": 940, "Instamojo": 941, + "AWSSessionKey": 942, } ) @@ -3274,7 +3277,7 @@ var file_detectors_proto_rawDesc = []byte{ 0x44, 0x65, 0x63, 0x6f, 0x64, 0x65, 0x72, 0x54, 0x79, 0x70, 0x65, 0x12, 0x0b, 0x0a, 0x07, 0x55, 0x4e, 0x4b, 0x4e, 0x4f, 0x57, 0x4e, 0x10, 0x00, 0x12, 0x09, 0x0a, 0x05, 0x50, 0x4c, 0x41, 0x49, 0x4e, 0x10, 0x01, 0x12, 0x0a, 0x0a, 0x06, 0x42, 0x41, 0x53, 0x45, 0x36, 0x34, 0x10, 0x02, 0x12, - 0x09, 0x0a, 0x05, 0x55, 0x54, 0x46, 0x31, 0x36, 0x10, 0x03, 0x2a, 0xf3, 0x75, 0x0a, 0x0c, 0x44, + 0x09, 0x0a, 0x05, 0x55, 0x54, 0x46, 0x31, 0x36, 0x10, 0x03, 0x2a, 0x87, 0x76, 0x0a, 0x0c, 0x44, 0x65, 0x74, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x54, 0x79, 0x70, 0x65, 0x12, 0x0b, 0x0a, 0x07, 0x41, 0x6c, 0x69, 0x62, 0x61, 0x62, 0x61, 0x10, 0x00, 0x12, 0x08, 0x0a, 0x04, 0x41, 0x4d, 0x51, 0x50, 0x10, 0x01, 0x12, 0x07, 0x0a, 0x03, 0x41, 0x57, 0x53, 0x10, 0x02, 0x12, 0x09, 0x0a, 0x05, 0x41, @@ -4218,11 +4221,12 @@ var file_detectors_proto_rawDesc = []byte{ 0x07, 0x12, 0x0b, 0x0a, 0x06, 0x49, 0x50, 0x49, 0x6e, 0x66, 0x6f, 0x10, 0xab, 0x07, 0x12, 0x10, 0x0a, 0x0b, 0x49, 0x70, 0x32, 0x6c, 0x6f, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x10, 0xac, 0x07, 0x12, 0x0e, 0x0a, 0x09, 0x49, 0x6e, 0x73, 0x74, 0x61, 0x6d, 0x6f, 0x6a, 0x6f, 0x10, 0xad, 0x07, - 0x42, 0x3d, 0x5a, 0x3b, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x74, - 0x72, 0x75, 0x66, 0x66, 0x6c, 0x65, 0x73, 0x65, 0x63, 0x75, 0x72, 0x69, 0x74, 0x79, 0x2f, 0x74, - 0x72, 0x75, 0x66, 0x66, 0x6c, 0x65, 0x68, 0x6f, 0x67, 0x2f, 0x76, 0x33, 0x2f, 0x70, 0x6b, 0x67, - 0x2f, 0x70, 0x62, 0x2f, 0x64, 0x65, 0x74, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x73, 0x70, 0x62, 0x62, - 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x12, 0x12, 0x0a, 0x0d, 0x41, 0x57, 0x53, 0x53, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x4b, 0x65, + 0x79, 0x10, 0xae, 0x07, 0x42, 0x3d, 0x5a, 0x3b, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, + 0x6f, 0x6d, 0x2f, 0x74, 0x72, 0x75, 0x66, 0x66, 0x6c, 0x65, 0x73, 0x65, 0x63, 0x75, 0x72, 0x69, + 0x74, 0x79, 0x2f, 0x74, 0x72, 0x75, 0x66, 0x66, 0x6c, 0x65, 0x68, 0x6f, 0x67, 0x2f, 0x76, 0x33, + 0x2f, 0x70, 0x6b, 0x67, 0x2f, 0x70, 0x62, 0x2f, 0x64, 0x65, 0x74, 0x65, 0x63, 0x74, 0x6f, 0x72, + 0x73, 0x70, 0x62, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( diff --git a/proto/detectors.proto b/proto/detectors.proto index aecb23639de2..9ff9e072213f 100644 --- a/proto/detectors.proto +++ b/proto/detectors.proto @@ -950,6 +950,7 @@ enum DetectorType { IPInfo = 939; Ip2location = 940; Instamojo = 941; + AWSSessionKey = 942; } message Result {