From bdc8b2d97d3b610f8cf91390d810e5257fec2918 Mon Sep 17 00:00:00 2001 From: Chair Date: Fri, 10 Nov 2023 14:44:06 -0800 Subject: [PATCH] adding postgres detector, and also changing our default username and password match --- pkg/common/patterns.go | 5 +- pkg/detectors/postgres/postgres.go | 170 +++++++++++++++++++++++++++++ pkg/engine/defaults.go | 2 + pkg/pb/detectorspb/detectors.pb.go | 16 ++- proto/detectors.proto | 1 + 5 files changed, 186 insertions(+), 8 deletions(-) create mode 100644 pkg/detectors/postgres/postgres.go diff --git a/pkg/common/patterns.go b/pkg/common/patterns.go index 774dbf3f6528..ed0812291b59 100644 --- a/pkg/common/patterns.go +++ b/pkg/common/patterns.go @@ -20,6 +20,7 @@ type RegexState struct { compiledRegex *regexp.Regexp } + // Custom Regex functions func BuildRegex(pattern string, specialChar string, length int) string { return fmt.Sprintf(`\b([%s%s]{%s})\b`, pattern, specialChar, strconv.Itoa(length)) @@ -58,14 +59,14 @@ func (r RegexState) Matches(data []byte) []string { // UsernameRegexCheck constructs an username usernameRegex pattern from a given pattern of excluded characters. func UsernameRegexCheck(pattern string) RegexState { - raw := fmt.Sprintf(`(?im)(?:user|usr)\S{0,40}?[:=\s]{1,3}[ '"=]{0,1}([^:%+v]{4,40})\b`, pattern) + raw := fmt.Sprintf(`(?im)(?:user|usr)\S{0,40}?[:=\s]{1,3}[ '"=]{0,1}([^:%+v'"\s]{4,40})`, pattern) return RegexState{regexp.MustCompile(raw)} } // PasswordRegexCheck constructs an username usernameRegex pattern from a given pattern of excluded characters. func PasswordRegexCheck(pattern string) RegexState { - raw := fmt.Sprintf(`(?im)(?:pass|password)\S{0,40}?[:=\s]{1,3}[ '"=]{0,1}([^:%+v]{4,40})`, pattern) + raw := fmt.Sprintf(`(?im)(?:pass)\S{0,40}?[:=\s]{1,3}[ '"=]{0,1}([^:%+v'"\s]{4,40})`, pattern) return RegexState{regexp.MustCompile(raw)} } diff --git a/pkg/detectors/postgres/postgres.go b/pkg/detectors/postgres/postgres.go new file mode 100644 index 000000000000..48dd6691f5e6 --- /dev/null +++ b/pkg/detectors/postgres/postgres.go @@ -0,0 +1,170 @@ +package postgres + +import ( + "context" + "database/sql" + "net/url" + "regexp" + "strings" + "fmt" + "time" + + "github.com/trufflesecurity/trufflehog/v3/pkg/common" + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" + "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" +) + +type Scanner struct{} + +var _ detectors.Detector = (*Scanner)(nil) + +var ( + // URI pattern for PostgreSQL connection string + uriPat = regexp.MustCompile(`\b(?i)postgresql://[\S]+\b`) + + // Separate patterns for username, password, and hostname + hostnamePat = regexp.MustCompile(`(?i)(?:host|server).{0,40}?(\b[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?(?:\.[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?)*\b)`) + + + // You might want to customize these patterns based on common practices in your codebases +) + +func (s Scanner) Keywords() []string { + return []string{"postgres", "psql", "pghost"} +} + +func verifyPostgres(pgURL *url.URL) error { + // Extract the necessary components + username := "" + password := "" + if pgURL.User != nil { + username = pgURL.User.Username() + password, _ = pgURL.User.Password() + } + hostname := pgURL.Hostname() + + // Handle custom port + port := pgURL.Port() + if port == "" { + port = "5432" // Default PostgreSQL port + } + + // Handle SSL mode + sslmode := "disable" // Default to disable + queryParams := pgURL.Query() + if sslQuery, ok := queryParams["sslmode"]; ok && len(sslQuery) > 0 { + sslmode = sslQuery[0] + } + + // Construct the PostgreSQL connection string + connStr := fmt.Sprintf("user=%s password=%s host=%s port=%s sslmode=%s", username, password, hostname, port, sslmode) + + // Open a connection to the database + db, err := sql.Open("postgres", connStr) + if err != nil { + return err + } + defer db.Close() + + ctx, cancel := context.WithTimeout(context.Background(), 1*time.Second) + defer cancel() + + + // Try to establish a connection + err = db.PingContext(ctx) + if err != nil { + return err + } + + // If we reach here, the credentials are valid + return nil +} + +func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) { + dataStr := string(data) + + // Check for inline connection strings + uriMatches := uriPat.FindAllString(dataStr, -1) + for _, uri := range uriMatches { + pgURL, err := url.Parse(uri) + if err != nil { + continue + } + + // PostgreSQL URLs might not always have the userinfo (username:password) part + if pgURL.User != nil { + username := pgURL.User.Username() + password, _ := pgURL.User.Password() + hostname := pgURL.Hostname() + s1 := detectors.Result{ + DetectorType: detectorspb.DetectorType_Postgres, + Raw: []byte(strings.Join([]string{hostname, username, password}, "\t")), + } + if verify { + verificationErr:= verifyPostgres(pgURL) + s1.Verified = verificationErr == nil + } + results = append(results, s1) + } + } + + // Check for separate components + usernameRegexState := common.UsernameRegexCheck("") + usernameMatches := usernameRegexState.Matches(data) + + passwordRegexState := common.PasswordRegexCheck("") // No explicit character exclusions by Snowflake for passwords + passwordMatches := passwordRegexState.Matches(data) + hostnameMatches := hostnamePat.FindAllStringSubmatch(dataStr, -1) + + // Combine the separate components into potential credentials + for _, username := range usernameMatches { + if len(username) < 2 { + continue + } + for _, hostname := range hostnameMatches { + if len(hostname) < 2 { + continue + } + result := false + s1 := detectors.Result{ + DetectorType: detectorspb.DetectorType_Postgres, + } + for _, password := range passwordMatches { + if len(password) < 2 { + continue + } + + // Since we're combining these, we should probably also ensure that the total length does not exceed the 255 character limit for hostnames + combinedLength := len(username) + len(password) + len(hostname[1]) + if combinedLength > 255 { + continue // Skip if the combined length is too long + } + s1.Raw = []byte(strings.Join([]string{hostname[1], username, password}, "\t")) + result = true + postgresURL := url.URL{ + Scheme: "postgresql", + User: url.UserPassword(username, password), + Host: fmt.Sprintf("%s:%s", hostname[1], "5432"), + } + if verify { + verificationErr:= verifyPostgres(&postgresURL) + s1.Verified = verificationErr == nil + break + } + } + if result { + results = append(results, s1) + } + } + } + + + // Verification could be done here if necessary + + return results, nil +} + +func (s Scanner) Type() detectorspb.DetectorType { + return detectorspb.DetectorType_Postgres +} + diff --git a/pkg/engine/defaults.go b/pkg/engine/defaults.go index 4ef289af853b..3a6068ab62a2 100644 --- a/pkg/engine/defaults.go +++ b/pkg/engine/defaults.go @@ -517,6 +517,7 @@ import ( "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/positionstack" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/postageapp" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/postbacks" + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/postgres" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/posthog" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/postman" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/postmark" @@ -1491,6 +1492,7 @@ func DefaultDetectors() []detectors.Detector { speechtextai.Scanner{}, databox.Scanner{}, postbacks.Scanner{}, + postgres.Scanner{}, collect2.Scanner{}, uclassify.Scanner{}, holistic.Scanner{}, diff --git a/pkg/pb/detectorspb/detectors.pb.go b/pkg/pb/detectorspb/detectors.pb.go index d743b8e83d8c..24590079e3c7 100644 --- a/pkg/pb/detectorspb/detectors.pb.go +++ b/pkg/pb/detectorspb/detectors.pb.go @@ -1050,6 +1050,7 @@ const ( DetectorType_Overloop DetectorType = 965 DetectorType_Ngrok DetectorType = 966 DetectorType_Replicate DetectorType = 967 + DetectorType_Postgres DetectorType = 968 ) // Enum value maps for DetectorType. @@ -2019,6 +2020,7 @@ var ( 965: "Overloop", 966: "Ngrok", 967: "Replicate", + 968: "Postgres", } DetectorType_value = map[string]int32{ "Alibaba": 0, @@ -2985,6 +2987,7 @@ var ( "Overloop": 965, "Ngrok": 966, "Replicate": 967, + "Postgres": 968, } ) @@ -3363,7 +3366,7 @@ var file_detectors_proto_rawDesc = []byte{ 0x44, 0x65, 0x63, 0x6f, 0x64, 0x65, 0x72, 0x54, 0x79, 0x70, 0x65, 0x12, 0x0b, 0x0a, 0x07, 0x55, 0x4e, 0x4b, 0x4e, 0x4f, 0x57, 0x4e, 0x10, 0x00, 0x12, 0x09, 0x0a, 0x05, 0x50, 0x4c, 0x41, 0x49, 0x4e, 0x10, 0x01, 0x12, 0x0a, 0x0a, 0x06, 0x42, 0x41, 0x53, 0x45, 0x36, 0x34, 0x10, 0x02, 0x12, - 0x09, 0x0a, 0x05, 0x55, 0x54, 0x46, 0x31, 0x36, 0x10, 0x03, 0x2a, 0xe7, 0x79, 0x0a, 0x0c, 0x44, + 0x09, 0x0a, 0x05, 0x55, 0x54, 0x46, 0x31, 0x36, 0x10, 0x03, 0x2a, 0xf6, 0x79, 0x0a, 0x0c, 0x44, 0x65, 0x74, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x54, 0x79, 0x70, 0x65, 0x12, 0x0b, 0x0a, 0x07, 0x41, 0x6c, 0x69, 0x62, 0x61, 0x62, 0x61, 0x10, 0x00, 0x12, 0x08, 0x0a, 0x04, 0x41, 0x4d, 0x51, 0x50, 0x10, 0x01, 0x12, 0x07, 0x0a, 0x03, 0x41, 0x57, 0x53, 0x10, 0x02, 0x12, 0x09, 0x0a, 0x05, 0x41, @@ -4338,11 +4341,12 @@ var file_detectors_proto_rawDesc = []byte{ 0x69, 0x6e, 0x61, 0x6e, 0x63, 0x65, 0x10, 0xc4, 0x07, 0x12, 0x0d, 0x0a, 0x08, 0x4f, 0x76, 0x65, 0x72, 0x6c, 0x6f, 0x6f, 0x70, 0x10, 0xc5, 0x07, 0x12, 0x0a, 0x0a, 0x05, 0x4e, 0x67, 0x72, 0x6f, 0x6b, 0x10, 0xc6, 0x07, 0x12, 0x0e, 0x0a, 0x09, 0x52, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, - 0x65, 0x10, 0xc7, 0x07, 0x42, 0x3d, 0x5a, 0x3b, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, - 0x6f, 0x6d, 0x2f, 0x74, 0x72, 0x75, 0x66, 0x66, 0x6c, 0x65, 0x73, 0x65, 0x63, 0x75, 0x72, 0x69, - 0x74, 0x79, 0x2f, 0x74, 0x72, 0x75, 0x66, 0x66, 0x6c, 0x65, 0x68, 0x6f, 0x67, 0x2f, 0x76, 0x33, - 0x2f, 0x70, 0x6b, 0x67, 0x2f, 0x70, 0x62, 0x2f, 0x64, 0x65, 0x74, 0x65, 0x63, 0x74, 0x6f, 0x72, - 0x73, 0x70, 0x62, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x65, 0x10, 0xc7, 0x07, 0x12, 0x0d, 0x0a, 0x08, 0x50, 0x6f, 0x73, 0x74, 0x67, 0x72, 0x65, 0x73, + 0x10, 0xc8, 0x07, 0x42, 0x3d, 0x5a, 0x3b, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, + 0x6d, 0x2f, 0x74, 0x72, 0x75, 0x66, 0x66, 0x6c, 0x65, 0x73, 0x65, 0x63, 0x75, 0x72, 0x69, 0x74, + 0x79, 0x2f, 0x74, 0x72, 0x75, 0x66, 0x66, 0x6c, 0x65, 0x68, 0x6f, 0x67, 0x2f, 0x76, 0x33, 0x2f, + 0x70, 0x6b, 0x67, 0x2f, 0x70, 0x62, 0x2f, 0x64, 0x65, 0x74, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x73, + 0x70, 0x62, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( diff --git a/proto/detectors.proto b/proto/detectors.proto index 83b34f3f95b7..dd3015de8a0a 100644 --- a/proto/detectors.proto +++ b/proto/detectors.proto @@ -976,6 +976,7 @@ enum DetectorType { Overloop = 965; Ngrok = 966; Replicate = 967; + Postgres = 968; } message Result {