From 72b1603a0dc8c3798b8bdf7c08f2f6c5ec4023e3 Mon Sep 17 00:00:00 2001 From: Zubair Khan Date: Tue, 22 Aug 2023 17:12:06 -0400 Subject: [PATCH] get verified test case working --- pkg/common/patterns.go | 1 + pkg/detectors/snowflake/snowflake.go | 211 +++++++++++++--------- pkg/detectors/snowflake/snowflake_test.go | 29 +-- 3 files changed, 141 insertions(+), 100 deletions(-) diff --git a/pkg/common/patterns.go b/pkg/common/patterns.go index 774dbf3f6528..af1c652fb975 100644 --- a/pkg/common/patterns.go +++ b/pkg/common/patterns.go @@ -59,6 +59,7 @@ func (r RegexState) Matches(data []byte) []string { // UsernameRegexCheck constructs an username usernameRegex pattern from a given pattern of excluded characters. func UsernameRegexCheck(pattern string) RegexState { raw := fmt.Sprintf(`(?im)(?:user|usr)\S{0,40}?[:=\s]{1,3}[ '"=]{0,1}([^:%+v]{4,40})\b`, pattern) + fmt.Printf("raw: %s \n", raw) return RegexState{regexp.MustCompile(raw)} } diff --git a/pkg/detectors/snowflake/snowflake.go b/pkg/detectors/snowflake/snowflake.go index 79df283c54d6..611236e47846 100644 --- a/pkg/detectors/snowflake/snowflake.go +++ b/pkg/detectors/snowflake/snowflake.go @@ -5,11 +5,13 @@ import ( "database/sql" "fmt" "github.com/snowflakedb/gosnowflake" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" "log" "net/http" "regexp" + "strings" "unicode" ) @@ -21,10 +23,15 @@ type Scanner struct { var _ detectors.Detector = (*Scanner)(nil) var ( - accountIdentifierPat = regexp.MustCompile(detectors.PrefixRegex([]string{"account"}) + `\b[a-zA-Z]{7}-[0-9a-zA-Z]{7}\b`) + accountIdentifierPat = regexp.MustCompile(detectors.PrefixRegex([]string{"account"}) + `\b([a-zA-Z]{7}-[0-9a-zA-Z]{7})\b`) usernameExclusionPat = `!@#$%^&*{}:<>,.;?()/\+=\s\n` ) +const ( + database = "SNOWFLAKE" + retrieveAllDatabasesQuery = "SHOW DATABASES" +) + // Keywords are used for efficiently pre-filtering chunks. // Use identifiers in the secret preferably, or the provider name. func (s Scanner) Keywords() []string { @@ -60,102 +67,130 @@ func meetsSnowflakePasswordRequirements(password string) (string, bool) { // FromData will find and optionally verify Snowflake secrets in a given set of bytes. func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) { + dataStr := string(data) - //usernameRegexState := common.UsernameRegexCheck("") - //usernameMatches := usernameRegexState.Matches(data) - // - //passwordRegexState := common.PasswordRegexCheck("") // No explicit character exclusions by Snowflake for passwords - //passwordMatches := passwordRegexState.Matches(data) - - config := &gosnowflake.Config{ - Account: accountName, - User: username, - Password: password, - Database: database, - //Schema: schema, - } - uri := fmt.Sprintf("%s:%s@%s/%s", config.User, config.Password, config.Account, config.Database) - // Open a connection to Snowflake - db, err := sql.Open("snowflake", uri) // Needs the snowflake driver from gosnowflake - if err != nil { - log.Fatal(err) - } - defer db.Close() + accountMatches := accountIdentifierPat.FindAllStringSubmatch(dataStr, -1) - err = db.Ping() - if err != nil { - log.Fatal(err) - } + fmt.Println("accountMatches: ", accountMatches) + regexPat := detectors.PrefixRegex([]string{"account"}) + `\b([a-zA-Z]{7}-[0-9a-zA-Z]{7})\b` + fmt.Println("regexPat", regexPat) - fmt.Println("Connected to Snowflake!") + usernameRegexState := common.UsernameRegexCheck(usernameExclusionPat) + usernameMatches := usernameRegexState.Matches(data) - query := "SHOW TABLES IN DATABASE " + database - rows, err := db.Query(query) - if err != nil { - log.Fatal(err) - } - defer rows.Close() - - fmt.Println("Tables in database:") - for rows.Next() { - var tableName string - err := rows.Scan(&tableName) - if err != nil { - log.Fatal(err) + passwordRegexState := common.PasswordRegexCheck(" ") // No explicit character exclusions by Snowflake for passwords + passwordMatches := passwordRegexState.Matches(data) + + for _, accountMatch := range accountMatches { + fmt.Println("accountMatch: ", accountMatch) + if len(accountMatch) != 2 { + continue + } + resAccountMatch := strings.TrimSpace(accountMatch[1]) + + for _, resUsernameMatch := range usernameMatches { + + for _, resPasswordMatch := range passwordMatches { + _, metPasswordRequirements := meetsSnowflakePasswordRequirements(resPasswordMatch) + + if !metPasswordRequirements { + continue + } + + uri := fmt.Sprintf("%s:%s@%s/%s", resUsernameMatch, resPasswordMatch, resAccountMatch, database) + + s1 := detectors.Result{ + DetectorType: detectorspb.DetectorType_Snowflake, + Raw: []byte(resPasswordMatch), + ExtraData: map[string]string{ + "account": resAccountMatch, + "username": resUsernameMatch, + }, + } + + if verify { + config := &gosnowflake.Config{ + Account: resAccountMatch, + User: resUsernameMatch, + Password: resPasswordMatch, + Database: database, + } + + fmt.Println("config: ", config) + // Open a connection to Snowflake + db, err := sql.Open("snowflake", uri) // Needs the snowflake driver from gosnowflake + if err != nil { + log.Fatal(err) + } + defer db.Close() + + err = db.Ping() + if err != nil { + log.Fatal(err) + } + + s1.Verified = true + + rows, err := db.Query(retrieveAllDatabasesQuery) + if err != nil { + log.Fatal(err) + } + defer rows.Close() + + var databases []string + for rows.Next() { + var name, createdOn, is_default, isCurrent, origin, owner, comment, option, retention_time, kind string + err := rows.Scan(&createdOn, &name, &is_default, &isCurrent, &origin, &owner, &comment, &option, &retention_time, &kind) + if err != nil { + log.Fatal(err) + } + databases = append(databases, name) + } + fmt.Println(databases) + s1.ExtraData["databases"] = strings.Join(databases, ", ") + + } + + results = append(results, s1) + } } - fmt.Println(tableName) } - - //dataStr := string(data) - // - //matches := keyPat.FindAllStringSubmatch(dataStr, -1) - // - //for _, match := range matches { - // if len(match) != 2 { - // continue - // } - // resMatch := strings.TrimSpace(match[1]) - // - // s1 := detectors.Result{ - // DetectorType: detectorspb.DetectorType_Snowflake, - // Raw: []byte(resMatch), - // } - // - // if verify { - // client := s.client - // if client == nil { - // client = defaultClient - // } - // req, err := http.NewRequestWithContext(ctx, "GET", "https://eth-mainnet.g.snowflake.com/v2/"+resMatch+"/getNFTs/?owner=vitalik.eth", nil) - // if err != nil { - // continue - // } - // res, err := client.Do(req) - // if err == nil { - // defer res.Body.Close() - // if res.StatusCode >= 200 && res.StatusCode < 300 { - // s1.Verified = true - // } else if res.StatusCode == 401 { - // // The secret is determinately not verified (nothing to do) - // } else { - // s1.VerificationError = fmt.Errorf("unexpected HTTP response status %d", res.StatusCode) - // } - // } else { - // s1.VerificationError = err - // } - // } - // - // // This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key. - // if !s1.Verified && detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) { - // continue - // } - // - // results = append(results, s1) - //} - return results, nil } +// +// if verify { +// client := s.client +// if client == nil { +// client = defaultClient +// } +// req, err := http.NewRequestWithContext(ctx, "GET", "https://eth-mainnet.g.snowflake.com/v2/"+resMatch+"/getNFTs/?owner=vitalik.eth", nil) +// if err != nil { +// continue +// } +// res, err := client.Do(req) +// if err == nil { +// defer res.Body.Close() +// if res.StatusCode >= 200 && res.StatusCode < 300 { +// s1.Verified = true +// } else if res.StatusCode == 401 { +// // The secret is determinately not verified (nothing to do) +// } else { +// s1.VerificationError = fmt.Errorf("unexpected HTTP response status %d", res.StatusCode) +// } +// } else { +// s1.VerificationError = err +// } +// } +// +// // This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key. +// if !s1.Verified && detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) { +// continue +// } +// +// results = append(results, s1) +//} + func (s Scanner) Type() detectorspb.DetectorType { return detectorspb.DetectorType_Snowflake } diff --git a/pkg/detectors/snowflake/snowflake_test.go b/pkg/detectors/snowflake/snowflake_test.go index b9b86c97ac22..c089286b9491 100644 --- a/pkg/detectors/snowflake/snowflake_test.go +++ b/pkg/detectors/snowflake/snowflake_test.go @@ -18,17 +18,17 @@ import ( ) func TestSnowflake_FromChunk(t *testing.T) { - //ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) - //defer cancel() - //testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors4") - //if err != nil { - // t.Fatalf("could not get test secrets from GCP: %s", err) - //} - //secret := testSecrets.MustGetField("SNOWFLAKE") - //inactiveSecret := testSecrets.MustGetField("SNOWFLAKE_INACTIVE") + ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) + defer cancel() + testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors5") + if err != nil { + t.Fatalf("could not get test secrets from GCP: %s", err) + } + + secret := testSecrets.MustGetField("SNOWFLAKE_PASS") + inactiveSecret := testSecrets.MustGetField("SNOWFLAKE_PASS_INACTIVE") - secret := "secret" - inactiveSecret := "fake_secret" + fmt.Println("secret: ", secret) type args struct { ctx context.Context @@ -48,13 +48,18 @@ func TestSnowflake_FromChunk(t *testing.T) { s: Scanner{}, args: args{ ctx: context.Background(), - data: []byte(fmt.Sprintf("You can find a snowflake secret %s within", secret)), + data: []byte(fmt.Sprintf("snowflake: \n account=tuacoip-zt74995 \n username=zubairkhan14 \n password=%s \n database=SNOWFLAKE", secret)), verify: true, }, want: []detectors.Result{ { DetectorType: detectorspb.DetectorType_Snowflake, Verified: true, + ExtraData: map[string]string{ + "account": "tuacoip-zt74995", + "databases": "SNOWFLAKE, SNOWFLAKE_SAMPLE_DATA", + "username": "zubairkhan14", + }, }, }, wantErr: false, @@ -65,7 +70,7 @@ func TestSnowflake_FromChunk(t *testing.T) { s: Scanner{}, args: args{ ctx: context.Background(), - data: []byte(fmt.Sprintf("You can find a snowflake secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation + data: []byte(fmt.Sprintf("snowflake: \n account=tuacoip-zt74995 \n username=zubairkhan14 \n password=%s \n database=SNOWFLAKE", inactiveSecret)), verify: true, }, want: []detectors.Result{