correct logging output for github comments and add oss flags (#1632)

* correct logging output * add flags * respect oss cli flags for github comment scanning * improve copy
trufflesecurity · Aug 16, 2023 · db89e34 · db89e34
1 parent e0db575
commit db89e34
Show file tree

Hide file tree

Showing 4 changed files with 47 additions and 29 deletions.
diff --git a/main.go b/main.go
@@ -73,17 +73,20 @@ var (
 	_                   = gitScan.Flag("entropy", "No-op flag for backwards compat.").Bool()
 	_                   = gitScan.Flag("regex", "No-op flag for backwards compat.").Bool()
 
-	githubScan             = cli.Command("github", "Find credentials in GitHub repositories.")
-	githubScanEndpoint     = githubScan.Flag("endpoint", "GitHub endpoint.").Default("https://api.github.com").String()
-	githubScanRepos        = githubScan.Flag("repo", `GitHub repository to scan. You can repeat this flag. Example: "https://github.com/dustin-decker/secretsandstuff"`).Strings()
-	githubScanOrgs         = githubScan.Flag("org", `GitHub organization to scan. You can repeat this flag. Example: "trufflesecurity"`).Strings()
-	githubScanToken        = githubScan.Flag("token", "GitHub token. Can be provided with environment variable GITHUB_TOKEN.").Envar("GITHUB_TOKEN").String()
-	githubIncludeForks     = githubScan.Flag("include-forks", "Include forks in scan.").Bool()
-	githubIncludeMembers   = githubScan.Flag("include-members", "Include organization member repositories in scan.").Bool()
-	githubIncludeRepos     = githubScan.Flag("include-repos", `Repositories to include in an org scan. This can also be a glob pattern. You can repeat this flag. Must use Github repo full name. Example: "trufflesecurity/trufflehog", "trufflesecurity/t*"`).Strings()
-	githubExcludeRepos     = githubScan.Flag("exclude-repos", `Repositories to exclude in an org scan. This can also be a glob pattern. You can repeat this flag. Must use Github repo full name. Example: "trufflesecurity/driftwood", "trufflesecurity/d*"`).Strings()
-	githubScanIncludePaths = githubScan.Flag("include-paths", "Path to file with newline separated regexes for files to include in scan.").Short('i').String()
-	githubScanExcludePaths = githubScan.Flag("exclude-paths", "Path to file with newline separated regexes for files to exclude in scan.").Short('x').String()
+	githubScan              = cli.Command("github", "Find credentials in GitHub repositories.")
+	githubScanEndpoint      = githubScan.Flag("endpoint", "GitHub endpoint.").Default("https://api.github.com").String()
+	githubScanRepos         = githubScan.Flag("repo", `GitHub repository to scan. You can repeat this flag. Example: "https://github.com/dustin-decker/secretsandstuff"`).Strings()
+	githubScanOrgs          = githubScan.Flag("org", `GitHub organization to scan. You can repeat this flag. Example: "trufflesecurity"`).Strings()
+	githubScanToken         = githubScan.Flag("token", "GitHub token. Can be provided with environment variable GITHUB_TOKEN.").Envar("GITHUB_TOKEN").String()
+	githubIncludeForks      = githubScan.Flag("include-forks", "Include forks in scan.").Bool()
+	githubIncludeMembers    = githubScan.Flag("include-members", "Include organization member repositories in scan.").Bool()
+	githubIncludeRepos      = githubScan.Flag("include-repos", `Repositories to include in an org scan. This can also be a glob pattern. You can repeat this flag. Must use Github repo full name. Example: "trufflesecurity/trufflehog", "trufflesecurity/t*"`).Strings()
+	githubExcludeRepos      = githubScan.Flag("exclude-repos", `Repositories to exclude in an org scan. This can also be a glob pattern. You can repeat this flag. Must use Github repo full name. Example: "trufflesecurity/driftwood", "trufflesecurity/d*"`).Strings()
+	githubScanIncludePaths  = githubScan.Flag("include-paths", "Path to file with newline separated regexes for files to include in scan.").Short('i').String()
+	githubScanExcludePaths  = githubScan.Flag("exclude-paths", "Path to file with newline separated regexes for files to exclude in scan.").Short('x').String()
+	githubScanIssueComments = githubScan.Flag("issue-comments", "Include issue comments in scan.").Bool()
+	githubScanPRComments    = githubScan.Flag("pr-comments", "Include pull request comments in scan.").Bool()
+	githubScanGistComments  = githubScan.Flag("gist-comments", "Include gist comments in scan.").Bool()
 
 	gitlabScan = cli.Command("gitlab", "Find credentials in GitLab repositories.")
 	// TODO: Add more GitLab options
@@ -409,16 +412,19 @@ func run(state overseer.State) {
 		}
 
 		cfg := sources.GithubConfig{
-			Endpoint:       *githubScanEndpoint,
-			Token:          *githubScanToken,
-			IncludeForks:   *githubIncludeForks,
-			IncludeMembers: *githubIncludeMembers,
-			Concurrency:    *concurrency,
-			ExcludeRepos:   *githubExcludeRepos,
-			IncludeRepos:   *githubIncludeRepos,
-			Repos:          *githubScanRepos,
-			Orgs:           *githubScanOrgs,
-			Filter:         filter,
+			Endpoint:                   *githubScanEndpoint,
+			Token:                      *githubScanToken,
+			IncludeForks:               *githubIncludeForks,
+			IncludeMembers:             *githubIncludeMembers,
+			Concurrency:                *concurrency,
+			ExcludeRepos:               *githubExcludeRepos,
+			IncludeRepos:               *githubIncludeRepos,
+			Repos:                      *githubScanRepos,
+			Orgs:                       *githubScanOrgs,
+			IncludeIssueComments:       *githubScanIssueComments,
+			IncludePullRequestComments: *githubScanPRComments,
+			IncludeGistComments:        *githubScanGistComments,
+			Filter:                     filter,
 		}
 		if err := e.ScanGitHub(ctx, cfg); err != nil {
 			logFatal(err, "Failed to scan Github.")

diff --git a/pkg/engine/github.go b/pkg/engine/github.go
@@ -15,12 +15,15 @@ import (
 // ScanGitHub scans Github with the provided options.
 func (e *Engine) ScanGitHub(ctx context.Context, c sources.GithubConfig) error {
 	connection := sourcespb.GitHub{
-		Endpoint:      c.Endpoint,
-		Organizations: c.Orgs,
-		Repositories:  c.Repos,
-		ScanUsers:     c.IncludeMembers,
-		IgnoreRepos:   c.ExcludeRepos,
-		IncludeRepos:  c.IncludeRepos,
+		Endpoint:                   c.Endpoint,
+		Organizations:              c.Orgs,
+		Repositories:               c.Repos,
+		ScanUsers:                  c.IncludeMembers,
+		IgnoreRepos:                c.ExcludeRepos,
+		IncludeRepos:               c.IncludeRepos,
+		IncludeIssueComments:       c.IncludeIssueComments,
+		IncludePullRequestComments: c.IncludePullRequestComments,
+		IncludeGistComments:        c.IncludeGistComments,
 	}
 	if len(c.Token) > 0 {
 		connection.Credential = &sourcespb.GitHub_Token{

diff --git a/pkg/sources/github/github.go b/pkg/sources/github/github.go
@@ -978,8 +978,6 @@ func (s *Source) setProgressCompleteWithRepo(index int, offset int, repoURL stri
 }
 
 func (s *Source) scanComments(ctx context.Context, repoPath string, chunksChan chan *sources.Chunk) error {
-	s.log.Info("scanning comments", "repository", repoPath)
-
 	// Support ssh and https URLs
 	repoURL, err := git.GitURLParse(repoPath)
 	if err != nil {
@@ -988,6 +986,7 @@ func (s *Source) scanComments(ctx context.Context, repoPath string, chunksChan c
 
 	trimmedURL := removeURLAndSplit(repoURL.String())
 	if repoURL.Host == "gist.github.com" && s.includeGistComments {
+		s.log.Info("scanning github gist comments", "repository", repoPath)
 		// GitHub Gist URL.
 		var gistId string
 		if len(trimmedURL) == 2 {
@@ -1036,6 +1035,8 @@ func (s *Source) scanComments(ctx context.Context, repoPath string, chunksChan c
 
 		if s.includeIssueComments {
 
+			s.log.Info("scanning github issue comments", "repository", repoPath)
+
 			issueOpts := &github.IssueListCommentsOptions{
 				Sort:      &sortType,
 				Direction: &directionType,
@@ -1070,6 +1071,8 @@ func (s *Source) scanComments(ctx context.Context, repoPath string, chunksChan c
 		}
 
 		if s.includePRComments {
+			s.log.Info("scanning github pull request comments", "repository", repoPath)
+
 			prOpts := &github.PullRequestListCommentsOptions{
 				Sort:      sortType,
 				Direction: directionType,

diff --git a/pkg/sources/sources.go b/pkg/sources/sources.go
@@ -172,6 +172,12 @@ type GithubConfig struct {
 	IncludeRepos []string
 	// Filter is the filter to use to scan the source.
 	Filter *common.Filter
+	// IncludeIssueComments indicates whether to include GitHub issue comments in the scan.
+	IncludeIssueComments,
+	// IncludePullRequestComments indicates whether to include GitHub pull request comments in the scan.
+	IncludePullRequestComments,
+	// IncludeGistComments indicates whether to include GitHub gist comments in the scan.
+	IncludeGistComments bool
 }
 
 // GitlabConfig defines the optional configuration for a gitlab source.