From db89e345d758258c608267a6ea6102d1b2a97505 Mon Sep 17 00:00:00 2001 From: Zubair Khan Date: Wed, 16 Aug 2023 18:23:59 -0400 Subject: [PATCH] correct logging output for github comments and add oss flags (#1632) * correct logging output * add flags * respect oss cli flags for github comment scanning * improve copy --- main.go | 48 ++++++++++++++++++++---------------- pkg/engine/github.go | 15 ++++++----- pkg/sources/github/github.go | 7 ++++-- pkg/sources/sources.go | 6 +++++ 4 files changed, 47 insertions(+), 29 deletions(-) diff --git a/main.go b/main.go index c680b7c08d9f..c31a1d1ac6e7 100644 --- a/main.go +++ b/main.go @@ -73,17 +73,20 @@ var ( _ = gitScan.Flag("entropy", "No-op flag for backwards compat.").Bool() _ = gitScan.Flag("regex", "No-op flag for backwards compat.").Bool() - githubScan = cli.Command("github", "Find credentials in GitHub repositories.") - githubScanEndpoint = githubScan.Flag("endpoint", "GitHub endpoint.").Default("https://api.github.com").String() - githubScanRepos = githubScan.Flag("repo", `GitHub repository to scan. You can repeat this flag. Example: "https://github.com/dustin-decker/secretsandstuff"`).Strings() - githubScanOrgs = githubScan.Flag("org", `GitHub organization to scan. You can repeat this flag. Example: "trufflesecurity"`).Strings() - githubScanToken = githubScan.Flag("token", "GitHub token. Can be provided with environment variable GITHUB_TOKEN.").Envar("GITHUB_TOKEN").String() - githubIncludeForks = githubScan.Flag("include-forks", "Include forks in scan.").Bool() - githubIncludeMembers = githubScan.Flag("include-members", "Include organization member repositories in scan.").Bool() - githubIncludeRepos = githubScan.Flag("include-repos", `Repositories to include in an org scan. This can also be a glob pattern. You can repeat this flag. Must use Github repo full name. Example: "trufflesecurity/trufflehog", "trufflesecurity/t*"`).Strings() - githubExcludeRepos = githubScan.Flag("exclude-repos", `Repositories to exclude in an org scan. This can also be a glob pattern. You can repeat this flag. Must use Github repo full name. Example: "trufflesecurity/driftwood", "trufflesecurity/d*"`).Strings() - githubScanIncludePaths = githubScan.Flag("include-paths", "Path to file with newline separated regexes for files to include in scan.").Short('i').String() - githubScanExcludePaths = githubScan.Flag("exclude-paths", "Path to file with newline separated regexes for files to exclude in scan.").Short('x').String() + githubScan = cli.Command("github", "Find credentials in GitHub repositories.") + githubScanEndpoint = githubScan.Flag("endpoint", "GitHub endpoint.").Default("https://api.github.com").String() + githubScanRepos = githubScan.Flag("repo", `GitHub repository to scan. You can repeat this flag. Example: "https://github.com/dustin-decker/secretsandstuff"`).Strings() + githubScanOrgs = githubScan.Flag("org", `GitHub organization to scan. You can repeat this flag. Example: "trufflesecurity"`).Strings() + githubScanToken = githubScan.Flag("token", "GitHub token. Can be provided with environment variable GITHUB_TOKEN.").Envar("GITHUB_TOKEN").String() + githubIncludeForks = githubScan.Flag("include-forks", "Include forks in scan.").Bool() + githubIncludeMembers = githubScan.Flag("include-members", "Include organization member repositories in scan.").Bool() + githubIncludeRepos = githubScan.Flag("include-repos", `Repositories to include in an org scan. This can also be a glob pattern. You can repeat this flag. Must use Github repo full name. Example: "trufflesecurity/trufflehog", "trufflesecurity/t*"`).Strings() + githubExcludeRepos = githubScan.Flag("exclude-repos", `Repositories to exclude in an org scan. This can also be a glob pattern. You can repeat this flag. Must use Github repo full name. Example: "trufflesecurity/driftwood", "trufflesecurity/d*"`).Strings() + githubScanIncludePaths = githubScan.Flag("include-paths", "Path to file with newline separated regexes for files to include in scan.").Short('i').String() + githubScanExcludePaths = githubScan.Flag("exclude-paths", "Path to file with newline separated regexes for files to exclude in scan.").Short('x').String() + githubScanIssueComments = githubScan.Flag("issue-comments", "Include issue comments in scan.").Bool() + githubScanPRComments = githubScan.Flag("pr-comments", "Include pull request comments in scan.").Bool() + githubScanGistComments = githubScan.Flag("gist-comments", "Include gist comments in scan.").Bool() gitlabScan = cli.Command("gitlab", "Find credentials in GitLab repositories.") // TODO: Add more GitLab options @@ -409,16 +412,19 @@ func run(state overseer.State) { } cfg := sources.GithubConfig{ - Endpoint: *githubScanEndpoint, - Token: *githubScanToken, - IncludeForks: *githubIncludeForks, - IncludeMembers: *githubIncludeMembers, - Concurrency: *concurrency, - ExcludeRepos: *githubExcludeRepos, - IncludeRepos: *githubIncludeRepos, - Repos: *githubScanRepos, - Orgs: *githubScanOrgs, - Filter: filter, + Endpoint: *githubScanEndpoint, + Token: *githubScanToken, + IncludeForks: *githubIncludeForks, + IncludeMembers: *githubIncludeMembers, + Concurrency: *concurrency, + ExcludeRepos: *githubExcludeRepos, + IncludeRepos: *githubIncludeRepos, + Repos: *githubScanRepos, + Orgs: *githubScanOrgs, + IncludeIssueComments: *githubScanIssueComments, + IncludePullRequestComments: *githubScanPRComments, + IncludeGistComments: *githubScanGistComments, + Filter: filter, } if err := e.ScanGitHub(ctx, cfg); err != nil { logFatal(err, "Failed to scan Github.") diff --git a/pkg/engine/github.go b/pkg/engine/github.go index 0442c4756287..4e83acbdd208 100644 --- a/pkg/engine/github.go +++ b/pkg/engine/github.go @@ -15,12 +15,15 @@ import ( // ScanGitHub scans Github with the provided options. func (e *Engine) ScanGitHub(ctx context.Context, c sources.GithubConfig) error { connection := sourcespb.GitHub{ - Endpoint: c.Endpoint, - Organizations: c.Orgs, - Repositories: c.Repos, - ScanUsers: c.IncludeMembers, - IgnoreRepos: c.ExcludeRepos, - IncludeRepos: c.IncludeRepos, + Endpoint: c.Endpoint, + Organizations: c.Orgs, + Repositories: c.Repos, + ScanUsers: c.IncludeMembers, + IgnoreRepos: c.ExcludeRepos, + IncludeRepos: c.IncludeRepos, + IncludeIssueComments: c.IncludeIssueComments, + IncludePullRequestComments: c.IncludePullRequestComments, + IncludeGistComments: c.IncludeGistComments, } if len(c.Token) > 0 { connection.Credential = &sourcespb.GitHub_Token{ diff --git a/pkg/sources/github/github.go b/pkg/sources/github/github.go index c4b0df8517ff..dea631e5cb42 100644 --- a/pkg/sources/github/github.go +++ b/pkg/sources/github/github.go @@ -978,8 +978,6 @@ func (s *Source) setProgressCompleteWithRepo(index int, offset int, repoURL stri } func (s *Source) scanComments(ctx context.Context, repoPath string, chunksChan chan *sources.Chunk) error { - s.log.Info("scanning comments", "repository", repoPath) - // Support ssh and https URLs repoURL, err := git.GitURLParse(repoPath) if err != nil { @@ -988,6 +986,7 @@ func (s *Source) scanComments(ctx context.Context, repoPath string, chunksChan c trimmedURL := removeURLAndSplit(repoURL.String()) if repoURL.Host == "gist.github.com" && s.includeGistComments { + s.log.Info("scanning github gist comments", "repository", repoPath) // GitHub Gist URL. var gistId string if len(trimmedURL) == 2 { @@ -1036,6 +1035,8 @@ func (s *Source) scanComments(ctx context.Context, repoPath string, chunksChan c if s.includeIssueComments { + s.log.Info("scanning github issue comments", "repository", repoPath) + issueOpts := &github.IssueListCommentsOptions{ Sort: &sortType, Direction: &directionType, @@ -1070,6 +1071,8 @@ func (s *Source) scanComments(ctx context.Context, repoPath string, chunksChan c } if s.includePRComments { + s.log.Info("scanning github pull request comments", "repository", repoPath) + prOpts := &github.PullRequestListCommentsOptions{ Sort: sortType, Direction: directionType, diff --git a/pkg/sources/sources.go b/pkg/sources/sources.go index 4ecc46e803f6..af6fa62d147e 100644 --- a/pkg/sources/sources.go +++ b/pkg/sources/sources.go @@ -172,6 +172,12 @@ type GithubConfig struct { IncludeRepos []string // Filter is the filter to use to scan the source. Filter *common.Filter + // IncludeIssueComments indicates whether to include GitHub issue comments in the scan. + IncludeIssueComments, + // IncludePullRequestComments indicates whether to include GitHub pull request comments in the scan. + IncludePullRequestComments, + // IncludeGistComments indicates whether to include GitHub gist comments in the scan. + IncludeGistComments bool } // GitlabConfig defines the optional configuration for a gitlab source.