From e95152f796308c25aaaa6cf75b2e7a81b3ab4388 Mon Sep 17 00:00:00 2001 From: Nikita Pivkin Date: Fri, 2 Aug 2024 13:34:57 +0700 Subject: [PATCH] refactor(misconf): remove file filtering from parsers (#7289) Signed-off-by: nikpivkin --- pkg/iac/detection/detect.go | 11 +++- pkg/iac/detection/detect_test.go | 64 +++++++++++++++++++ pkg/iac/rego/scanner.go | 4 -- pkg/iac/scanners/azure/arm/parser/parser.go | 44 ++----------- .../scanners/azure/arm/parser/parser_test.go | 5 -- pkg/iac/scanners/azure/arm/scanner.go | 12 ++-- .../scanners/cloudformation/parser/parser.go | 29 --------- pkg/iac/scanners/cloudformation/scanner.go | 15 ++--- pkg/iac/scanners/dockerfile/parser/parser.go | 19 +----- pkg/iac/scanners/dockerfile/scanner.go | 33 +++++----- pkg/iac/scanners/helm/parser/parser.go | 21 ------ pkg/iac/scanners/helm/scanner.go | 6 +- pkg/iac/scanners/json/parser/parser.go | 20 +----- pkg/iac/scanners/json/scanner.go | 29 ++++----- pkg/iac/scanners/kubernetes/parser/parser.go | 29 +-------- pkg/iac/scanners/kubernetes/scanner.go | 29 ++++----- pkg/iac/scanners/options/parser.go | 7 -- pkg/iac/scanners/options/scanner.go | 7 -- pkg/iac/scanners/terraform/parser/parser.go | 5 -- pkg/iac/scanners/terraform/scanner.go | 12 ++-- .../scanners/terraformplan/tfjson/scanner.go | 12 ++-- pkg/iac/scanners/toml/parser/parser.go | 19 +----- pkg/iac/scanners/toml/scanner.go | 29 ++++----- pkg/iac/scanners/yaml/parser/parser.go | 19 +----- pkg/iac/scanners/yaml/scanner.go | 29 ++++----- pkg/misconf/scanner.go | 1 - 26 files changed, 173 insertions(+), 337 deletions(-) diff --git a/pkg/iac/detection/detect.go b/pkg/iac/detection/detect.go index 596d8da1851d..787b82cb0b6a 100644 --- a/pkg/iac/detection/detect.go +++ b/pkg/iac/detection/detect.go @@ -135,15 +135,20 @@ func init() { } sniff := struct { - ContentType string `json:"contentType"` - Parameters map[string]any `json:"parameters"` - Resources []any `json:"resources"` + Schema string `json:"$schema"` + Parameters map[string]any `json:"parameters"` + Resources []any `json:"resources"` }{} metadata := types.NewUnmanagedMetadata() if err := armjson.UnmarshalFromReader(r, &sniff, &metadata); err != nil { return false } + // schema is required https://learn.microsoft.com/en-us/azure/azure-resource-manager/templates/syntax + if !strings.HasPrefix(sniff.Schema, "https://schema.management.azure.com/schemas") { + return false + } + return (sniff.Parameters != nil && len(sniff.Parameters) > 0) || (sniff.Resources != nil && len(sniff.Resources) > 0) } diff --git a/pkg/iac/detection/detect_test.go b/pkg/iac/detection/detect_test.go index 5a0d9f876b16..20998427d99d 100644 --- a/pkg/iac/detection/detect_test.go +++ b/pkg/iac/detection/detect_test.go @@ -355,6 +355,70 @@ rules: FileTypeYAML, }, }, + { + name: "Azure ARM template with resources", + path: "test.json", + r: strings.NewReader(` +{ + "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#", + "contentVersion": "1.0.0.0", + "resources": [ + { + "type": "Microsoft.Storage/storageAccounts", + "apiVersion": "2021-09-01", + "name": "{provide-unique-name}", + "location": "eastus", + "sku": { + "name": "Standard_LRS" + }, + "kind": "StorageV2", + "properties": { + "supportsHttpsTrafficOnly": true + } + } + ] +} +`), + expected: []FileType{ + FileTypeJSON, + FileTypeAzureARM, + }, + }, + { + name: "Azure ARM template with parameters", + path: "test.json", + r: strings.NewReader(` +{ + "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#", + "contentVersion": "1.0.0.0", + "parameters": { + "storageName": { + "type": "string", + "minLength": 3, + "maxLength": 24 + } + } +} +`), + expected: []FileType{ + FileTypeJSON, + FileTypeAzureARM, + }, + }, + { + name: "empty Azure ARM template", + path: "test.json", + r: strings.NewReader(` +{ + "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#", + "contentVersion": "1.0.0.0", + "resources": [] +} +`), + expected: []FileType{ + FileTypeJSON, + }, + }, } for _, test := range tests { diff --git a/pkg/iac/rego/scanner.go b/pkg/iac/rego/scanner.go index 2e0516761a02..f7293c46a0c5 100644 --- a/pkg/iac/rego/scanner.go +++ b/pkg/iac/rego/scanner.go @@ -138,10 +138,6 @@ func (s *Scanner) SetPolicyNamespaces(namespaces ...string) { } } -func (s *Scanner) SetSkipRequiredCheck(_ bool) { - // NOTE: Skip required option not applicable for rego. -} - func (s *Scanner) SetRegoErrorLimit(limit int) { s.regoErrorLimit = limit } diff --git a/pkg/iac/scanners/azure/arm/parser/parser.go b/pkg/iac/scanners/azure/arm/parser/parser.go index e5e171914b42..9fbf7b7019a8 100644 --- a/pkg/iac/scanners/azure/arm/parser/parser.go +++ b/pkg/iac/scanners/azure/arm/parser/parser.go @@ -5,8 +5,6 @@ import ( "fmt" "io" "io/fs" - "path/filepath" - "strings" "github.com/aquasecurity/trivy/pkg/iac/debug" azure2 "github.com/aquasecurity/trivy/pkg/iac/scanners/azure" @@ -17,19 +15,14 @@ import ( ) type Parser struct { - targetFS fs.FS - skipRequired bool - debug debug.Logger + targetFS fs.FS + debug debug.Logger } func (p *Parser) SetDebugWriter(writer io.Writer) { p.debug = debug.New(writer, "azure", "arm") } -func (p *Parser) SetSkipRequiredCheck(b bool) { - p.skipRequired = b -} - func New(targetFS fs.FS, opts ...options.ParserOption) *Parser { p := &Parser{ targetFS: targetFS, @@ -56,14 +49,13 @@ func (p *Parser) ParseFS(ctx context.Context, dir string) ([]azure2.Deployment, if entry.IsDir() { return nil } - if !p.Required(path) { - return nil - } + f, err := p.targetFS.Open(path) if err != nil { return err } defer f.Close() + deployment, err := p.parseFile(f, path) if err != nil { return err @@ -77,34 +69,6 @@ func (p *Parser) ParseFS(ctx context.Context, dir string) ([]azure2.Deployment, return deployments, nil } -func (p *Parser) Required(path string) bool { - if p.skipRequired { - return true - } - if !strings.HasSuffix(path, ".json") { - return false - } - data, err := fs.ReadFile(p.targetFS, path) - if err != nil { - return false - } - var template Template - root := types.NewMetadata( - types.NewRange(filepath.Base(path), 0, 0, "", p.targetFS), - "", - ) - if err := armjson.Unmarshal(data, &template, &root); err != nil { - p.debug.Log("Error scanning %s: %s", path, err) - return false - } - - if template.Schema.Kind != azure2.KindString { - return false - } - - return strings.HasPrefix(template.Schema.AsString(), "https://schema.management.azure.com") -} - func (p *Parser) parseFile(r io.Reader, filename string) (*azure2.Deployment, error) { var template Template data, err := io.ReadAll(r) diff --git a/pkg/iac/scanners/azure/arm/parser/parser_test.go b/pkg/iac/scanners/azure/arm/parser/parser_test.go index 3273c0dd6596..b16213d9bebd 100644 --- a/pkg/iac/scanners/azure/arm/parser/parser_test.go +++ b/pkg/iac/scanners/azure/arm/parser/parser_test.go @@ -36,11 +36,6 @@ func TestParser_Parse(t *testing.T) { want func() azure2.Deployment wantDeployment bool }{ - { - name: "invalid code", - input: `blah`, - wantDeployment: false, - }, { name: "basic param", input: `{ diff --git a/pkg/iac/scanners/azure/arm/scanner.go b/pkg/iac/scanners/azure/arm/scanner.go index b4bcfc539486..871b58df3f3d 100644 --- a/pkg/iac/scanners/azure/arm/scanner.go +++ b/pkg/iac/scanners/azure/arm/scanner.go @@ -24,12 +24,12 @@ import ( var _ scanners.FSScanner = (*Scanner)(nil) var _ options.ConfigurableScanner = (*Scanner)(nil) -type Scanner struct { // nolint: gocritic +type Scanner struct { + mu sync.Mutex scannerOptions []options.ScannerOption parserOptions []options.ParserOption debug debug.Logger frameworks []framework.Framework - skipRequired bool regoOnly bool loadEmbeddedPolicies bool loadEmbeddedLibraries bool @@ -37,7 +37,6 @@ type Scanner struct { // nolint: gocritic policyReaders []io.Reader regoScanner *rego.Scanner spec string - sync.Mutex } func (s *Scanner) SetIncludeDeprecatedChecks(b bool) {} @@ -73,9 +72,6 @@ func (s *Scanner) SetPolicyDirs(dirs ...string) { s.policyDirs = dirs } -func (s *Scanner) SetSkipRequiredCheck(skipRequired bool) { - s.skipRequired = skipRequired -} func (s *Scanner) SetPolicyReaders(readers []io.Reader) { s.policyReaders = readers } @@ -106,8 +102,8 @@ func (s *Scanner) SetPolicyNamespaces(...string) {} func (s *Scanner) SetRegoErrorLimit(_ int) {} func (s *Scanner) initRegoScanner(srcFS fs.FS) error { - s.Lock() - defer s.Unlock() + s.mu.Lock() + defer s.mu.Unlock() if s.regoScanner != nil { return nil } diff --git a/pkg/iac/scanners/cloudformation/parser/parser.go b/pkg/iac/scanners/cloudformation/parser/parser.go index 5aa760e19882..d281b585035e 100644 --- a/pkg/iac/scanners/cloudformation/parser/parser.go +++ b/pkg/iac/scanners/cloudformation/parser/parser.go @@ -1,7 +1,6 @@ package parser import ( - "bytes" "context" "encoding/json" "errors" @@ -15,7 +14,6 @@ import ( "gopkg.in/yaml.v3" "github.com/aquasecurity/trivy/pkg/iac/debug" - "github.com/aquasecurity/trivy/pkg/iac/detection" "github.com/aquasecurity/trivy/pkg/iac/ignore" "github.com/aquasecurity/trivy/pkg/iac/scanners/options" ) @@ -24,7 +22,6 @@ var _ options.ConfigurableParser = (*Parser)(nil) type Parser struct { debug debug.Logger - skipRequired bool parameterFiles []string parameters map[string]any overridedParameters Parameters @@ -59,10 +56,6 @@ func (p *Parser) SetDebugWriter(writer io.Writer) { p.debug = debug.New(writer, "cloudformation", "parser") } -func (p *Parser) SetSkipRequiredCheck(b bool) { - p.skipRequired = b -} - func New(opts ...options.ParserOption) *Parser { p := &Parser{} for _, option := range opts { @@ -86,11 +79,6 @@ func (p *Parser) ParseFS(ctx context.Context, fsys fs.FS, dir string) (FileConte return nil } - if !p.Required(fsys, path) { - p.debug.Log("not a CloudFormation file, skipping %s", path) - return nil - } - c, err := p.ParseFile(ctx, fsys, path) if err != nil { p.debug.Log("Error parsing file '%s': %s", path, err) @@ -104,23 +92,6 @@ func (p *Parser) ParseFS(ctx context.Context, fsys fs.FS, dir string) (FileConte return contexts, nil } -func (p *Parser) Required(fsys fs.FS, path string) bool { - if p.skipRequired { - return true - } - - f, err := fsys.Open(filepath.ToSlash(path)) - if err != nil { - return false - } - defer func() { _ = f.Close() }() - if data, err := io.ReadAll(f); err == nil { - return detection.IsType(path, bytes.NewReader(data), detection.FileTypeCloudFormation) - } - return false - -} - func (p *Parser) ParseFile(ctx context.Context, fsys fs.FS, path string) (fctx *FileContext, err error) { defer func() { if e := recover(); e != nil { diff --git a/pkg/iac/scanners/cloudformation/scanner.go b/pkg/iac/scanners/cloudformation/scanner.go index 1bbbe39f2117..20b96ce947fd 100644 --- a/pkg/iac/scanners/cloudformation/scanner.go +++ b/pkg/iac/scanners/cloudformation/scanner.go @@ -47,13 +47,13 @@ func WithConfigsFS(fsys fs.FS) options.ScannerOption { var _ scanners.FSScanner = (*Scanner)(nil) var _ options.ConfigurableScanner = (*Scanner)(nil) -type Scanner struct { // nolint: gocritic +type Scanner struct { + mu sync.Mutex debug debug.Logger policyDirs []string policyReaders []io.Reader parser *parser.Parser regoScanner *rego.Scanner - skipRequired bool regoOnly bool loadEmbeddedPolicies bool loadEmbeddedLibraries bool @@ -61,7 +61,6 @@ type Scanner struct { // nolint: gocritic parserOptions []options.ParserOption frameworks []framework.Framework spec string - sync.Mutex } func (s *Scanner) SetIncludeDeprecatedChecks(bool) {} @@ -98,12 +97,9 @@ func (s *Scanner) SetPolicyReaders(readers []io.Reader) { s.policyReaders = readers } -func (s *Scanner) SetSkipRequiredCheck(skip bool) { - s.skipRequired = skip -} - func (s *Scanner) SetDebugWriter(writer io.Writer) { s.debug = debug.New(writer, "cloudformation", "scanner") + s.parserOptions = append(s.parserOptions, options.ParserWithDebug(writer)) } func (s *Scanner) SetPolicyDirs(dirs ...string) { @@ -132,14 +128,13 @@ func New(opts ...options.ScannerOption) *Scanner { for _, opt := range opts { opt(s) } - s.addParserOptions(options.ParserWithSkipRequiredCheck(s.skipRequired)) s.parser = parser.New(s.parserOptions...) return s } func (s *Scanner) initRegoScanner(srcFS fs.FS) (*rego.Scanner, error) { - s.Lock() - defer s.Unlock() + s.mu.Lock() + defer s.mu.Unlock() if s.regoScanner != nil { return s.regoScanner, nil } diff --git a/pkg/iac/scanners/dockerfile/parser/parser.go b/pkg/iac/scanners/dockerfile/parser/parser.go index d6ff7b4df21a..4255b4609b46 100644 --- a/pkg/iac/scanners/dockerfile/parser/parser.go +++ b/pkg/iac/scanners/dockerfile/parser/parser.go @@ -12,7 +12,6 @@ import ( "github.com/moby/buildkit/frontend/dockerfile/parser" "github.com/aquasecurity/trivy/pkg/iac/debug" - "github.com/aquasecurity/trivy/pkg/iac/detection" "github.com/aquasecurity/trivy/pkg/iac/providers/dockerfile" "github.com/aquasecurity/trivy/pkg/iac/scanners/options" ) @@ -20,18 +19,13 @@ import ( var _ options.ConfigurableParser = (*Parser)(nil) type Parser struct { - debug debug.Logger - skipRequired bool + debug debug.Logger } func (p *Parser) SetDebugWriter(writer io.Writer) { p.debug = debug.New(writer, "dockerfile", "parser") } -func (p *Parser) SetSkipRequiredCheck(b bool) { - p.skipRequired = b -} - // New creates a new Dockerfile parser func New(opts ...options.ParserOption) *Parser { p := &Parser{} @@ -56,9 +50,7 @@ func (p *Parser) ParseFS(ctx context.Context, target fs.FS, path string) (map[st if entry.IsDir() { return nil } - if !p.Required(path) { - return nil - } + df, err := p.ParseFile(ctx, target, path) if err != nil { // TODO add debug for parse errors @@ -82,13 +74,6 @@ func (p *Parser) ParseFile(_ context.Context, fsys fs.FS, path string) (*dockerf return p.parse(path, f) } -func (p *Parser) Required(path string) bool { - if p.skipRequired { - return true - } - return detection.IsType(path, nil, detection.FileTypeDockerfile) -} - func (p *Parser) parse(path string, r io.Reader) (*dockerfile.Dockerfile, error) { parsed, err := parser.Parse(r) if err != nil { diff --git a/pkg/iac/scanners/dockerfile/scanner.go b/pkg/iac/scanners/dockerfile/scanner.go index 29df54634d58..561872c70636 100644 --- a/pkg/iac/scanners/dockerfile/scanner.go +++ b/pkg/iac/scanners/dockerfile/scanner.go @@ -19,17 +19,17 @@ import ( var _ scanners.FSScanner = (*Scanner)(nil) var _ options.ConfigurableScanner = (*Scanner)(nil) -type Scanner struct { // nolint: gocritic - debug debug.Logger - policyDirs []string - policyReaders []io.Reader - parser *parser.Parser - regoScanner *rego.Scanner - skipRequired bool - options []options.ScannerOption - frameworks []framework.Framework - spec string - sync.Mutex +type Scanner struct { + mu sync.Mutex + debug debug.Logger + policyDirs []string + policyReaders []io.Reader + parser *parser.Parser + regoScanner *rego.Scanner + options []options.ScannerOption + parserOpts []options.ParserOption + frameworks []framework.Framework + spec string loadEmbeddedLibraries bool loadEmbeddedPolicies bool } @@ -63,12 +63,9 @@ func (s *Scanner) SetPolicyReaders(readers []io.Reader) { s.policyReaders = readers } -func (s *Scanner) SetSkipRequiredCheck(skip bool) { - s.skipRequired = skip -} - func (s *Scanner) SetDebugWriter(writer io.Writer) { s.debug = debug.New(writer, "dockerfile", "scanner") + s.parserOpts = append(s.parserOpts, options.ParserWithDebug(writer)) } func (s *Scanner) SetTraceWriter(_ io.Writer) { @@ -110,7 +107,7 @@ func NewScanner(opts ...options.ScannerOption) *Scanner { for _, opt := range opts { opt(s) } - s.parser = parser.New(options.ParserWithSkipRequiredCheck(s.skipRequired)) + s.parser = parser.New() return s } @@ -154,8 +151,8 @@ func (s *Scanner) ScanFile(ctx context.Context, fsys fs.FS, path string) (scan.R } func (s *Scanner) initRegoScanner(srcFS fs.FS) (*rego.Scanner, error) { - s.Lock() - defer s.Unlock() + s.mu.Lock() + defer s.mu.Unlock() if s.regoScanner != nil { return s.regoScanner, nil } diff --git a/pkg/iac/scanners/helm/parser/parser.go b/pkg/iac/scanners/helm/parser/parser.go index 8768ac459867..bf4a9fc91721 100644 --- a/pkg/iac/scanners/helm/parser/parser.go +++ b/pkg/iac/scanners/helm/parser/parser.go @@ -34,7 +34,6 @@ type Parser struct { ChartSource string filepaths []string debug debug.Logger - skipRequired bool workingFS fs.FS valuesFiles []string values []string @@ -53,10 +52,6 @@ func (p *Parser) SetDebugWriter(writer io.Writer) { p.debug = debug.New(writer, "helm", "parser") } -func (p *Parser) SetSkipRequiredCheck(b bool) { - p.skipRequired = b -} - func (p *Parser) SetValuesFile(s ...string) { p.valuesFiles = s } @@ -129,10 +124,6 @@ func (p *Parser) ParseFS(ctx context.Context, target fs.FS, path string) error { return nil } - if !p.required(path, p.workingFS) { - return nil - } - if detection.IsArchive(path) { tarFS, err := p.addTarToFS(path) if errors.Is(err, errSkipFS) { @@ -310,15 +301,3 @@ func getManifestPath(manifest string) string { } return manifestFilePathParts[0] } - -func (p *Parser) required(path string, workingFS fs.FS) bool { - if p.skipRequired { - return true - } - content, err := fs.ReadFile(workingFS, path) - if err != nil { - return false - } - - return detection.IsType(path, bytes.NewReader(content), detection.FileTypeHelm) -} diff --git a/pkg/iac/scanners/helm/scanner.go b/pkg/iac/scanners/helm/scanner.go index fc54af44781f..fe74911c51bc 100644 --- a/pkg/iac/scanners/helm/scanner.go +++ b/pkg/iac/scanners/helm/scanner.go @@ -36,7 +36,6 @@ type Scanner struct { loadEmbeddedLibraries bool loadEmbeddedPolicies bool policyFS fs.FS - skipRequired bool frameworks []framework.Framework spec string regoScanner *rego.Scanner @@ -88,12 +87,9 @@ func (s *Scanner) SetPolicyReaders(readers []io.Reader) { s.policyReaders = readers } -func (s *Scanner) SetSkipRequiredCheck(skip bool) { - s.skipRequired = skip -} - func (s *Scanner) SetDebugWriter(writer io.Writer) { s.debug = debug.New(writer, "helm", "scanner") + s.parserOptions = append(s.parserOptions, options.ParserWithDebug(writer)) } func (s *Scanner) SetTraceWriter(_ io.Writer) { diff --git a/pkg/iac/scanners/json/parser/parser.go b/pkg/iac/scanners/json/parser/parser.go index 340f23c9d11c..8f35794229ef 100644 --- a/pkg/iac/scanners/json/parser/parser.go +++ b/pkg/iac/scanners/json/parser/parser.go @@ -8,25 +8,19 @@ import ( "path/filepath" "github.com/aquasecurity/trivy/pkg/iac/debug" - "github.com/aquasecurity/trivy/pkg/iac/detection" "github.com/aquasecurity/trivy/pkg/iac/scanners/options" ) var _ options.ConfigurableParser = (*Parser)(nil) type Parser struct { - debug debug.Logger - skipRequired bool + debug debug.Logger } func (p *Parser) SetDebugWriter(writer io.Writer) { p.debug = debug.New(writer, "json", "parser") } -func (p *Parser) SetSkipRequiredCheck(b bool) { - p.skipRequired = b -} - // New creates a new parser func New(opts ...options.ParserOption) *Parser { p := &Parser{} @@ -51,14 +45,13 @@ func (p *Parser) ParseFS(ctx context.Context, target fs.FS, path string) (map[st if entry.IsDir() { return nil } - if !p.Required(path) { - return nil - } + df, err := p.ParseFile(ctx, target, path) if err != nil { p.debug.Log("Parse error in '%s': %s", path, err) return nil } + files[path] = df return nil }); err != nil { @@ -80,10 +73,3 @@ func (p *Parser) ParseFile(_ context.Context, fsys fs.FS, path string) (any, err } return target, nil } - -func (p *Parser) Required(path string) bool { - if p.skipRequired { - return true - } - return detection.IsType(path, nil, detection.FileTypeJSON) -} diff --git a/pkg/iac/scanners/json/scanner.go b/pkg/iac/scanners/json/scanner.go index 3d563c34c790..3aa0dffdb485 100644 --- a/pkg/iac/scanners/json/scanner.go +++ b/pkg/iac/scanners/json/scanner.go @@ -19,15 +19,15 @@ import ( var _ scanners.FSScanner = (*Scanner)(nil) var _ options.ConfigurableScanner = (*Scanner)(nil) -type Scanner struct { // nolint: gocritic - debug debug.Logger - policyDirs []string - policyReaders []io.Reader - parser *parser.Parser - regoScanner *rego.Scanner - skipRequired bool - options []options.ScannerOption - sync.Mutex +type Scanner struct { + mu sync.Mutex + debug debug.Logger + policyDirs []string + policyReaders []io.Reader + parser *parser.Parser + regoScanner *rego.Scanner + options []options.ScannerOption + parserOpts []options.ParserOption frameworks []framework.Framework spec string loadEmbeddedPolicies bool @@ -61,6 +61,7 @@ func (s *Scanner) SetPolicyReaders(readers []io.Reader) { func (s *Scanner) SetDebugWriter(writer io.Writer) { s.debug = debug.New(writer, "json", "scanner") + s.parserOpts = append(s.parserOpts, options.ParserWithDebug(writer)) } func (s *Scanner) SetTraceWriter(_ io.Writer) { @@ -76,10 +77,6 @@ func (s *Scanner) SetPolicyDirs(dirs ...string) { func (s *Scanner) SetDataDirs(_ ...string) {} func (s *Scanner) SetPolicyNamespaces(_ ...string) {} -func (s *Scanner) SetSkipRequiredCheck(skip bool) { - s.skipRequired = skip -} - func (s *Scanner) SetPolicyFilesystem(_ fs.FS) { // handled by rego when option is passed on } @@ -96,7 +93,7 @@ func NewScanner(opts ...options.ScannerOption) *Scanner { for _, opt := range opts { opt(s) } - s.parser = parser.New(options.ParserWithSkipRequiredCheck(s.skipRequired)) + s.parser = parser.New(s.parserOpts...) return s } @@ -144,8 +141,8 @@ func (s *Scanner) ScanFile(ctx context.Context, fsys fs.FS, path string) (scan.R } func (s *Scanner) initRegoScanner(srcFS fs.FS) (*rego.Scanner, error) { - s.Lock() - defer s.Unlock() + s.mu.Lock() + defer s.mu.Unlock() if s.regoScanner != nil { return s.regoScanner, nil } diff --git a/pkg/iac/scanners/kubernetes/parser/parser.go b/pkg/iac/scanners/kubernetes/parser/parser.go index a1c5ecf46962..e2f225a9fb86 100644 --- a/pkg/iac/scanners/kubernetes/parser/parser.go +++ b/pkg/iac/scanners/kubernetes/parser/parser.go @@ -1,7 +1,6 @@ package parser import ( - "bytes" "context" "encoding/json" "fmt" @@ -15,25 +14,19 @@ import ( kyaml "sigs.k8s.io/yaml" "github.com/aquasecurity/trivy/pkg/iac/debug" - "github.com/aquasecurity/trivy/pkg/iac/detection" "github.com/aquasecurity/trivy/pkg/iac/scanners/options" ) var _ options.ConfigurableParser = (*Parser)(nil) type Parser struct { - debug debug.Logger - skipRequired bool + debug debug.Logger } func (p *Parser) SetDebugWriter(writer io.Writer) { p.debug = debug.New(writer, "kubernetes", "parser") } -func (p *Parser) SetSkipRequiredCheck(b bool) { - p.skipRequired = b -} - // New creates a new K8s parser func New(opts ...options.ParserOption) *Parser { p := &Parser{} @@ -57,14 +50,13 @@ func (p *Parser) ParseFS(ctx context.Context, target fs.FS, path string) (map[st if entry.IsDir() { return nil } - if !p.required(target, path) { - return nil - } + parsed, err := p.ParseFile(ctx, target, path) if err != nil { p.debug.Log("Parse error in '%s': %s", path, err) return nil } + files[path] = parsed return nil }); err != nil { @@ -83,21 +75,6 @@ func (p *Parser) ParseFile(_ context.Context, fsys fs.FS, path string) ([]any, e return p.Parse(f, path) } -func (p *Parser) required(fsys fs.FS, path string) bool { - if p.skipRequired { - return true - } - f, err := fsys.Open(filepath.ToSlash(path)) - if err != nil { - return false - } - defer func() { _ = f.Close() }() - if data, err := io.ReadAll(f); err == nil { - return detection.IsType(path, bytes.NewReader(data), detection.FileTypeKubernetes) - } - return false -} - func (p *Parser) Parse(r io.Reader, path string) ([]any, error) { contents, err := io.ReadAll(r) diff --git a/pkg/iac/scanners/kubernetes/scanner.go b/pkg/iac/scanners/kubernetes/scanner.go index 44f13ce5b003..c5437f292d85 100644 --- a/pkg/iac/scanners/kubernetes/scanner.go +++ b/pkg/iac/scanners/kubernetes/scanner.go @@ -23,15 +23,15 @@ import ( var _ scanners.FSScanner = (*Scanner)(nil) var _ options.ConfigurableScanner = (*Scanner)(nil) -type Scanner struct { // nolint: gocritic - debug debug.Logger - options []options.ScannerOption - policyDirs []string - policyReaders []io.Reader - regoScanner *rego.Scanner - parser *parser.Parser - skipRequired bool - sync.Mutex +type Scanner struct { + mu sync.Mutex + debug debug.Logger + options []options.ScannerOption + parserOpts []options.ParserOption + policyDirs []string + policyReaders []io.Reader + regoScanner *rego.Scanner + parser *parser.Parser loadEmbeddedPolicies bool frameworks []framework.Framework spec string @@ -62,12 +62,9 @@ func (s *Scanner) SetPolicyReaders(readers []io.Reader) { s.policyReaders = readers } -func (s *Scanner) SetSkipRequiredCheck(skip bool) { - s.skipRequired = skip -} - func (s *Scanner) SetDebugWriter(writer io.Writer) { s.debug = debug.New(writer, "kubernetes", "scanner") + s.parserOpts = append(s.parserOpts, options.ParserWithDebug(writer)) } func (s *Scanner) SetTraceWriter(_ io.Writer) { @@ -100,7 +97,7 @@ func NewScanner(opts ...options.ScannerOption) *Scanner { for _, opt := range opts { opt(s) } - s.parser = parser.New(options.ParserWithSkipRequiredCheck(s.skipRequired)) + s.parser = parser.New(s.parserOpts...) return s } @@ -109,8 +106,8 @@ func (s *Scanner) Name() string { } func (s *Scanner) initRegoScanner(srcFS fs.FS) (*rego.Scanner, error) { - s.Lock() - defer s.Unlock() + s.mu.Lock() + defer s.mu.Unlock() if s.regoScanner != nil { return s.regoScanner, nil } diff --git a/pkg/iac/scanners/options/parser.go b/pkg/iac/scanners/options/parser.go index 65ec41bb825d..e411126a6329 100644 --- a/pkg/iac/scanners/options/parser.go +++ b/pkg/iac/scanners/options/parser.go @@ -4,17 +4,10 @@ import "io" type ConfigurableParser interface { SetDebugWriter(io.Writer) - SetSkipRequiredCheck(bool) } type ParserOption func(s ConfigurableParser) -func ParserWithSkipRequiredCheck(skip bool) ParserOption { - return func(s ConfigurableParser) { - s.SetSkipRequiredCheck(skip) - } -} - // ParserWithDebug specifies an io.Writer for debug logs - if not set, they are discarded func ParserWithDebug(w io.Writer) ParserOption { return func(s ConfigurableParser) { diff --git a/pkg/iac/scanners/options/scanner.go b/pkg/iac/scanners/options/scanner.go index 8e79b0c4a185..291400887037 100644 --- a/pkg/iac/scanners/options/scanner.go +++ b/pkg/iac/scanners/options/scanner.go @@ -14,7 +14,6 @@ type ConfigurableScanner interface { SetPolicyDirs(...string) SetDataDirs(...string) SetPolicyNamespaces(...string) - SetSkipRequiredCheck(bool) SetPolicyReaders([]io.Reader) SetPolicyFilesystem(fs.FS) SetDataFilesystem(fs.FS) @@ -104,12 +103,6 @@ func ScannerWithPolicyNamespaces(namespaces ...string) ScannerOption { } } -func ScannerWithSkipRequiredCheck(skip bool) ScannerOption { - return func(s ConfigurableScanner) { - s.SetSkipRequiredCheck(skip) - } -} - func ScannerWithPolicyFilesystem(f fs.FS) ScannerOption { return func(s ConfigurableScanner) { s.SetPolicyFilesystem(f) diff --git a/pkg/iac/scanners/terraform/parser/parser.go b/pkg/iac/scanners/terraform/parser/parser.go index fa511fed54c2..4f79c1fdf6f2 100644 --- a/pkg/iac/scanners/terraform/parser/parser.go +++ b/pkg/iac/scanners/terraform/parser/parser.go @@ -48,7 +48,6 @@ type Parser struct { allowDownloads bool skipCachedModules bool fsMap map[string]fs.FS - skipRequired bool configsFS fs.FS } @@ -76,10 +75,6 @@ func (p *Parser) SetSkipCachedModules(b bool) { p.skipCachedModules = b } -func (p *Parser) SetSkipRequiredCheck(b bool) { - p.skipRequired = b -} - func (p *Parser) SetConfigsFS(fsys fs.FS) { p.configsFS = fsys } diff --git a/pkg/iac/scanners/terraform/scanner.go b/pkg/iac/scanners/terraform/scanner.go index c999acf337f5..a64201e1fcc5 100644 --- a/pkg/iac/scanners/terraform/scanner.go +++ b/pkg/iac/scanners/terraform/scanner.go @@ -27,8 +27,8 @@ var _ scanners.FSScanner = (*Scanner)(nil) var _ options.ConfigurableScanner = (*Scanner)(nil) var _ ConfigurableTerraformScanner = (*Scanner)(nil) -type Scanner struct { // nolint: gocritic - sync.Mutex +type Scanner struct { + mu sync.Mutex options []options.ScannerOption parserOpt []options.ParserOption executorOpt []executor.Option @@ -87,10 +87,6 @@ func (s *Scanner) SetPolicyReaders(readers []io.Reader) { s.policyReaders = readers } -func (s *Scanner) SetSkipRequiredCheck(skip bool) { - s.parserOpt = append(s.parserOpt, options.ParserWithSkipRequiredCheck(skip)) -} - func (s *Scanner) SetDebugWriter(writer io.Writer) { s.parserOpt = append(s.parserOpt, options.ParserWithDebug(writer)) s.executorOpt = append(s.executorOpt, executor.OptionWithDebugWriter(writer)) @@ -131,8 +127,8 @@ func New(opts ...options.ScannerOption) *Scanner { } func (s *Scanner) initRegoScanner(srcFS fs.FS) (*rego.Scanner, error) { - s.Lock() - defer s.Unlock() + s.mu.Lock() + defer s.mu.Unlock() if s.regoScanner != nil { return s.regoScanner, nil } diff --git a/pkg/iac/scanners/terraformplan/tfjson/scanner.go b/pkg/iac/scanners/terraformplan/tfjson/scanner.go index 6f62d822177f..b25eed6ae42b 100644 --- a/pkg/iac/scanners/terraformplan/tfjson/scanner.go +++ b/pkg/iac/scanners/terraformplan/tfjson/scanner.go @@ -23,8 +23,8 @@ var tfPlanExts = []string{ } type Scanner struct { - parser parser.Parser - parserOpt []options.ParserOption + parser *parser.Parser + parserOpt []parser.Option debug debug.Logger options []options.ScannerOption @@ -68,12 +68,8 @@ func (s *Scanner) SetPolicyReaders(readers []io.Reader) { s.policyReaders = readers } -func (s *Scanner) SetSkipRequiredCheck(skip bool) { - s.parserOpt = append(s.parserOpt, options.ParserWithSkipRequiredCheck(skip)) -} - func (s *Scanner) SetDebugWriter(writer io.Writer) { - s.parserOpt = append(s.parserOpt, options.ParserWithDebug(writer)) + s.parserOpt = append(s.parserOpt, parser.OptionWithDebugWriter(writer)) s.executorOpt = append(s.executorOpt, executor.OptionWithDebugWriter(writer)) s.debug = debug.New(writer, "tfplan", "scanner") } @@ -128,12 +124,12 @@ func (s *Scanner) ScanFS(ctx context.Context, inputFS fs.FS, dir string) (scan.R func New(opts ...options.ScannerOption) *Scanner { scanner := &Scanner{ - parser: *parser.New(), options: opts, } for _, o := range opts { o(scanner) } + scanner.parser = parser.New(scanner.parserOpt...) return scanner } diff --git a/pkg/iac/scanners/toml/parser/parser.go b/pkg/iac/scanners/toml/parser/parser.go index a8cdcd730f86..6beed83b1908 100644 --- a/pkg/iac/scanners/toml/parser/parser.go +++ b/pkg/iac/scanners/toml/parser/parser.go @@ -9,25 +9,19 @@ import ( "github.com/BurntSushi/toml" "github.com/aquasecurity/trivy/pkg/iac/debug" - "github.com/aquasecurity/trivy/pkg/iac/detection" "github.com/aquasecurity/trivy/pkg/iac/scanners/options" ) var _ options.ConfigurableParser = (*Parser)(nil) type Parser struct { - debug debug.Logger - skipRequired bool + debug debug.Logger } func (p *Parser) SetDebugWriter(writer io.Writer) { p.debug = debug.New(writer, "toml", "parser") } -func (p *Parser) SetSkipRequiredCheck(b bool) { - p.skipRequired = b -} - // New creates a new parser func New(opts ...options.ParserOption) *Parser { p := &Parser{} @@ -52,9 +46,7 @@ func (p *Parser) ParseFS(ctx context.Context, target fs.FS, path string) (map[st if entry.IsDir() { return nil } - if !p.Required(path) { - return nil - } + df, err := p.ParseFile(ctx, target, path) if err != nil { p.debug.Log("Parse error in '%s': %s", path, err) @@ -81,10 +73,3 @@ func (p *Parser) ParseFile(_ context.Context, fsys fs.FS, path string) (any, err } return target, nil } - -func (p *Parser) Required(path string) bool { - if p.skipRequired { - return true - } - return detection.IsType(path, nil, detection.FileTypeTOML) -} diff --git a/pkg/iac/scanners/toml/scanner.go b/pkg/iac/scanners/toml/scanner.go index 0a05fdbac18f..37e1807ac254 100644 --- a/pkg/iac/scanners/toml/scanner.go +++ b/pkg/iac/scanners/toml/scanner.go @@ -17,15 +17,15 @@ import ( var _ options.ConfigurableScanner = (*Scanner)(nil) -type Scanner struct { // nolint: gocritic - debug debug.Logger - options []options.ScannerOption - policyDirs []string - policyReaders []io.Reader - parser *parser.Parser - regoScanner *rego.Scanner - skipRequired bool - sync.Mutex +type Scanner struct { + mu sync.Mutex + debug debug.Logger + options []options.ScannerOption + parserOptions []options.ParserOption + policyDirs []string + policyReaders []io.Reader + parser *parser.Parser + regoScanner *rego.Scanner frameworks []framework.Framework spec string loadEmbeddedPolicies bool @@ -60,12 +60,9 @@ func (s *Scanner) SetPolicyReaders(readers []io.Reader) { s.policyReaders = readers } -func (s *Scanner) SetSkipRequiredCheck(skip bool) { - s.skipRequired = skip -} - func (s *Scanner) SetDebugWriter(writer io.Writer) { s.debug = debug.New(writer, "toml", "scanner") + s.parserOptions = append(s.parserOptions, options.ParserWithDebug(writer)) } func (s *Scanner) SetTraceWriter(_ io.Writer) {} @@ -94,7 +91,7 @@ func NewScanner(opts ...options.ScannerOption) *Scanner { for _, opt := range opts { opt(s) } - s.parser = parser.New(options.ParserWithSkipRequiredCheck(s.skipRequired)) + s.parser = parser.New(s.parserOptions...) return s } @@ -138,8 +135,8 @@ func (s *Scanner) ScanFile(ctx context.Context, fsys fs.FS, path string) (scan.R } func (s *Scanner) initRegoScanner(srcFS fs.FS) (*rego.Scanner, error) { - s.Lock() - defer s.Unlock() + s.mu.Lock() + defer s.mu.Unlock() if s.regoScanner != nil { return s.regoScanner, nil } diff --git a/pkg/iac/scanners/yaml/parser/parser.go b/pkg/iac/scanners/yaml/parser/parser.go index 6f113635be4a..d8b50faf2437 100644 --- a/pkg/iac/scanners/yaml/parser/parser.go +++ b/pkg/iac/scanners/yaml/parser/parser.go @@ -11,25 +11,19 @@ import ( "gopkg.in/yaml.v3" "github.com/aquasecurity/trivy/pkg/iac/debug" - "github.com/aquasecurity/trivy/pkg/iac/detection" "github.com/aquasecurity/trivy/pkg/iac/scanners/options" ) var _ options.ConfigurableParser = (*Parser)(nil) type Parser struct { - debug debug.Logger - skipRequired bool + debug debug.Logger } func (p *Parser) SetDebugWriter(writer io.Writer) { p.debug = debug.New(writer, "yaml", "parser") } -func (p *Parser) SetSkipRequiredCheck(b bool) { - p.skipRequired = b -} - // New creates a new parser func New(opts ...options.ParserOption) *Parser { p := &Parser{} @@ -54,9 +48,7 @@ func (p *Parser) ParseFS(ctx context.Context, target fs.FS, path string) (map[st if entry.IsDir() { return nil } - if !p.Required(path) { - return nil - } + df, err := p.ParseFile(ctx, target, path) if err != nil { p.debug.Log("Parse error in '%s': %s", path, err) @@ -101,10 +93,3 @@ func (p *Parser) ParseFile(_ context.Context, fsys fs.FS, path string) ([]any, e return results, nil } - -func (p *Parser) Required(path string) bool { - if p.skipRequired { - return true - } - return detection.IsType(path, nil, detection.FileTypeYAML) -} diff --git a/pkg/iac/scanners/yaml/scanner.go b/pkg/iac/scanners/yaml/scanner.go index 0adc43bbd4cf..534ccbd8cb7b 100644 --- a/pkg/iac/scanners/yaml/scanner.go +++ b/pkg/iac/scanners/yaml/scanner.go @@ -17,15 +17,15 @@ import ( var _ options.ConfigurableScanner = (*Scanner)(nil) -type Scanner struct { // nolint: gocritic - options []options.ScannerOption - debug debug.Logger - policyDirs []string - policyReaders []io.Reader - parser *parser.Parser - regoScanner *rego.Scanner - skipRequired bool - sync.Mutex +type Scanner struct { + mu sync.Mutex + options []options.ScannerOption + parserOptions []options.ParserOption + debug debug.Logger + policyDirs []string + policyReaders []io.Reader + parser *parser.Parser + regoScanner *rego.Scanner frameworks []framework.Framework spec string loadEmbeddedLibraries bool @@ -60,12 +60,9 @@ func (s *Scanner) SetPolicyReaders(readers []io.Reader) { s.policyReaders = readers } -func (s *Scanner) SetSkipRequiredCheck(skip bool) { - s.skipRequired = skip -} - func (s *Scanner) SetDebugWriter(writer io.Writer) { s.debug = debug.New(writer, "yaml", "scanner") + s.parserOptions = append(s.parserOptions, options.ParserWithDebug(writer)) } func (s *Scanner) SetTraceWriter(_ io.Writer) {} @@ -93,7 +90,7 @@ func NewScanner(opts ...options.ScannerOption) *Scanner { for _, opt := range opts { opt(s) } - s.parser = parser.New(options.ParserWithSkipRequiredCheck(s.skipRequired)) + s.parser = parser.New(s.parserOptions...) return s } @@ -139,8 +136,8 @@ func (s *Scanner) ScanFile(ctx context.Context, fsys fs.FS, path string) (scan.R } func (s *Scanner) initRegoScanner(srcFS fs.FS) (*rego.Scanner, error) { - s.Lock() - defer s.Unlock() + s.mu.Lock() + defer s.mu.Unlock() if s.regoScanner != nil { return s.regoScanner, nil } diff --git a/pkg/misconf/scanner.go b/pkg/misconf/scanner.go index 90ee90cb4216..8730b03d2faf 100644 --- a/pkg/misconf/scanner.go +++ b/pkg/misconf/scanner.go @@ -215,7 +215,6 @@ func (s *Scanner) filterFS(fsys fs.FS) (fs.FS, error) { func scannerOptions(t detection.FileType, opt ScannerOption) ([]options.ScannerOption, error) { opts := []options.ScannerOption{ - options.ScannerWithSkipRequiredCheck(true), options.ScannerWithEmbeddedPolicies(!opt.DisableEmbeddedPolicies), options.ScannerWithEmbeddedLibraries(!opt.DisableEmbeddedLibraries), options.ScannerWithIncludeDeprecatedChecks(opt.IncludeDeprecatedChecks),