Skip to content

Commit

Permalink
feat: report unknowns in sbom (#2998)
Browse files Browse the repository at this point in the history
Signed-off-by: Keith Zantow <[email protected]>
Signed-off-by: Alex Goodman <[email protected]>
Co-authored-by: Alex Goodman <[email protected]>
  • Loading branch information
kzantow and wagoodman authored Oct 7, 2024
1 parent 4d7ed9f commit ccbee94
Show file tree
Hide file tree
Showing 145 changed files with 4,420 additions and 233 deletions.
12 changes: 12 additions & 0 deletions cmd/syft/internal/options/catalog.go
Original file line number Diff line number Diff line change
Expand Up @@ -53,6 +53,9 @@ type Catalog struct {
Platform string `yaml:"platform" json:"platform" mapstructure:"platform"`
Source sourceConfig `yaml:"source" json:"source" mapstructure:"source"`
Exclusions []string `yaml:"exclude" json:"exclude" mapstructure:"exclude"`

// configuration for inclusion of unknown information within elements
Unknowns unknownsConfig `yaml:"unknowns" mapstructure:"unknowns"`
}

var _ interface {
Expand All @@ -71,6 +74,7 @@ func DefaultCatalog() Catalog {
Java: defaultJavaConfig(),
File: defaultFileConfig(),
Relationships: defaultRelationshipsConfig(),
Unknowns: defaultUnknowns(),
Source: defaultSourceConfig(),
Parallelism: 1,
}
Expand All @@ -82,6 +86,7 @@ func (cfg Catalog) ToSBOMConfig(id clio.Identification) *syft.CreateSBOMConfig {
WithParallelism(cfg.Parallelism).
WithRelationshipsConfig(cfg.ToRelationshipsConfig()).
WithComplianceConfig(cfg.ToComplianceConfig()).
WithUnknownsConfig(cfg.ToUnknownsConfig()).
WithSearchConfig(cfg.ToSearchConfig()).
WithPackagesConfig(cfg.ToPackagesConfig()).
WithFilesConfig(cfg.ToFilesConfig()).
Expand Down Expand Up @@ -114,6 +119,13 @@ func (cfg Catalog) ToComplianceConfig() cataloging.ComplianceConfig {
}
}

func (cfg Catalog) ToUnknownsConfig() cataloging.UnknownsConfig {
return cataloging.UnknownsConfig{
IncludeExecutablesWithoutPackages: cfg.Unknowns.ExecutablesWithoutPackages,
IncludeUnexpandedArchives: cfg.Unknowns.UnexpandedArchives,
}
}

func (cfg Catalog) ToFilesConfig() filecataloging.Config {
hashers, err := intFile.Hashers(cfg.File.Metadata.Digests...)
if err != nil {
Expand Down
31 changes: 31 additions & 0 deletions cmd/syft/internal/options/unknowns.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,31 @@
package options

import (
"github.com/anchore/clio"
"github.com/anchore/syft/syft/cataloging"
)

type unknownsConfig struct {
RemoveWhenPackagesDefined bool `json:"remove-when-packages-defined" yaml:"remove-when-packages-defined" mapstructure:"remove-when-packages-defined"`
ExecutablesWithoutPackages bool `json:"executables-without-packages" yaml:"executables-without-packages" mapstructure:"executables-without-packages"`
UnexpandedArchives bool `json:"unexpanded-archives" yaml:"unexpanded-archives" mapstructure:"unexpanded-archives"`
}

var _ interface {
clio.FieldDescriber
} = (*unknownsConfig)(nil)

func (o *unknownsConfig) DescribeFields(descriptions clio.FieldDescriptionSet) {
descriptions.Add(&o.RemoveWhenPackagesDefined, `remove unknown errors on files with discovered packages`)
descriptions.Add(&o.ExecutablesWithoutPackages, `include executables without any identified packages`)
descriptions.Add(&o.UnexpandedArchives, `include archives which were not expanded and searched`)
}

func defaultUnknowns() unknownsConfig {
def := cataloging.DefaultUnknownsConfig()
return unknownsConfig{
RemoveWhenPackagesDefined: def.RemoveWhenPackagesDefined,
ExecutablesWithoutPackages: def.IncludeExecutablesWithoutPackages,
UnexpandedArchives: def.IncludeUnexpandedArchives,
}
}
2 changes: 1 addition & 1 deletion internal/constants.go
Original file line number Diff line number Diff line change
Expand Up @@ -3,5 +3,5 @@ package internal
const (
// JSONSchemaVersion is the current schema version output by the JSON encoder
// This is roughly following the "SchemaVer" guidelines for versioning the JSON schema. Please see schema/json/README.md for details on how to increment.
JSONSchemaVersion = "16.0.17"
JSONSchemaVersion = "16.0.18"
)
6 changes: 3 additions & 3 deletions internal/file/zip_file_manifest.go
Original file line number Diff line number Diff line change
@@ -1,7 +1,6 @@
package file

import (
"fmt"
"os"
"sort"
"strings"
Expand All @@ -19,12 +18,13 @@ func NewZipFileManifest(archivePath string) (ZipFileManifest, error) {
zipReader, err := OpenZip(archivePath)
manifest := make(ZipFileManifest)
if err != nil {
return manifest, fmt.Errorf("unable to open zip archive (%s): %w", archivePath, err)
log.Debugf("unable to open zip archive (%s): %v", archivePath, err)
return manifest, err
}
defer func() {
err = zipReader.Close()
if err != nil {
log.Warnf("unable to close zip archive (%s): %+v", archivePath, err)
log.Debugf("unable to close zip archive (%s): %+v", archivePath, err)
}
}()

Expand Down
8 changes: 6 additions & 2 deletions internal/file/zip_read_closer.go
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,8 @@ import (
"io"
"math"
"os"

"github.com/anchore/syft/internal/log"
)

// directoryEndLen, readByf, directoryEnd, and findSignatureInBlock were copied from the golang stdlib, specifically:
Expand Down Expand Up @@ -46,7 +48,8 @@ func OpenZip(filepath string) (*ZipReadCloser, error) {
// need to find the start of the archive and keep track of this offset.
offset, err := findArchiveStartOffset(f, fi.Size())
if err != nil {
return nil, fmt.Errorf("cannot find beginning of zip archive=%q : %w", filepath, err)
log.Debugf("cannot find beginning of zip archive=%q : %v", filepath, err)
return nil, err
}

if _, err := f.Seek(0, io.SeekStart); err != nil {
Expand All @@ -62,7 +65,8 @@ func OpenZip(filepath string) (*ZipReadCloser, error) {

r, err := zip.NewReader(io.NewSectionReader(f, offset64, size), size)
if err != nil {
return nil, fmt.Errorf("unable to open ZipReadCloser @ %q: %w", filepath, err)
log.Debugf("unable to open ZipReadCloser @ %q: %v", filepath, err)
return nil, err
}

return &ZipReadCloser{
Expand Down
34 changes: 31 additions & 3 deletions internal/task/executor.go
Original file line number Diff line number Diff line change
Expand Up @@ -11,8 +11,10 @@ import (

"github.com/anchore/syft/internal/log"
"github.com/anchore/syft/internal/sbomsync"
"github.com/anchore/syft/internal/unknown"
"github.com/anchore/syft/syft/event/monitor"
"github.com/anchore/syft/syft/file"
"github.com/anchore/syft/syft/sbom"
)

type Executor struct {
Expand All @@ -35,6 +37,12 @@ func NewTaskExecutor(tasks []Task, numWorkers int) *Executor {
}

func (p *Executor) Execute(ctx context.Context, resolver file.Resolver, s sbomsync.Builder, prog *monitor.CatalogerTaskProgress) error {
var lock sync.Mutex
withLock := func(fn func()) {
lock.Lock()
defer lock.Unlock()
fn()
}
var errs error
wg := &sync.WaitGroup{}
for i := 0; i < p.numWorkers; i++ {
Expand All @@ -48,9 +56,16 @@ func (p *Executor) Execute(ctx context.Context, resolver file.Resolver, s sbomsy
return
}

if err := runTaskSafely(ctx, tsk, resolver, s); err != nil {
errs = multierror.Append(errs, fmt.Errorf("failed to run task: %w", err))
prog.SetError(err)
err := runTaskSafely(ctx, tsk, resolver, s)
unknowns, err := unknown.ExtractCoordinateErrors(err)
if len(unknowns) > 0 {
appendUnknowns(s, tsk.Name(), unknowns)
}
if err != nil {
withLock(func() {
errs = multierror.Append(errs, fmt.Errorf("failed to run task: %w", err))
prog.SetError(err)
})
}
prog.Increment()
}
Expand All @@ -62,6 +77,19 @@ func (p *Executor) Execute(ctx context.Context, resolver file.Resolver, s sbomsy
return errs
}

func appendUnknowns(builder sbomsync.Builder, taskName string, unknowns []unknown.CoordinateError) {
if accessor, ok := builder.(sbomsync.Accessor); ok {
accessor.WriteToSBOM(func(sb *sbom.SBOM) {
for _, u := range unknowns {
if sb.Artifacts.Unknowns == nil {
sb.Artifacts.Unknowns = map[file.Coordinates][]string{}
}
sb.Artifacts.Unknowns[u.Coordinates] = append(sb.Artifacts.Unknowns[u.Coordinates], formatUnknown(u.Reason.Error(), taskName))
}
})
}
}

func runTaskSafely(ctx context.Context, t Task, resolver file.Resolver, s sbomsync.Builder) (err error) {
// handle individual cataloger panics
defer func() {
Expand Down
21 changes: 4 additions & 17 deletions internal/task/file_tasks.go
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,6 @@ package task
import (
"context"
"crypto"
"fmt"

"github.com/anchore/syft/internal/sbomsync"
"github.com/anchore/syft/syft/artifact"
Expand Down Expand Up @@ -32,15 +31,12 @@ func NewFileDigestCatalogerTask(selection file.Selection, hashers ...crypto.Hash
}

result, err := digestsCataloger.Catalog(ctx, resolver, coordinates...)
if err != nil {
return fmt.Errorf("unable to catalog file digests: %w", err)
}

accessor.WriteToSBOM(func(sbom *sbom.SBOM) {
sbom.Artifacts.FileDigests = result
})

return nil
return err
}

return NewTask("file-digest-cataloger", fn)
Expand All @@ -62,15 +58,12 @@ func NewFileMetadataCatalogerTask(selection file.Selection) Task {
}

result, err := metadataCataloger.Catalog(ctx, resolver, coordinates...)
if err != nil {
return err
}

accessor.WriteToSBOM(func(sbom *sbom.SBOM) {
sbom.Artifacts.FileMetadata = result
})

return nil
return err
}

return NewTask("file-metadata-cataloger", fn)
Expand All @@ -87,15 +80,12 @@ func NewFileContentCatalogerTask(cfg filecontent.Config) Task {
accessor := builder.(sbomsync.Accessor)

result, err := cat.Catalog(ctx, resolver)
if err != nil {
return err
}

accessor.WriteToSBOM(func(sbom *sbom.SBOM) {
sbom.Artifacts.FileContents = result
})

return nil
return err
}

return NewTask("file-content-cataloger", fn)
Expand All @@ -112,15 +102,12 @@ func NewExecutableCatalogerTask(selection file.Selection, cfg executable.Config)
accessor := builder.(sbomsync.Accessor)

result, err := cat.Catalog(resolver)
if err != nil {
return err
}

accessor.WriteToSBOM(func(sbom *sbom.SBOM) {
sbom.Artifacts.Executables = result
})

return nil
return err
}

return NewTask("file-executable-cataloger", fn)
Expand Down
5 changes: 1 addition & 4 deletions internal/task/package_task_factory.go
Original file line number Diff line number Diff line change
Expand Up @@ -103,9 +103,6 @@ func NewPackageTask(cfg CatalogingFactoryConfig, c pkg.Cataloger, tags ...string
t := bus.StartCatalogerTask(info, -1, "")

pkgs, relationships, err := c.Catalog(ctx, resolver)
if err != nil {
return fmt.Errorf("unable to catalog packages with %q: %w", catalogerName, err)
}

log.WithFields("cataloger", catalogerName).Debugf("discovered %d packages", len(pkgs))

Expand All @@ -120,7 +117,7 @@ func NewPackageTask(cfg CatalogingFactoryConfig, c pkg.Cataloger, tags ...string
t.SetCompleted()
log.WithFields("name", catalogerName).Trace("package cataloger completed")

return nil
return err
}
tags = append(tags, pkgcataloging.PackageTag)

Expand Down
Loading

0 comments on commit ccbee94

Please sign in to comment.