From 2e164aaf304b103a2a33853a221968b0f3c0810b Mon Sep 17 00:00:00 2001 From: Naimuddin Shaik Date: Tue, 15 Jun 2021 02:51:32 +0530 Subject: [PATCH] Changed trivy DB for below changes 1. New debian source will added in new buckets, called debain-salsa 7/ debian-salsa 8 etc 2. CVE and advisory relation is captured in advisory bucket. If a CVE is fixed inside a advisory, that advisory ID is saved along with CVE. 3. All advisory details are saved in vulnerability bucket. 4. In ubuntu needs-traiged status CVEs are also captured into bolt DB now. 5. Added advisory details for Amazon Linux --- pkg/db/advisory_detail.go | 39 +++++ pkg/db/db.go | 5 + pkg/db/security_advisory.go | 82 ++++++++++ pkg/types/types.go | 59 +++++--- pkg/vulnsrc/amazon/amazon.go | 29 ++++ pkg/vulnsrc/debian-salsa/debian.go | 167 +++++++++++++++++++++ pkg/vulnsrc/debian-salsa/types.go | 25 +++ pkg/vulnsrc/ubuntu/ubuntu.go | 2 +- pkg/vulnsrc/vulnerability/const.go | 1 + pkg/vulnsrc/vulnerability/vulnerability.go | 11 ++ pkg/vulnsrc/vulnsrc.go | 10 +- 11 files changed, 406 insertions(+), 24 deletions(-) create mode 100644 pkg/db/security_advisory.go create mode 100644 pkg/vulnsrc/debian-salsa/debian.go create mode 100644 pkg/vulnsrc/debian-salsa/types.go diff --git a/pkg/db/advisory_detail.go b/pkg/db/advisory_detail.go index baef2ed9..a60d4e7d 100644 --- a/pkg/db/advisory_detail.go +++ b/pkg/db/advisory_detail.go @@ -71,3 +71,42 @@ func (dbc Config) GetAdvisoryDetails(cveID string) ([]types.AdvisoryDetail, erro func (dbc Config) DeleteAdvisoryDetailBucket() error { return dbc.deleteBucket(advisoryDetailBucket) } + +func (dbc Config) GetAdvisoryDetail(tx *bolt.Tx, cveID string, platformName string, pkgName string) (types.AdvisoryDetail, error) { + advisory := types.AdvisoryDetail{} + root := tx.Bucket([]byte(advisoryDetailBucket)) + if root == nil { + return advisory, nil + } + cveBucket := root.Bucket([]byte(cveID)) + if cveBucket == nil { + return advisory, nil + } + err := cveBucket.ForEach(func(platform, v []byte) error { + packageBucket := cveBucket.Bucket(platform) + if packageBucket == nil { + return nil + } + err := packageBucket.ForEach(func(packageName, v []byte) error { + var detail types.Advisory + if err := json.Unmarshal(v, &detail); err != nil { + return xerrors.Errorf("failed to unmarshall advisory_detail: %w", err) + } + if string(packageName) == pkgName && string(platform) == platformName { + advisory = types.AdvisoryDetail{ + PlatformName: string(platform), + PackageName: string(packageName), + AdvisoryItem: detail, + } + + return nil + } + return nil + }) + return err + }) + if err != nil { + return advisory, xerrors.Errorf("error in db foreach: %w", err) + } + return advisory, nil +} diff --git a/pkg/db/db.go b/pkg/db/db.go index 7d9b5f96..eefd0dfc 100644 --- a/pkg/db/db.go +++ b/pkg/db/db.go @@ -55,7 +55,12 @@ type Operation interface { GetAdvisoryDetails(cveID string) ([]types.AdvisoryDetail, error) PutAdvisoryDetail(tx *bolt.Tx, vulnerabilityID string, source string, pkgName string, advisory interface{}) (err error) + GetAdvisoryDetail(tx *bolt.Tx, cveID string, platformName string, pkgName string) (types.AdvisoryDetail, error) DeleteAdvisoryDetailBucket() error + + GetSecurityAdvisoryDetails(cveId string) (types.SecurityAdvisories, error) + PutSecurityAdvisoryDetails(tx *bolt.Tx, platform string, advisoryId string, securityAdvisory map[string]types.SecurityAdvisory) error + DeleteSecurityAdvisoryBucket() error } type Metadata struct { diff --git a/pkg/db/security_advisory.go b/pkg/db/security_advisory.go new file mode 100644 index 00000000..74c3da0a --- /dev/null +++ b/pkg/db/security_advisory.go @@ -0,0 +1,82 @@ +package db + +import ( + "encoding/json" + + "github.com/aquasecurity/trivy-db/pkg/types" + bolt "go.etcd.io/bbolt" + "golang.org/x/xerrors" +) + +const ( + securityAdvisoryBucket = "security-advisory" +) + +func (dbc Config) GetSecurityAdvisoryDetails(cveId string) (types.SecurityAdvisories, error) { + SecurityAdvisories := types.SecurityAdvisories{} + err := db.View(func(tx *bolt.Tx) error { + root := tx.Bucket([]byte(securityAdvisoryBucket)) + if root == nil { + return nil + } + cveBucket := root.Bucket([]byte(cveId)) + if cveBucket == nil { + return nil + } + err := cveBucket.ForEach(func(platform, v []byte) error { + securityAdvisory := make(map[string]types.SecurityAdvisory) + advisoryBucket := cveBucket.Bucket(platform) + if advisoryBucket == nil { + return nil + } + err := advisoryBucket.ForEach(func(advisoryID, v []byte) error { + detail := types.SecurityAdvisory{} + if err := json.Unmarshal(v, &detail); err != nil { + return xerrors.Errorf("failed to unmarshall advisory_detail: %w", err) + } + securityAdvisory[string(advisoryID)] = detail + return nil + }) + SecurityAdvisories[string(platform)] = securityAdvisory + return err + }) + if err != nil { + return xerrors.Errorf("error in db foreach: %w", err) + } + return nil + }) + return SecurityAdvisories, err +} + +func (dbc Config) PutSecurityAdvisoryDetails(tx *bolt.Tx, cveId string, osName string, securityAdvisory map[string]types.SecurityAdvisory) error { + root, err := tx.CreateBucketIfNotExists([]byte(securityAdvisoryBucket)) + if err != nil { + return err + } + + cveBucket, err := root.CreateBucketIfNotExists([]byte(cveId)) + if err != nil { + return err + } + + osBucket, err := cveBucket.CreateBucketIfNotExists([]byte(osName)) + if err != nil { + return err + } + + for secAdvId, advisoryDetail := range securityAdvisory { + jsonVal, err := json.Marshal(advisoryDetail) + if err != nil { + return xerrors.Errorf("failed to marshal JSON: %w", err) + } + err = osBucket.Put([]byte(secAdvId), jsonVal) + if err != nil { + return err + } + } + return nil +} + +func (dbc Config) DeleteSecurityAdvisoryBucket() error { + return dbc.deleteBucket(securityAdvisoryBucket) +} diff --git a/pkg/types/types.go b/pkg/types/types.go index bf5a425a..3edf10e5 100644 --- a/pkg/types/types.go +++ b/pkg/types/types.go @@ -10,6 +10,7 @@ import ( type Severity int type VendorSeverity map[string]Severity +type SecurityAdvisories map[string]map[string]SecurityAdvisory type CVSS struct { V2Vector string `json:"V2Vector,omitempty"` @@ -82,19 +83,20 @@ type LastUpdated struct { Date time.Time } type VulnerabilityDetail struct { - ID string `json:",omitempty"` // e.g. CVE-2019-8331, OSVDB-104365 - CvssScore float64 `json:",omitempty"` - CvssVector string `json:",omitempty"` - CvssScoreV3 float64 `json:",omitempty"` - CvssVectorV3 string `json:",omitempty"` - Severity Severity `json:",omitempty"` - SeverityV3 Severity `json:",omitempty"` - CweIDs []string `json:",omitempty"` // e.g. CWE-78, CWE-89 - References []string `json:",omitempty"` - Title string `json:",omitempty"` - Description string `json:",omitempty"` - PublishedDate *time.Time `json:",omitempty"` - LastModifiedDate *time.Time `json:",omitempty"` + ID string `json:",omitempty"` // e.g. CVE-2019-8331, OSVDB-104365 + CvssScore float64 `json:",omitempty"` + CvssVector string `json:",omitempty"` + CvssScoreV3 float64 `json:",omitempty"` + CvssVectorV3 string `json:",omitempty"` + Severity Severity `json:",omitempty"` + SeverityV3 Severity `json:",omitempty"` + AdvisoryDetails SecurityAdvisories `json:",omitempty"` + CweIDs []string `json:",omitempty"` // e.g. CWE-78, CWE-89 + References []string `json:",omitempty"` + Title string `json:",omitempty"` + Description string `json:",omitempty"` + PublishedDate *time.Time `json:",omitempty"` + LastModifiedDate *time.Time `json:",omitempty"` } type AdvisoryDetail struct { @@ -103,6 +105,13 @@ type AdvisoryDetail struct { AdvisoryItem interface{} } +type SecurityAdvisory struct { + SecurityAdvisoryId string `json:"security_advisory_id,omitempty"` + Severity string `json:"severity,omitempty"` + PublishDate time.Time `json:"publish_date,omitempty"` + Description string `json:"description,omitempty"` +} + type Advisory struct { VulnerabilityID string `json:",omitempty"` @@ -110,23 +119,29 @@ type Advisory struct { FixedVersion string `json:",omitempty"` AffectedVersion string `json:",omitempty"` // Only for Arch Linux + WillNotFix bool `json:"will_not_fix,omitempty"` + // Version ranges for language-specific package // Some advisories provide VulnerableVersions only, others provide PatchedVersions and UnaffectedVersions VulnerableVersions []string `json:",omitempty"` PatchedVersions []string `json:",omitempty"` UnaffectedVersions []string `json:",omitempty"` + // Security Advisories + SecurityAdvisory []string `json:",omitempty"` } type Vulnerability struct { - Title string `json:",omitempty"` - Description string `json:",omitempty"` - Severity string `json:",omitempty"` // Selected from VendorSeverity, depending on a scan target - CweIDs []string `json:",omitempty"` // e.g. CWE-78, CWE-89 - VendorSeverity VendorSeverity `json:",omitempty"` - CVSS VendorCVSS `json:",omitempty"` - References []string `json:",omitempty"` - PublishedDate *time.Time `json:",omitempty"` - LastModifiedDate *time.Time `json:",omitempty"` + Title string `json:",omitempty"` + Description string `json:",omitempty"` + Severity string `json:",omitempty"` // Selected from VendorSeverity, depending on a scan target + CweIDs []string `json:",omitempty"` // e.g. CWE-78, CWE-89 + VendorSeverity VendorSeverity `json:",omitempty"` + CVSS VendorCVSS `json:",omitempty"` + AdvisoryDetails SecurityAdvisories `json:",omitempty"` + References []string `json:",omitempty"` + PublishedDate *time.Time `json:",omitempty"` + LastModifiedDate *time.Time `json:",omitempty"` + VendorURL string `json:",omitempty"` } type VulnSrc interface { diff --git a/pkg/vulnsrc/amazon/amazon.go b/pkg/vulnsrc/amazon/amazon.go index 77f6a49b..13f70ba3 100644 --- a/pkg/vulnsrc/amazon/amazon.go +++ b/pkg/vulnsrc/amazon/amazon.go @@ -7,6 +7,7 @@ import ( "log" "path/filepath" "strings" + "time" "github.com/aquasecurity/trivy-db/pkg/types" @@ -105,6 +106,22 @@ func (vs VulnSrc) commitFunc(tx *bolt.Tx) error { advisory := types.Advisory{ FixedVersion: constructVersion(pkg.Epoch, pkg.Version, pkg.Release), } + existingAdvisory, err := vs.dbc.GetAdvisoryDetail(tx, cveID, platformName, pkg.Name) + if err != nil { + return xerrors.Errorf("failed to get Amazon advisory: %w", err) + } + if existingAdvisory.AdvisoryItem != nil { + existingAdvisoryDetails := existingAdvisory.AdvisoryItem.(types.Advisory) + + if len(existingAdvisoryDetails.SecurityAdvisory) > 0 { + advisory.SecurityAdvisory = existingAdvisoryDetails.SecurityAdvisory + } + } + + if !utils.StringInSlice(alas.ID, advisory.SecurityAdvisory) { + advisory.SecurityAdvisory = append(advisory.SecurityAdvisory, alas.ID) + } + if err := vs.dbc.PutAdvisoryDetail(tx, cveID, platformName, pkg.Name, advisory); err != nil { return xerrors.Errorf("failed to save Amazon advisory: %w", err) } @@ -123,6 +140,18 @@ func (vs VulnSrc) commitFunc(tx *bolt.Tx) error { if err := vs.dbc.PutVulnerabilityDetail(tx, cveID, vulnerability.Amazon, vuln); err != nil { return xerrors.Errorf("failed to save Amazon vulnerability detail: %w", err) } + publishDate, err := time.Parse("2006-01-02 15:04", alas.Issued.Date) + if err != nil { + log.Println("Error in publish Date, %w", err) + } + securityAdvisory := map[string]types.SecurityAdvisory{alas.ID: types.SecurityAdvisory{ + Severity: alas.Severity, + PublishDate: publishDate, + Description: alas.Description, + }} + if err := vs.dbc.PutSecurityAdvisoryDetails(tx, cveID, vulnerability.Amazon, securityAdvisory); err != nil { + return xerrors.Errorf("failed to save Debian vulnerability: %w", err) + } // for light DB if err := vs.dbc.PutSeverity(tx, cveID, types.SeverityUnknown); err != nil { diff --git a/pkg/vulnsrc/debian-salsa/debian.go b/pkg/vulnsrc/debian-salsa/debian.go new file mode 100644 index 00000000..b357a33d --- /dev/null +++ b/pkg/vulnsrc/debian-salsa/debian.go @@ -0,0 +1,167 @@ +package debian + +import ( + "encoding/json" + "fmt" + "io" + "log" + "path/filepath" + "strings" + "time" + + "github.com/aquasecurity/trivy-db/pkg/types" + + "github.com/aquasecurity/trivy-db/pkg/db" + "github.com/aquasecurity/trivy-db/pkg/utils" + "github.com/aquasecurity/trivy-db/pkg/vulnsrc/vulnerability" + bolt "go.etcd.io/bbolt" + "golang.org/x/xerrors" +) + +const ( + debianDir = "debian-salsa" +) + +var ( + // e.g. debian 8 + platformFormat = "debian-salsa %s" + DebianReleasesMapping = map[string]string{ + // Code names + "squeeze": "6", + "wheezy": "7", + "jessie": "8", + "stretch": "9", + "buster": "10", + "unstable": "unstable", + "bullseye": "bullseye", + } +) + +type VulnSrc struct { + dbc db.Operation +} + +func NewVulnSrc() VulnSrc { + return VulnSrc{ + dbc: db.Config{}, + } +} + +func (vs VulnSrc) Update(dir string) error { + rootDir := filepath.Join(dir, "vuln-list", debianDir) + var cves []DebianCVE + err := utils.FileWalk(rootDir, func(r io.Reader, path string) error { + var cve DebianCVE + if err := json.NewDecoder(r).Decode(&cve); err != nil { + return xerrors.Errorf("failed to decode Debian JSON: %w", err) + } + cve.VulnerabilityID = strings.TrimSuffix(filepath.Base(path), ".json") + cve.Package = filepath.Base(filepath.Dir(path)) + cves = append(cves, cve) + + return nil + }) + if err != nil { + return xerrors.Errorf("error in Debian walk: %w", err) + } + + if err = vs.save(cves); err != nil { + return xerrors.Errorf("error in Debian save: %w", err) + } + + return nil +} + +func (vs VulnSrc) save(cves []DebianCVE) error { + log.Println("Saving Debian DB") + err := vs.dbc.BatchUpdate(func(tx *bolt.Tx) error { + return vs.commit(tx, cves) + }) + if err != nil { + return xerrors.Errorf("error in batch update: %w", err) + } + + return nil +} + +func (vs VulnSrc) commit(tx *bolt.Tx, cves []DebianCVE) error { + for _, cve := range cves { + for releaseName, release := range cve.Releases { + majorVersion, ok := DebianReleasesMapping[releaseName] + if !ok { + continue + } + platformName := fmt.Sprintf(platformFormat, majorVersion) + // Advisory IDs + var advisoryIds []string + for secAdvId, _ := range release.SecurityAdvisory { + advisoryIds = append(advisoryIds, secAdvId) + } + advisory := types.Advisory{ + FixedVersion: release.FixVersion, + WillNotFix: release.WillNotFix, + SecurityAdvisory: advisoryIds, + } + if err := vs.dbc.PutAdvisoryDetail(tx, cve.VulnerabilityID, platformName, cve.Package, advisory); err != nil { + return xerrors.Errorf("failed to save Debian advisory: %w", err) + } + + vuln := types.VulnerabilityDetail{ + Severity: severityFromUrgency(release.Severity), + Description: cve.Description, + } + if err := vs.dbc.PutVulnerabilityDetail(tx, cve.VulnerabilityID, vulnerability.Debian, vuln); err != nil { + return xerrors.Errorf("failed to save Debian vulnerability: %w", err) + } + // Save Security Advisory details + if len(release.SecurityAdvisory) > 0 { + securityAdvisories := make(map[string]types.SecurityAdvisory) + for advId, advisory := range release.SecurityAdvisory { + secAdvisory := types.SecurityAdvisory{} + publishDate, err := time.Parse("2006-01-02 15:04:05 +0000 UTC", advisory.PublishDate) + if err != nil { + log.Println("Error in publish Date, %w", err) + } + secAdvisory.PublishDate = publishDate + secAdvisory.Description = advisory.Description + securityAdvisories[advId] = secAdvisory + } + if err := vs.dbc.PutSecurityAdvisoryDetails(tx, cve.VulnerabilityID, vulnerability.Debian, securityAdvisories); err != nil { + return xerrors.Errorf("failed to save Debian vulnerability: %w", err) + } + } + // for light DB + if err := vs.dbc.PutSeverity(tx, cve.VulnerabilityID, types.SeverityUnknown); err != nil { + return xerrors.Errorf("failed to save Debian vulnerability severity: %w", err) + } + } + } + return nil +} + +func (vs VulnSrc) Get(release string, pkgName string) ([]types.Advisory, error) { + bucket := fmt.Sprintf(platformFormat, release) + advisories, err := vs.dbc.GetAdvisories(bucket, pkgName) + if err != nil { + return nil, xerrors.Errorf("failed to get Debian advisories: %w", err) + } + return advisories, nil +} + +func severityFromUrgency(urgency string) types.Severity { + switch urgency { + case "not yet assigned", "end-of-life": + return types.SeverityUnknown + + case "unimportant", "low", "low*", "low**": + return types.SeverityLow + + case "medium", "medium*", "medium**": + return types.SeverityMedium + + case "high", "high*", "high**": + return types.SeverityHigh + default: + return types.SeverityUnknown + } +} diff --git a/pkg/vulnsrc/debian-salsa/types.go b/pkg/vulnsrc/debian-salsa/types.go new file mode 100644 index 00000000..33670eaa --- /dev/null +++ b/pkg/vulnsrc/debian-salsa/types.go @@ -0,0 +1,25 @@ +package debian + +type DebianCVE struct { + Description string `json:"description,omitempty"` + Releases map[string]Release `json:"releases,omitempty"` + Package string + VulnerabilityID string +} + +type Release struct { + FixVersion string `json:"fix_version"` + WillNotFix bool `json:"will_not_fix"` + Severity string `json:"severity"` + Statement string `json:"statement"` + SecurityAdvisory map[string]SecurityAdvisoryDebian `json:"security_advisory"` + ClassificationID int64 `json:"classification_id"` + SeverityClassification string `json:"severity_classification"` +} + +type SecurityAdvisoryDebian struct { + SecurityAdvisoryId string `json:"security_advisory_id,omitempty"` + Severity string `json:"severity,omitempty"` + PublishDate string `json:"publish_date,omitempty"` + Description string `json:"description,omitempty"` +} diff --git a/pkg/vulnsrc/ubuntu/ubuntu.go b/pkg/vulnsrc/ubuntu/ubuntu.go index 845e36fc..66474ab4 100644 --- a/pkg/vulnsrc/ubuntu/ubuntu.go +++ b/pkg/vulnsrc/ubuntu/ubuntu.go @@ -22,7 +22,7 @@ const ( ) var ( - targetStatus = []string{"needed", "deferred", "released"} + targetStatus = []string{"needed", "deferred", "released", "needs-triage"} UbuntuReleasesMapping = map[string]string{ "precise": "12.04", "quantal": "12.10", diff --git a/pkg/vulnsrc/vulnerability/const.go b/pkg/vulnsrc/vulnerability/const.go index aaba8931..5bf62a8f 100644 --- a/pkg/vulnsrc/vulnerability/const.go +++ b/pkg/vulnsrc/vulnerability/const.go @@ -30,6 +30,7 @@ const ( GHSARubygems = "ghsa-rubygems" GLAD = "glad" GoVulnDB = "vulndb" + DebianSalsa = "debian-salsa" // Ecosystem Npm = "npm" diff --git a/pkg/vulnsrc/vulnerability/vulnerability.go b/pkg/vulnsrc/vulnerability/vulnerability.go index 30eb39a7..223945f4 100644 --- a/pkg/vulnsrc/vulnerability/vulnerability.go +++ b/pkg/vulnsrc/vulnerability/vulnerability.go @@ -222,3 +222,14 @@ func scoreToSeverity(score float64) types.Severity { return types.SeverityUnknown } } + +func (v Vulnerability) GetSecurityAdvisoryDetails(vulnID string) types.SecurityAdvisories { + securityAdvisories, err := v.dbc.GetSecurityAdvisoryDetails(vulnID) + if err != nil { + log.Println(err) + return nil + } else if len(securityAdvisories) == 0 { + return nil + } + return securityAdvisories +} diff --git a/pkg/vulnsrc/vulnsrc.go b/pkg/vulnsrc/vulnsrc.go index 6d8e3918..9c308627 100644 --- a/pkg/vulnsrc/vulnsrc.go +++ b/pkg/vulnsrc/vulnsrc.go @@ -21,6 +21,7 @@ import ( "github.com/aquasecurity/trivy-db/pkg/vulnsrc/composer" "github.com/aquasecurity/trivy-db/pkg/vulnsrc/debian" debianoval "github.com/aquasecurity/trivy-db/pkg/vulnsrc/debian-oval" + debiansalsa "github.com/aquasecurity/trivy-db/pkg/vulnsrc/debian-salsa" "github.com/aquasecurity/trivy-db/pkg/vulnsrc/ghsa" "github.com/aquasecurity/trivy-db/pkg/vulnsrc/glad" "github.com/aquasecurity/trivy-db/pkg/vulnsrc/govulndb" @@ -70,6 +71,7 @@ var ( vulnerability.GLAD: glad.NewVulnSrc(), vulnerability.GoVulnDB: govulndb.NewVulnSrc(), vulnerability.ArchLinux: archlinux.NewVulnSrc(), + vulnerability.DebianSalsa: debiansalsa.NewVulnSrc(), } ) @@ -184,6 +186,10 @@ func (o fullOptimizer) Optimize() error { return xerrors.Errorf("failed to delete advisory detail bucket: %w", err) } + if err := o.dbc.DeleteSecurityAdvisoryBucket(); err != nil { + return xerrors.Errorf("failed to delete advisory detail bucket: %w", err) + } + return nil } @@ -193,12 +199,14 @@ func (o fullOptimizer) fullOptimize(tx *bolt.Tx, cveID string) error { if o.vulnClient.IsRejected(details) { return nil } + // No need to check for reject here. + securityAdvisories := o.vulnClient.GetSecurityAdvisoryDetails(cveID) if err := o.vulnClient.SaveAdvisoryDetails(tx, cveID); err != nil { return xerrors.Errorf("failed to save advisories: %w", err) } - vuln := o.vulnClient.Normalize(details) + vuln.AdvisoryDetails = securityAdvisories if err := o.dbc.PutVulnerability(tx, cveID, vuln); err != nil { return xerrors.Errorf("failed to put vulnerability: %w", err) }