From 8e907467e9ab0047bab497f3a18b04b58745eeae Mon Sep 17 00:00:00 2001 From: Teppei Fukuda Date: Mon, 1 Jul 2024 14:34:00 +0400 Subject: [PATCH] refactor(metadata): pass db dir (#412) Signed-off-by: knqyf263 --- .gitignore | 5 ++-- Makefile | 57 +++++++++++++++++++++------------------- pkg/app.go | 5 ++++ pkg/build.go | 8 +++--- pkg/db/db.go | 18 ++++--------- pkg/dbtest/init.go | 15 +++-------- pkg/metadata/metadata.go | 11 +++----- pkg/vulndb/db.go | 4 +-- pkg/vulndb/db_test.go | 30 +++++++++++---------- 9 files changed, 72 insertions(+), 81 deletions(-) diff --git a/.gitignore b/.gitignore index 1b6054a2..9338acc7 100644 --- a/.gitignore +++ b/.gitignore @@ -17,8 +17,9 @@ coverage.txt **/.DS_Store -assets -cache +/assets +/cache +/out # trivy-db Outputs trivy-db diff --git a/Makefile b/Makefile index 9f92533e..a73f157f 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,8 @@ SHELL=/bin/bash LDFLAGS=-ldflags "-s -w" +CACHE_DIR=cache +OUT_DIR=out +ASSET_DIR=assets GOPATH=$(shell go env GOPATH) GOBIN=$(GOPATH)/bin @@ -60,44 +63,44 @@ trivy-db: .PHONY: db-fetch-langs db-fetch-langs: - mkdir -p cache/{ruby-advisory-db,php-security-advisories,nodejs-security-wg,ghsa,cocoapods-specs,bitnami-vulndb,govulndb} - wget -qO - https://github.com/rubysec/ruby-advisory-db/archive/master.tar.gz | tar xz -C cache/ruby-advisory-db --strip-components=1 - wget -qO - https://github.com/FriendsOfPHP/security-advisories/archive/master.tar.gz | tar xz -C cache/php-security-advisories --strip-components=1 - wget -qO - https://github.com/nodejs/security-wg/archive/main.tar.gz | tar xz -C cache/nodejs-security-wg --strip-components=1 - wget -qO - https://github.com/bitnami/vulndb/archive/main.tar.gz | tar xz -C cache/bitnami-vulndb --strip-components=1 - wget -qO - https://github.com/github/advisory-database/archive/refs/heads/main.tar.gz | tar xz -C cache/ghsa --strip-components=1 - wget -qO - https://github.com/golang/vulndb/archive/refs/heads/master.tar.gz | tar xz -C cache/govulndb --strip-components=1 + mkdir -p $(CACHE_DIR)/{ruby-advisory-db,php-security-advisories,nodejs-security-wg,ghsa,cocoapods-specs,bitnami-vulndb,govulndb} + wget -qO - https://github.com/rubysec/ruby-advisory-db/archive/master.tar.gz | tar xz -C $(CACHE_DIR)/ruby-advisory-db --strip-components=1 + wget -qO - https://github.com/FriendsOfPHP/security-advisories/archive/master.tar.gz | tar xz -C $(CACHE_DIR)/php-security-advisories --strip-components=1 + wget -qO - https://github.com/nodejs/security-wg/archive/main.tar.gz | tar xz -C $(CACHE_DIR)/nodejs-security-wg --strip-components=1 + wget -qO - https://github.com/bitnami/vulndb/archive/main.tar.gz | tar xz -C $(CACHE_DIR)/bitnami-vulndb --strip-components=1 + wget -qO - https://github.com/github/advisory-database/archive/refs/heads/main.tar.gz | tar xz -C $(CACHE_DIR)/ghsa --strip-components=1 + wget -qO - https://github.com/golang/vulndb/archive/refs/heads/master.tar.gz | tar xz -C $(CACHE_DIR)/govulndb --strip-components=1 ## required to convert GHSA Swift repo links to Cocoapods package names - wget -qO - https://github.com/CocoaPods/Specs/archive/master.tar.gz | tar xz -C cache/cocoapods-specs --strip-components=1 + wget -qO - https://github.com/CocoaPods/Specs/archive/master.tar.gz | tar xz -C $(CACHE_DIR)/cocoapods-specs --strip-components=1 .PHONY: db-build db-build: trivy-db - ./trivy-db build --cache-dir cache --update-interval 6h + ./trivy-db build --cache-dir ./$(CACHE_DIR) --output-dir ./$(OUT_DIR) --update-interval 6h .PHONY: db-compact -db-compact: $(GOBIN)/bbolt cache/db/trivy.db - mkdir -p assets/ - $(GOBIN)/bbolt compact -o ./assets/trivy.db cache/db/trivy.db - cp cache/db/metadata.json ./assets/metadata.json - rm -rf cache/db +db-compact: $(GOBIN)/bbolt out/trivy.db + mkdir -p ./$(ASSET_DIR) + $(GOBIN)/bbolt compact -o ./$(ASSET_DIR)/trivy.db ./$(OUT_DIR)/trivy.db + cp ./$(OUT_DIR)/metadata.json ./$(ASSET_DIR)/metadata.json + rm -rf ./$(OUT_DIR) .PHONY: db-compress -db-compress: assets/trivy.db assets/metadata.json - tar cvzf assets/db.tar.gz -C assets/ trivy.db metadata.json +db-compress: $(ASSET_DIR)/trivy.db $(ASSET_DIR)/metadata.json + tar cvzf ./$(ASSET_DIR)/db.tar.gz -C $(ASSET_DIR) trivy.db metadata.json .PHONY: db-clean db-clean: - rm -rf cache assets + rm -rf $(CACHE_DIR) $(OUT_DIR) $(ASSET_DIR) .PHONY: db-fetch-vuln-list db-fetch-vuln-list: - mkdir -p cache/vuln-list - wget -qO - https://github.com/$(REPO_OWNER)/vuln-list/archive/main.tar.gz | tar xz -C cache/vuln-list --strip-components=1 - mkdir -p cache/vuln-list-redhat - wget -qO - https://github.com/$(REPO_OWNER)/vuln-list-redhat/archive/main.tar.gz | tar xz -C cache/vuln-list-redhat --strip-components=1 - mkdir -p cache/vuln-list-debian - wget -qO - https://github.com/$(REPO_OWNER)/vuln-list-debian/archive/main.tar.gz | tar xz -C cache/vuln-list-debian --strip-components=1 - mkdir -p cache/vuln-list-nvd - wget -qO - https://github.com/$(REPO_OWNER)/vuln-list-nvd/archive/main.tar.gz | tar xz -C cache/vuln-list-nvd --strip-components=1 - mkdir -p cache/vuln-list-k8s - wget -qO - https://github.com/$(REPO_OWNER)/vuln-list-k8s/archive/main.tar.gz | tar xz -C cache/vuln-list-k8s --strip-components=1 + mkdir -p $(CACHE_DIR)/vuln-list + wget -qO - https://github.com/$(REPO_OWNER)/vuln-list/archive/main.tar.gz | tar xz -C $(CACHE_DIR)/vuln-list --strip-components=1 + mkdir -p $(CACHE_DIR)/vuln-list-redhat + wget -qO - https://github.com/$(REPO_OWNER)/vuln-list-redhat/archive/main.tar.gz | tar xz -C $(CACHE_DIR)/vuln-list-redhat --strip-components=1 + mkdir -p $(CACHE_DIR)/vuln-list-debian + wget -qO - https://github.com/$(REPO_OWNER)/vuln-list-debian/archive/main.tar.gz | tar xz -C $(CACHE_DIR)/vuln-list-debian --strip-components=1 + mkdir -p $(CACHE_DIR)/vuln-list-nvd + wget -qO - https://github.com/$(REPO_OWNER)/vuln-list-nvd/archive/main.tar.gz | tar xz -C $(CACHE_DIR)/vuln-list-nvd --strip-components=1 + mkdir -p $(CACHE_DIR)/vuln-list-k8s + wget -qO - https://github.com/$(REPO_OWNER)/vuln-list-k8s/archive/main.tar.gz | tar xz -C $(CACHE_DIR)/vuln-list-k8s --strip-components=1 diff --git a/pkg/app.go b/pkg/app.go index 6973a332..5053253f 100644 --- a/pkg/app.go +++ b/pkg/app.go @@ -39,6 +39,11 @@ func (ac *AppConfig) NewApp(version string) *cli.App { Usage: "cache directory path", Value: utils.CacheDir(), }, + cli.StringFlag{ + Name: "output-dir", + Usage: "output directory path", + Value: "out", + }, cli.DurationFlag{ Name: "update-interval", Usage: "update interval", diff --git a/pkg/build.go b/pkg/build.go index 591f1c98..8575ef6a 100644 --- a/pkg/build.go +++ b/pkg/build.go @@ -9,19 +9,19 @@ import ( ) func build(c *cli.Context) error { - cacheDir := c.String("cache-dir") - if err := db.Init(cacheDir); err != nil { + outputDir := c.String("output-dir") + if err := db.Init(outputDir); err != nil { return xerrors.Errorf("db initialize error: %w", err) } + cacheDir := c.String("cache-dir") targets := c.StringSlice("only-update") updateInterval := c.Duration("update-interval") - vdb := vulndb.New(cacheDir, updateInterval) + vdb := vulndb.New(cacheDir, outputDir, updateInterval) if err := vdb.Build(targets); err != nil { return xerrors.Errorf("build error: %w", err) } return nil - } diff --git a/pkg/db/db.go b/pkg/db/db.go index 40f0d41d..e9603dd6 100644 --- a/pkg/db/db.go +++ b/pkg/db/db.go @@ -19,10 +19,7 @@ type CustomPut func(dbc Operation, tx *bolt.Tx, adv interface{}) error const SchemaVersion = 2 -var ( - db *bolt.DB - dbDir string -) +var db *bolt.DB type Operation interface { BatchUpdate(fn func(*bolt.Tx) error) (err error) @@ -58,12 +55,11 @@ type Operation interface { type Config struct { } -func Init(cacheDir string) (err error) { - dbPath := Path(cacheDir) - dbDir = filepath.Dir(dbPath) +func Init(dbDir string) (err error) { if err = os.MkdirAll(dbDir, 0700); err != nil { return xerrors.Errorf("failed to mkdir: %w", err) } + dbPath := Path(dbDir) // bbolt sometimes occurs the fatal error of "unexpected fault address". // In that case, the local DB should be broken and needs to be removed. @@ -85,12 +81,8 @@ func Init(cacheDir string) (err error) { return nil } -func Dir(cacheDir string) string { - return filepath.Join(cacheDir, "db") -} - -func Path(cacheDir string) string { - dbPath := filepath.Join(Dir(cacheDir), "trivy.db") +func Path(dbDir string) string { + dbPath := filepath.Join(dbDir, "trivy.db") return dbPath } diff --git a/pkg/dbtest/init.go b/pkg/dbtest/init.go index d480ee9a..2b266bb4 100644 --- a/pkg/dbtest/init.go +++ b/pkg/dbtest/init.go @@ -1,8 +1,6 @@ package dbtest import ( - "os" - "path/filepath" "testing" "github.com/stretchr/testify/require" @@ -15,13 +13,8 @@ func InitDB(t *testing.T, fixtureFiles []string) string { t.Helper() // Create a temp dir - dir := t.TempDir() - - // Create the database dir - dbPath := db.Path(dir) - dbDir := filepath.Dir(dbPath) - err := os.MkdirAll(dbDir, 0700) - require.NoError(t, err) + dbDir := t.TempDir() + dbPath := db.Path(dbDir) // Load testdata into BoltDB loader, err := fixtures.New(dbPath, fixtureFiles) @@ -30,7 +23,7 @@ func InitDB(t *testing.T, fixtureFiles []string) string { require.NoError(t, loader.Close()) // Initialize DB - require.NoError(t, db.Init(dir)) + require.NoError(t, db.Init(dbDir)) - return dir + return dbDir } diff --git a/pkg/metadata/metadata.go b/pkg/metadata/metadata.go index 4930c3c1..3cbe2074 100644 --- a/pkg/metadata/metadata.go +++ b/pkg/metadata/metadata.go @@ -7,8 +7,6 @@ import ( "time" "golang.org/x/xerrors" - - "github.com/aquasecurity/trivy-db/pkg/db" ) const metadataFile = "metadata.json" @@ -26,16 +24,13 @@ type Client struct { } // NewClient is the factory method for the metadata Client -func NewClient(cacheDir string) Client { - filePath := Path(cacheDir) +func NewClient(dbDir string) Client { return Client{ - filePath: filePath, + filePath: Path(dbDir), } } -// Path returns the metaData file path -func Path(cacheDir string) string { - dbDir := db.Dir(cacheDir) +func Path(dbDir string) string { return filepath.Join(dbDir, metadataFile) } diff --git a/pkg/vulndb/db.go b/pkg/vulndb/db.go index 30a8d569..4630d41a 100644 --- a/pkg/vulndb/db.go +++ b/pkg/vulndb/db.go @@ -43,7 +43,7 @@ func WithVulnSrcs(srcs map[types.SourceID]vulnsrc.VulnSrc) Option { } } -func New(cacheDir string, updateInterval time.Duration, opts ...Option) *TrivyDB { +func New(cacheDir, outputDir string, updateInterval time.Duration, opts ...Option) *TrivyDB { // Initialize map vulnSrcs := map[types.SourceID]vulnsrc.VulnSrc{} for _, v := range vulnsrc.All { @@ -53,7 +53,7 @@ func New(cacheDir string, updateInterval time.Duration, opts ...Option) *TrivyDB dbc := db.Config{} tdb := &TrivyDB{ dbc: dbc, - metadata: metadata.NewClient(cacheDir), + metadata: metadata.NewClient(outputDir), vulnClient: vulnerability.New(dbc), vulnSrcs: vulnSrcs, cacheDir: cacheDir, diff --git a/pkg/vulndb/db_test.go b/pkg/vulndb/db_test.go index 1657239a..b56de912 100644 --- a/pkg/vulndb/db_test.go +++ b/pkg/vulndb/db_test.go @@ -1,8 +1,6 @@ package vulndb_test import ( - "encoding/json" - "os" "path/filepath" "strings" "testing" @@ -94,11 +92,12 @@ func TestTrivyDB_Insert(t *testing.T) { "fake": fakeVulnSrc{}, } cacheDir := filepath.Join(t.TempDir(), tt.fields.cacheDir) + outputDir := t.TempDir() require.NoError(t, db.Init(cacheDir)) defer db.Close() - c := vulndb.New(cacheDir, 12*time.Hour, vulndb.WithClock(tt.fields.clock), vulndb.WithVulnSrcs(vulnsrcs)) + c := vulndb.New(cacheDir, outputDir, 12*time.Hour, vulndb.WithClock(tt.fields.clock), vulndb.WithVulnSrcs(vulnsrcs)) err := c.Insert(tt.args.targets) if tt.wantErr != "" { require.NotNil(t, err) @@ -107,14 +106,9 @@ func TestTrivyDB_Insert(t *testing.T) { } require.NoError(t, err) - f, err := os.Open(metadata.Path(cacheDir)) - require.NoError(t, err) - // Compare metadata JSON file - var got metadata.Metadata - err = json.NewDecoder(f).Decode(&got) + got, err := metadata.NewClient(outputDir).Get() require.NoError(t, err) - assert.Equal(t, tt.want, got) }) } @@ -143,13 +137,20 @@ func TestTrivyDB_Build(t *testing.T) { }, wantValues: []wantKV{ { - key: []string{"Red Hat Enterprise Linux 8", "python-jinja2", "CVE-2019-10906"}, + key: []string{ + "Red Hat Enterprise Linux 8", + "python-jinja2", + "CVE-2019-10906", + }, value: types.Advisory{ FixedVersion: "2.10.1-2.el8_0", }, }, { - key: []string{"vulnerability", "CVE-2019-10906"}, + key: []string{ + "vulnerability", + "CVE-2019-10906", + }, value: types.Vulnerability{ Title: "python-jinja2: str.format_map allows sandbox escape", Description: "In Pallets Jinja before 2.10.1, str.format_map allows a sandbox escape.", @@ -185,10 +186,11 @@ func TestTrivyDB_Build(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - cacheDir := dbtest.InitDB(t, tt.fixtures) + cacheDir := t.TempDir() + dbDir := dbtest.InitDB(t, tt.fixtures) defer db.Close() - full := vulndb.New(cacheDir, 12*time.Hour) + full := vulndb.New(cacheDir, dbDir, 12*time.Hour) err := full.Build(nil) if tt.wantErr != "" { require.NotNil(t, err) @@ -199,7 +201,7 @@ func TestTrivyDB_Build(t *testing.T) { // Compare DB entries require.NoError(t, db.Close()) - dbPath := db.Path(cacheDir) + dbPath := db.Path(dbDir) for _, want := range tt.wantValues { dbtest.JSONEq(t, dbPath, want.key, want.value) }