refactor(metadata): pass db dir (#412)

Signed-off-by: knqyf263 <[email protected]>

.gitignore

@@ -17,8 +17,9 @@ coverage.txt
 
 **/.DS_Store
 
-assets
-cache
+/assets
+/cache
+/out
 
 # trivy-db Outputs
 trivy-db

Makefile

@@ -1,5 +1,8 @@
 SHELL=/bin/bash
 LDFLAGS=-ldflags "-s -w"
+CACHE_DIR=cache
+OUT_DIR=out
+ASSET_DIR=assets
 
 GOPATH=$(shell go env GOPATH)
 GOBIN=$(GOPATH)/bin
@@ -60,44 +63,44 @@ trivy-db:
 
 .PHONY: db-fetch-langs
 db-fetch-langs:
-	mkdir -p cache/{ruby-advisory-db,php-security-advisories,nodejs-security-wg,ghsa,cocoapods-specs,bitnami-vulndb,govulndb}
-	wget -qO - https://github.com/rubysec/ruby-advisory-db/archive/master.tar.gz | tar xz -C cache/ruby-advisory-db --strip-components=1
-	wget -qO - https://github.com/FriendsOfPHP/security-advisories/archive/master.tar.gz | tar xz -C cache/php-security-advisories --strip-components=1
-	wget -qO - https://github.com/nodejs/security-wg/archive/main.tar.gz | tar xz -C cache/nodejs-security-wg --strip-components=1
-	wget -qO - https://github.com/bitnami/vulndb/archive/main.tar.gz | tar xz -C cache/bitnami-vulndb --strip-components=1
-	wget -qO - https://github.com/github/advisory-database/archive/refs/heads/main.tar.gz | tar xz -C cache/ghsa --strip-components=1
-	wget -qO - https://github.com/golang/vulndb/archive/refs/heads/master.tar.gz | tar xz -C cache/govulndb --strip-components=1
+	mkdir -p $(CACHE_DIR)/{ruby-advisory-db,php-security-advisories,nodejs-security-wg,ghsa,cocoapods-specs,bitnami-vulndb,govulndb}
+	wget -qO - https://github.com/rubysec/ruby-advisory-db/archive/master.tar.gz | tar xz -C $(CACHE_DIR)/ruby-advisory-db --strip-components=1
+	wget -qO - https://github.com/FriendsOfPHP/security-advisories/archive/master.tar.gz | tar xz -C $(CACHE_DIR)/php-security-advisories --strip-components=1
+	wget -qO - https://github.com/nodejs/security-wg/archive/main.tar.gz | tar xz -C $(CACHE_DIR)/nodejs-security-wg --strip-components=1
+	wget -qO - https://github.com/bitnami/vulndb/archive/main.tar.gz | tar xz -C $(CACHE_DIR)/bitnami-vulndb --strip-components=1
+	wget -qO - https://github.com/github/advisory-database/archive/refs/heads/main.tar.gz | tar xz -C $(CACHE_DIR)/ghsa --strip-components=1
+	wget -qO - https://github.com/golang/vulndb/archive/refs/heads/master.tar.gz | tar xz -C $(CACHE_DIR)/govulndb --strip-components=1
 	## required to convert GHSA Swift repo links to Cocoapods package names
-	wget -qO - https://github.com/CocoaPods/Specs/archive/master.tar.gz | tar xz -C cache/cocoapods-specs --strip-components=1
+	wget -qO - https://github.com/CocoaPods/Specs/archive/master.tar.gz | tar xz -C $(CACHE_DIR)/cocoapods-specs --strip-components=1
 
 .PHONY: db-build
 db-build: trivy-db
-	./trivy-db build --cache-dir cache --update-interval 6h
+	./trivy-db build --cache-dir ./$(CACHE_DIR) --output-dir ./$(OUT_DIR) --update-interval 6h
 
 .PHONY: db-compact
-db-compact: $(GOBIN)/bbolt cache/db/trivy.db
-	mkdir -p assets/
-	$(GOBIN)/bbolt compact -o ./assets/trivy.db cache/db/trivy.db
-	cp cache/db/metadata.json ./assets/metadata.json
-	rm -rf cache/db
+db-compact: $(GOBIN)/bbolt out/trivy.db
+	mkdir -p ./$(ASSET_DIR)
+	$(GOBIN)/bbolt compact -o ./$(ASSET_DIR)/trivy.db ./$(OUT_DIR)/trivy.db
+	cp ./$(OUT_DIR)/metadata.json ./$(ASSET_DIR)/metadata.json
+	rm -rf ./$(OUT_DIR)
 
 .PHONY: db-compress
-db-compress: assets/trivy.db assets/metadata.json
-	tar cvzf assets/db.tar.gz -C assets/ trivy.db metadata.json
+db-compress: $(ASSET_DIR)/trivy.db $(ASSET_DIR)/metadata.json
+	tar cvzf ./$(ASSET_DIR)/db.tar.gz -C $(ASSET_DIR) trivy.db metadata.json
 
 .PHONY: db-clean
 db-clean:
-	rm -rf cache assets
+	rm -rf $(CACHE_DIR) $(OUT_DIR) $(ASSET_DIR)
 
 .PHONY: db-fetch-vuln-list
 db-fetch-vuln-list:
-	mkdir -p cache/vuln-list
-	wget -qO - https://github.com/$(REPO_OWNER)/vuln-list/archive/main.tar.gz | tar xz -C cache/vuln-list --strip-components=1
-	mkdir -p cache/vuln-list-redhat
-	wget -qO - https://github.com/$(REPO_OWNER)/vuln-list-redhat/archive/main.tar.gz | tar xz -C cache/vuln-list-redhat --strip-components=1
-	mkdir -p cache/vuln-list-debian
-	wget -qO - https://github.com/$(REPO_OWNER)/vuln-list-debian/archive/main.tar.gz | tar xz -C cache/vuln-list-debian --strip-components=1
-	mkdir -p cache/vuln-list-nvd
-	wget -qO - https://github.com/$(REPO_OWNER)/vuln-list-nvd/archive/main.tar.gz | tar xz -C cache/vuln-list-nvd --strip-components=1
-	mkdir -p cache/vuln-list-k8s
-	wget -qO - https://github.com/$(REPO_OWNER)/vuln-list-k8s/archive/main.tar.gz | tar xz -C cache/vuln-list-k8s --strip-components=1
+	mkdir -p $(CACHE_DIR)/vuln-list
+	wget -qO - https://github.com/$(REPO_OWNER)/vuln-list/archive/main.tar.gz | tar xz -C $(CACHE_DIR)/vuln-list --strip-components=1
+	mkdir -p $(CACHE_DIR)/vuln-list-redhat
+	wget -qO - https://github.com/$(REPO_OWNER)/vuln-list-redhat/archive/main.tar.gz | tar xz -C $(CACHE_DIR)/vuln-list-redhat --strip-components=1
+	mkdir -p $(CACHE_DIR)/vuln-list-debian
+	wget -qO - https://github.com/$(REPO_OWNER)/vuln-list-debian/archive/main.tar.gz | tar xz -C $(CACHE_DIR)/vuln-list-debian --strip-components=1
+	mkdir -p $(CACHE_DIR)/vuln-list-nvd
+	wget -qO - https://github.com/$(REPO_OWNER)/vuln-list-nvd/archive/main.tar.gz | tar xz -C $(CACHE_DIR)/vuln-list-nvd --strip-components=1
+	mkdir -p $(CACHE_DIR)/vuln-list-k8s
+	wget -qO - https://github.com/$(REPO_OWNER)/vuln-list-k8s/archive/main.tar.gz | tar xz -C $(CACHE_DIR)/vuln-list-k8s --strip-components=1

pkg/app.go

@@ -39,6 +39,11 @@ func (ac *AppConfig) NewApp(version string) *cli.App {
 					Usage: "cache directory path",
 					Value: utils.CacheDir(),
 				},
+				cli.StringFlag{
+					Name:  "output-dir",
+					Usage: "output directory path",
+					Value: "out",
+				},
 				cli.DurationFlag{
 					Name:   "update-interval",
 					Usage:  "update interval",

pkg/build.go

@@ -9,19 +9,19 @@ import (
 )
 
 func build(c *cli.Context) error {
-	cacheDir := c.String("cache-dir")
-	if err := db.Init(cacheDir); err != nil {
+	outputDir := c.String("output-dir")
+	if err := db.Init(outputDir); err != nil {
 		return xerrors.Errorf("db initialize error: %w", err)
 	}
 
+	cacheDir := c.String("cache-dir")
 	targets := c.StringSlice("only-update")
 	updateInterval := c.Duration("update-interval")
 
-	vdb := vulndb.New(cacheDir, updateInterval)
+	vdb := vulndb.New(cacheDir, outputDir, updateInterval)
 	if err := vdb.Build(targets); err != nil {
 		return xerrors.Errorf("build error: %w", err)
 	}
 
 	return nil
-
 }

pkg/db/db.go

@@ -19,10 +19,7 @@ type CustomPut func(dbc Operation, tx *bolt.Tx, adv interface{}) error
 
 const SchemaVersion = 2
 
-var (
-	db    *bolt.DB
-	dbDir string
-)
+var db *bolt.DB
 
 type Operation interface {
 	BatchUpdate(fn func(*bolt.Tx) error) (err error)
@@ -58,12 +55,11 @@ type Operation interface {
 type Config struct {
 }
 
-func Init(cacheDir string) (err error) {
-	dbPath := Path(cacheDir)
-	dbDir = filepath.Dir(dbPath)
+func Init(dbDir string) (err error) {
 	if err = os.MkdirAll(dbDir, 0700); err != nil {
 		return xerrors.Errorf("failed to mkdir: %w", err)
 	}
+	dbPath := Path(dbDir)
 
 	// bbolt sometimes occurs the fatal error of "unexpected fault address".
 	// In that case, the local DB should be broken and needs to be removed.
@@ -85,12 +81,8 @@ func Init(cacheDir string) (err error) {
 	return nil
 }
 
-func Dir(cacheDir string) string {
-	return filepath.Join(cacheDir, "db")
-}
-
-func Path(cacheDir string) string {
-	dbPath := filepath.Join(Dir(cacheDir), "trivy.db")
+func Path(dbDir string) string {
+	dbPath := filepath.Join(dbDir, "trivy.db")
 	return dbPath
 }
 

pkg/dbtest/init.go

@@ -1,8 +1,6 @@
 package dbtest
 
 import (
-	"os"
-	"path/filepath"
 	"testing"
 
 	"github.com/stretchr/testify/require"
@@ -15,13 +13,8 @@ func InitDB(t *testing.T, fixtureFiles []string) string {
 	t.Helper()
 
 	// Create a temp dir
-	dir := t.TempDir()
-
-	// Create the database dir
-	dbPath := db.Path(dir)
-	dbDir := filepath.Dir(dbPath)
-	err := os.MkdirAll(dbDir, 0700)
-	require.NoError(t, err)
+	dbDir := t.TempDir()
+	dbPath := db.Path(dbDir)
 
 	// Load testdata into BoltDB
 	loader, err := fixtures.New(dbPath, fixtureFiles)
@@ -30,7 +23,7 @@ func InitDB(t *testing.T, fixtureFiles []string) string {
 	require.NoError(t, loader.Close())
 
 	// Initialize DB
-	require.NoError(t, db.Init(dir))
+	require.NoError(t, db.Init(dbDir))
 
-	return dir
+	return dbDir
 }

pkg/metadata/metadata.go

@@ -7,8 +7,6 @@ import (
 	"time"
 
 	"golang.org/x/xerrors"
-
-	"github.com/aquasecurity/trivy-db/pkg/db"
 )
 
 const metadataFile = "metadata.json"
@@ -26,16 +24,13 @@ type Client struct {
 }
 
 // NewClient is the factory method for the metadata Client
-func NewClient(cacheDir string) Client {
-	filePath := Path(cacheDir)
+func NewClient(dbDir string) Client {
 	return Client{
-		filePath: filePath,
+		filePath: Path(dbDir),
 	}
 }
 
-// Path returns the metaData file path
-func Path(cacheDir string) string {
-	dbDir := db.Dir(cacheDir)
+func Path(dbDir string) string {
 	return filepath.Join(dbDir, metadataFile)
 }
 

pkg/vulndb/db.go

@@ -43,7 +43,7 @@ func WithVulnSrcs(srcs map[types.SourceID]vulnsrc.VulnSrc) Option {
 	}
 }
 
-func New(cacheDir string, updateInterval time.Duration, opts ...Option) *TrivyDB {
+func New(cacheDir, outputDir string, updateInterval time.Duration, opts ...Option) *TrivyDB {
 	// Initialize map
 	vulnSrcs := map[types.SourceID]vulnsrc.VulnSrc{}
 	for _, v := range vulnsrc.All {
@@ -53,7 +53,7 @@ func New(cacheDir string, updateInterval time.Duration, opts ...Option) *TrivyDB
 	dbc := db.Config{}
 	tdb := &TrivyDB{
 		dbc:            dbc,
-		metadata:       metadata.NewClient(cacheDir),
+		metadata:       metadata.NewClient(outputDir),
 		vulnClient:     vulnerability.New(dbc),
 		vulnSrcs:       vulnSrcs,
 		cacheDir:       cacheDir,

pkg/vulndb/db_test.go

@@ -1,8 +1,6 @@
 package vulndb_test
 
 import (
-	"encoding/json"
-	"os"
 	"path/filepath"
 	"strings"
 	"testing"
@@ -94,11 +92,12 @@ func TestTrivyDB_Insert(t *testing.T) {
 				"fake": fakeVulnSrc{},
 			}
 			cacheDir := filepath.Join(t.TempDir(), tt.fields.cacheDir)
+			outputDir := t.TempDir()
 
 			require.NoError(t, db.Init(cacheDir))
 			defer db.Close()
 
-			c := vulndb.New(cacheDir, 12*time.Hour, vulndb.WithClock(tt.fields.clock), vulndb.WithVulnSrcs(vulnsrcs))
+			c := vulndb.New(cacheDir, outputDir, 12*time.Hour, vulndb.WithClock(tt.fields.clock), vulndb.WithVulnSrcs(vulnsrcs))
 			err := c.Insert(tt.args.targets)
 			if tt.wantErr != "" {
 				require.NotNil(t, err)
@@ -107,14 +106,9 @@ func TestTrivyDB_Insert(t *testing.T) {
 			}
 			require.NoError(t, err)
 
-			f, err := os.Open(metadata.Path(cacheDir))
-			require.NoError(t, err)
-
 			// Compare metadata JSON file
-			var got metadata.Metadata
-			err = json.NewDecoder(f).Decode(&got)
+			got, err := metadata.NewClient(outputDir).Get()
 			require.NoError(t, err)
-
 			assert.Equal(t, tt.want, got)
 		})
 	}
@@ -143,13 +137,20 @@ func TestTrivyDB_Build(t *testing.T) {
 			},
 			wantValues: []wantKV{
 				{
-					key: []string{"Red Hat Enterprise Linux 8", "python-jinja2", "CVE-2019-10906"},
+					key: []string{
+						"Red Hat Enterprise Linux 8",
+						"python-jinja2",
+						"CVE-2019-10906",
+					},
 					value: types.Advisory{
 						FixedVersion: "2.10.1-2.el8_0",
 					},
 				},
 				{
-					key: []string{"vulnerability", "CVE-2019-10906"},
+					key: []string{
+						"vulnerability",
+						"CVE-2019-10906",
+					},
 					value: types.Vulnerability{
 						Title:       "python-jinja2: str.format_map allows sandbox escape",
 						Description: "In Pallets Jinja before 2.10.1, str.format_map allows a sandbox escape.",
@@ -185,10 +186,11 @@ func TestTrivyDB_Build(t *testing.T) {
 
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			cacheDir := dbtest.InitDB(t, tt.fixtures)
+			cacheDir := t.TempDir()
+			dbDir := dbtest.InitDB(t, tt.fixtures)
 			defer db.Close()
 
-			full := vulndb.New(cacheDir, 12*time.Hour)
+			full := vulndb.New(cacheDir, dbDir, 12*time.Hour)
 			err := full.Build(nil)
 			if tt.wantErr != "" {
 				require.NotNil(t, err)
@@ -199,7 +201,7 @@ func TestTrivyDB_Build(t *testing.T) {
 
 			// Compare DB entries
 			require.NoError(t, db.Close())
-			dbPath := db.Path(cacheDir)
+			dbPath := db.Path(dbDir)
 			for _, want := range tt.wantValues {
 				dbtest.JSONEq(t, dbPath, want.key, want.value)
 			}