From 258d1534614a98811de657eb196051728529defd Mon Sep 17 00:00:00 2001 From: DmitriyLewen <91113035+DmitriyLewen@users.noreply.github.com> Date: Wed, 27 Mar 2024 12:08:58 +0600 Subject: [PATCH] fix(nodejs): merge `Indirect`, `Dev`, `ExternalReferences` fields for same deps from `package-lock.json` files v2 or later (#6356) Signed-off-by: knqyf263 Co-authored-by: knqyf263 --- .../go.sum | 1 + pkg/dependency/parser/nodejs/npm/parse.go | 48 ++++++++--- .../parser/nodejs/npm/parse_test.go | 6 ++ .../parser/nodejs/npm/parse_testcase.go | 85 +++++++++++++++++++ ...age-lock_v3_with-same-dev-and-non-dev.json | 74 ++++++++++++++++ 5 files changed, 202 insertions(+), 12 deletions(-) create mode 100644 pkg/dependency/parser/nodejs/npm/testdata/package-lock_v3_with-same-dev-and-non-dev.json diff --git a/pkg/dependency/parser/golang/mod/testdata/replaced-with-local-path-and-version-mismatch/go.sum b/pkg/dependency/parser/golang/mod/testdata/replaced-with-local-path-and-version-mismatch/go.sum index 3d1d7c0e3913..8a219a39d474 100644 --- a/pkg/dependency/parser/golang/mod/testdata/replaced-with-local-path-and-version-mismatch/go.sum +++ b/pkg/dependency/parser/golang/mod/testdata/replaced-with-local-path-and-version-mismatch/go.sum @@ -50,6 +50,7 @@ golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgw golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= diff --git a/pkg/dependency/parser/nodejs/npm/parse.go b/pkg/dependency/parser/nodejs/npm/parse.go index e289720b89a2..b74cfa5ce2f5 100644 --- a/pkg/dependency/parser/nodejs/npm/parse.go +++ b/pkg/dependency/parser/nodejs/npm/parse.go @@ -4,6 +4,7 @@ import ( "fmt" "io" "path" + "slices" "sort" "strings" @@ -115,28 +116,42 @@ func (p *Parser) parseV2(packages map[string]Package) ([]types.Library, []types. EndLine: pkg.EndLine, } + var ref types.ExternalRef + if pkg.Resolved != "" { + ref = types.ExternalRef{ + Type: types.RefOther, + URL: pkg.Resolved, + } + } + + pkgIndirect := isIndirectLib(pkgPath, directDeps) + // There are cases when similar libraries use same dependencies // we need to add location for each these dependencies if savedLib, ok := libs[pkgID]; ok { + savedLib.Dev = savedLib.Dev && pkg.Dev + savedLib.Indirect = savedLib.Indirect && pkgIndirect + + if ref.URL != "" && !slices.Contains(savedLib.ExternalReferences, ref) { + savedLib.ExternalReferences = append(savedLib.ExternalReferences, ref) + sortExternalReferences(savedLib.ExternalReferences) + } + savedLib.Locations = append(savedLib.Locations, location) sort.Sort(savedLib.Locations) + libs[pkgID] = savedLib continue } lib := types.Library{ - ID: pkgID, - Name: pkgName, - Version: pkg.Version, - Indirect: isIndirectLib(pkgPath, directDeps), - Dev: pkg.Dev, - ExternalReferences: []types.ExternalRef{ - { - Type: types.RefOther, - URL: pkg.Resolved, - }, - }, - Locations: []types.Location{location}, + ID: pkgID, + Name: pkgName, + Version: pkg.Version, + Indirect: pkgIndirect, + Dev: pkg.Dev, + ExternalReferences: lo.Ternary(ref.URL != "", []types.ExternalRef{ref}, nil), + Locations: []types.Location{location}, } libs[pkgID] = lib @@ -385,3 +400,12 @@ func (t *Package) UnmarshalJSONWithMetadata(node jfather.Node) error { func packageID(name, version string) string { return dependency.ID(ftypes.Npm, name, version) } + +func sortExternalReferences(refs []types.ExternalRef) { + sort.Slice(refs, func(i, j int) bool { + if refs[i].Type != refs[j].Type { + return refs[i].Type < refs[j].Type + } + return refs[i].URL < refs[j].URL + }) +} diff --git a/pkg/dependency/parser/nodejs/npm/parse_test.go b/pkg/dependency/parser/nodejs/npm/parse_test.go index c67055a71628..786fe643dfde 100644 --- a/pkg/dependency/parser/nodejs/npm/parse_test.go +++ b/pkg/dependency/parser/nodejs/npm/parse_test.go @@ -41,6 +41,12 @@ func TestParse(t *testing.T) { want: npmV3WithWorkspaceLibs, wantDeps: npmV3WithWorkspaceDeps, }, + { + name: "lock file v3 contains same dev and non-dev dependencies", + file: "testdata/package-lock_v3_with-same-dev-and-non-dev.json", + want: npmV3WithSameDevAndNonDevLibs, + wantDeps: npmV3WithSameDevAndNonDevDeps, + }, { name: "lock version v3 with workspace and without direct deps field", file: "testdata/package-lock_v3_without_root_deps_field.json", diff --git a/pkg/dependency/parser/nodejs/npm/parse_testcase.go b/pkg/dependency/parser/nodejs/npm/parse_testcase.go index e68addd15219..29b9e63d8f1c 100644 --- a/pkg/dependency/parser/nodejs/npm/parse_testcase.go +++ b/pkg/dependency/parser/nodejs/npm/parse_testcase.go @@ -1516,4 +1516,89 @@ var ( DependsOn: []string{"debug@2.6.9"}, }, } + + npmV3WithSameDevAndNonDevLibs = []types.Library{ + { + ID: "fsevents@1.2.9", + Name: "fsevents", + Version: "1.2.9", + Dev: true, + ExternalReferences: []types.ExternalRef{ + { + Type: types.RefOther, + URL: "https://registry.npmjs.org/fsevents/-/fsevents-1.2.9.tgz", + }, + }, + Locations: []types.Location{ + { + StartLine: 18, + EndLine: 37, + }, + }, + }, + { + ID: "minimist@0.0.8", + Name: "minimist", + Version: "0.0.8", + Indirect: false, + Dev: false, + ExternalReferences: []types.ExternalRef{ + { + Type: types.RefOther, + URL: "https://registry.npmjs.org/minimist/-/minimist-0.0.8.tgz", + }, + }, + Locations: []types.Location{ + { + StartLine: 38, + EndLine: 43, + }, + { + StartLine: 68, + EndLine: 72, + }, + }, + }, + { + ID: "mkdirp@0.5.1", + Name: "mkdirp", + Version: "0.5.1", + Indirect: true, + Dev: true, + Locations: []types.Location{ + { + StartLine: 44, + EndLine: 55, + }, + }, + }, + { + ID: "node-pre-gyp@0.12.0", + Name: "node-pre-gyp", + Version: "0.12.0", + Indirect: true, + Dev: true, + Locations: []types.Location{ + { + StartLine: 56, + EndLine: 67, + }, + }, + }, + } + + npmV3WithSameDevAndNonDevDeps = []types.Dependency{ + { + ID: "fsevents@1.2.9", + DependsOn: []string{"node-pre-gyp@0.12.0"}, + }, + { + ID: "mkdirp@0.5.1", + DependsOn: []string{"minimist@0.0.8"}, + }, + { + ID: "node-pre-gyp@0.12.0", + DependsOn: []string{"mkdirp@0.5.1"}, + }, + } ) diff --git a/pkg/dependency/parser/nodejs/npm/testdata/package-lock_v3_with-same-dev-and-non-dev.json b/pkg/dependency/parser/nodejs/npm/testdata/package-lock_v3_with-same-dev-and-non-dev.json new file mode 100644 index 000000000000..4fe518b82bcd --- /dev/null +++ b/pkg/dependency/parser/nodejs/npm/testdata/package-lock_v3_with-same-dev-and-non-dev.json @@ -0,0 +1,74 @@ +{ + "name": "5139", + "version": "1.0.0", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "5139", + "version": "1.0.0", + "license": "ISC", + "dependencies": { + "minimist": "^0.0.8" + }, + "devDependencies": { + "fsevents": "^1.2.9" + } + }, + "node_modules/fsevents": { + "version": "1.2.9", + "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-1.2.9.tgz", + "integrity": "sha512-oeyj2H3EjjonWcFjD5NvZNE9Rqe4UW+nQBU2HNeKw0koVLEFIhtyETyAakeAM3de7Z/SW5kcA+fZUait9EApnw==", + "bundleDependencies": [ + "node-pre-gyp" + ], + "deprecated": "The v1 package contains DANGEROUS / INSECURE binaries. Upgrade to safe fsevents v2", + "dev": true, + "hasInstallScript": true, + "os": [ + "darwin" + ], + "dependencies": { + "node-pre-gyp": "^0.12.0" + }, + "engines": { + "node": ">=4.0" + } + }, + "node_modules/fsevents/node_modules/minimist": { + "version": "0.0.8", + "dev": true, + "inBundle": true, + "license": "MIT" + }, + "node_modules/fsevents/node_modules/mkdirp": { + "version": "0.5.1", + "dev": true, + "inBundle": true, + "license": "MIT", + "dependencies": { + "minimist": "0.0.8" + }, + "bin": { + "mkdirp": "bin/cmd.js" + } + }, + "node_modules/fsevents/node_modules/node-pre-gyp": { + "version": "0.12.0", + "dev": true, + "inBundle": true, + "license": "BSD-3-Clause", + "dependencies": { + "mkdirp": "^0.5.1" + }, + "bin": { + "node-pre-gyp": "bin/node-pre-gyp" + } + }, + "node_modules/minimist": { + "version": "0.0.8", + "resolved": "https://registry.npmjs.org/minimist/-/minimist-0.0.8.tgz", + "integrity": "sha512-miQKw5Hv4NS1Psg2517mV4e4dYNaO3++hjAvLOAzKqZ61rH8NS1SK+vbfBWZ5PY/Me/bEWhUwqMghEW5Fb9T7Q==" + } + } +}