diff --git a/src/packagedcode/models.py b/src/packagedcode/models.py index 7d6d223ffb3..062eecb0a4e 100644 --- a/src/packagedcode/models.py +++ b/src/packagedcode/models.py @@ -371,6 +371,14 @@ class DependentPackage(ModelMixin): 'been resolved and this dependency url points to an ' 'exact version.') + is_direct = Boolean( + default=True, + label='is direct flag', + help='True if this dependency version requirement is ' + 'a direct dependency relation between two packages ' + 'as opposed to a transitive dependency relation, ' + 'which are present in lockfiles/dependency list.') + resolved_package = Mapping( label='resolved package data', help='A mapping of resolved package data for this dependent package, ' @@ -1072,6 +1080,17 @@ def is_datafile(cls, location, filetypes=tuple(), _bare_filename=False): actual_type = T.filetype_file.lower() return any(ft in actual_type for ft in filetypes) + @classmethod + def is_lockfile(cls): + """ + Return True if this is a lockfile, False otherwise. + + This has to be implemented by datafile handlers classes + of lockfiles, to return True, in contrast to the default + value False. + """ + return False + @classmethod def parse(cls, location, package_only=False): """ diff --git a/src/packagedcode/npm.py b/src/packagedcode/npm.py index 95f198b060a..1808e041c4f 100644 --- a/src/packagedcode/npm.py +++ b/src/packagedcode/npm.py @@ -204,6 +204,7 @@ def update_dependencies_by_purl( is_runtime=False, is_optional=False, is_resolved=False, + is_direct=True, ): metadata_deps = ['peerDependenciesMeta', 'dependenciesMeta'] @@ -221,6 +222,7 @@ def update_dependencies_by_purl( is_runtime=is_runtime, is_optional=is_optional, is_resolved=is_resolved, + is_direct=is_direct, ) dependecies_by_purl[dep_purl] = dep_package @@ -244,6 +246,7 @@ def update_dependencies_by_purl( is_runtime=is_runtime, is_optional=metadata.get("optional"), is_resolved=is_resolved, + is_direct=is_direct, ) dependecies_by_purl[dep_purl] = dep_package continue @@ -264,6 +267,7 @@ def update_dependencies_by_purl( is_runtime=is_runtime, is_optional=is_optional, is_resolved=is_resolved, + is_direct=is_direct, ) dependecies_by_purl[dep_purl] = dep_package @@ -476,6 +480,10 @@ def parse(cls, location, package_only=False): class BaseNpmLockHandler(BaseNpmHandler): + @classmethod + def is_lockfile(cls): + return True + @classmethod def parse(cls, location, package_only=False): @@ -590,6 +598,7 @@ def parse(cls, location, package_only=False): is_runtime=is_runtime, is_optional=is_optional, is_resolved=True, + is_direct=False, ) # URLs and checksums @@ -638,6 +647,7 @@ def parse(cls, location, package_only=False): is_runtime=is_runtime, is_optional=is_optional, is_resolved=False, + is_direct=True, ) resolved_package.dependencies = [ @@ -723,6 +733,10 @@ class YarnLockV2Handler(BaseNpmHandler): def is_datafile(cls, location, filetypes=tuple()): return super().is_datafile(location, filetypes=filetypes) and is_yarn_v2(location) + @classmethod + def is_lockfile(cls): + return True + @classmethod def parse(cls, location, package_only=False): """ @@ -833,6 +847,10 @@ class YarnLockV1Handler(BaseNpmHandler): description = 'yarn.lock lockfile v1 format' documentation_url = 'https://classic.yarnpkg.com/lang/en/docs/yarn-lock/' + @classmethod + def is_lockfile(cls): + return True + @classmethod def is_datafile(cls, location, filetypes=tuple()): return super().is_datafile(location, filetypes=filetypes) and not is_yarn_v2(location) @@ -953,6 +971,7 @@ def parse(cls, location, package_only=False): scope='dependencies', is_optional=False, is_runtime=True, + is_direct=True, ) resolved_package_data.dependencies.append(subdep) @@ -972,6 +991,7 @@ def parse(cls, location, package_only=False): scope='dependencies', is_optional=False, is_runtime=True, + is_direct=False, resolved_package=resolved_package_data.to_dict(), ) dependencies.append(dep.to_dict()) @@ -988,6 +1008,10 @@ def parse(cls, location, package_only=False): class BasePnpmLockHandler(BaseNpmHandler): + @classmethod + def is_lockfile(cls): + return True + @classmethod def parse(cls, location, package_only=False): """ @@ -1063,12 +1087,14 @@ def parse(cls, location, package_only=False): scope='dependencies', dependecies_by_purl=deps_for_resolved_by_purl, is_resolved=True, + is_direct=False, ) cls.update_dependencies_by_purl( dependencies=peer_dependencies, scope='peerDependencies', dependecies_by_purl=deps_for_resolved_by_purl, is_optional=True, + is_direct=False, ) cls.update_dependencies_by_purl( dependencies=optional_dependencies, @@ -1076,6 +1102,7 @@ def parse(cls, location, package_only=False): dependecies_by_purl=deps_for_resolved_by_purl, is_resolved=True, is_optional=True, + is_direct=False, ) cls.update_dependencies_by_purl( dependencies=peer_dependencies_meta, @@ -1122,6 +1149,7 @@ def parse(cls, location, package_only=False): is_optional=is_optional, is_runtime=is_runtime, is_resolved=True, + is_direct=True, resolved_package=resolved_package.to_dict(), extra_data=extra_data_deps, ) @@ -1577,7 +1605,7 @@ def bundle_deps_mapper(bundle_deps, package): return package -def deps_mapper(deps, package, field_name): +def deps_mapper(deps, package, field_name, is_direct=True): """ Handle deps such as dependencies, devDependencies, peerDependencies, optionalDependencies return a tuple of (dep type, list of deps) @@ -1630,6 +1658,7 @@ def deps_mapper(deps, package, field_name): purl=purl, scope=field_name, extracted_requirement=requirement, + is_direct=is_direct, **dependency_attributes ) dependencies.append(dep) diff --git a/tests/packagedcode/data/npm/yarn-lock/resolve-deps.expected.json b/tests/packagedcode/data/npm/yarn-lock/resolve-deps.expected.json new file mode 100644 index 00000000000..12e2a25e0a2 --- /dev/null +++ b/tests/packagedcode/data/npm/yarn-lock/resolve-deps.expected.json @@ -0,0 +1,1152 @@ +{ + "packages": [ + { + "type": "npm", + "namespace": null, + "name": "create-athena-partition", + "version": "1.0.0", + "qualifiers": {}, + "subpath": null, + "primary_language": "JavaScript", + "description": null, + "release_date": null, + "parties": [], + "keywords": [], + "homepage_url": null, + "download_url": "https://registry.npmjs.org/create-athena-partition/-/create-athena-partition-1.0.0.tgz", + "size": null, + "sha1": null, + "md5": null, + "sha256": null, + "sha512": null, + "bug_tracking_url": null, + "code_view_url": null, + "vcs_url": null, + "copyright": null, + "holder": null, + "declared_license_expression": "unknown", + "declared_license_expression_spdx": "LicenseRef-scancode-unknown", + "license_detections": [ + { + "license_expression": "unknown", + "license_expression_spdx": "LicenseRef-scancode-unknown", + "matches": [ + { + "license_expression": "unknown", + "spdx_license_expression": "LicenseRef-scancode-unknown", + "from_file": "resolve-deps/package.json", + "start_line": 1, + "end_line": 1, + "matcher": "5-undetected", + "score": 100.0, + "matched_length": 2, + "match_coverage": 100.0, + "rule_relevance": 100, + "rule_identifier": "package-manifest-unknown-cb9ea49fe36cb2e1ba6d87c68e1195a492f762cf", + "rule_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/rules/package-manifest-unknown-cb9ea49fe36cb2e1ba6d87c68e1195a492f762cf", + "matched_text": "license - UNLICENSED" + } + ], + "identifier": "unknown-0669ac45-20f6-defd-ec9f-2b6aafc9f944" + } + ], + "other_license_expression": null, + "other_license_expression_spdx": null, + "other_license_detections": [], + "extracted_license_statement": "- UNLICENSED\n", + "notice_text": null, + "source_packages": [], + "is_private": true, + "extra_data": {}, + "repository_homepage_url": null, + "repository_download_url": null, + "api_data_url": null, + "package_uid": "pkg:npm/create-athena-partition@1.0.0?uuid=fixed-uid-done-for-testing-5642512d1758", + "datafile_paths": [ + "resolve-deps/package.json" + ], + "datasource_ids": [ + "npm_package_json" + ], + "purl": "pkg:npm/create-athena-partition@1.0.0" + } + ], + "dependencies": [ + { + "purl": "pkg:npm/athena-express", + "extracted_requirement": "^6.0.4", + "scope": "dependencies", + "is_runtime": true, + "is_optional": false, + "is_resolved": false, + "is_direct": true, + "resolved_package": {}, + "extra_data": {}, + "dependency_uid": "pkg:npm/athena-express?uuid=fixed-uid-done-for-testing-5642512d1758", + "for_package_uid": "pkg:npm/create-athena-partition@1.0.0?uuid=fixed-uid-done-for-testing-5642512d1758", + "datafile_path": "resolve-deps/package.json", + "datasource_id": "npm_package_json" + }, + { + "purl": "pkg:npm/athena-express@6.0.4", + "extracted_requirement": "^6.0.4", + "scope": "dependencies", + "is_runtime": true, + "is_optional": false, + "is_resolved": true, + "is_direct": false, + "resolved_package": { + "type": "npm", + "namespace": "", + "name": "athena-express", + "version": "6.0.4", + "qualifiers": {}, + "subpath": null, + "primary_language": "JavaScript", + "description": null, + "release_date": null, + "parties": [], + "keywords": [], + "homepage_url": null, + "download_url": "https://registry.yarnpkg.com/athena-express/-/athena-express-6.0.4.tgz", + "size": null, + "sha1": "bb457dcc967686faea2138f934828a331b98c8f6", + "md5": null, + "sha256": null, + "sha512": "02d0272ee0a30050812b2780926a607d4ccbd0d0c0bda64d82992e914ac1f68e7e53ffb0cdcac7bdc2a946e7c2f905c9ed6ffc512f5b3c28fc275b28d2c8ff7c", + "bug_tracking_url": null, + "code_view_url": null, + "vcs_url": null, + "copyright": null, + "holder": null, + "declared_license_expression": null, + "declared_license_expression_spdx": null, + "license_detections": [], + "other_license_expression": null, + "other_license_expression_spdx": null, + "other_license_detections": [], + "extracted_license_statement": null, + "notice_text": null, + "source_packages": [], + "file_references": [], + "is_private": false, + "extra_data": {}, + "dependencies": [ + { + "purl": "pkg:npm/csvtojson", + "extracted_requirement": "^2.0.10", + "scope": "dependencies", + "is_runtime": true, + "is_optional": false, + "is_resolved": true, + "is_direct": true, + "resolved_package": {}, + "extra_data": {} + } + ], + "repository_homepage_url": "https://www.npmjs.com/package/athena-express", + "repository_download_url": "https://registry.npmjs.org/athena-express/-/athena-express-6.0.4.tgz", + "api_data_url": "https://registry.npmjs.org/athena-express/6.0.4", + "datasource_id": "yarn_lock_v1", + "purl": "pkg:npm/athena-express@6.0.4" + }, + "extra_data": {}, + "dependency_uid": "pkg:npm/athena-express@6.0.4?uuid=fixed-uid-done-for-testing-5642512d1758", + "for_package_uid": "pkg:npm/create-athena-partition@1.0.0?uuid=fixed-uid-done-for-testing-5642512d1758", + "datafile_path": "resolve-deps/yarn.lock", + "datasource_id": "yarn_lock_v1" + }, + { + "purl": "pkg:npm/bluebird@3.7.2", + "extracted_requirement": "^3.5.1", + "scope": "dependencies", + "is_runtime": true, + "is_optional": false, + "is_resolved": true, + "is_direct": false, + "resolved_package": { + "type": "npm", + "namespace": "", + "name": "bluebird", + "version": "3.7.2", + "qualifiers": {}, + "subpath": null, + "primary_language": "JavaScript", + "description": null, + "release_date": null, + "parties": [], + "keywords": [], + "homepage_url": null, + "download_url": "https://registry.yarnpkg.com/bluebird/-/bluebird-3.7.2.tgz", + "size": null, + "sha1": "9f229c15be272454ffa973ace0dbee79a1b0c36f", + "md5": null, + "sha256": null, + "sha512": "5e9363e860d0cdd7d6fabd969e7ef189201ded33378f39311970464ed58ab925efd71515f9acf1026f2375664dd3a413424fb63765c1f6344392f6e6426711b6", + "bug_tracking_url": null, + "code_view_url": null, + "vcs_url": null, + "copyright": null, + "holder": null, + "declared_license_expression": null, + "declared_license_expression_spdx": null, + "license_detections": [], + "other_license_expression": null, + "other_license_expression_spdx": null, + "other_license_detections": [], + "extracted_license_statement": null, + "notice_text": null, + "source_packages": [], + "file_references": [], + "is_private": false, + "extra_data": {}, + "dependencies": [], + "repository_homepage_url": "https://www.npmjs.com/package/bluebird", + "repository_download_url": "https://registry.npmjs.org/bluebird/-/bluebird-3.7.2.tgz", + "api_data_url": "https://registry.npmjs.org/bluebird/3.7.2", + "datasource_id": "yarn_lock_v1", + "purl": "pkg:npm/bluebird@3.7.2" + }, + "extra_data": {}, + "dependency_uid": "pkg:npm/bluebird@3.7.2?uuid=fixed-uid-done-for-testing-5642512d1758", + "for_package_uid": "pkg:npm/create-athena-partition@1.0.0?uuid=fixed-uid-done-for-testing-5642512d1758", + "datafile_path": "resolve-deps/yarn.lock", + "datasource_id": "yarn_lock_v1" + }, + { + "purl": "pkg:npm/csvtojson@2.0.10", + "extracted_requirement": "^2.0.10", + "scope": "dependencies", + "is_runtime": true, + "is_optional": false, + "is_resolved": true, + "is_direct": false, + "resolved_package": { + "type": "npm", + "namespace": "", + "name": "csvtojson", + "version": "2.0.10", + "qualifiers": {}, + "subpath": null, + "primary_language": "JavaScript", + "description": null, + "release_date": null, + "parties": [], + "keywords": [], + "homepage_url": null, + "download_url": "https://registry.yarnpkg.com/csvtojson/-/csvtojson-2.0.10.tgz", + "size": null, + "sha1": "11e7242cc630da54efce7958a45f443210357574", + "md5": null, + "sha256": null, + "sha512": "954585c462b286b68a096f10821cfa6747f697f3ea0755b700ed07289cc6210e49452951eb9d5e9090e21896c14f8b1134dbf975d9d2195127b313fd5d1095a5", + "bug_tracking_url": null, + "code_view_url": null, + "vcs_url": null, + "copyright": null, + "holder": null, + "declared_license_expression": null, + "declared_license_expression_spdx": null, + "license_detections": [], + "other_license_expression": null, + "other_license_expression_spdx": null, + "other_license_detections": [], + "extracted_license_statement": null, + "notice_text": null, + "source_packages": [], + "file_references": [], + "is_private": false, + "extra_data": {}, + "dependencies": [ + { + "purl": "pkg:npm/bluebird", + "extracted_requirement": "^3.5.1", + "scope": "dependencies", + "is_runtime": true, + "is_optional": false, + "is_resolved": true, + "is_direct": true, + "resolved_package": {}, + "extra_data": {} + }, + { + "purl": "pkg:npm/lodash", + "extracted_requirement": "^4.17.3", + "scope": "dependencies", + "is_runtime": true, + "is_optional": false, + "is_resolved": true, + "is_direct": true, + "resolved_package": {}, + "extra_data": {} + }, + { + "purl": "pkg:npm/strip-bom", + "extracted_requirement": "^2.0.0", + "scope": "dependencies", + "is_runtime": true, + "is_optional": false, + "is_resolved": true, + "is_direct": true, + "resolved_package": {}, + "extra_data": {} + } + ], + "repository_homepage_url": "https://www.npmjs.com/package/csvtojson", + "repository_download_url": "https://registry.npmjs.org/csvtojson/-/csvtojson-2.0.10.tgz", + "api_data_url": "https://registry.npmjs.org/csvtojson/2.0.10", + "datasource_id": "yarn_lock_v1", + "purl": "pkg:npm/csvtojson@2.0.10" + }, + "extra_data": {}, + "dependency_uid": "pkg:npm/csvtojson@2.0.10?uuid=fixed-uid-done-for-testing-5642512d1758", + "for_package_uid": "pkg:npm/create-athena-partition@1.0.0?uuid=fixed-uid-done-for-testing-5642512d1758", + "datafile_path": "resolve-deps/yarn.lock", + "datasource_id": "yarn_lock_v1" + }, + { + "purl": "pkg:npm/is-utf8@0.1.1", + "extracted_requirement": "^0.1.0", + "scope": "dependencies", + "is_runtime": true, + "is_optional": false, + "is_resolved": true, + "is_direct": false, + "resolved_package": { + "type": "npm", + "namespace": "", + "name": "is-utf8", + "version": "0.1.1", + "qualifiers": {}, + "subpath": null, + "primary_language": "JavaScript", + "description": null, + "release_date": null, + "parties": [], + "keywords": [], + "homepage_url": null, + "download_url": "https://registry.yarnpkg.com/is-utf8/-/is-utf8-0.1.1.tgz", + "size": null, + "sha1": "4b0da1442104d1b336340e80797e865cf39f7d72", + "md5": null, + "sha256": null, + "sha512": null, + "bug_tracking_url": null, + "code_view_url": null, + "vcs_url": null, + "copyright": null, + "holder": null, + "declared_license_expression": null, + "declared_license_expression_spdx": null, + "license_detections": [], + "other_license_expression": null, + "other_license_expression_spdx": null, + "other_license_detections": [], + "extracted_license_statement": null, + "notice_text": null, + "source_packages": [], + "file_references": [], + "is_private": false, + "extra_data": {}, + "dependencies": [], + "repository_homepage_url": "https://www.npmjs.com/package/is-utf8", + "repository_download_url": "https://registry.npmjs.org/is-utf8/-/is-utf8-0.1.1.tgz", + "api_data_url": "https://registry.npmjs.org/is-utf8/0.1.1", + "datasource_id": "yarn_lock_v1", + "purl": "pkg:npm/is-utf8@0.1.1" + }, + "extra_data": {}, + "dependency_uid": "pkg:npm/is-utf8@0.1.1?uuid=fixed-uid-done-for-testing-5642512d1758", + "for_package_uid": "pkg:npm/create-athena-partition@1.0.0?uuid=fixed-uid-done-for-testing-5642512d1758", + "datafile_path": "resolve-deps/yarn.lock", + "datasource_id": "yarn_lock_v1" + }, + { + "purl": "pkg:npm/is-utf8@0.2.1", + "extracted_requirement": "^0.2.0", + "scope": "dependencies", + "is_runtime": true, + "is_optional": false, + "is_resolved": true, + "is_direct": false, + "resolved_package": { + "type": "npm", + "namespace": "", + "name": "is-utf8", + "version": "0.2.1", + "qualifiers": {}, + "subpath": null, + "primary_language": "JavaScript", + "description": null, + "release_date": null, + "parties": [], + "keywords": [], + "homepage_url": null, + "download_url": "https://registry.yarnpkg.com/is-utf8/-/is-utf8-0.2.1.tgz", + "size": null, + "sha1": "4b0da1442104d1b336340e80797e865cf39f7d72", + "md5": null, + "sha256": null, + "sha512": null, + "bug_tracking_url": null, + "code_view_url": null, + "vcs_url": null, + "copyright": null, + "holder": null, + "declared_license_expression": null, + "declared_license_expression_spdx": null, + "license_detections": [], + "other_license_expression": null, + "other_license_expression_spdx": null, + "other_license_detections": [], + "extracted_license_statement": null, + "notice_text": null, + "source_packages": [], + "file_references": [], + "is_private": false, + "extra_data": {}, + "dependencies": [], + "repository_homepage_url": "https://www.npmjs.com/package/is-utf8", + "repository_download_url": "https://registry.npmjs.org/is-utf8/-/is-utf8-0.2.1.tgz", + "api_data_url": "https://registry.npmjs.org/is-utf8/0.2.1", + "datasource_id": "yarn_lock_v1", + "purl": "pkg:npm/is-utf8@0.2.1" + }, + "extra_data": {}, + "dependency_uid": "pkg:npm/is-utf8@0.2.1?uuid=fixed-uid-done-for-testing-5642512d1758", + "for_package_uid": "pkg:npm/create-athena-partition@1.0.0?uuid=fixed-uid-done-for-testing-5642512d1758", + "datafile_path": "resolve-deps/yarn.lock", + "datasource_id": "yarn_lock_v1" + }, + { + "purl": "pkg:npm/lodash@4.17.21", + "extracted_requirement": "^4.17.3", + "scope": "dependencies", + "is_runtime": true, + "is_optional": false, + "is_resolved": true, + "is_direct": false, + "resolved_package": { + "type": "npm", + "namespace": "", + "name": "lodash", + "version": "4.17.21", + "qualifiers": {}, + "subpath": null, + "primary_language": "JavaScript", + "description": null, + "release_date": null, + "parties": [], + "keywords": [], + "homepage_url": null, + "download_url": "https://registry.yarnpkg.com/lodash/-/lodash-4.17.21.tgz", + "size": null, + "sha1": "679591c564c3bffaae8454cf0b3df370c3d6911c", + "md5": null, + "sha256": null, + "sha512": "bf690311ee7b95e713ba568322e3533f2dd1cb880b189e99d4edef13592b81764daec43e2c54c61d5c558dc5cfb35ecb85b65519e74026ff17675b6f8f916f4a", + "bug_tracking_url": null, + "code_view_url": null, + "vcs_url": null, + "copyright": null, + "holder": null, + "declared_license_expression": null, + "declared_license_expression_spdx": null, + "license_detections": [], + "other_license_expression": null, + "other_license_expression_spdx": null, + "other_license_detections": [], + "extracted_license_statement": null, + "notice_text": null, + "source_packages": [], + "file_references": [], + "is_private": false, + "extra_data": {}, + "dependencies": [ + { + "purl": "pkg:npm/is-utf8", + "extracted_requirement": "^0.1.0", + "scope": "dependencies", + "is_runtime": true, + "is_optional": false, + "is_resolved": true, + "is_direct": true, + "resolved_package": {}, + "extra_data": {} + } + ], + "repository_homepage_url": "https://www.npmjs.com/package/lodash", + "repository_download_url": "https://registry.npmjs.org/lodash/-/lodash-4.17.21.tgz", + "api_data_url": "https://registry.npmjs.org/lodash/4.17.21", + "datasource_id": "yarn_lock_v1", + "purl": "pkg:npm/lodash@4.17.21" + }, + "extra_data": {}, + "dependency_uid": "pkg:npm/lodash@4.17.21?uuid=fixed-uid-done-for-testing-5642512d1758", + "for_package_uid": "pkg:npm/create-athena-partition@1.0.0?uuid=fixed-uid-done-for-testing-5642512d1758", + "datafile_path": "resolve-deps/yarn.lock", + "datasource_id": "yarn_lock_v1" + }, + { + "purl": "pkg:npm/strip-bom@2.0.0", + "extracted_requirement": "^2.0.0", + "scope": "dependencies", + "is_runtime": true, + "is_optional": false, + "is_resolved": true, + "is_direct": false, + "resolved_package": { + "type": "npm", + "namespace": "", + "name": "strip-bom", + "version": "2.0.0", + "qualifiers": {}, + "subpath": null, + "primary_language": "JavaScript", + "description": null, + "release_date": null, + "parties": [], + "keywords": [], + "homepage_url": null, + "download_url": "https://registry.yarnpkg.com/strip-bom/-/strip-bom-2.0.0.tgz", + "size": null, + "sha1": "6219a85616520491f35788bdbf1447a99c7e6b0e", + "md5": null, + "sha256": null, + "sha512": null, + "bug_tracking_url": null, + "code_view_url": null, + "vcs_url": null, + "copyright": null, + "holder": null, + "declared_license_expression": null, + "declared_license_expression_spdx": null, + "license_detections": [], + "other_license_expression": null, + "other_license_expression_spdx": null, + "other_license_detections": [], + "extracted_license_statement": null, + "notice_text": null, + "source_packages": [], + "file_references": [], + "is_private": false, + "extra_data": {}, + "dependencies": [ + { + "purl": "pkg:npm/is-utf8", + "extracted_requirement": "^0.2.0", + "scope": "dependencies", + "is_runtime": true, + "is_optional": false, + "is_resolved": true, + "is_direct": true, + "resolved_package": {}, + "extra_data": {} + } + ], + "repository_homepage_url": "https://www.npmjs.com/package/strip-bom", + "repository_download_url": "https://registry.npmjs.org/strip-bom/-/strip-bom-2.0.0.tgz", + "api_data_url": "https://registry.npmjs.org/strip-bom/2.0.0", + "datasource_id": "yarn_lock_v1", + "purl": "pkg:npm/strip-bom@2.0.0" + }, + "extra_data": {}, + "dependency_uid": "pkg:npm/strip-bom@2.0.0?uuid=fixed-uid-done-for-testing-5642512d1758", + "for_package_uid": "pkg:npm/create-athena-partition@1.0.0?uuid=fixed-uid-done-for-testing-5642512d1758", + "datafile_path": "resolve-deps/yarn.lock", + "datasource_id": "yarn_lock_v1" + } + ], + "files": [ + { + "path": "resolve-deps", + "type": "directory", + "package_data": [], + "for_packages": [], + "scan_errors": [] + }, + { + "path": "resolve-deps/package.json", + "type": "file", + "package_data": [ + { + "type": "npm", + "namespace": null, + "name": "create-athena-partition", + "version": "1.0.0", + "qualifiers": {}, + "subpath": null, + "primary_language": "JavaScript", + "description": null, + "release_date": null, + "parties": [], + "keywords": [], + "homepage_url": null, + "download_url": "https://registry.npmjs.org/create-athena-partition/-/create-athena-partition-1.0.0.tgz", + "size": null, + "sha1": null, + "md5": null, + "sha256": null, + "sha512": null, + "bug_tracking_url": null, + "code_view_url": null, + "vcs_url": null, + "copyright": null, + "holder": null, + "declared_license_expression": "unknown", + "declared_license_expression_spdx": "LicenseRef-scancode-unknown", + "license_detections": [ + { + "license_expression": "unknown", + "license_expression_spdx": "LicenseRef-scancode-unknown", + "matches": [ + { + "license_expression": "unknown", + "spdx_license_expression": "LicenseRef-scancode-unknown", + "from_file": "resolve-deps/package.json", + "start_line": 1, + "end_line": 1, + "matcher": "5-undetected", + "score": 100.0, + "matched_length": 2, + "match_coverage": 100.0, + "rule_relevance": 100, + "rule_identifier": "package-manifest-unknown-cb9ea49fe36cb2e1ba6d87c68e1195a492f762cf", + "rule_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/rules/package-manifest-unknown-cb9ea49fe36cb2e1ba6d87c68e1195a492f762cf", + "matched_text": "license - UNLICENSED" + } + ], + "identifier": "unknown-0669ac45-20f6-defd-ec9f-2b6aafc9f944" + } + ], + "other_license_expression": null, + "other_license_expression_spdx": null, + "other_license_detections": [], + "extracted_license_statement": "- UNLICENSED\n", + "notice_text": null, + "source_packages": [], + "file_references": [], + "is_private": true, + "extra_data": {}, + "dependencies": [ + { + "purl": "pkg:npm/athena-express", + "extracted_requirement": "^6.0.4", + "scope": "dependencies", + "is_runtime": true, + "is_optional": false, + "is_resolved": false, + "is_direct": true, + "resolved_package": {}, + "extra_data": {} + } + ], + "repository_homepage_url": null, + "repository_download_url": null, + "api_data_url": null, + "datasource_id": "npm_package_json", + "purl": "pkg:npm/create-athena-partition@1.0.0" + } + ], + "for_packages": [ + "pkg:npm/create-athena-partition@1.0.0?uuid=fixed-uid-done-for-testing-5642512d1758" + ], + "scan_errors": [] + }, + { + "path": "resolve-deps/yarn.lock", + "type": "file", + "package_data": [ + { + "type": "npm", + "namespace": null, + "name": null, + "version": null, + "qualifiers": {}, + "subpath": null, + "primary_language": "JavaScript", + "description": null, + "release_date": null, + "parties": [], + "keywords": [], + "homepage_url": null, + "download_url": null, + "size": null, + "sha1": null, + "md5": null, + "sha256": null, + "sha512": null, + "bug_tracking_url": null, + "code_view_url": null, + "vcs_url": null, + "copyright": null, + "holder": null, + "declared_license_expression": null, + "declared_license_expression_spdx": null, + "license_detections": [], + "other_license_expression": null, + "other_license_expression_spdx": null, + "other_license_detections": [], + "extracted_license_statement": null, + "notice_text": null, + "source_packages": [], + "file_references": [], + "is_private": false, + "extra_data": {}, + "dependencies": [ + { + "purl": "pkg:npm/athena-express@6.0.4", + "extracted_requirement": "^6.0.4", + "scope": "dependencies", + "is_runtime": true, + "is_optional": false, + "is_resolved": true, + "is_direct": false, + "resolved_package": { + "type": "npm", + "namespace": "", + "name": "athena-express", + "version": "6.0.4", + "qualifiers": {}, + "subpath": null, + "primary_language": "JavaScript", + "description": null, + "release_date": null, + "parties": [], + "keywords": [], + "homepage_url": null, + "download_url": "https://registry.yarnpkg.com/athena-express/-/athena-express-6.0.4.tgz", + "size": null, + "sha1": "bb457dcc967686faea2138f934828a331b98c8f6", + "md5": null, + "sha256": null, + "sha512": "02d0272ee0a30050812b2780926a607d4ccbd0d0c0bda64d82992e914ac1f68e7e53ffb0cdcac7bdc2a946e7c2f905c9ed6ffc512f5b3c28fc275b28d2c8ff7c", + "bug_tracking_url": null, + "code_view_url": null, + "vcs_url": null, + "copyright": null, + "holder": null, + "declared_license_expression": null, + "declared_license_expression_spdx": null, + "license_detections": [], + "other_license_expression": null, + "other_license_expression_spdx": null, + "other_license_detections": [], + "extracted_license_statement": null, + "notice_text": null, + "source_packages": [], + "file_references": [], + "is_private": false, + "extra_data": {}, + "dependencies": [ + { + "purl": "pkg:npm/csvtojson", + "extracted_requirement": "^2.0.10", + "scope": "dependencies", + "is_runtime": true, + "is_optional": false, + "is_resolved": true, + "is_direct": true, + "resolved_package": {}, + "extra_data": {} + } + ], + "repository_homepage_url": "https://www.npmjs.com/package/athena-express", + "repository_download_url": "https://registry.npmjs.org/athena-express/-/athena-express-6.0.4.tgz", + "api_data_url": "https://registry.npmjs.org/athena-express/6.0.4", + "datasource_id": "yarn_lock_v1", + "purl": "pkg:npm/athena-express@6.0.4" + }, + "extra_data": {} + }, + { + "purl": "pkg:npm/bluebird@3.7.2", + "extracted_requirement": "^3.5.1", + "scope": "dependencies", + "is_runtime": true, + "is_optional": false, + "is_resolved": true, + "is_direct": false, + "resolved_package": { + "type": "npm", + "namespace": "", + "name": "bluebird", + "version": "3.7.2", + "qualifiers": {}, + "subpath": null, + "primary_language": "JavaScript", + "description": null, + "release_date": null, + "parties": [], + "keywords": [], + "homepage_url": null, + "download_url": "https://registry.yarnpkg.com/bluebird/-/bluebird-3.7.2.tgz", + "size": null, + "sha1": "9f229c15be272454ffa973ace0dbee79a1b0c36f", + "md5": null, + "sha256": null, + "sha512": "5e9363e860d0cdd7d6fabd969e7ef189201ded33378f39311970464ed58ab925efd71515f9acf1026f2375664dd3a413424fb63765c1f6344392f6e6426711b6", + "bug_tracking_url": null, + "code_view_url": null, + "vcs_url": null, + "copyright": null, + "holder": null, + "declared_license_expression": null, + "declared_license_expression_spdx": null, + "license_detections": [], + "other_license_expression": null, + "other_license_expression_spdx": null, + "other_license_detections": [], + "extracted_license_statement": null, + "notice_text": null, + "source_packages": [], + "file_references": [], + "is_private": false, + "extra_data": {}, + "dependencies": [], + "repository_homepage_url": "https://www.npmjs.com/package/bluebird", + "repository_download_url": "https://registry.npmjs.org/bluebird/-/bluebird-3.7.2.tgz", + "api_data_url": "https://registry.npmjs.org/bluebird/3.7.2", + "datasource_id": "yarn_lock_v1", + "purl": "pkg:npm/bluebird@3.7.2" + }, + "extra_data": {} + }, + { + "purl": "pkg:npm/csvtojson@2.0.10", + "extracted_requirement": "^2.0.10", + "scope": "dependencies", + "is_runtime": true, + "is_optional": false, + "is_resolved": true, + "is_direct": false, + "resolved_package": { + "type": "npm", + "namespace": "", + "name": "csvtojson", + "version": "2.0.10", + "qualifiers": {}, + "subpath": null, + "primary_language": "JavaScript", + "description": null, + "release_date": null, + "parties": [], + "keywords": [], + "homepage_url": null, + "download_url": "https://registry.yarnpkg.com/csvtojson/-/csvtojson-2.0.10.tgz", + "size": null, + "sha1": "11e7242cc630da54efce7958a45f443210357574", + "md5": null, + "sha256": null, + "sha512": "954585c462b286b68a096f10821cfa6747f697f3ea0755b700ed07289cc6210e49452951eb9d5e9090e21896c14f8b1134dbf975d9d2195127b313fd5d1095a5", + "bug_tracking_url": null, + "code_view_url": null, + "vcs_url": null, + "copyright": null, + "holder": null, + "declared_license_expression": null, + "declared_license_expression_spdx": null, + "license_detections": [], + "other_license_expression": null, + "other_license_expression_spdx": null, + "other_license_detections": [], + "extracted_license_statement": null, + "notice_text": null, + "source_packages": [], + "file_references": [], + "is_private": false, + "extra_data": {}, + "dependencies": [ + { + "purl": "pkg:npm/bluebird", + "extracted_requirement": "^3.5.1", + "scope": "dependencies", + "is_runtime": true, + "is_optional": false, + "is_resolved": true, + "is_direct": true, + "resolved_package": {}, + "extra_data": {} + }, + { + "purl": "pkg:npm/lodash", + "extracted_requirement": "^4.17.3", + "scope": "dependencies", + "is_runtime": true, + "is_optional": false, + "is_resolved": true, + "is_direct": true, + "resolved_package": {}, + "extra_data": {} + }, + { + "purl": "pkg:npm/strip-bom", + "extracted_requirement": "^2.0.0", + "scope": "dependencies", + "is_runtime": true, + "is_optional": false, + "is_resolved": true, + "is_direct": true, + "resolved_package": {}, + "extra_data": {} + } + ], + "repository_homepage_url": "https://www.npmjs.com/package/csvtojson", + "repository_download_url": "https://registry.npmjs.org/csvtojson/-/csvtojson-2.0.10.tgz", + "api_data_url": "https://registry.npmjs.org/csvtojson/2.0.10", + "datasource_id": "yarn_lock_v1", + "purl": "pkg:npm/csvtojson@2.0.10" + }, + "extra_data": {} + }, + { + "purl": "pkg:npm/is-utf8@0.1.1", + "extracted_requirement": "^0.1.0", + "scope": "dependencies", + "is_runtime": true, + "is_optional": false, + "is_resolved": true, + "is_direct": false, + "resolved_package": { + "type": "npm", + "namespace": "", + "name": "is-utf8", + "version": "0.1.1", + "qualifiers": {}, + "subpath": null, + "primary_language": "JavaScript", + "description": null, + "release_date": null, + "parties": [], + "keywords": [], + "homepage_url": null, + "download_url": "https://registry.yarnpkg.com/is-utf8/-/is-utf8-0.1.1.tgz", + "size": null, + "sha1": "4b0da1442104d1b336340e80797e865cf39f7d72", + "md5": null, + "sha256": null, + "sha512": null, + "bug_tracking_url": null, + "code_view_url": null, + "vcs_url": null, + "copyright": null, + "holder": null, + "declared_license_expression": null, + "declared_license_expression_spdx": null, + "license_detections": [], + "other_license_expression": null, + "other_license_expression_spdx": null, + "other_license_detections": [], + "extracted_license_statement": null, + "notice_text": null, + "source_packages": [], + "file_references": [], + "is_private": false, + "extra_data": {}, + "dependencies": [], + "repository_homepage_url": "https://www.npmjs.com/package/is-utf8", + "repository_download_url": "https://registry.npmjs.org/is-utf8/-/is-utf8-0.1.1.tgz", + "api_data_url": "https://registry.npmjs.org/is-utf8/0.1.1", + "datasource_id": "yarn_lock_v1", + "purl": "pkg:npm/is-utf8@0.1.1" + }, + "extra_data": {} + }, + { + "purl": "pkg:npm/is-utf8@0.2.1", + "extracted_requirement": "^0.2.0", + "scope": "dependencies", + "is_runtime": true, + "is_optional": false, + "is_resolved": true, + "is_direct": false, + "resolved_package": { + "type": "npm", + "namespace": "", + "name": "is-utf8", + "version": "0.2.1", + "qualifiers": {}, + "subpath": null, + "primary_language": "JavaScript", + "description": null, + "release_date": null, + "parties": [], + "keywords": [], + "homepage_url": null, + "download_url": "https://registry.yarnpkg.com/is-utf8/-/is-utf8-0.2.1.tgz", + "size": null, + "sha1": "4b0da1442104d1b336340e80797e865cf39f7d72", + "md5": null, + "sha256": null, + "sha512": null, + "bug_tracking_url": null, + "code_view_url": null, + "vcs_url": null, + "copyright": null, + "holder": null, + "declared_license_expression": null, + "declared_license_expression_spdx": null, + "license_detections": [], + "other_license_expression": null, + "other_license_expression_spdx": null, + "other_license_detections": [], + "extracted_license_statement": null, + "notice_text": null, + "source_packages": [], + "file_references": [], + "is_private": false, + "extra_data": {}, + "dependencies": [], + "repository_homepage_url": "https://www.npmjs.com/package/is-utf8", + "repository_download_url": "https://registry.npmjs.org/is-utf8/-/is-utf8-0.2.1.tgz", + "api_data_url": "https://registry.npmjs.org/is-utf8/0.2.1", + "datasource_id": "yarn_lock_v1", + "purl": "pkg:npm/is-utf8@0.2.1" + }, + "extra_data": {} + }, + { + "purl": "pkg:npm/lodash@4.17.21", + "extracted_requirement": "^4.17.3", + "scope": "dependencies", + "is_runtime": true, + "is_optional": false, + "is_resolved": true, + "is_direct": false, + "resolved_package": { + "type": "npm", + "namespace": "", + "name": "lodash", + "version": "4.17.21", + "qualifiers": {}, + "subpath": null, + "primary_language": "JavaScript", + "description": null, + "release_date": null, + "parties": [], + "keywords": [], + "homepage_url": null, + "download_url": "https://registry.yarnpkg.com/lodash/-/lodash-4.17.21.tgz", + "size": null, + "sha1": "679591c564c3bffaae8454cf0b3df370c3d6911c", + "md5": null, + "sha256": null, + "sha512": "bf690311ee7b95e713ba568322e3533f2dd1cb880b189e99d4edef13592b81764daec43e2c54c61d5c558dc5cfb35ecb85b65519e74026ff17675b6f8f916f4a", + "bug_tracking_url": null, + "code_view_url": null, + "vcs_url": null, + "copyright": null, + "holder": null, + "declared_license_expression": null, + "declared_license_expression_spdx": null, + "license_detections": [], + "other_license_expression": null, + "other_license_expression_spdx": null, + "other_license_detections": [], + "extracted_license_statement": null, + "notice_text": null, + "source_packages": [], + "file_references": [], + "is_private": false, + "extra_data": {}, + "dependencies": [ + { + "purl": "pkg:npm/is-utf8", + "extracted_requirement": "^0.1.0", + "scope": "dependencies", + "is_runtime": true, + "is_optional": false, + "is_resolved": true, + "is_direct": true, + "resolved_package": {}, + "extra_data": {} + } + ], + "repository_homepage_url": "https://www.npmjs.com/package/lodash", + "repository_download_url": "https://registry.npmjs.org/lodash/-/lodash-4.17.21.tgz", + "api_data_url": "https://registry.npmjs.org/lodash/4.17.21", + "datasource_id": "yarn_lock_v1", + "purl": "pkg:npm/lodash@4.17.21" + }, + "extra_data": {} + }, + { + "purl": "pkg:npm/strip-bom@2.0.0", + "extracted_requirement": "^2.0.0", + "scope": "dependencies", + "is_runtime": true, + "is_optional": false, + "is_resolved": true, + "is_direct": false, + "resolved_package": { + "type": "npm", + "namespace": "", + "name": "strip-bom", + "version": "2.0.0", + "qualifiers": {}, + "subpath": null, + "primary_language": "JavaScript", + "description": null, + "release_date": null, + "parties": [], + "keywords": [], + "homepage_url": null, + "download_url": "https://registry.yarnpkg.com/strip-bom/-/strip-bom-2.0.0.tgz", + "size": null, + "sha1": "6219a85616520491f35788bdbf1447a99c7e6b0e", + "md5": null, + "sha256": null, + "sha512": null, + "bug_tracking_url": null, + "code_view_url": null, + "vcs_url": null, + "copyright": null, + "holder": null, + "declared_license_expression": null, + "declared_license_expression_spdx": null, + "license_detections": [], + "other_license_expression": null, + "other_license_expression_spdx": null, + "other_license_detections": [], + "extracted_license_statement": null, + "notice_text": null, + "source_packages": [], + "file_references": [], + "is_private": false, + "extra_data": {}, + "dependencies": [ + { + "purl": "pkg:npm/is-utf8", + "extracted_requirement": "^0.2.0", + "scope": "dependencies", + "is_runtime": true, + "is_optional": false, + "is_resolved": true, + "is_direct": true, + "resolved_package": {}, + "extra_data": {} + } + ], + "repository_homepage_url": "https://www.npmjs.com/package/strip-bom", + "repository_download_url": "https://registry.npmjs.org/strip-bom/-/strip-bom-2.0.0.tgz", + "api_data_url": "https://registry.npmjs.org/strip-bom/2.0.0", + "datasource_id": "yarn_lock_v1", + "purl": "pkg:npm/strip-bom@2.0.0" + }, + "extra_data": {} + } + ], + "repository_homepage_url": null, + "repository_download_url": null, + "api_data_url": null, + "datasource_id": "yarn_lock_v1", + "purl": null + } + ], + "for_packages": [ + "pkg:npm/create-athena-partition@1.0.0?uuid=fixed-uid-done-for-testing-5642512d1758" + ], + "scan_errors": [] + } + ] +} \ No newline at end of file diff --git a/tests/packagedcode/data/npm/yarn-lock/resolve-deps/package.json b/tests/packagedcode/data/npm/yarn-lock/resolve-deps/package.json new file mode 100644 index 00000000000..818a6f03b03 --- /dev/null +++ b/tests/packagedcode/data/npm/yarn-lock/resolve-deps/package.json @@ -0,0 +1,10 @@ +{ + "name": "create-athena-partition", + "private": true, + "license": "UNLICENSED", + "version": "1.0.0", + "main": "index.js", + "dependencies": { + "athena-express": "^6.0.4" + } +} diff --git a/tests/packagedcode/data/npm/yarn-lock/resolve-deps/yarn.lock b/tests/packagedcode/data/npm/yarn-lock/resolve-deps/yarn.lock new file mode 100644 index 00000000000..aa8c78f0f07 --- /dev/null +++ b/tests/packagedcode/data/npm/yarn-lock/resolve-deps/yarn.lock @@ -0,0 +1,48 @@ +# THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY. +# yarn lockfile v1 + + +athena-express@^6.0.4: + version "6.0.4" + resolved "https://registry.yarnpkg.com/athena-express/-/athena-express-6.0.4.tgz#bb457dcc967686faea2138f934828a331b98c8f6" + integrity sha512-AtAnLuCjAFCBKyeAkmpgfUzL0NDAvaZNgpkukUrB9o5+U/+wzcrHvcKpRufC+QXJ7W/8US9bPCj8J1so0sj/fA== + dependencies: + csvtojson "^2.0.10" + +bluebird@^3.5.1: + version "3.7.2" + resolved "https://registry.yarnpkg.com/bluebird/-/bluebird-3.7.2.tgz#9f229c15be272454ffa973ace0dbee79a1b0c36f" + integrity sha512-XpNj6GDQzdfW+r2Wnn7xiSAd7TM3jzkxGXBGTtWKuSXv1xUV+azxAm8jdWZN06QTQk+2N2XB9jRDkvbmQmcRtg== + +csvtojson@^2.0.10: + version "2.0.10" + resolved "https://registry.yarnpkg.com/csvtojson/-/csvtojson-2.0.10.tgz#11e7242cc630da54efce7958a45f443210357574" + integrity sha512-lUWFxGKyhraKCW8Qghz6Z0f2l/PqB1W3AO0HKJzGIQ5JRSlR651ekJDiGJbBT4sRNNv5ddnSGVEnsxP9XRCVpQ== + dependencies: + bluebird "^3.5.1" + lodash "^4.17.3" + strip-bom "^2.0.0" + +is-utf8@^0.1.0: + version "0.1.1" + resolved "https://registry.yarnpkg.com/is-utf8/-/is-utf8-0.1.1.tgz#4b0da1442104d1b336340e80797e865cf39f7d72" + integrity sha1-Sw2hRCEE0bM2NA6AeX6GXPOffXI= + +is-utf8@^0.2.0: + version "0.2.1" + resolved "https://registry.yarnpkg.com/is-utf8/-/is-utf8-0.2.1.tgz#4b0da1442104d1b336340e80797e865cf39f7d72" + integrity sha1-Sw2hRCEE0bM2NA6AeX6GXPOffXI= + +lodash@^4.17.3: + version "4.17.21" + resolved "https://registry.yarnpkg.com/lodash/-/lodash-4.17.21.tgz#679591c564c3bffaae8454cf0b3df370c3d6911c" + integrity sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg== + dependencies: + is-utf8 "^0.1.0" + +strip-bom@^2.0.0: + version "2.0.0" + resolved "https://registry.yarnpkg.com/strip-bom/-/strip-bom-2.0.0.tgz#6219a85616520491f35788bdbf1447a99c7e6b0e" + integrity sha1-YhmoVhZSBJHzV4i9vxRHqZx+aw4= + dependencies: + is-utf8 "^0.2.0" diff --git a/tests/packagedcode/test_npm.py b/tests/packagedcode/test_npm.py index 46dd558585c..d0312417004 100644 --- a/tests/packagedcode/test_npm.py +++ b/tests/packagedcode/test_npm.py @@ -330,6 +330,15 @@ def test_parse_yarn_lock_v2(self): packages = npm.YarnLockV2Handler.parse(test_file) self.check_packages_data(packages, expected_loc, regen=REGEN_TEST_FIXTURES) + def test_npm_yarn_with_package_json_resolve_dependencies(self): + test_folder = self.get_test_loc('npm/yarn-lock/resolve-deps/') + expected_file = self.get_test_loc('npm/yarn-lock/resolve-deps.expected.json') + result_file = self.get_temp_file('results.json') + run_scan_click(['--package', test_folder, '--json', result_file]) + check_json_scan( + expected_file, result_file, remove_uuid=True, regen=REGEN_TEST_FIXTURES + ) + def test_is_datafile_pnpm_shrinkwrap_yaml(self): test_file = self.get_test_loc('npm/pnpm/shrinkwrap/v3/vuepack/shrinkwrap.yaml') assert npm.PnpmShrinkwrapYamlHandler.is_datafile(test_file)