diff --git a/src/packagedcode/npm.py b/src/packagedcode/npm.py index d4a5563063d..2bf6ded56bc 100644 --- a/src/packagedcode/npm.py +++ b/src/packagedcode/npm.py @@ -23,6 +23,7 @@ from packagedcode.utils import normalize_vcs_url from packagedcode.utils import yield_dependencies_from_package_data from packagedcode.utils import yield_dependencies_from_package_resource +from packagedcode.utils import update_dependencies_as_resolved import saneyaml """ @@ -420,8 +421,9 @@ def parse(cls, location, package_only=False): ) resolved_package.dependencies = sub_deps dependency.resolved_package = resolved_package.to_dict() - dependencies.append(dependency) + dependencies.append(dependency.to_dict()) + update_dependencies_as_resolved(dependencies=dependencies) root_package_data.dependencies = dependencies yield root_package_data @@ -551,8 +553,9 @@ def parse(cls, location, package_only=False): is_optional=False, is_runtime=True, ) - top_dependencies.append(dependency) + top_dependencies.append(dependency.to_dict()) + update_dependencies_as_resolved(dependencies=top_dependencies) package_data = dict( datasource_id=cls.datasource_id, type=cls.default_package_type, @@ -714,8 +717,9 @@ def parse(cls, location, package_only=False): is_runtime=True, resolved_package=resolved_package_data.to_dict(), ) - dependencies.append(dep) + dependencies.append(dep.to_dict()) + update_dependencies_as_resolved(dependencies=dependencies) package_data = dict( datasource_id=cls.datasource_id, type=cls.default_package_type, @@ -811,7 +815,11 @@ def parse(cls, location, package_only=False): ) dependencies_by_purl[purl] = dependency_data - dependencies = list(dependencies_by_purl.values()) + dependencies = [ + dep.to_dict() + for dep in list(dependencies_by_purl.values()) + ] + update_dependencies_as_resolved(dependencies=dependencies) root_package_data = dict( datasource_id=cls.datasource_id, type=cls.default_package_type, diff --git a/src/packagedcode/utils.py b/src/packagedcode/utils.py index a954b882d3d..bcc813a202a 100644 --- a/src/packagedcode/utils.py +++ b/src/packagedcode/utils.py @@ -7,6 +7,8 @@ # See https://aboutcode.org for more information about nexB OSS projects. # +from packageurl import PackageURL + try: from license_expression import Licensing from license_expression import combine_expressions as le_combine_expressions @@ -215,3 +217,57 @@ def yield_dependencies_from_package_resource(resource, package_uid=None): for pkg_data in resource.package_data: pkg_data = models.PackageData.from_dict(pkg_data) yield from yield_dependencies_from_package_data(pkg_data, resource.path, package_uid) + + +def update_dependencies_as_resolved(dependencies): + """ + For a list of dependency mappings with their respective + resolved packages, update in place the dependencies for those + resolved packages as resolved (update `is_resolved` as True), + if the requirement is also present as a resolved package. + """ + #TODO: Use vers to mark update `is_resolved` even in the case + # of incomplete resolution/partially pinned dependencies + + # These are only type, namespace and name (without version and qualifiers) + base_resolved_purls = [] + base_purl_fields = ["type", "namespace", "name"] + try: + resolved_packages = [ + dep.get("resolved_package") + for dep in dependencies + if dep.get("resolved_package") + ] + except AttributeError: + raise Exception(dependencies) + + # No resolved packages are present for dependencies + if not resolved_packages: + return + + for pkg in resolved_packages: + purl_mapping = PackageURL.from_string(purl=pkg.get("purl")).to_dict() + base_purl_mapping = { + purl_field: purl_value + for purl_field, purl_value in purl_mapping.items() + if purl_field in base_purl_fields + } + base_resolved_purls.append( + PackageURL(**base_purl_mapping).to_string() + ) + + for dependency in dependencies: + resolved_package = dependency.get("resolved_package") + dependencies_from_resolved = [] + if resolved_package: + dependencies_from_resolved = resolved_package.get("dependencies") + + if not dependencies_from_resolved: + continue + + for dep in dependencies_from_resolved: + dep_purl = dep.get("purl") + if dep_purl in base_resolved_purls: + dep["is_resolved"] = True + + diff --git a/tests/packagedcode/data/npm/package-lock-v1/package-lock.json-expected b/tests/packagedcode/data/npm/package-lock-v1/package-lock.json-expected index 73daf7f8819..f58d30a8913 100644 --- a/tests/packagedcode/data/npm/package-lock-v1/package-lock.json-expected +++ b/tests/packagedcode/data/npm/package-lock-v1/package-lock.json-expected @@ -161,7 +161,7 @@ "scope": "devDependencies", "is_runtime": false, "is_optional": true, - "is_resolved": false, + "is_resolved": true, "resolved_package": {}, "extra_data": {} }, @@ -171,7 +171,7 @@ "scope": "devDependencies", "is_runtime": false, "is_optional": true, - "is_resolved": false, + "is_resolved": true, "resolved_package": {}, "extra_data": {} } @@ -245,7 +245,7 @@ "scope": "devDependencies", "is_runtime": false, "is_optional": true, - "is_resolved": false, + "is_resolved": true, "resolved_package": {}, "extra_data": {} } @@ -319,7 +319,7 @@ "scope": "devDependencies", "is_runtime": false, "is_optional": true, - "is_resolved": false, + "is_resolved": true, "resolved_package": {}, "extra_data": {} } @@ -467,7 +467,7 @@ "scope": "devDependencies", "is_runtime": false, "is_optional": true, - "is_resolved": false, + "is_resolved": true, "resolved_package": {}, "extra_data": {} }, diff --git a/tests/packagedcode/data/npm/yarn-lock/v1-complex2/yarn.lock-expected b/tests/packagedcode/data/npm/yarn-lock/v1-complex2/yarn.lock-expected index 91d390b1ed0..163a365b28d 100644 --- a/tests/packagedcode/data/npm/yarn-lock/v1-complex2/yarn.lock-expected +++ b/tests/packagedcode/data/npm/yarn-lock/v1-complex2/yarn.lock-expected @@ -625,7 +625,7 @@ "scope": "dependencies", "is_runtime": true, "is_optional": false, - "is_resolved": false, + "is_resolved": true, "resolved_package": {}, "extra_data": {} } @@ -707,7 +707,7 @@ "scope": "dependencies", "is_runtime": true, "is_optional": false, - "is_resolved": false, + "is_resolved": true, "resolved_package": {}, "extra_data": {} } @@ -893,7 +893,7 @@ "scope": "dependencies", "is_runtime": true, "is_optional": false, - "is_resolved": false, + "is_resolved": true, "resolved_package": {}, "extra_data": {} }, @@ -903,7 +903,7 @@ "scope": "dependencies", "is_runtime": true, "is_optional": false, - "is_resolved": false, + "is_resolved": true, "resolved_package": {}, "extra_data": {} }, @@ -913,7 +913,7 @@ "scope": "dependencies", "is_runtime": true, "is_optional": false, - "is_resolved": false, + "is_resolved": true, "resolved_package": {}, "extra_data": {} } @@ -975,7 +975,7 @@ "scope": "dependencies", "is_runtime": true, "is_optional": false, - "is_resolved": false, + "is_resolved": true, "resolved_package": {}, "extra_data": {} }, @@ -985,7 +985,7 @@ "scope": "dependencies", "is_runtime": true, "is_optional": false, - "is_resolved": false, + "is_resolved": true, "resolved_package": {}, "extra_data": {} }, @@ -995,7 +995,7 @@ "scope": "dependencies", "is_runtime": true, "is_optional": false, - "is_resolved": false, + "is_resolved": true, "resolved_package": {}, "extra_data": {} } diff --git a/tests/packagedcode/data/npm/yarn-lock/v1/yarn.lock-expected b/tests/packagedcode/data/npm/yarn-lock/v1/yarn.lock-expected index b1e8dee3875..720f609bc1b 100644 --- a/tests/packagedcode/data/npm/yarn-lock/v1/yarn.lock-expected +++ b/tests/packagedcode/data/npm/yarn-lock/v1/yarn.lock-expected @@ -475,7 +475,7 @@ "scope": "dependencies", "is_runtime": true, "is_optional": false, - "is_resolved": false, + "is_resolved": true, "resolved_package": {}, "extra_data": {} }