Merge pull request #2261 from nexB/fix-npm-dist-issue

Fix `dist_mapper()` in packagedcode/npm.py

src/packagedcode/npm.py

@@ -1,5 +1,5 @@
 
-# Copyright (c) 2018-2020 nexB Inc. and others. All rights reserved.
+# Copyright (c) nexB Inc. and others. All rights reserved.
 # http://nexb.com and https://github.com/nexB/scancode-toolkit/
 # The ScanCode software is licensed under the Apache License version 2.0.
 # Data generated with ScanCode require an acknowledgment.
@@ -26,6 +26,7 @@
 from __future__ import print_function
 from __future__ import unicode_literals
 
+import base64
 from collections import defaultdict
 from collections import OrderedDict
 from functools import partial
@@ -37,6 +38,7 @@
 import attr
 from packageurl import PackageURL
 from six import string_types
+from six import binary_type
 
 from commoncode import filetype
 from commoncode import fileutils
@@ -575,10 +577,16 @@ def dist_mapper(dist, package):
     integrity = dist.get('integrity') or None
     if integrity:
         algo, _, b64value = integrity.partition('-')
-        assert 'sha512' == algo
         algo = algo.lower()
-        sha512 = b64value.decode('base64').encode('hex')
+        assert 'sha512' == algo
+
+        decoded_b64value = base64.b64decode(b64value)
+        if isinstance(decoded_b64value, string_types):
+            sha512 = decoded_b64value.encode('hex')
+        elif isinstance(decoded_b64value, binary_type):
+            sha512 = decoded_b64value.hex()
         package.sha512 = sha512
+
 
     sha1 = dist.get('shasum')
     if sha1:

tests/packagedcode/data/npm/dist/package.json

@@ -0,0 +1,45 @@
+{
+    "name": "angular-compare-validator",
+    "version": "0.1.1",
+    "license": "MIT",
+    "repository": {
+        "type": "git",
+        "url": "git+https://github.com/GeorgDangl/angular-compare-validator.git"
+    },
+    "description": "Angular form validation directive to compare two inputs",
+    "main": "./dist/index.js",
+    "typings": "./dist/index",
+    "dependencies": {
+        "@angular/core": ">=2.0.0",
+        "@angular/forms": ">=2.0.0",
+        "@angular/common": ">=2.0.0",
+        "rxjs": ">=5.0.0-beta.12",
+        "zone.js": ">=0.6.21"
+    },
+    "bugs": {
+        "url": "https://github.com/GeorgDangl/angular-compare-validator/issues"
+    },
+    "homepage": "https://github.com/GeorgDangl/angular-compare-validator#readme",
+    "_id": "[email protected]",
+    "_npmVersion": "5.2.0",
+    "_nodeVersion": "7.7.1",
+    "_npmUser": {
+        "name": "georgdangl",
+        "email": "[email protected]"
+    },
+    "dist": {
+        "integrity": "sha512-j3DtXjUTGFrVj7KjEUdprJPd1og2zokUblhvwD4DrJPc+x8RNUrCb0CLdcDr9RZj1eTo4nw4dSo8Br3edJp8Aw==",
+        "shasum": "d35a0754c8587b0502874e3636cf0f19565d09b7",
+        "tarball": "https://registry.npmjs.org/angular-compare-validator/-/angular-compare-validator-0.1.1.tgz"
+    },
+    "maintainers": [
+        {
+            "name": "georgdangl",
+            "email": "[email protected]"
+        }
+    ],
+    "_npmOperationalInternal": {
+        "host": "s3://npm-registry-packages",
+        "tmp": "tmp/angular-compare-validator-0.1.1.tgz_1506372149543_0.2524787310976535"
+    }
+}

tests/packagedcode/data/npm/dist/package.json.expected

@@ -0,0 +1,88 @@
+[
+  {
+    "type": "npm",
+    "namespace": null,
+    "name": "angular-compare-validator",
+    "version": "0.1.1",
+    "qualifiers": {},
+    "subpath": null,
+    "primary_language": "JavaScript",
+    "description": "Angular form validation directive to compare two inputs",
+    "release_date": null,
+    "parties": [
+      {
+        "type": "person",
+        "role": "maintainer",
+        "name": "georgdangl",
+        "email": "[email protected]",
+        "url": null
+      }
+    ],
+    "keywords": [],
+    "homepage_url": "https://github.com/GeorgDangl/angular-compare-validator#readme",
+    "download_url": "https://registry.npmjs.org/angular-compare-validator/-/angular-compare-validator-0.1.1.tgz",
+    "size": null,
+    "sha1": "d35a0754c8587b0502874e3636cf0f19565d09b7",
+    "md5": null,
+    "sha256": null,
+    "sha512": "8f70ed5e3513185ad58fb2a3114769ac93ddd68836ce89146e586fc03e03ac93dcfb1f11354ac26f408b75c0ebf51663d5e4e8e27c38752a3c06bdde749a7c03",
+    "bug_tracking_url": "https://github.com/GeorgDangl/angular-compare-validator/issues",
+    "code_view_url": null,
+    "vcs_url": "git+https://github.com/GeorgDangl/angular-compare-validator.git",
+    "copyright": null,
+    "license_expression": "mit",
+    "declared_license": [
+      "MIT"
+    ],
+    "notice_text": null,
+    "root_path": null,
+    "dependencies": [
+      {
+        "purl": "pkg:npm/%40angular/core",
+        "requirement": ">=2.0.0",
+        "scope": "dependencies",
+        "is_runtime": true,
+        "is_optional": false,
+        "is_resolved": false
+      },
+      {
+        "purl": "pkg:npm/%40angular/forms",
+        "requirement": ">=2.0.0",
+        "scope": "dependencies",
+        "is_runtime": true,
+        "is_optional": false,
+        "is_resolved": false
+      },
+      {
+        "purl": "pkg:npm/%40angular/common",
+        "requirement": ">=2.0.0",
+        "scope": "dependencies",
+        "is_runtime": true,
+        "is_optional": false,
+        "is_resolved": false
+      },
+      {
+        "purl": "pkg:npm/rxjs",
+        "requirement": ">=5.0.0-beta.12",
+        "scope": "dependencies",
+        "is_runtime": true,
+        "is_optional": false,
+        "is_resolved": false
+      },
+      {
+        "purl": "pkg:npm/zone.js",
+        "requirement": ">=0.6.21",
+        "scope": "dependencies",
+        "is_runtime": true,
+        "is_optional": false,
+        "is_resolved": false
+      }
+    ],
+    "contains_source_code": null,
+    "source_packages": [],
+    "purl": "pkg:npm/[email protected]",
+    "repository_homepage_url": "https://www.npmjs.com/package/angular-compare-validator",
+    "repository_download_url": "https://registry.npmjs.org/angular-compare-validator/-/angular-compare-validator-0.1.1.tgz",
+    "api_data_url": "https://registry.npmjs.org/angular-compare-validator/0.1.1"
+  }
+]

tests/packagedcode/test_npm.py

@@ -76,6 +76,12 @@ def test_parse_person_dict4(self):
                 'url': 'http://example.com'}
         assert ('Isaac Z. Schlueter', '[email protected]' , 'http://example.com') == npm.parse_person(test)
 
+    def test_parse_dist_with_string_values(self):
+        test_file = self.get_test_loc('npm/dist/package.json')
+        expected_loc = self.get_test_loc('npm/dist/package.json.expected')
+        packages = npm.parse(test_file)
+        self.check_packages(packages, expected_loc, regen=False)
+
     def test_parse_as_installed(self):
         test_file = self.get_test_loc('npm/as_installed/package.json')
         expected_loc = self.get_test_loc('npm/as_installed/package.json.expected')