From cad7f2e5601eae69fc4a3be9931ee248a50b34be Mon Sep 17 00:00:00 2001 From: tdruez <489057+tdruez@users.noreply.github.com> Date: Tue, 11 Jul 2023 18:26:15 +0400 Subject: [PATCH] Add policies and compliance alert support for Discovered Packages #151 (#805) Signed-off-by: Thomas Druez --- CHANGELOG.rst | 3 + scanpipe/api/serializers.py | 2 + scanpipe/filters.py | 6 + ...overedpackage_compliance_alert_and_more.py | 50 +++++ scanpipe/models.py | 186 ++++++++++-------- scanpipe/templates/scanpipe/package_list.html | 7 + .../templates/scanpipe/resource_list.html | 2 +- .../data/scancode/is-npm-1.0.0_summary.json | 1 + scanpipe/tests/test_api.py | 2 +- scanpipe/tests/test_models.py | 25 +++ scanpipe/views.py | 12 +- 11 files changed, 211 insertions(+), 85 deletions(-) create mode 100644 scanpipe/migrations/0039_discoveredpackage_compliance_alert_and_more.py diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 6673a22c5..689c7c943 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -4,6 +4,9 @@ Changelog v32.4.0 (unreleased) -------------------- +- Add support for license policies and complaince alert for Discovered Packages. + https://github.com/nexB/scancode.io/issues/151 + - Refine the details views and tabs: - Add a "Relations" tab in the Resource details view - Disable empty tabs by default diff --git a/scanpipe/api/serializers.py b/scanpipe/api/serializers.py index 96aa6c430..b171c2bdd 100644 --- a/scanpipe/api/serializers.py +++ b/scanpipe/api/serializers.py @@ -273,6 +273,7 @@ class Meta: class DiscoveredPackageSerializer(serializers.ModelSerializer): purl = serializers.CharField(source="package_url") + compliance_alert = serializers.CharField() class Meta: model = DiscoveredPackage @@ -311,6 +312,7 @@ class Meta: "other_license_expression_spdx", "other_license_detections", "extracted_license_statement", + "compliance_alert", "notice_text", "source_packages", "extra_data", diff --git a/scanpipe/filters.py b/scanpipe/filters.py index 0b38ab426..75b4970c8 100644 --- a/scanpipe/filters.py +++ b/scanpipe/filters.py @@ -466,6 +466,7 @@ class PackageFilterSet(FilterSetUtilsMixin, django_filters.FilterSet): fields=[ "declared_license_expression", "other_license_expression", + "compliance_alert", "copyright", "primary_language", ], @@ -475,6 +476,10 @@ class PackageFilterSet(FilterSetUtilsMixin, django_filters.FilterSet): field_name="affected_by_vulnerabilities", widget=BulmaDropdownWidget, ) + compliance_alert = django_filters.ChoiceFilter( + choices=[(EMPTY_VAR, "None")] + CodebaseResource.Compliance.choices, + widget=BulmaDropdownWidget, + ) class Meta: model = DiscoveredPackage @@ -506,6 +511,7 @@ class Meta: "extracted_license_statement", "copyright", "is_vulnerable", + "compliance_alert", ] def __init__(self, *args, **kwargs): diff --git a/scanpipe/migrations/0039_discoveredpackage_compliance_alert_and_more.py b/scanpipe/migrations/0039_discoveredpackage_compliance_alert_and_more.py new file mode 100644 index 000000000..3b4d07ed9 --- /dev/null +++ b/scanpipe/migrations/0039_discoveredpackage_compliance_alert_and_more.py @@ -0,0 +1,50 @@ +# Generated by Django 4.2.3 on 2023-07-11 12:22 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + dependencies = [ + ("scanpipe", "0038_migrate_vulnerability_data"), + ] + + operations = [ + migrations.AddField( + model_name="discoveredpackage", + name="compliance_alert", + field=models.CharField( + blank=True, + choices=[ + ("ok", "Ok"), + ("warning", "Warning"), + ("error", "Error"), + ("missing", "Missing"), + ], + editable=False, + help_text="Indicates how the license expression complies with provided policies.", + max_length=10, + ), + ), + migrations.AlterField( + model_name="codebaseresource", + name="compliance_alert", + field=models.CharField( + blank=True, + choices=[ + ("ok", "Ok"), + ("warning", "Warning"), + ("error", "Error"), + ("missing", "Missing"), + ], + editable=False, + help_text="Indicates how the license expression complies with provided policies.", + max_length=10, + ), + ), + migrations.AddIndex( + model_name="discoveredpackage", + index=models.Index( + fields=["compliance_alert"], name="scanpipe_di_complia_ccf329_idx" + ), + ), + ] diff --git a/scanpipe/models.py b/scanpipe/models.py index 58b086cae..02eda7ae9 100644 --- a/scanpipe/models.py +++ b/scanpipe/models.py @@ -1692,6 +1692,103 @@ def copy_scan_results(self, from_instance): self.save(update_fields=updated_fields) +class ComplianceAlertMixin(models.Model): + """ + Include the ``compliance_alert`` field and related code to compute its value. + Add the db `indexes` in Meta of the concrete model: + + class Meta: + indexes = [ + models.Index(fields=["compliance_alert"]), + ] + """ + + license_expression_field = None + + class Compliance(models.TextChoices): + OK = "ok" + WARNING = "warning" + ERROR = "error" + MISSING = "missing" + + compliance_alert = models.CharField( + max_length=10, + blank=True, + choices=Compliance.choices, + editable=False, + help_text=_( + "Indicates how the license expression complies with provided policies." + ), + ) + + class Meta: + abstract = True + + @classmethod + def from_db(cls, db, field_names, values): + """ + Store the ``license_expression_field`` on loading this instance from the + database value. + The cached value is then used to detect changes on `save()`. + """ + new = super().from_db(db, field_names, values) + + if cls.license_expression_field in field_names: + field_index = field_names.index(cls.license_expression_field) + new._loaded_license_expression = values[field_index] + + return new + + def save(self, codebase=None, *args, **kwargs): + """ + Injects policies, if the feature is enabled, when the + ``license_expression_field`` field value has changed. + + `codebase` is not used in this context but required for compatibility + with the commoncode.resource.Codebase class API. + """ + if scanpipe_app.policies_enabled: + loaded_license_expression = getattr(self, "_loaded_license_expression", "") + license_expression = getattr(self, self.license_expression_field, "") + if license_expression != loaded_license_expression: + self.compliance_alert = self.compute_compliance_alert() + if "update_fields" in kwargs: + kwargs["update_fields"].append("compliance_alert") + + super().save(*args, **kwargs) + + def compute_compliance_alert(self): + """Compute and return the compliance_alert value from the licenses policies.""" + license_expression = getattr(self, self.license_expression_field, "") + if not license_expression: + return "" + + alerts = [] + policy_index = scanpipe_app.license_policies_index + + licensing = get_licensing() + parsed = licensing.parse(license_expression, simple=True) + license_keys = licensing.license_keys(parsed) + + for license_key in license_keys: + if policy := policy_index.get(license_key): + alerts.append(policy.get("compliance_alert") or self.Compliance.OK) + else: + alerts.append(self.Compliance.MISSING) + + compliance_ordered_by_severity = [ + self.Compliance.ERROR, + self.Compliance.WARNING, + self.Compliance.MISSING, + ] + + for compliance_severity in compliance_ordered_by_severity: + if compliance_severity in alerts: + return compliance_severity + + return self.Compliance.OK + + class CodebaseResource( ProjectRelatedModel, ScanFieldsModelMixin, @@ -1699,6 +1796,7 @@ class CodebaseResource( SaveProjectErrorMixin, UpdateFromDataMixin, HashFieldsMixin, + ComplianceAlertMixin, models.Model, ): """ @@ -1708,6 +1806,8 @@ class CodebaseResource( These model fields should be kept in line with `commoncode.resource.Resource`. """ + license_expression_field = "detected_license_expression" + path = models.CharField( max_length=2000, help_text=_( @@ -1788,25 +1888,6 @@ class Type(models.TextChoices): is_archive = models.BooleanField(default=False) is_key_file = models.BooleanField(default=False) is_media = models.BooleanField(default=False) - - class Compliance(models.TextChoices): - """List of compliance alert values.""" - - OK = "ok" - WARNING = "warning" - ERROR = "error" - MISSING = "missing" - - compliance_alert = models.CharField( - max_length=10, - blank=True, - choices=Compliance.choices, - editable=False, - help_text=_( - "Indicates how the detected licenses in a codebase resource complies with " - "provided policies." - ), - ) package_data = models.JSONField( default=list, blank=True, @@ -1843,39 +1924,6 @@ class Meta: def __str__(self): return self.path - @classmethod - def from_db(cls, db, field_names, values): - """ - Store the `detected_license_expression` field on loading this instance from the - database value. - The cached value is then used to detect changes on `save()`. - """ - new = super().from_db(db, field_names, values) - - if "detected_license_expression" in field_names: - field_index = field_names.index("detected_license_expression") - new._loaded_license_expression = values[field_index] - - return new - - def save(self, codebase=None, *args, **kwargs): - """ - Save the current resource instance. - Injects policies, if the feature is enabled, when the - `detected_license_expression` field value is changed. - - `codebase` is not used in this context but required for compatibility - with the commoncode.resource.Codebase class API. - """ - if scanpipe_app.policies_enabled: - loaded_license_expression = getattr(self, "_loaded_license_expression", "") - if self.detected_license_expression != loaded_license_expression: - self.compliance_alert = self.compute_compliance_alert() - if "update_fields" in kwargs: - kwargs["update_fields"].append("compliance_alert") - - super().save(*args, **kwargs) - @property def location_path(self): """Return the location of the resource as a Path instance.""" @@ -1927,36 +1975,6 @@ def get_path_segments_with_subpath(self): return part_and_subpath - def compute_compliance_alert(self): - """Compute and return the compliance_alert value from the licenses policies.""" - if not self.detected_license_expression: - return "" - - alerts = [] - policy_index = scanpipe_app.license_policies_index - - licensing = get_licensing() - parsed = licensing.parse(self.detected_license_expression, simple=True) - license_keys = licensing.license_keys(parsed) - - for license_key in license_keys: - if policy := policy_index.get(license_key): - alerts.append(policy.get("compliance_alert") or self.Compliance.OK) - else: - alerts.append(self.Compliance.MISSING) - - compliance_ordered_by_severity = [ - self.Compliance.ERROR, - self.Compliance.WARNING, - self.Compliance.MISSING, - ] - - for compliance_severity in compliance_ordered_by_severity: - if compliance_severity in alerts: - return compliance_severity - - return self.Compliance.OK - def parent_path(self): """Return the parent path for this CodebaseResource or None.""" return parent_directory(self.path, with_trail=False) @@ -2431,6 +2449,7 @@ class DiscoveredPackage( HashFieldsMixin, PackageURLMixin, VulnerabilityMixin, + ComplianceAlertMixin, AbstractPackage, ): """ @@ -2442,6 +2461,8 @@ class DiscoveredPackage( See https://github.com/package-url for more details. """ + license_expression_field = "declared_license_expression" + uuid = models.UUIDField( verbose_name=_("UUID"), default=uuid.uuid4, unique=True, editable=False ) @@ -2476,6 +2497,7 @@ class Meta: models.Index(fields=["sha1"]), models.Index(fields=["sha256"]), models.Index(fields=["sha512"]), + models.Index(fields=["compliance_alert"]), ] constraints = [ models.UniqueConstraint( diff --git a/scanpipe/templates/scanpipe/package_list.html b/scanpipe/templates/scanpipe/package_list.html index 30d8678ac..daeeef2e0 100644 --- a/scanpipe/templates/scanpipe/package_list.html +++ b/scanpipe/templates/scanpipe/package_list.html @@ -34,6 +34,13 @@ {{ package.declared_license_expression }} + {% if display_compliance_alert %} + + + {{ package.compliance_alert }} + + + {% endif %} {{ package.copyright|truncatechars:150|linebreaksbr }} diff --git a/scanpipe/templates/scanpipe/resource_list.html b/scanpipe/templates/scanpipe/resource_list.html index f9dd4cf95..24c6ac8dd 100644 --- a/scanpipe/templates/scanpipe/resource_list.html +++ b/scanpipe/templates/scanpipe/resource_list.html @@ -57,7 +57,7 @@ {{ resource.detected_license_expression }} - {% if include_compliance_alert %} + {% if display_compliance_alert %} {{ resource.compliance_alert }} diff --git a/scanpipe/tests/data/scancode/is-npm-1.0.0_summary.json b/scanpipe/tests/data/scancode/is-npm-1.0.0_summary.json index 687d467e9..16b8bfb41 100644 --- a/scanpipe/tests/data/scancode/is-npm-1.0.0_summary.json +++ b/scanpipe/tests/data/scancode/is-npm-1.0.0_summary.json @@ -200,6 +200,7 @@ "other_license_expression_spdx": "", "other_license_detections": [], "extracted_license_statement": "- MIT\n", + "compliance_alert": "", "notice_text": "", "source_packages": [], "extra_data": {}, diff --git a/scanpipe/tests/test_api.py b/scanpipe/tests/test_api.py index e5d0485cc..d7bf13343 100644 --- a/scanpipe/tests/test_api.py +++ b/scanpipe/tests/test_api.py @@ -726,7 +726,7 @@ def test_scanpipe_api_serializer_get_model_serializer(self): get_model_serializer(None) def test_scanpipe_api_serializer_get_serializer_fields(self): - self.assertEqual(43, len(get_serializer_fields(DiscoveredPackage))) + self.assertEqual(44, len(get_serializer_fields(DiscoveredPackage))) self.assertEqual(11, len(get_serializer_fields(DiscoveredDependency))) self.assertEqual(33, len(get_serializer_fields(CodebaseResource))) self.assertEqual(3, len(get_serializer_fields(CodebaseRelation))) diff --git a/scanpipe/tests/test_models.py b/scanpipe/tests/test_models.py index e53dcb0ed..7301678f1 100644 --- a/scanpipe/tests/test_models.py +++ b/scanpipe/tests/test_models.py @@ -1689,6 +1689,30 @@ def test_scanpipe_discovered_package_model_as_cyclonedx(self): self.assertEqual("vcs", external_references[0].type) self.assertEqual("https://packages.vcs.url", external_references[0].url) + def test_scanpipe_discovered_package_model_compliance_alert(self): + scanpipe_app.license_policies_index = license_policies_index + package_data = package_data1.copy() + package_data["declared_license_expression"] = "" + package = DiscoveredPackage.create_from_data(self.project1, package_data) + self.assertEqual("", package.compliance_alert) + + license_expression = "bsd-new" + self.assertNotIn(license_expression, scanpipe_app.license_policies_index) + package.update(declared_license_expression=license_expression) + self.assertEqual("missing", package.compliance_alert) + + license_expression = "apache-2.0" + self.assertIn(license_expression, scanpipe_app.license_policies_index) + package.update(declared_license_expression=license_expression) + self.assertEqual("ok", package.compliance_alert) + + license_expression = "apache-2.0 AND mpl-2.0 OR gpl-3.0" + package.update(declared_license_expression=license_expression) + self.assertEqual("error", package.compliance_alert) + + # Reset the index value + scanpipe_app.license_policies_index = None + def test_scanpipe_model_create_user_creates_auth_token(self): basic_user = User.objects.create_user(username="basic_user") self.assertTrue(basic_user.auth_token.key) @@ -1998,6 +2022,7 @@ def test_scanpipe_package_model_integrity_with_toolkit_package_model(self): "package_uid", "filename", "affected_by_vulnerabilities", + "compliance_alert", ] discovered_package_fields = [ diff --git a/scanpipe/views.py b/scanpipe/views.py index f0d76fe88..f80ebfa15 100644 --- a/scanpipe/views.py +++ b/scanpipe/views.py @@ -1072,7 +1072,7 @@ class CodebaseResourceListView( def get_context_data(self, **kwargs): context = super().get_context_data(**kwargs) - context["include_compliance_alert"] = scanpipe_app.policies_enabled + context["display_compliance_alert"] = scanpipe_app.policies_enabled return context @@ -1098,6 +1098,11 @@ class DiscoveredPackageListView( "field_name": "declared_license_expression", "filter_fieldname": "declared_license_expression", }, + { + "field_name": "compliance_alert", + "condition": scanpipe_app.policies_enabled, + "filter_fieldname": "compliance_alert", + }, { "field_name": "copyright", "filter_fieldname": "copyright", @@ -1106,6 +1111,11 @@ class DiscoveredPackageListView( "resources", ] + def get_context_data(self, **kwargs): + context = super().get_context_data(**kwargs) + context["display_compliance_alert"] = scanpipe_app.policies_enabled + return context + class DiscoveredDependencyListView( ConditionalLoginRequired,