Add policies and compliance alert support for Discovered Packages #151

Signed-off-by: Thomas Druez <[email protected]>

CHANGELOG.rst

@@ -4,6 +4,9 @@ Changelog
 v32.4.0 (unreleased)
 --------------------
 
+- Add support for license policies and complaince alert for Discovered Packages.
+  https://github.com/nexB/scancode.io/issues/151
+
 - Refine the details views and tabs:
   - Add a "Relations" tab in the Resource details view
   - Disable empty tabs by default

scanpipe/api/serializers.py

@@ -273,6 +273,7 @@ class Meta:
 
 class DiscoveredPackageSerializer(serializers.ModelSerializer):
     purl = serializers.CharField(source="package_url")
+    compliance_alert = serializers.CharField()
 
     class Meta:
         model = DiscoveredPackage
@@ -311,6 +312,7 @@ class Meta:
             "other_license_expression_spdx",
             "other_license_detections",
             "extracted_license_statement",
+            "compliance_alert",
             "notice_text",
             "source_packages",
             "extra_data",

scanpipe/filters.py

@@ -466,6 +466,7 @@ class PackageFilterSet(FilterSetUtilsMixin, django_filters.FilterSet):
         fields=[
             "declared_license_expression",
             "other_license_expression",
+            "compliance_alert",
             "copyright",
             "primary_language",
         ],
@@ -475,6 +476,10 @@ class PackageFilterSet(FilterSetUtilsMixin, django_filters.FilterSet):
         field_name="affected_by_vulnerabilities",
         widget=BulmaDropdownWidget,
     )
+    compliance_alert = django_filters.ChoiceFilter(
+        choices=[(EMPTY_VAR, "None")] + CodebaseResource.Compliance.choices,
+        widget=BulmaDropdownWidget,
+    )
 
     class Meta:
         model = DiscoveredPackage
@@ -506,6 +511,7 @@ class Meta:
             "extracted_license_statement",
             "copyright",
             "is_vulnerable",
+            "compliance_alert",
         ]
 
     def __init__(self, *args, **kwargs):

scanpipe/migrations/0039_discoveredpackage_compliance_alert_and_more.py

@@ -0,0 +1,50 @@
+# Generated by Django 4.2.3 on 2023-07-11 12:22
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("scanpipe", "0038_migrate_vulnerability_data"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="discoveredpackage",
+            name="compliance_alert",
+            field=models.CharField(
+                blank=True,
+                choices=[
+                    ("ok", "Ok"),
+                    ("warning", "Warning"),
+                    ("error", "Error"),
+                    ("missing", "Missing"),
+                ],
+                editable=False,
+                help_text="Indicates how the license expression complies with provided policies.",
+                max_length=10,
+            ),
+        ),
+        migrations.AlterField(
+            model_name="codebaseresource",
+            name="compliance_alert",
+            field=models.CharField(
+                blank=True,
+                choices=[
+                    ("ok", "Ok"),
+                    ("warning", "Warning"),
+                    ("error", "Error"),
+                    ("missing", "Missing"),
+                ],
+                editable=False,
+                help_text="Indicates how the license expression complies with provided policies.",
+                max_length=10,
+            ),
+        ),
+        migrations.AddIndex(
+            model_name="discoveredpackage",
+            index=models.Index(
+                fields=["compliance_alert"], name="scanpipe_di_complia_ccf329_idx"
+            ),
+        ),
+    ]

scanpipe/models.py

@@ -1692,13 +1692,111 @@ def copy_scan_results(self, from_instance):
             self.save(update_fields=updated_fields)
 
 
+class ComplianceAlertMixin(models.Model):
+    """
+    Include the ``compliance_alert`` field and related code to compute its value.
+    Add the db `indexes` in Meta of the concrete model:
+
+    class Meta:
+        indexes = [
+            models.Index(fields=["compliance_alert"]),
+        ]
+    """
+
+    license_expression_field = None
+
+    class Compliance(models.TextChoices):
+        OK = "ok"
+        WARNING = "warning"
+        ERROR = "error"
+        MISSING = "missing"
+
+    compliance_alert = models.CharField(
+        max_length=10,
+        blank=True,
+        choices=Compliance.choices,
+        editable=False,
+        help_text=_(
+            "Indicates how the license expression complies with provided policies."
+        ),
+    )
+
+    class Meta:
+        abstract = True
+
+    @classmethod
+    def from_db(cls, db, field_names, values):
+        """
+        Store the ``license_expression_field`` on loading this instance from the
+        database value.
+        The cached value is then used to detect changes on `save()`.
+        """
+        new = super().from_db(db, field_names, values)
+
+        if cls.license_expression_field in field_names:
+            field_index = field_names.index(cls.license_expression_field)
+            new._loaded_license_expression = values[field_index]
+
+        return new
+
+    def save(self, codebase=None, *args, **kwargs):
+        """
+        Injects policies, if the feature is enabled, when the
+        ``license_expression_field`` field value has changed.
+
+        `codebase` is not used in this context but required for compatibility
+        with the commoncode.resource.Codebase class API.
+        """
+        if scanpipe_app.policies_enabled:
+            loaded_license_expression = getattr(self, "_loaded_license_expression", "")
+            license_expression = getattr(self, self.license_expression_field, "")
+            if license_expression != loaded_license_expression:
+                self.compliance_alert = self.compute_compliance_alert()
+                if "update_fields" in kwargs:
+                    kwargs["update_fields"].append("compliance_alert")
+
+        super().save(*args, **kwargs)
+
+    def compute_compliance_alert(self):
+        """Compute and return the compliance_alert value from the licenses policies."""
+        license_expression = getattr(self, self.license_expression_field, "")
+        if not license_expression:
+            return ""
+
+        alerts = []
+        policy_index = scanpipe_app.license_policies_index
+
+        licensing = get_licensing()
+        parsed = licensing.parse(license_expression, simple=True)
+        license_keys = licensing.license_keys(parsed)
+
+        for license_key in license_keys:
+            if policy := policy_index.get(license_key):
+                alerts.append(policy.get("compliance_alert") or self.Compliance.OK)
+            else:
+                alerts.append(self.Compliance.MISSING)
+
+        compliance_ordered_by_severity = [
+            self.Compliance.ERROR,
+            self.Compliance.WARNING,
+            self.Compliance.MISSING,
+        ]
+
+        for compliance_severity in compliance_ordered_by_severity:
+            if compliance_severity in alerts:
+                return compliance_severity
+
+        return self.Compliance.OK
+
+
 class CodebaseResource(
     ProjectRelatedModel,
     ScanFieldsModelMixin,
     ExtraDataFieldMixin,
     SaveProjectErrorMixin,
     UpdateFromDataMixin,
     HashFieldsMixin,
+    ComplianceAlertMixin,
     models.Model,
 ):
     """
@@ -1708,6 +1806,8 @@ class CodebaseResource(
     These model fields should be kept in line with `commoncode.resource.Resource`.
     """
 
+    license_expression_field = "detected_license_expression"
+
     path = models.CharField(
         max_length=2000,
         help_text=_(
@@ -1788,25 +1888,6 @@ class Type(models.TextChoices):
     is_archive = models.BooleanField(default=False)
     is_key_file = models.BooleanField(default=False)
     is_media = models.BooleanField(default=False)
-
-    class Compliance(models.TextChoices):
-        """List of compliance alert values."""
-
-        OK = "ok"
-        WARNING = "warning"
-        ERROR = "error"
-        MISSING = "missing"
-
-    compliance_alert = models.CharField(
-        max_length=10,
-        blank=True,
-        choices=Compliance.choices,
-        editable=False,
-        help_text=_(
-            "Indicates how the detected licenses in a codebase resource complies with "
-            "provided policies."
-        ),
-    )
     package_data = models.JSONField(
         default=list,
         blank=True,
@@ -1843,39 +1924,6 @@ class Meta:
     def __str__(self):
         return self.path
 
-    @classmethod
-    def from_db(cls, db, field_names, values):
-        """
-        Store the `detected_license_expression` field on loading this instance from the
-        database value.
-        The cached value is then used to detect changes on `save()`.
-        """
-        new = super().from_db(db, field_names, values)
-
-        if "detected_license_expression" in field_names:
-            field_index = field_names.index("detected_license_expression")
-            new._loaded_license_expression = values[field_index]
-
-        return new
-
-    def save(self, codebase=None, *args, **kwargs):
-        """
-        Save the current resource instance.
-        Injects policies, if the feature is enabled, when the
-        `detected_license_expression` field value is changed.
-
-        `codebase` is not used in this context but required for compatibility
-        with the commoncode.resource.Codebase class API.
-        """
-        if scanpipe_app.policies_enabled:
-            loaded_license_expression = getattr(self, "_loaded_license_expression", "")
-            if self.detected_license_expression != loaded_license_expression:
-                self.compliance_alert = self.compute_compliance_alert()
-                if "update_fields" in kwargs:
-                    kwargs["update_fields"].append("compliance_alert")
-
-        super().save(*args, **kwargs)
-
     @property
     def location_path(self):
         """Return the location of the resource as a Path instance."""
@@ -1927,36 +1975,6 @@ def get_path_segments_with_subpath(self):
 
         return part_and_subpath
 
-    def compute_compliance_alert(self):
-        """Compute and return the compliance_alert value from the licenses policies."""
-        if not self.detected_license_expression:
-            return ""
-
-        alerts = []
-        policy_index = scanpipe_app.license_policies_index
-
-        licensing = get_licensing()
-        parsed = licensing.parse(self.detected_license_expression, simple=True)
-        license_keys = licensing.license_keys(parsed)
-
-        for license_key in license_keys:
-            if policy := policy_index.get(license_key):
-                alerts.append(policy.get("compliance_alert") or self.Compliance.OK)
-            else:
-                alerts.append(self.Compliance.MISSING)
-
-        compliance_ordered_by_severity = [
-            self.Compliance.ERROR,
-            self.Compliance.WARNING,
-            self.Compliance.MISSING,
-        ]
-
-        for compliance_severity in compliance_ordered_by_severity:
-            if compliance_severity in alerts:
-                return compliance_severity
-
-        return self.Compliance.OK
-
     def parent_path(self):
         """Return the parent path for this CodebaseResource or None."""
         return parent_directory(self.path, with_trail=False)
@@ -2431,6 +2449,7 @@ class DiscoveredPackage(
     HashFieldsMixin,
     PackageURLMixin,
     VulnerabilityMixin,
+    ComplianceAlertMixin,
     AbstractPackage,
 ):
     """
@@ -2442,6 +2461,8 @@ class DiscoveredPackage(
     See https://github.com/package-url for more details.
     """
 
+    license_expression_field = "declared_license_expression"
+
     uuid = models.UUIDField(
         verbose_name=_("UUID"), default=uuid.uuid4, unique=True, editable=False
     )
@@ -2476,6 +2497,7 @@ class Meta:
             models.Index(fields=["sha1"]),
             models.Index(fields=["sha256"]),
             models.Index(fields=["sha512"]),
+            models.Index(fields=["compliance_alert"]),
         ]
         constraints = [
             models.UniqueConstraint(

scanpipe/templates/scanpipe/package_list.html

@@ -34,6 +34,13 @@
                 {{ package.declared_license_expression }}
               </a>
             </td>
+            {% if display_compliance_alert %}
+              <td>
+                <a href="?compliance_alert={{ package.compliance_alert }}" class="is-black-link">
+                  {{ package.compliance_alert }}
+                </a>
+              </td>
+            {% endif %}
             <td title="{{ package.copyright }}">
               {{ package.copyright|truncatechars:150|linebreaksbr }}
             </td>

scanpipe/templates/scanpipe/resource_list.html

@@ -57,7 +57,7 @@
             <td>
                <a href="?detected_license_expression={{ resource.detected_license_expression }}" class="is-black-link">{{ resource.detected_license_expression }}</a>
             </td>
-            {% if include_compliance_alert %}
+            {% if display_compliance_alert %}
               <td>
                 <a href="?compliance_alert={{ resource.compliance_alert }}" class="is-black-link">{{ resource.compliance_alert }}</a>
               </td>

scanpipe/tests/data/scancode/is-npm-1.0.0_summary.json

@@ -200,6 +200,7 @@
       "other_license_expression_spdx": "",
       "other_license_detections": [],
       "extracted_license_statement": "- MIT\n",
+      "compliance_alert": "",
       "notice_text": "",
       "source_packages": [],
       "extra_data": {},

scanpipe/tests/test_api.py

@@ -726,7 +726,7 @@ def test_scanpipe_api_serializer_get_model_serializer(self):
             get_model_serializer(None)
 
     def test_scanpipe_api_serializer_get_serializer_fields(self):
-        self.assertEqual(43, len(get_serializer_fields(DiscoveredPackage)))
+        self.assertEqual(44, len(get_serializer_fields(DiscoveredPackage)))
         self.assertEqual(11, len(get_serializer_fields(DiscoveredDependency)))
         self.assertEqual(33, len(get_serializer_fields(CodebaseResource)))
         self.assertEqual(3, len(get_serializer_fields(CodebaseRelation)))