From c15fedd99a7fad534f1c35ceeccb5280b6050e28 Mon Sep 17 00:00:00 2001 From: Thomas Druez Date: Wed, 12 Jul 2023 16:35:36 +0400 Subject: [PATCH] Migrate the ProjectError model to a more global ProjectMessage #338 Signed-off-by: Thomas Druez --- docs/data-models.rst | 6 +- scanpipe/api/serializers.py | 22 +++-- scanpipe/api/views.py | 8 +- scanpipe/filters.py | 22 +++-- scanpipe/management/commands/__init__.py | 4 +- scanpipe/migrations/0001_initial.py | 4 +- ...022_create_discovereddependencies_model.py | 2 +- scanpipe/migrations/0040_projectmessage.py | 85 +++++++++++++++++++ ...041_projecterror_to_projectmessage_data.py | 32 +++++++ .../migrations/0042_delete_projecterror.py | 15 ++++ scanpipe/models.py | 77 +++++++++++------ scanpipe/pipelines/__init__.py | 2 +- scanpipe/pipes/output.py | 8 +- scanpipe/pipes/scancode.py | 2 +- .../scanpipe/includes/project_list_table.html | 6 +- .../includes/project_summary_level.html | 8 +- ...or_list.html => project_message_list.html} | 29 ++++--- scanpipe/tests/pipes/test_output.py | 12 ++- scanpipe/tests/pipes/test_pipes.py | 6 +- scanpipe/tests/pipes/test_scancode.py | 28 +++--- scanpipe/tests/test_api.py | 33 ++++--- scanpipe/tests/test_auth.py | 2 +- scanpipe/tests/test_commands.py | 2 +- scanpipe/tests/test_models.py | 50 +++++------ scanpipe/tests/test_pipelines.py | 20 ++--- scanpipe/urls.py | 6 +- scanpipe/views.py | 26 +++--- 27 files changed, 354 insertions(+), 163 deletions(-) create mode 100644 scanpipe/migrations/0040_projectmessage.py create mode 100644 scanpipe/migrations/0041_projecterror_to_projectmessage_data.py create mode 100644 scanpipe/migrations/0042_delete_projecterror.py rename scanpipe/templates/scanpipe/{error_list.html => project_message_list.html} (66%) diff --git a/docs/data-models.rst b/docs/data-models.rst index 9b2db4e80..acf19b5f9 100644 --- a/docs/data-models.rst +++ b/docs/data-models.rst @@ -42,9 +42,9 @@ CodebaseRelation :undoc-members: :member-order: groupwise -ProjectError ------------- -.. autoclass:: scanpipe.models.ProjectError() +ProjectMessage +-------------- +.. autoclass:: scanpipe.models.ProjectMessage() :members: :undoc-members: :member-order: groupwise diff --git a/scanpipe/api/serializers.py b/scanpipe/api/serializers.py index b171c2bdd..7df892c8a 100644 --- a/scanpipe/api/serializers.py +++ b/scanpipe/api/serializers.py @@ -30,7 +30,7 @@ from scanpipe.models import DiscoveredDependency from scanpipe.models import DiscoveredPackage from scanpipe.models import Project -from scanpipe.models import ProjectError +from scanpipe.models import ProjectMessage from scanpipe.models import Run from scanpipe.pipes import count_group_by from scanpipe.pipes.fetch import fetch_urls @@ -139,7 +139,7 @@ class Meta: "next_run", "runs", "extra_data", - "error_count", + "message_count", "resource_count", "package_count", "dependency_count", @@ -155,7 +155,7 @@ class Meta: "input_root", "output_root", "extra_data", - "error_count", + "message_count", "resource_count", "package_count", "dependency_count", @@ -361,12 +361,20 @@ class Meta: ] -class ProjectErrorSerializer(serializers.ModelSerializer): +class ProjectMessageSerializer(serializers.ModelSerializer): traceback = serializers.SerializerMethodField() class Meta: - model = ProjectError - fields = ["uuid", "model", "message", "details", "traceback", "created_date"] + model = ProjectMessage + fields = [ + "uuid", + "severity", + "description", + "model", + "details", + "traceback", + "created_date", + ] def get_traceback(self, project_error): return project_error.traceback.splitlines() @@ -397,7 +405,7 @@ def get_model_serializer(model_class): DiscoveredPackage: DiscoveredPackageSerializer, DiscoveredDependency: DiscoveredDependencySerializer, CodebaseRelation: CodebaseRelationSerializer, - ProjectError: ProjectErrorSerializer, + ProjectMessage: ProjectMessageSerializer, }.get(model_class, None) if not serializer: diff --git a/scanpipe/api/views.py b/scanpipe/api/views.py index cb6093736..84f3858fe 100644 --- a/scanpipe/api/views.py +++ b/scanpipe/api/views.py @@ -40,7 +40,7 @@ from scanpipe.api.serializers import DiscoveredDependencySerializer from scanpipe.api.serializers import DiscoveredPackageSerializer from scanpipe.api.serializers import PipelineSerializer -from scanpipe.api.serializers import ProjectErrorSerializer +from scanpipe.api.serializers import ProjectMessageSerializer from scanpipe.api.serializers import ProjectSerializer from scanpipe.api.serializers import RunSerializer from scanpipe.models import Project @@ -201,12 +201,12 @@ def relations(self, request, *args, **kwargs): return self.get_paginated_response(serializer.data) @action(detail=True) - def errors(self, request, *args, **kwargs): + def messages(self, request, *args, **kwargs): project = self.get_object() - queryset = project.projecterrors.all() + queryset = project.projectmessages.all() paginated_qs = self.paginate_queryset(queryset) - serializer = ProjectErrorSerializer(paginated_qs, many=True) + serializer = ProjectMessageSerializer(paginated_qs, many=True) return self.get_paginated_response(serializer.data) diff --git a/scanpipe/filters.py b/scanpipe/filters.py index 75b4970c8..877474e9f 100644 --- a/scanpipe/filters.py +++ b/scanpipe/filters.py @@ -36,7 +36,7 @@ from scanpipe.models import DiscoveredDependency from scanpipe.models import DiscoveredPackage from scanpipe.models import Project -from scanpipe.models import ProjectError +from scanpipe.models import ProjectMessage from scanpipe.models import Run scanpipe_app = apps.get_app_config("scanpipe") @@ -199,7 +199,7 @@ class ProjectFilterSet(FilterSetUtilsMixin, django_filters.FilterSet): "discoveredpackages_count", "discovereddependencies_count", "codebaseresources_count", - "projecterrors_count", + "projectmessages_count", ], empty_label="Newest", choices=( @@ -212,8 +212,8 @@ class ProjectFilterSet(FilterSetUtilsMixin, django_filters.FilterSet): ("discovereddependencies_count", "Dependencies (-)"), ("-codebaseresources_count", "Resources (+)"), ("codebaseresources_count", "Resources (-)"), - ("-projecterrors_count", "Errors (+)"), - ("projecterrors_count", "Errors (-)"), + ("-projectmessages_count", "Messages (+)"), + ("projectmessages_count", "Messages (-)"), ), widget=BulmaDropdownWidget, ) @@ -547,21 +547,27 @@ class Meta: ] -class ErrorFilterSet(FilterSetUtilsMixin, django_filters.FilterSet): +class ProjectMessageFilterSet(FilterSetUtilsMixin, django_filters.FilterSet): search = django_filters.CharFilter( - label="Search", field_name="message", lookup_expr="icontains" + label="Search", field_name="description", lookup_expr="icontains" ) sort = django_filters.OrderingFilter( label="Sort", fields=[ + "severity", "model", ], ) class Meta: - model = ProjectError + model = ProjectMessage fields = [ "search", + "severity", "model", - "message", + "description", ] + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.filters["severity"].extra["widget"] = BulmaDropdownWidget() diff --git a/scanpipe/management/commands/__init__.py b/scanpipe/management/commands/__init__.py index 5f6ff2f7a..95e483c7a 100644 --- a/scanpipe/management/commands/__init__.py +++ b/scanpipe/management/commands/__init__.py @@ -32,7 +32,7 @@ from scanpipe.models import CodebaseResource from scanpipe.models import DiscoveredPackage from scanpipe.models import Project -from scanpipe.models import ProjectError +from scanpipe.models import ProjectMessage from scanpipe.pipes import count_group_by from scanpipe.pipes.fetch import fetch_urls @@ -93,7 +93,7 @@ def get_run_status_messages(self, project): def get_queryset_objects_messages(self, project): messages = [] - for model_class in [CodebaseResource, DiscoveredPackage, ProjectError]: + for model_class in [CodebaseResource, DiscoveredPackage, ProjectMessage]: queryset = model_class.objects.project(project) messages.append(f" - {model_class.__name__}: {queryset.count()}") diff --git a/scanpipe/migrations/0001_initial.py b/scanpipe/migrations/0001_initial.py index 03723d8bd..1540333ed 100644 --- a/scanpipe/migrations/0001_initial.py +++ b/scanpipe/migrations/0001_initial.py @@ -44,7 +44,7 @@ class Migration(migrations.Migration): options={ 'ordering': ('project', 'path'), }, - bases=(scanpipe.models.SaveProjectErrorMixin, models.Model), + bases=(scanpipe.models.SaveProjectMessageMixin, models.Model), ), migrations.CreateModel( name='Project', @@ -132,7 +132,7 @@ class Migration(migrations.Migration): options={ 'ordering': ['uuid'], }, - bases=(scanpipe.models.SaveProjectErrorMixin, models.Model), + bases=(scanpipe.models.SaveProjectMessageMixin, models.Model), ), migrations.AddField( model_name='codebaseresource', diff --git a/scanpipe/migrations/0022_create_discovereddependencies_model.py b/scanpipe/migrations/0022_create_discovereddependencies_model.py index 7cb522c47..37dde64ce 100644 --- a/scanpipe/migrations/0022_create_discovereddependencies_model.py +++ b/scanpipe/migrations/0022_create_discovereddependencies_model.py @@ -153,7 +153,7 @@ class Migration(migrations.Migration): ], }, bases=( - scanpipe.models.SaveProjectErrorMixin, + scanpipe.models.SaveProjectMessageMixin, scanpipe.models.UpdateFromDataMixin, models.Model, ), diff --git a/scanpipe/migrations/0040_projectmessage.py b/scanpipe/migrations/0040_projectmessage.py new file mode 100644 index 000000000..1c73e132a --- /dev/null +++ b/scanpipe/migrations/0040_projectmessage.py @@ -0,0 +1,85 @@ +# Generated by Django 4.2.3 on 2023-07-12 12:13 + +import django.core.serializers.json +from django.db import migrations, models +import django.db.models.deletion +import scanpipe.models +import uuid + + +class Migration(migrations.Migration): + dependencies = [ + ("scanpipe", "0039_discoveredpackage_compliance_alert_and_more"), + ] + + operations = [ + migrations.CreateModel( + name="ProjectMessage", + fields=[ + ( + "uuid", + models.UUIDField( + db_index=True, + default=uuid.uuid4, + editable=False, + primary_key=True, + serialize=False, + verbose_name="UUID", + ), + ), + ( + "severity", + models.CharField( + choices=[ + ("info", "Info"), + ("warning", "Warning"), + ("error", "Error"), + ], + editable=False, + help_text="Severity level of the message.", + max_length=10, + ), + ), + ("description", models.TextField(blank=True, help_text="Description.")), + ( + "model", + models.CharField( + help_text="Name of the model class.", max_length=100 + ), + ), + ( + "details", + models.JSONField( + blank=True, + default=dict, + encoder=django.core.serializers.json.DjangoJSONEncoder, + help_text="Data that caused the error.", + ), + ), + ( + "traceback", + models.TextField(blank=True, help_text="Exception traceback."), + ), + ("created_date", models.DateTimeField(auto_now_add=True)), + ( + "project", + models.ForeignKey( + editable=False, + on_delete=django.db.models.deletion.CASCADE, + related_name="%(class)ss", + to="scanpipe.project", + ), + ), + ], + options={ + "ordering": ["created_date"], + "indexes": [ + models.Index( + fields=["severity"], name="scanpipe_pr_severit_9e1b35_idx" + ), + models.Index(fields=["model"], name="scanpipe_pr_model_013dd4_idx"), + ], + }, + bases=(scanpipe.models.UpdateMixin, models.Model), + ), + ] diff --git a/scanpipe/migrations/0041_projecterror_to_projectmessage_data.py b/scanpipe/migrations/0041_projecterror_to_projectmessage_data.py new file mode 100644 index 000000000..128b312f8 --- /dev/null +++ b/scanpipe/migrations/0041_projecterror_to_projectmessage_data.py @@ -0,0 +1,32 @@ +# Generated by Django 4.2.3 on 2023-07-12 12:13 + +from django.db import migrations + + +def migrate_error_to_message_data(apps, schema_editor): + ProjectError = apps.get_model("scanpipe", "ProjectError") + ProjectMessage = apps.get_model("scanpipe", "ProjectMessage") + ERROR = "error" + + for project_error in ProjectError.objects.all(): + ProjectMessage.objects.create( + project=project_error.project, + severity=ERROR, + description=project_error.message, + model=project_error.model, + details=project_error.details, + traceback=project_error.traceback, + ) + + +class Migration(migrations.Migration): + dependencies = [ + ("scanpipe", "0040_projectmessage"), + ] + + operations = [ + migrations.RunPython( + migrate_error_to_message_data, + reverse_code=migrations.RunPython.noop, + ), + ] diff --git a/scanpipe/migrations/0042_delete_projecterror.py b/scanpipe/migrations/0042_delete_projecterror.py new file mode 100644 index 000000000..bfc4d2f31 --- /dev/null +++ b/scanpipe/migrations/0042_delete_projecterror.py @@ -0,0 +1,15 @@ +# Generated by Django 4.2.3 on 2023-07-12 12:14 + +from django.db import migrations + + +class Migration(migrations.Migration): + dependencies = [ + ("scanpipe", "0041_projecterror_to_projectmessage_data"), + ] + + operations = [ + migrations.DeleteModel( + name="ProjectError", + ), + ] diff --git a/scanpipe/models.py b/scanpipe/models.py index 02eda7ae9..2f2e3ea78 100644 --- a/scanpipe/models.py +++ b/scanpipe/models.py @@ -588,7 +588,7 @@ def delete_related_objects(self): _, deleted_counter = self.discoveredpackages.all().delete() relationships = [ - self.projecterrors, + self.projectmessages, self.codebaserelations, self.discovereddependencies, self.codebaseresources, @@ -952,24 +952,38 @@ def get_latest_failed_run(self): with suppress(ObjectDoesNotExist): return self.runs.failed().latest("created_date") - def add_error(self, error, model, details=None): + def add_message(self, severity, description, model="", details=None, traceback=""): """ - Create a "ProjectError" record from the provided `error` Exception for this - project. - The `model` attribute can be provided as a string or as a Model class. + Create a ProjectMessage record for this Project. + + The ``model`` attribute can be provided as a string or as a Model class. """ if inspect.isclass(model): model = model.__name__ + return ProjectMessage.objects.create( + project=self, + severity=severity, + description=description, + model=model, + details=details or {}, + traceback=traceback, + ) + + def add_error(self, error, model, details=None): + """ + Create a ProjectMessage record from the provided `error` Exception for this + project. + """ traceback = "" if hasattr(error, "__traceback__"): traceback = "".join(format_tb(error.__traceback__)) - return ProjectError.objects.create( - project=self, + return self.add_message( + severity=ProjectMessage.Severity.ERROR, + description=str(error), model=model, - details=details or {}, - message=str(error), + details=details, traceback=traceback, ) @@ -1019,9 +1033,9 @@ def dependency_count(self): return self.discovereddependencies.count() @cached_property - def error_count(self): - """Return the number of errors related to this project.""" - return self.projecterrors.count() + def message_count(self): + """Return the number of messages related to this project.""" + return self.projectmessages.count() @cached_property def relation_count(self): @@ -1159,10 +1173,21 @@ def model_fields(cls): return [field.name for field in cls._meta.get_fields()] -class ProjectError(UUIDPKModel, ProjectRelatedModel): - """Stores errors and exceptions raised during a pipeline run.""" +class ProjectMessage(UUIDPKModel, ProjectRelatedModel): + """Stores messages such as errors and exceptions raised during a pipeline run.""" - created_date = models.DateTimeField(auto_now_add=True, editable=False) + class Severity(models.TextChoices): + INFO = "info" + WARNING = "warning" + ERROR = "error" + + severity = models.CharField( + max_length=10, + choices=Severity.choices, + editable=False, + help_text=_("Severity level of the message."), + ) + description = models.TextField(blank=True, help_text=_("Description.")) model = models.CharField(max_length=100, help_text=_("Name of the model class.")) details = models.JSONField( default=dict, @@ -1170,19 +1195,23 @@ class ProjectError(UUIDPKModel, ProjectRelatedModel): encoder=DjangoJSONEncoder, help_text=_("Data that caused the error."), ) - message = models.TextField(blank=True, help_text=_("Error message.")) traceback = models.TextField(blank=True, help_text=_("Exception traceback.")) + created_date = models.DateTimeField(auto_now_add=True, editable=False) class Meta: ordering = ["created_date"] + indexes = [ + models.Index(fields=["severity"]), + models.Index(fields=["model"]), + ] def __str__(self): - return f"[{self.pk}] {self.model}: {self.message}" + return f"[{self.pk}] {self.model}: {self.description}" -class SaveProjectErrorMixin: +class SaveProjectMessageMixin: """ - Uses `SaveProjectErrorMixin` on a model to create a "ProjectError" entry + Uses `SaveProjectMessageMixin` on a model to create a "ProjectError" entry from a raised exception during `save()` instead of stopping the analysis process. The creation of a "ProjectError" can be skipped providing False for the `save_error` @@ -1211,7 +1240,7 @@ def _check_project_field(cls, **kwargs): if "project" not in fields: return [ checks.Error( - "'project' field is required when using SaveProjectErrorMixin.", + "'project' field is required when using SaveProjectMessageMixin.", obj=cls, id="scanpipe.models.E001", ) @@ -1220,7 +1249,7 @@ def _check_project_field(cls, **kwargs): return [] def add_error(self, error): - """Create a "ProjectError" record from a given `error` Exception instance.""" + """Create a "ProjectMessage" record from a given `error` Exception instance.""" return self.project.add_error( error=error, model=self.__class__, @@ -1793,7 +1822,7 @@ class CodebaseResource( ProjectRelatedModel, ScanFieldsModelMixin, ExtraDataFieldMixin, - SaveProjectErrorMixin, + SaveProjectMessageMixin, UpdateFromDataMixin, HashFieldsMixin, ComplianceAlertMixin, @@ -2444,7 +2473,7 @@ class Meta: class DiscoveredPackage( ProjectRelatedModel, ExtraDataFieldMixin, - SaveProjectErrorMixin, + SaveProjectMessageMixin, UpdateFromDataMixin, HashFieldsMixin, PackageURLMixin, @@ -2741,7 +2770,7 @@ def prefetch_for_serializer(self): class DiscoveredDependency( ProjectRelatedModel, - SaveProjectErrorMixin, + SaveProjectMessageMixin, UpdateFromDataMixin, PackageURLMixin, ): diff --git a/scanpipe/pipelines/__init__.py b/scanpipe/pipelines/__init__.py index 0a8d44425..1c36dd504 100644 --- a/scanpipe/pipelines/__init__.py +++ b/scanpipe/pipelines/__init__.py @@ -132,7 +132,7 @@ def execute(self): return 0, "" def add_error(self, error): - """Create a `ProjectError` record on the current `project`.""" + """Create a ``ProjectMessage`` ERROR record on the current `project`.""" self.project.add_error(error, model=self.pipeline_name) @contextmanager diff --git a/scanpipe/pipes/output.py b/scanpipe/pipes/output.py index 136acec17..5bda85947 100644 --- a/scanpipe/pipes/output.py +++ b/scanpipe/pipes/output.py @@ -86,7 +86,7 @@ def get_queryset(project, model_name): "codebaserelation": ( project.codebaserelations.select_related("from_resource", "to_resource") ), - "projecterror": project.projecterrors.all(), + "projectmessage": project.projectmessages.all(), } return querysets.get(model_name) @@ -138,7 +138,7 @@ def to_csv(project): "discoveredpackage", "discovereddependency", "codebaseresource", - "projecterror", + "projectmessage", ] output_files = [] @@ -285,7 +285,7 @@ def to_json(project): "discovereddependency": "DEPENDENCIES", "codebaseresource": "RESOURCES", "codebaserelation": "RELATIONS", - "projecterror": "ERRORS", + "projectmessage": "MESSAGES", } @@ -460,7 +460,7 @@ def to_xlsx(project): "discovereddependency", "codebaseresource", "codebaserelation", - "projecterror", + "projectmessage", ] with xlsxwriter.Workbook(output_file) as workbook: diff --git a/scanpipe/pipes/scancode.py b/scanpipe/pipes/scancode.py index 6bbda440d..bb154e853 100644 --- a/scanpipe/pipes/scancode.py +++ b/scanpipe/pipes/scancode.py @@ -365,7 +365,7 @@ def add_resource_to_package(package_uid, resource, project): """ Relate a DiscoveredPackage to `resource` from `project` using `package_uid`. - Add a ProjectError when the DiscoveredPackage could not be fetched using the + Add a ProjectMessage when the DiscoveredPackage could not be fetched using the provided `package_uid`. """ if not package_uid: diff --git a/scanpipe/templates/scanpipe/includes/project_list_table.html b/scanpipe/templates/scanpipe/includes/project_list_table.html index eb27f86ac..b2031a112 100644 --- a/scanpipe/templates/scanpipe/includes/project_list_table.html +++ b/scanpipe/templates/scanpipe/includes/project_list_table.html @@ -36,9 +36,9 @@ {% endif %} - {% if project.projecterrors_count %} - - {{ project.projecterrors_count|intcomma }} + {% if project.projectmessages_count %} + + {{ project.projectmessages_count|intcomma }} {% else %} 0 diff --git a/scanpipe/templates/scanpipe/includes/project_summary_level.html b/scanpipe/templates/scanpipe/includes/project_summary_level.html index 945591f52..9431f6e60 100644 --- a/scanpipe/templates/scanpipe/includes/project_summary_level.html +++ b/scanpipe/templates/scanpipe/includes/project_summary_level.html @@ -62,11 +62,11 @@ {% endif %}
-

Errors

+

Messages

- {% if project.error_count %} - - {{ project.error_count|intcomma }} + {% if project.message_count %} + + {{ project.message_count|intcomma }} {% else %} 0 diff --git a/scanpipe/templates/scanpipe/error_list.html b/scanpipe/templates/scanpipe/project_message_list.html similarity index 66% rename from scanpipe/templates/scanpipe/error_list.html rename to scanpipe/templates/scanpipe/project_message_list.html index 7cd7953ca..61a7ecb18 100644 --- a/scanpipe/templates/scanpipe/error_list.html +++ b/scanpipe/templates/scanpipe/project_message_list.html @@ -1,13 +1,13 @@ {% extends "scanpipe/base.html" %} -{% block title %}ScanCode.io: {{ project.name }} - Errors{% endblock %} +{% block title %}ScanCode.io: {{ project.name }} - Messages{% endblock %} {% block content %}

{% include 'scanpipe/includes/navbar_header.html' %}
- {% include 'scanpipe/includes/breadcrumb.html' with linked_project=True current="Errors" %} + {% include 'scanpipe/includes/breadcrumb.html' with linked_project=True current="Messages" %} {% include 'scanpipe/includes/search_field.html' with extra_class="is-small" %}
{% include 'scanpipe/includes/pagination_header.html' %} @@ -19,43 +19,46 @@ {% include 'scanpipe/includes/list_view_thead.html' %} - {% for error in object_list %} + {% for message in object_list %} + {% empty %} {% endfor %} diff --git a/scanpipe/tests/pipes/test_output.py b/scanpipe/tests/pipes/test_output.py index eb003cbfa..0779d8488 100644 --- a/scanpipe/tests/pipes/test_output.py +++ b/scanpipe/tests/pipes/test_output.py @@ -40,7 +40,7 @@ from scanpipe import pipes from scanpipe.models import CodebaseResource from scanpipe.models import Project -from scanpipe.models import ProjectError +from scanpipe.models import ProjectMessage from scanpipe.pipes import output from scanpipe.tests import FIXTURES_REGEN from scanpipe.tests import mocked_now @@ -161,7 +161,7 @@ def test_scanpipe_pipes_outputs_to_csv(self): "codebaseresource-2010-10-10-10-10-10.csv", "discovereddependency-2010-10-10-10-10-10.csv", "discoveredpackage-2010-10-10-10-10-10.csv", - "projecterror-2010-10-10-10-10-10.csv", + "projectmessage-2010-10-10-10-10-10.csv", ] for csv_file in csv_files: @@ -204,8 +204,12 @@ def test_scanpipe_pipes_outputs_to_xlsx(self): call_command("loaddata", fixtures, **{"verbosity": 0}) project = Project.objects.get(name="asgiref") - ProjectError.objects.create( - project=project, model="Model", details={}, message="Error" + ProjectMessage.objects.create( + project=project, + severity=ProjectMessage.Severity.ERROR, + description="Error", + model="Model", + details={}, ) output_file = output.to_xlsx(project=project) diff --git a/scanpipe/tests/pipes/test_pipes.py b/scanpipe/tests/pipes/test_pipes.py index f199b595b..0a89bb3ab 100644 --- a/scanpipe/tests/pipes/test_pipes.py +++ b/scanpipe/tests/pipes/test_pipes.py @@ -236,7 +236,7 @@ def test_scanpipe_pipes_make_codebase_resource(self): # Duplicated path: skip the creation and no project error added pipes.make_codebase_resource(p1, resource_location) self.assertEqual(1, p1.codebaseresources.count()) - self.assertEqual(0, p1.projecterrors.count()) + self.assertEqual(0, p1.projectmessages.count()) def test_scanpipe_add_resource_to_package(self): project1 = Project.objects.create(name="Analysis") @@ -249,8 +249,8 @@ def test_scanpipe_add_resource_to_package(self): scancode.add_resource_to_package("not_available", resource1, project1) self.assertFalse(resource1.for_packages) - self.assertEqual(1, project1.projecterrors.count()) - error = project1.projecterrors.get() + self.assertEqual(1, project1.projectmessages.count()) + error = project1.projectmessages.get() self.assertEqual("assemble_package", error.model) expected = {"resource": "filename.ext", "package_uid": "not_available"} self.assertEqual(expected, error.details) diff --git a/scanpipe/tests/pipes/test_scancode.py b/scanpipe/tests/pipes/test_scancode.py index 13f01f88a..8187dc9c5 100644 --- a/scanpipe/tests/pipes/test_scancode.py +++ b/scanpipe/tests/pipes/test_scancode.py @@ -205,13 +205,13 @@ def test_scanpipe_pipes_scancode_scan_file_and_save_results(self): project=project1, path="not available" ) - self.assertEqual(0, project1.projecterrors.count()) + self.assertEqual(0, project1.projectmessages.count()) scan_results, scan_errors = scancode.scan_file(codebase_resource1.location) scancode.save_scan_file_results(codebase_resource1, scan_results, scan_errors) codebase_resource1.refresh_from_db() self.assertEqual("scanned-with-error", codebase_resource1.status) - self.assertEqual(4, project1.projecterrors.count()) + self.assertEqual(4, project1.projectmessages.count()) copy_input(self.data_location / "notice.NOTICE", project1.codebase_path) codebase_resource2 = CodebaseResource.objects.create( @@ -238,15 +238,15 @@ def test_scanpipe_pipes_scancode_scan_file_and_save_results_timeout_error(self): codebase_resource.refresh_from_db() self.assertEqual("scanned-with-error", codebase_resource.status) - self.assertEqual(1, project1.projecterrors.count()) - error = project1.projecterrors.latest("created_date") - self.assertEqual("CodebaseResource", error.model) - self.assertEqual("", error.traceback) - expected_message = ( + self.assertEqual(1, project1.projectmessages.count()) + message = project1.projectmessages.latest("created_date") + self.assertEqual("CodebaseResource", message.model) + self.assertEqual("", message.traceback) + expected_description = ( "ERROR: for scanner: copyrights:\n" "ERROR: Processing interrupted: timeout after 120 seconds." ) - self.assertEqual(expected_message, error.message) + self.assertEqual(expected_description, message.description) @mock.patch("scanpipe.pipes.scancode._scan_resource") def test_scanpipe_pipes_scancode_scan_for_files(self, mock_scan_resource): @@ -330,15 +330,15 @@ def test_scanpipe_pipes_scancode_scan_package_and_save_results_timeout_error(sel codebase_resource.refresh_from_db() self.assertEqual("scanned-with-error", codebase_resource.status) - self.assertEqual(1, project1.projecterrors.count()) - error = project1.projecterrors.latest("created_date") - self.assertEqual("CodebaseResource", error.model) - self.assertEqual("", error.traceback) - expected_message = ( + self.assertEqual(1, project1.projectmessages.count()) + message = project1.projectmessages.latest("created_date") + self.assertEqual("CodebaseResource", message.model) + self.assertEqual("", message.traceback) + expected_description = ( "ERROR: for scanner: package_data:\n" "ERROR: Processing interrupted: timeout after 120 seconds." ) - self.assertEqual(expected_message, error.message) + self.assertEqual(expected_description, message.description) def test_scanpipe_pipes_scancode_scan_and_save_multiprocessing_with_threading(self): def noop(*args, **kwargs): diff --git a/scanpipe/tests/test_api.py b/scanpipe/tests/test_api.py index d7bf13343..522bae397 100644 --- a/scanpipe/tests/test_api.py +++ b/scanpipe/tests/test_api.py @@ -40,7 +40,7 @@ from scanpipe.api.serializers import CodebaseResourceSerializer from scanpipe.api.serializers import DiscoveredDependencySerializer from scanpipe.api.serializers import DiscoveredPackageSerializer -from scanpipe.api.serializers import ProjectErrorSerializer +from scanpipe.api.serializers import ProjectMessageSerializer from scanpipe.api.serializers import get_model_serializer from scanpipe.api.serializers import get_serializer_fields from scanpipe.models import CodebaseRelation @@ -48,7 +48,7 @@ from scanpipe.models import DiscoveredDependency from scanpipe.models import DiscoveredPackage from scanpipe.models import Project -from scanpipe.models import ProjectError +from scanpipe.models import ProjectMessage from scanpipe.models import Run from scanpipe.pipes.input import copy_input from scanpipe.pipes.output import JSONResultsGenerator @@ -102,7 +102,7 @@ def test_scanpipe_api_project_list(self): self.assertEqual(1, response.data["count"]) self.assertNotContains(response, "input_root") self.assertNotContains(response, "extra_data") - self.assertNotContains(response, "error_count") + self.assertNotContains(response, "message_count") self.assertNotContains(response, "resource_count") self.assertNotContains(response, "package_count") self.assertNotContains(response, "dependency_count") @@ -174,7 +174,7 @@ def test_scanpipe_api_project_detail(self): self.assertEqual([], response.data["input_sources"]) self.assertIn("input_root", response.data) self.assertIn("extra_data", response.data) - self.assertEqual(0, response.data["error_count"]) + self.assertEqual(0, response.data["message_count"]) self.assertEqual(1, response.data["resource_count"]) self.assertEqual(1, response.data["package_count"]) self.assertEqual(1, response.data["dependency_count"]) @@ -423,10 +423,14 @@ def test_scanpipe_api_project_action_relations(self): } self.assertEqual(expected, relation) - def test_scanpipe_api_project_action_errors(self): - url = reverse("project-errors", args=[self.project1.uuid]) - ProjectError.objects.create( - project=self.project1, model="ModelName", details={}, message="Error" + def test_scanpipe_api_project_action_messages(self): + url = reverse("project-messages", args=[self.project1.uuid]) + ProjectMessage.objects.create( + project=self.project1, + severity=ProjectMessage.Severity.ERROR, + description="Error", + model="ModelName", + details={}, ) response = self.csrf_client.get(url) @@ -435,10 +439,11 @@ def test_scanpipe_api_project_action_errors(self): self.assertIsNone(response.data["previous"]) self.assertEqual(1, len(response.data["results"])) - error = response.data["results"][0] - self.assertEqual("ModelName", error["model"]) - self.assertEqual({}, error["details"]) - self.assertEqual("Error", error["message"]) + message = response.data["results"][0] + self.assertEqual("error", message["severity"]) + self.assertEqual("Error", message["description"]) + self.assertEqual("ModelName", message["model"]) + self.assertEqual({}, message["details"]) def test_scanpipe_api_project_action_file_content(self): url = reverse("project-file-content", args=[self.project1.uuid]) @@ -720,7 +725,7 @@ def test_scanpipe_api_serializer_get_model_serializer(self): self.assertEqual( CodebaseRelationSerializer, get_model_serializer(CodebaseRelation) ) - self.assertEqual(ProjectErrorSerializer, get_model_serializer(ProjectError)) + self.assertEqual(ProjectMessageSerializer, get_model_serializer(ProjectMessage)) with self.assertRaises(LookupError): get_model_serializer(None) @@ -730,7 +735,7 @@ def test_scanpipe_api_serializer_get_serializer_fields(self): self.assertEqual(11, len(get_serializer_fields(DiscoveredDependency))) self.assertEqual(33, len(get_serializer_fields(CodebaseResource))) self.assertEqual(3, len(get_serializer_fields(CodebaseRelation))) - self.assertEqual(6, len(get_serializer_fields(ProjectError))) + self.assertEqual(7, len(get_serializer_fields(ProjectMessage))) with self.assertRaises(LookupError): get_serializer_fields(None) diff --git a/scanpipe/tests/test_auth.py b/scanpipe/tests/test_auth.py index 100325698..80c8af6ad 100644 --- a/scanpipe/tests/test_auth.py +++ b/scanpipe/tests/test_auth.py @@ -124,7 +124,7 @@ def test_scancodeio_auth_views_are_protected(self): ("project_list", None), ("project_resources", [a_uuid]), ("project_packages", [a_uuid]), - ("project_errors", [a_uuid]), + ("project_messages", [a_uuid]), ("project_archive", [a_uuid]), ("project_delete", [a_uuid]), ("project_reset", [a_uuid]), diff --git a/scanpipe/tests/test_commands.py b/scanpipe/tests/test_commands.py index b7f4f4bf4..8f8d88be6 100644 --- a/scanpipe/tests/test_commands.py +++ b/scanpipe/tests/test_commands.py @@ -375,7 +375,7 @@ def test_scanpipe_management_command_status(self): self.assertIn("Project: my_project", output) self.assertIn("- CodebaseResource: 0", output) self.assertIn("- DiscoveredPackage: 0", output) - self.assertIn("- ProjectError: 0", output) + self.assertIn("- ProjectMessage: 0", output) self.assertIn("[NOT_STARTED] docker", output) run.task_id = uuid.uuid4() diff --git a/scanpipe/tests/test_models.py b/scanpipe/tests/test_models.py index 7301678f1..632c96ce4 100644 --- a/scanpipe/tests/test_models.py +++ b/scanpipe/tests/test_models.py @@ -55,7 +55,7 @@ from scanpipe.models import DiscoveredDependency from scanpipe.models import DiscoveredPackage from scanpipe.models import Project -from scanpipe.models import ProjectError +from scanpipe.models import ProjectMessage from scanpipe.models import Run from scanpipe.models import RunInProgressError from scanpipe.models import get_project_work_directory @@ -159,7 +159,7 @@ def test_scanpipe_project_model_delete_related_objects(self): expected = { "scanpipe.DiscoveredPackage_codebase_resources": 1, "scanpipe.DiscoveredPackage": 1, - "scanpipe.ProjectError": 0, + "scanpipe.ProjectMessage": 0, "scanpipe.CodebaseRelation": 0, "scanpipe.DiscoveredDependency": 0, "scanpipe.CodebaseResource": 1, @@ -199,7 +199,7 @@ def test_scanpipe_project_model_reset(self): self.project1.reset() self.assertTrue(Project.objects.filter(name=self.project1.name).exists()) - self.assertEqual(0, self.project1.projecterrors.count()) + self.assertEqual(0, self.project1.projectmessages.count()) self.assertEqual(0, self.project1.runs.count()) self.assertEqual(0, self.project1.discoveredpackages.count()) self.assertEqual(0, self.project1.codebaseresources.count()) @@ -488,7 +488,7 @@ def test_scanpipe_project_queryset_with_counts(self): project_qs = Project.objects.with_counts( "codebaseresources", "discoveredpackages", - "projecterrors", + "projectmessages", ) project = project_qs.get(pk=self.project_asgiref.pk) @@ -496,8 +496,8 @@ def test_scanpipe_project_queryset_with_counts(self): self.assertEqual(18, project.codebaseresources.count()) self.assertEqual(2, project.discoveredpackages_count) self.assertEqual(2, project.discoveredpackages.count()) - self.assertEqual(2, project.projecterrors_count) - self.assertEqual(2, project.projecterrors.count()) + self.assertEqual(2, project.projectmessages_count) + self.assertEqual(2, project.projectmessages.count()) def test_scanpipe_project_related_queryset_get_or_none(self): self.assertIsNone(CodebaseResource.objects.get_or_none(path="path/")) @@ -1801,10 +1801,10 @@ def test_scanpipe_project_model_add_error(self): model="Package", details=details, ) - self.assertEqual(error, ProjectError.objects.get()) + self.assertEqual(error, ProjectMessage.objects.get()) self.assertEqual("Package", error.model) self.assertEqual(details, error.details) - self.assertEqual("Error message", error.message) + self.assertEqual("Error message", error.description) self.assertEqual("", error.traceback) def test_scanpipe_project_model_update_extra_data(self): @@ -1841,10 +1841,10 @@ def test_scanpipe_codebase_resource_model_add_error(self): codebase_resource = CodebaseResource.objects.create(project=project1) error = codebase_resource.add_error(Exception("Error message")) - self.assertEqual(error, ProjectError.objects.get()) + self.assertEqual(error, ProjectMessage.objects.get()) self.assertEqual("CodebaseResource", error.model) self.assertTrue(error.details) - self.assertEqual("Error message", error.message) + self.assertEqual("Error message", error.description) self.assertEqual("", error.traceback) def test_scanpipe_codebase_resource_model_add_errors(self): @@ -1852,7 +1852,7 @@ def test_scanpipe_codebase_resource_model_add_errors(self): codebase_resource = CodebaseResource.objects.create(project=project1) codebase_resource.add_error(Exception("Error1")) codebase_resource.add_error(Exception("Error2")) - self.assertEqual(2, ProjectError.objects.count()) + self.assertEqual(2, ProjectMessage.objects.count()) @skipIf(connection.vendor == "sqlite", "No max_length constraints on SQLite.") def test_scanpipe_project_error_model_save_non_valid_related_object(self): @@ -1865,14 +1865,14 @@ def test_scanpipe_project_error_model_save_non_valid_related_object(self): # The DiscoveredPackage was not created self.assertIsNone(package.id) self.assertEqual(0, DiscoveredPackage.objects.count()) - # A ProjectError was saved instead - self.assertEqual(1, project1.projecterrors.count()) + # A ProjectMessage was saved instead + self.assertEqual(1, project1.projectmessages.count()) - error = project1.projecterrors.get() + error = project1.projectmessages.get() self.assertEqual("DiscoveredPackage", error.model) self.assertEqual(long_value, error.details["filename"]) self.assertEqual( - "value too long for type character varying(255)", error.message + "value too long for type character varying(255)", error.description ) codebase_resource = CodebaseResource.objects.create( @@ -1880,7 +1880,7 @@ def test_scanpipe_project_error_model_save_non_valid_related_object(self): ) self.assertIsNone(codebase_resource.id) self.assertEqual(0, CodebaseResource.objects.count()) - self.assertEqual(2, project1.projecterrors.count()) + self.assertEqual(2, project1.projectmessages.count()) @skipIf(connection.vendor == "sqlite", "No max_length constraints on SQLite.") def test_scanpipe_discovered_package_model_create_from_data(self): @@ -1904,26 +1904,26 @@ def test_scanpipe_discovered_package_model_create_from_data(self): incomplete_data["name"] = "" self.assertIsNone(DiscoveredPackage.create_from_data(project1, incomplete_data)) self.assertEqual(package_count, DiscoveredPackage.objects.count()) - error = project1.projecterrors.latest("created_date") + error = project1.projectmessages.latest("created_date") self.assertEqual("DiscoveredPackage", error.model) expected_message = "No values for the following required fields: name" - self.assertEqual(expected_message, error.message) + self.assertEqual(expected_message, error.description) self.assertEqual(package_data1["purl"], error.details["purl"]) self.assertEqual("", error.details["name"]) self.assertEqual("", error.traceback) package_count = DiscoveredPackage.objects.count() - project_error_count = ProjectError.objects.count() + project_message_count = ProjectMessage.objects.count() bad_data = dict(package_data1) bad_data["version"] = "a" * 200 # The exception are not capture at the DiscoveredPackage.create_from_data but # rather in the CodebaseResource.create_and_add_package method so resource data - # can be injected in the ProjectError record. + # can be injected in the ProjectMessage record. with self.assertRaises(DataError): DiscoveredPackage.create_from_data(project1, bad_data) self.assertEqual(package_count, DiscoveredPackage.objects.count()) - self.assertEqual(project_error_count, ProjectError.objects.count()) + self.assertEqual(project_message_count, ProjectMessage.objects.count()) @skipIf(connection.vendor == "sqlite", "No max_length constraints on SQLite.") def test_scanpipe_discovered_dependency_model_create_from_data(self): @@ -1965,10 +1965,10 @@ def test_scanpipe_discovered_dependency_model_create_from_data(self): DiscoveredDependency.create_from_data(project1, incomplete_data) ) self.assertEqual(dependency_count, DiscoveredDependency.objects.count()) - error = project1.projecterrors.latest("created_date") + error = project1.projectmessages.latest("created_date") self.assertEqual("DiscoveredDependency", error.model) expected_message = "No values for the following required fields: dependency_uid" - self.assertEqual(expected_message, error.message) + self.assertEqual(expected_message, error.description) self.assertEqual(dependency_data1["purl"], error.details["purl"]) self.assertEqual("", error.details["dependency_uid"]) self.assertEqual("", error.traceback) @@ -2002,10 +2002,10 @@ def test_scanpipe_codebase_resource_create_and_add_package_errors(self): package = resource.create_and_add_package(bad_data) self.assertIsNone(package) self.assertEqual(package_count, DiscoveredPackage.objects.count()) - error = project1.projecterrors.latest("created_date") + error = project1.projectmessages.latest("created_date") self.assertEqual("DiscoveredPackage", error.model) expected_message = "value too long for type character varying(100)" - self.assertEqual(expected_message, error.message) + self.assertEqual(expected_message, error.description) self.assertEqual(bad_data["version"], error.details["version"]) self.assertTrue(error.details["codebase_resource_pk"]) self.assertEqual(resource.path, error.details["codebase_resource_path"]) diff --git a/scanpipe/tests/test_pipelines.py b/scanpipe/tests/test_pipelines.py index 8a6c771fa..275cca84e 100644 --- a/scanpipe/tests/test_pipelines.py +++ b/scanpipe/tests/test_pipelines.py @@ -141,11 +141,11 @@ def test_scanpipe_pipeline_class_save_errors_context_manager(self): with pipeline.save_errors(Exception): raise Exception("Error message") - error = project1.projecterrors.get() - self.assertEqual("do_nothing", error.model) - self.assertEqual({}, error.details) - self.assertEqual("Error message", error.message) - self.assertIn('raise Exception("Error message")', error.traceback) + message = project1.projectmessages.get() + self.assertEqual("do_nothing", message.model) + self.assertEqual({}, message.details) + self.assertEqual("Error message", message.description) + self.assertIn('raise Exception("Error message")', message.traceback) def test_scanpipe_pipelines_is_pipeline(self): self.assertFalse(is_pipeline(None)) @@ -246,8 +246,8 @@ def test_scanpipe_rootfs_pipeline_extract_input_files_errors(self): extract_archive.return_value = ["Error"] pipeline_instance.extract_input_files_to_codebase_directory() - error = project1.projecterrors.get() - self.assertEqual("Error\nError", error.message) + error = project1.projectmessages.get() + self.assertEqual("Error\nError", error.description) def sort_scanned_files_by_path(scan_data): @@ -509,9 +509,9 @@ def test_scanpipe_docker_pipeline_does_not_report_errors_for_broken_symlinks(sel exitcode, out = pipeline.execute() self.assertEqual(0, exitcode, msg=out) - project_errors = [pe.message for pe in project1.projecterrors.all()] - self.assertEqual(1, len(project_errors)) - self.assertEqual("Distro not found.", project_errors[0]) + project_messages = project1.projectmessages.all() + self.assertEqual(1, len(project_messages)) + self.assertEqual("Distro not found.", project_messages[0].description) result_file = output.to_json(project1) expected_file = ( diff --git a/scanpipe/urls.py b/scanpipe/urls.py index 05b5028ab..fb0edbfe7 100644 --- a/scanpipe/urls.py +++ b/scanpipe/urls.py @@ -72,9 +72,9 @@ name="project_relations", ), path( - "project//errors/", - views.ProjectErrorListView.as_view(), - name="project_errors", + "project//messages/", + views.ProjectMessageListView.as_view(), + name="project_messages", ), path( "project//archive/", diff --git a/scanpipe/views.py b/scanpipe/views.py index f80ebfa15..1290657c6 100644 --- a/scanpipe/views.py +++ b/scanpipe/views.py @@ -59,9 +59,9 @@ from scanpipe.api.serializers import DiscoveredDependencySerializer from scanpipe.filters import PAGE_VAR from scanpipe.filters import DependencyFilterSet -from scanpipe.filters import ErrorFilterSet from scanpipe.filters import PackageFilterSet from scanpipe.filters import ProjectFilterSet +from scanpipe.filters import ProjectMessageFilterSet from scanpipe.filters import ResourceFilterSet from scanpipe.forms import AddInputsForm from scanpipe.forms import AddPipelineForm @@ -72,7 +72,7 @@ from scanpipe.models import DiscoveredDependency from scanpipe.models import DiscoveredPackage from scanpipe.models import Project -from scanpipe.models import ProjectError +from scanpipe.models import ProjectMessage from scanpipe.models import Run from scanpipe.models import RunInProgressError from scanpipe.pipes import count_group_by @@ -484,9 +484,9 @@ class ProjectListView( "sort_name": "codebaseresources_count", }, { - "field_name": "projecterrors", - "label": "Errors", - "sort_name": "projecterrors_count", + "field_name": "projectmessages", + "label": "Messages", + "sort_name": "projectmessages_count", }, { "field_name": "runs", @@ -506,7 +506,7 @@ def get_queryset(self): "codebaseresources", "discoveredpackages", "discovereddependencies", - "projecterrors", + "projectmessages", ) ) @@ -1144,20 +1144,24 @@ class DiscoveredDependencyListView( ] -class ProjectErrorListView( +class ProjectMessageListView( ConditionalLoginRequired, ProjectRelatedViewMixin, TableColumnsMixin, ExportXLSXMixin, FilterView, ): - model = ProjectError - filterset_class = ErrorFilterSet - template_name = "scanpipe/error_list.html" + model = ProjectMessage + filterset_class = ProjectMessageFilterSet + template_name = "scanpipe/project_message_list.html" paginate_by = settings.SCANCODEIO_PAGINATE_BY.get("error", 50) table_columns = [ + { + "field_name": "severity", + "filter_fieldname": "severity", + }, "model", - "message", + "description", "details", "traceback", ]
- {{ error.model }} + {{ message.severity }} + + {{ message.model }}
- {% if error.message|length < 100 %} - {{ error.message }} + {% if message.description|length < 100 %} + {{ message.description }} {% else %} - {{ error.message }} + {{ message.description }} {% endif %}
- {% if error.details.codebase_resource_pk and error.details.codebase_resource_path %} + {% if message.details.codebase_resource_pk and message.details.codebase_resource_path %} {% endif %} - {% for key, value in error.details.items %} + {% for key, value in message.details.items %} {{ key }}: {{ value }}
{% endfor %}
- 
{{ error.traceback }}
+ 
{{ message.traceback }}
- No Errors found. Clear search and filters + No messages found. Clear search and filters