Migrate the ProjectError model to a more global ProjectMessage #338

Signed-off-by: Thomas Druez <[email protected]>

docs/data-models.rst

@@ -42,9 +42,9 @@ CodebaseRelation
     :undoc-members:
     :member-order: groupwise
 
-ProjectError
-------------
-.. autoclass:: scanpipe.models.ProjectError()
+ProjectMessage
+--------------
+.. autoclass:: scanpipe.models.ProjectMessage()
     :members:
     :undoc-members:
     :member-order: groupwise

scanpipe/api/serializers.py

@@ -30,7 +30,7 @@
 from scanpipe.models import DiscoveredDependency
 from scanpipe.models import DiscoveredPackage
 from scanpipe.models import Project
-from scanpipe.models import ProjectError
+from scanpipe.models import ProjectMessage
 from scanpipe.models import Run
 from scanpipe.pipes import count_group_by
 from scanpipe.pipes.fetch import fetch_urls
@@ -139,7 +139,7 @@ class Meta:
             "next_run",
             "runs",
             "extra_data",
-            "error_count",
+            "message_count",
             "resource_count",
             "package_count",
             "dependency_count",
@@ -155,7 +155,7 @@ class Meta:
             "input_root",
             "output_root",
             "extra_data",
-            "error_count",
+            "message_count",
             "resource_count",
             "package_count",
             "dependency_count",
@@ -361,12 +361,20 @@ class Meta:
         ]
 
 
-class ProjectErrorSerializer(serializers.ModelSerializer):
+class ProjectMessageSerializer(serializers.ModelSerializer):
     traceback = serializers.SerializerMethodField()
 
     class Meta:
-        model = ProjectError
-        fields = ["uuid", "model", "message", "details", "traceback", "created_date"]
+        model = ProjectMessage
+        fields = [
+            "uuid",
+            "severity",
+            "description",
+            "model",
+            "details",
+            "traceback",
+            "created_date",
+        ]
 
     def get_traceback(self, project_error):
         return project_error.traceback.splitlines()
@@ -397,7 +405,7 @@ def get_model_serializer(model_class):
         DiscoveredPackage: DiscoveredPackageSerializer,
         DiscoveredDependency: DiscoveredDependencySerializer,
         CodebaseRelation: CodebaseRelationSerializer,
-        ProjectError: ProjectErrorSerializer,
+        ProjectMessage: ProjectMessageSerializer,
     }.get(model_class, None)
 
     if not serializer:

scanpipe/api/views.py

@@ -40,7 +40,7 @@
 from scanpipe.api.serializers import DiscoveredDependencySerializer
 from scanpipe.api.serializers import DiscoveredPackageSerializer
 from scanpipe.api.serializers import PipelineSerializer
-from scanpipe.api.serializers import ProjectErrorSerializer
+from scanpipe.api.serializers import ProjectMessageSerializer
 from scanpipe.api.serializers import ProjectSerializer
 from scanpipe.api.serializers import RunSerializer
 from scanpipe.models import Project
@@ -201,12 +201,12 @@ def relations(self, request, *args, **kwargs):
         return self.get_paginated_response(serializer.data)
 
     @action(detail=True)
-    def errors(self, request, *args, **kwargs):
+    def messages(self, request, *args, **kwargs):
         project = self.get_object()
-        queryset = project.projecterrors.all()
+        queryset = project.projectmessages.all()
 
         paginated_qs = self.paginate_queryset(queryset)
-        serializer = ProjectErrorSerializer(paginated_qs, many=True)
+        serializer = ProjectMessageSerializer(paginated_qs, many=True)
 
         return self.get_paginated_response(serializer.data)
 

scanpipe/filters.py

@@ -36,7 +36,7 @@
 from scanpipe.models import DiscoveredDependency
 from scanpipe.models import DiscoveredPackage
 from scanpipe.models import Project
-from scanpipe.models import ProjectError
+from scanpipe.models import ProjectMessage
 from scanpipe.models import Run
 
 scanpipe_app = apps.get_app_config("scanpipe")
@@ -199,7 +199,7 @@ class ProjectFilterSet(FilterSetUtilsMixin, django_filters.FilterSet):
             "discoveredpackages_count",
             "discovereddependencies_count",
             "codebaseresources_count",
-            "projecterrors_count",
+            "projectmessages_count",
         ],
         empty_label="Newest",
         choices=(
@@ -212,8 +212,8 @@ class ProjectFilterSet(FilterSetUtilsMixin, django_filters.FilterSet):
             ("discovereddependencies_count", "Dependencies (-)"),
             ("-codebaseresources_count", "Resources (+)"),
             ("codebaseresources_count", "Resources (-)"),
-            ("-projecterrors_count", "Errors (+)"),
-            ("projecterrors_count", "Errors (-)"),
+            ("-projectmessages_count", "Messages (+)"),
+            ("projectmessages_count", "Messages (-)"),
         ),
         widget=BulmaDropdownWidget,
     )
@@ -547,21 +547,27 @@ class Meta:
         ]
 
 
-class ErrorFilterSet(FilterSetUtilsMixin, django_filters.FilterSet):
+class ProjectMessageFilterSet(FilterSetUtilsMixin, django_filters.FilterSet):
     search = django_filters.CharFilter(
-        label="Search", field_name="message", lookup_expr="icontains"
+        label="Search", field_name="description", lookup_expr="icontains"
     )
     sort = django_filters.OrderingFilter(
         label="Sort",
         fields=[
+            "severity",
             "model",
         ],
     )
 
     class Meta:
-        model = ProjectError
+        model = ProjectMessage
         fields = [
             "search",
+            "severity",
             "model",
-            "message",
+            "description",
         ]
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.filters["severity"].extra["widget"] = BulmaDropdownWidget()

scanpipe/management/commands/__init__.py

@@ -32,7 +32,7 @@
 from scanpipe.models import CodebaseResource
 from scanpipe.models import DiscoveredPackage
 from scanpipe.models import Project
-from scanpipe.models import ProjectError
+from scanpipe.models import ProjectMessage
 from scanpipe.pipes import count_group_by
 from scanpipe.pipes.fetch import fetch_urls
 
@@ -93,7 +93,7 @@ def get_run_status_messages(self, project):
     def get_queryset_objects_messages(self, project):
         messages = []
 
-        for model_class in [CodebaseResource, DiscoveredPackage, ProjectError]:
+        for model_class in [CodebaseResource, DiscoveredPackage, ProjectMessage]:
             queryset = model_class.objects.project(project)
             messages.append(f" - {model_class.__name__}: {queryset.count()}")
 

scanpipe/migrations/0001_initial.py

@@ -44,7 +44,7 @@ class Migration(migrations.Migration):
             options={
                 'ordering': ('project', 'path'),
             },
-            bases=(scanpipe.models.SaveProjectErrorMixin, models.Model),
+            bases=(scanpipe.models.SaveProjectMessageMixin, models.Model),
         ),
         migrations.CreateModel(
             name='Project',
@@ -132,7 +132,7 @@ class Migration(migrations.Migration):
             options={
                 'ordering': ['uuid'],
             },
-            bases=(scanpipe.models.SaveProjectErrorMixin, models.Model),
+            bases=(scanpipe.models.SaveProjectMessageMixin, models.Model),
         ),
         migrations.AddField(
             model_name='codebaseresource',

scanpipe/migrations/0022_create_discovereddependencies_model.py

@@ -153,7 +153,7 @@ class Migration(migrations.Migration):
                 ],
             },
             bases=(
-                scanpipe.models.SaveProjectErrorMixin,
+                scanpipe.models.SaveProjectMessageMixin,
                 scanpipe.models.UpdateFromDataMixin,
                 models.Model,
             ),

scanpipe/migrations/0040_projectmessage.py

@@ -0,0 +1,85 @@
+# Generated by Django 4.2.3 on 2023-07-12 12:13
+
+import django.core.serializers.json
+from django.db import migrations, models
+import django.db.models.deletion
+import scanpipe.models
+import uuid
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("scanpipe", "0039_discoveredpackage_compliance_alert_and_more"),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name="ProjectMessage",
+            fields=[
+                (
+                    "uuid",
+                    models.UUIDField(
+                        db_index=True,
+                        default=uuid.uuid4,
+                        editable=False,
+                        primary_key=True,
+                        serialize=False,
+                        verbose_name="UUID",
+                    ),
+                ),
+                (
+                    "severity",
+                    models.CharField(
+                        choices=[
+                            ("info", "Info"),
+                            ("warning", "Warning"),
+                            ("error", "Error"),
+                        ],
+                        editable=False,
+                        help_text="Severity level of the message.",
+                        max_length=10,
+                    ),
+                ),
+                ("description", models.TextField(blank=True, help_text="Description.")),
+                (
+                    "model",
+                    models.CharField(
+                        help_text="Name of the model class.", max_length=100
+                    ),
+                ),
+                (
+                    "details",
+                    models.JSONField(
+                        blank=True,
+                        default=dict,
+                        encoder=django.core.serializers.json.DjangoJSONEncoder,
+                        help_text="Data that caused the error.",
+                    ),
+                ),
+                (
+                    "traceback",
+                    models.TextField(blank=True, help_text="Exception traceback."),
+                ),
+                ("created_date", models.DateTimeField(auto_now_add=True)),
+                (
+                    "project",
+                    models.ForeignKey(
+                        editable=False,
+                        on_delete=django.db.models.deletion.CASCADE,
+                        related_name="%(class)ss",
+                        to="scanpipe.project",
+                    ),
+                ),
+            ],
+            options={
+                "ordering": ["created_date"],
+                "indexes": [
+                    models.Index(
+                        fields=["severity"], name="scanpipe_pr_severit_9e1b35_idx"
+                    ),
+                    models.Index(fields=["model"], name="scanpipe_pr_model_013dd4_idx"),
+                ],
+            },
+            bases=(scanpipe.models.UpdateMixin, models.Model),
+        ),
+    ]

scanpipe/migrations/0041_projecterror_to_projectmessage_data.py

@@ -0,0 +1,32 @@
+# Generated by Django 4.2.3 on 2023-07-12 12:13
+
+from django.db import migrations
+
+
+def migrate_error_to_message_data(apps, schema_editor):
+    ProjectError = apps.get_model("scanpipe", "ProjectError")
+    ProjectMessage = apps.get_model("scanpipe", "ProjectMessage")
+    ERROR = "error"
+
+    for project_error in ProjectError.objects.all():
+        ProjectMessage.objects.create(
+            project=project_error.project,
+            severity=ERROR,
+            description=project_error.message,
+            model=project_error.model,
+            details=project_error.details,
+            traceback=project_error.traceback,
+        )
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("scanpipe", "0040_projectmessage"),
+    ]
+
+    operations = [
+        migrations.RunPython(
+            migrate_error_to_message_data,
+            reverse_code=migrations.RunPython.noop,
+        ),
+    ]

scanpipe/migrations/0042_delete_projecterror.py

@@ -0,0 +1,15 @@
+# Generated by Django 4.2.3 on 2023-07-12 12:14
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("scanpipe", "0041_projecterror_to_projectmessage_data"),
+    ]
+
+    operations = [
+        migrations.DeleteModel(
+            name="ProjectError",
+        ),
+    ]