add xls validattion

pyproject.toml

@@ -16,6 +16,8 @@ dependencies = [
     "jsonpickle>=3.2.2",
     "xlsxwriter>=3.2.0",
     "django-adminactions>=2.3.0",
+    "duckdb>=1.0.0",
+    "python-calamine>=0.2.0",
 ]
 requires-python = ">=3.11"
 readme = "README.md"

src/hope_flex_fields/admin/datachecker.py

@@ -1,15 +1,45 @@
+from pathlib import Path
+
+from django import forms
 from django.contrib import messages
 from django.contrib.admin import ModelAdmin, TabularInline, register
+from django.core.exceptions import ValidationError
 from django.http import HttpResponse
 from django.shortcuts import render
+from django.utils.deconstruct import deconstructible
+from django.utils.translation import gettext as _
 
 from admin_extra_buttons.decorators import button
 from admin_extra_buttons.mixins import ExtraButtonsMixin
 
+from ..file_handlers import HANDLERS
 from ..models import DataChecker, DataCheckerFieldset, Fieldset
 from ..models.datachecker import create_xls_importer
 
 
+@deconstructible
+class ValidatableFileValidator(object):
+    error_messages = {
+        "invalid_file": _("Unsupported file format '%s'"),
+    }
+
+    def __call__(self, f):
+        if Path(f.name).suffix not in HANDLERS.keys():
+            raise ValidationError(
+                self.error_messages["invalid_file"] % Path(f.name).suffix
+            )
+
+
+class FileForm(forms.Form):
+    include_success = forms.BooleanField(required=False)
+    fail_if_alien = forms.BooleanField(required=False)
+    file = forms.FileField(
+        validators=[
+            ValidatableFileValidator(),
+        ]
+    )
+
+
 class DataCheckerFieldsetTabularInline(TabularInline):
     model = DataCheckerFieldset
     fields = ("fieldset", "prefix", "order")
@@ -26,7 +56,27 @@ class DataCheckerAdmin(ExtraButtonsMixin, ModelAdmin):
     @button()
     def inspect(self, request, pk):
         ctx = self.get_common_context(request, pk, title="Inspect")
-        return render(request, "flex_fields/datachecker/inspect.html", ctx)
+        return render(request, "flex_fields/inspect.html", ctx)
+
+    @button()
+    def validate(self, request, pk):
+        ctx = self.get_common_context(request, pk, title="Validate file")
+        if request.method == "POST":
+            form = FileForm(request.POST, request.FILES)
+            if form.is_valid():
+                dc: DataChecker = ctx["original"]
+                f = form.cleaned_data["file"]
+                parser = HANDLERS[Path(f.name).suffix]
+                ret = dc.validate(parser(f), True)
+                ctx["results"] = ret
+                self.message_user(request, "Data looks valid", messages.SUCCESS)
+            else:
+                self.message_user(request, "Some data did not validate", messages.ERROR)
+
+        else:
+            form = FileForm()
+        ctx["form"] = form
+        return render(request, "flex_fields/datachecker/validate.html", ctx)
 
     @button()
     def create_xls_importer(self, request, pk):

src/hope_flex_fields/admin/fieldset.py

@@ -18,6 +18,11 @@ class FieldsetAdmin(ExtraButtonsMixin, ModelAdmin):
     inlines = [FieldsetFieldTabularInline]
     form = FieldsetForm
 
+    @button()
+    def inspect(self, request, pk):
+        ctx = self.get_common_context(request, pk, title="Inspect")
+        return render(request, "flex_fields/inspect.html", ctx)
+
     @button()
     def test(self, request, pk):
         ctx = self.get_common_context(request, pk, title="Test")

src/hope_flex_fields/file_handlers.py

@@ -0,0 +1,14 @@
+from django.core.files.uploadedfile import UploadedFile
+
+
+def parse_xlsx(f: UploadedFile):
+    import python_calamine
+
+    workbook = python_calamine.CalamineWorkbook.from_filelike(f)  # type: ignore[arg-type]
+    rows = iter(workbook.get_sheet_by_index(0).to_python())
+    headers = list(map(str, next(rows)))
+    for row in rows:
+        yield dict(zip(headers, row))
+
+
+HANDLERS = {".xlsx": parse_xlsx}

src/hope_flex_fields/models/base.py

@@ -1,4 +1,6 @@
 import logging
+from types import GeneratorType
+from typing import Iterable
 
 from django import forms
 from django.db import models
@@ -13,7 +15,7 @@ def get_default_attrs():
     return {"required": False, "help_text": ""}
 
 
-class TestForm(forms.Form):
+class FlexForm(forms.Form):
     fieldset = None
 
 
@@ -26,3 +28,29 @@ class AbstractField(models.Model):
 
     class Meta:
         abstract = True
+
+
+class ValidatorMixin:
+
+    def validate(
+        self, data: Iterable, include_success: bool = False, fail_if_alien: bool = False
+    ):
+        if not isinstance(data, (list, tuple, GeneratorType)):
+            data = [data]
+        form_class: type[FlexForm] = self.get_form()
+        known_fields = set(sorted(form_class.declared_fields.keys()))
+        ret = {}
+        for i, row in enumerate(data, 1):
+            form: "FlexForm" = form_class(data=row)
+            posted_fields = set(sorted(row.keys()))
+            row_errors = {}
+            if fail_if_alien and (diff := posted_fields.difference(known_fields)):
+                row_errors["-"] = [f"Alien values found {diff}"]
+            if not form.is_valid():
+                row_errors.update(**form.errors)
+
+            if row_errors:
+                ret[i] = row_errors
+            elif include_success:
+                ret[i] = "Ok"
+        return ret

src/hope_flex_fields/models/datachecker.py

@@ -2,16 +2,15 @@
 from typing import TYPE_CHECKING
 
 from django import forms
-from django.core.exceptions import ValidationError
 from django.db import models
 from django.utils.translation import gettext as _
 
+from ..fields import FlexFormMixin
 from ..xlsx import get_format_for_field, get_validation_for_field
-from .base import TestForm
+from .base import FlexForm, ValidatorMixin
 from .fieldset import Fieldset
 
 if TYPE_CHECKING:
-    from ..forms import FieldDefinitionForm
     from .flexfield import FLexField
 
 
@@ -74,11 +73,11 @@ class DataCheckerFieldset(models.Model):
         "DataChecker", on_delete=models.CASCADE, related_name="members"
     )
     fieldset = models.ForeignKey(Fieldset, on_delete=models.CASCADE)
-    prefix = models.CharField(max_length=30)
+    prefix = models.CharField(max_length=30, blank=True, default="")
     order = models.PositiveSmallIntegerField(default=0)
 
 
-class DataChecker(models.Model):
+class DataChecker(ValidatorMixin, models.Model):
     name = models.CharField(max_length=255, unique=True)
     description = models.TextField(blank=True)
     fieldsets = models.ManyToManyField(Fieldset, through=DataCheckerFieldset)
@@ -99,31 +98,13 @@ def get_fields(self):
             for field in fs.fieldset.fields.filter():
                 yield field
 
-    def get_form(self) -> "type[FieldDefinitionForm]":
+    def get_form(self) -> "type[FlexForm]":
         fields: dict[str, forms.Field] = {}
         field: "FLexField"
         for fs in self.members.all():
             for field in fs.fieldset.fields.filter():
-                fld = field.get_field()
-                fields[f"{fs.prefix}_{field.name}"] = fld
+                fld: FlexFormMixin = field.get_field()
+                fld.label = f"{fs.prefix}_{field.name}"
+                fields[f"{fs.prefix}{field.name}"] = fld
         form_class_attrs = {"DataChecker": self, **fields}
-        return type(f"{self.name}DataChecker", (TestForm,), form_class_attrs)
-
-    def validate(self, data):
-        form_class = self.get_form()
-        form: "FieldDefinitionForm" = form_class(data=data)
-        if form.is_valid():
-            return True
-        else:
-            self.errors = form.errors
-            raise ValidationError(form.errors)
-
-    def validate_many(self, data):
-        ret = []
-        for r in data:
-            try:
-                self.validate(r)
-                ret.append("Ok")
-            except ValidationError as e:
-                ret.append(e.message_dict)
-        return ret
+        return type(f"{self.name}DataChecker", (FlexForm,), form_class_attrs)