From fb5b4f5467bfc7d089c50d9f3cd96f7229a625b0 Mon Sep 17 00:00:00 2001 From: saxix Date: Tue, 9 Jul 2024 11:30:57 +0200 Subject: [PATCH] add sync API --- pyproject.toml | 3 + src/hope_flex_fields/admin/definition.py | 33 ++------- src/hope_flex_fields/api/serializers.py | 60 ++++++++++++++-- src/hope_flex_fields/api/views.py | 44 ++++++++++-- src/hope_flex_fields/config.py | 13 ++++ src/hope_flex_fields/models/base.py | 1 + src/hope_flex_fields/models/datachecker.py | 2 + src/hope_flex_fields/models/definition.py | 1 + src/hope_flex_fields/models/fieldset.py | 1 + src/hope_flex_fields/utils.py | 39 ++++++++++ tests/admin/data.json | 7 +- tests/extra/demoapp/demo/api.py | 0 tests/extra/demoapp/demo/urls.py | 14 +++- tests/test_api.py | 84 ++++++++++++++++++++-- 14 files changed, 256 insertions(+), 46 deletions(-) create mode 100644 src/hope_flex_fields/config.py delete mode 100644 tests/extra/demoapp/demo/api.py diff --git a/pyproject.toml b/pyproject.toml index 820ea6d..055d36e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -18,6 +18,8 @@ dependencies = [ "django-adminactions>=2.3.0", "duckdb>=1.0.0", "python-calamine>=0.2.0", + "requests>=2.32.3", + "responses>=0.25.3", ] requires-python = ">=3.11" readme = "README.md" @@ -79,6 +81,7 @@ dev = [ "responses>=0.25.3", "django-csp>=3.8", "bandit>=1.7.9", + "pdm-bump>=0.9.0", ] [tool.pdm.scripts] diff --git a/src/hope_flex_fields/admin/definition.py b/src/hope_flex_fields/admin/definition.py index c5218d8..eecd086 100644 --- a/src/hope_flex_fields/admin/definition.py +++ b/src/hope_flex_fields/admin/definition.py @@ -1,15 +1,10 @@ -import io import json -import tempfile -from io import StringIO from json import JSONDecodeError -from pathlib import Path from django import forms from django.contrib import messages from django.contrib.admin import ModelAdmin, register from django.core.exceptions import ValidationError -from django.core.management import call_command from django.core.serializers.base import DeserializationError from django.core.validators import FileExtensionValidator from django.http import HttpResponse, HttpResponseRedirect @@ -22,6 +17,7 @@ from ..forms import FieldDefinitionForm from ..models import FieldDefinition, get_default_attrs +from ..utils import dumpdata_to_buffer, loaddata_from_buffer @deconstructible @@ -76,16 +72,8 @@ def js_validation(self, obj): @view() def export_all(self, request): - buf = StringIO() - call_command( - "dumpdata", - "hope_flex_fields", - use_natural_primary_keys=True, - use_natural_foreign_keys=True, - stdout=buf, - ) - buf.seek(0) - return HttpResponse(buf.getvalue(), content_type="application/json") + data = dumpdata_to_buffer() + return HttpResponse(data, content_type="application/json") @view() def import_all(self, request): @@ -93,24 +81,11 @@ def import_all(self, request): if request.method == "POST": form = ImportConfigurationForm(request.POST, request.FILES) if form.is_valid(): - workdir = Path(".").absolute() - kwargs = { - "dir": workdir, - "prefix": "~LOADDATA", - "suffix": ".json", - "delete": False, - } - with tempfile.NamedTemporaryFile(**kwargs) as fdst: - fdst.write(form.files["file"].file.read()) - fixture = (workdir / fdst.name).absolute() - out = io.StringIO() try: - call_command("loaddata", fixture, stdout=out, verbosity=3) + loaddata_from_buffer(form.files["file"].file) self.message_user(request, "Data successfully imported.") except DeserializationError as e: self.message_user(request, str(e), messages.ERROR) - finally: - fixture.unlink() return HttpResponseRedirect("..") else: form = ImportConfigurationForm() diff --git a/src/hope_flex_fields/api/serializers.py b/src/hope_flex_fields/api/serializers.py index d7b372f..e2cfd5f 100644 --- a/src/hope_flex_fields/api/serializers.py +++ b/src/hope_flex_fields/api/serializers.py @@ -1,15 +1,65 @@ from rest_framework import serializers from strategy_field.utils import fqn -from hope_flex_fields.models import FieldDefinition +from hope_flex_fields.models import DataChecker, FieldDefinition, Fieldset, FlexField -class FieldDefinitionSerializer(serializers.HyperlinkedModelSerializer): +class BaseSerializer(serializers.ModelSerializer): + pass + + +class FieldDefinitionSerializer(BaseSerializer): field_type = serializers.SerializerMethodField() class Meta: model = FieldDefinition - fields = ["name", "description", "field_type", "attrs", "regex", "validation"] + fields = [ + "last_modified", + "name", + "description", + "field_type", + "attrs", + "regex", + "validation", + ] + + def get_field_type(self, obj: FieldDefinition): + return fqn(obj.field_type) + + +class FlexFieldSerializer(BaseSerializer): + field = serializers.SlugRelatedField(read_only=True, slug_field="name") + + class Meta: + model = FlexField + fields = [ + "last_modified", + "name", + "description", + "field", + "fieldset", + "attrs", + "regex", + "validation", + ] - def get_field_type(self, obj): - return fqn(obj) + +class FieldsetSerializer(BaseSerializer): + # fields = serializers.SlugRelatedField( + # many=True, read_only=True, slug_field="name" + # ) + fields = FlexFieldSerializer(many=True) + + class Meta: + model = Fieldset + fields = ["last_modified", "name", "description", "extends", "fields"] + + +class DataCheckerSerializer(BaseSerializer): + fieldsets = serializers.SlugRelatedField( + many=True, read_only=True, slug_field="name" + ) + + class Meta: + model = DataChecker + fields = ["last_modified", "name", "description", "fieldsets"] diff --git a/src/hope_flex_fields/api/views.py b/src/hope_flex_fields/api/views.py index d010316..9020b27 100644 --- a/src/hope_flex_fields/api/views.py +++ b/src/hope_flex_fields/api/views.py @@ -1,10 +1,44 @@ -from rest_framework import permissions, viewsets +from django.http import HttpResponse -from ..models import FieldDefinition -from .serializers import FieldDefinitionSerializer +from rest_framework import viewsets +from ..config import CONFIG +from ..models import DataChecker, FieldDefinition, Fieldset, FlexField +from ..utils import dumpdata_to_buffer +from .serializers import ( + DataCheckerSerializer, + FieldDefinitionSerializer, + FieldsetSerializer, + FlexFieldSerializer, +) -class FieldDefinitionViewSet(viewsets.ModelViewSet): + +class Base: + permission_classes = CONFIG["API_PERMISSION_CLASSES"] + authentication_classes = CONFIG["API_AUTHENTICATION_CLASSES"] + + +class FieldDefinitionViewSet(Base, viewsets.ModelViewSet): queryset = FieldDefinition.objects.all().order_by("pk") serializer_class = FieldDefinitionSerializer - permission_classes = [permissions.IsAuthenticated] + + +class FieldsetViewSet(Base, viewsets.ModelViewSet): + queryset = Fieldset.objects.all().order_by("pk") + serializer_class = FieldsetSerializer + + +class FlexFieldViewSet(Base, viewsets.ModelViewSet): + queryset = FlexField.objects.all().order_by("pk") + serializer_class = FlexFieldSerializer + + +class DataCheckerViewSet(Base, viewsets.ModelViewSet): + queryset = DataChecker.objects.all().order_by("pk") + serializer_class = DataCheckerSerializer + + +class SyncViewSet(Base, viewsets.ModelViewSet): + + def list(self, request, *args, **kwargs): + return HttpResponse(dumpdata_to_buffer(), content_type="application/json") diff --git a/src/hope_flex_fields/config.py b/src/hope_flex_fields/config.py new file mode 100644 index 0000000..6799c81 --- /dev/null +++ b/src/hope_flex_fields/config.py @@ -0,0 +1,13 @@ +from django.conf import settings +from django.utils.module_loading import import_string + +CONFIG = { + "API_PERMISSION_CLASSES": ["rest_framework.permissions.IsAuthenticated"], + "API_AUTHENTICATION_CLASSES": ["rest_framework.authentication.BasicAuthentication"], + "API_TOKEN": "", + "MASTER_URL": None, +} + +CONFIG.update(**getattr(settings, "FLEX_FIELDS_CONFIG", {})) +for entry in ["API_AUTHENTICATION_CLASSES", "API_PERMISSION_CLASSES"]: + CONFIG[entry] = [import_string(m) for m in CONFIG[entry]] diff --git a/src/hope_flex_fields/models/base.py b/src/hope_flex_fields/models/base.py index ee3abff..87231fd 100644 --- a/src/hope_flex_fields/models/base.py +++ b/src/hope_flex_fields/models/base.py @@ -20,6 +20,7 @@ class FlexForm(forms.Form): class AbstractField(models.Model): + last_modified = models.DateTimeField(auto_now=True) name = models.CharField(max_length=255) description = models.TextField(max_length=500, blank=True, null=True, default="") attrs = models.JSONField(default=dict, blank=True, null=False) diff --git a/src/hope_flex_fields/models/datachecker.py b/src/hope_flex_fields/models/datachecker.py index c2ad9ca..d216815 100644 --- a/src/hope_flex_fields/models/datachecker.py +++ b/src/hope_flex_fields/models/datachecker.py @@ -69,6 +69,7 @@ def get_by_natural_key(self, name): class DataCheckerFieldset(models.Model): + last_modified = models.DateTimeField(auto_now=True) checker = models.ForeignKey( "DataChecker", on_delete=models.CASCADE, related_name="members" ) @@ -78,6 +79,7 @@ class DataCheckerFieldset(models.Model): class DataChecker(ValidatorMixin, models.Model): + last_modified = models.DateTimeField(auto_now=True) name = models.CharField(max_length=255, unique=True) description = models.TextField(blank=True) fieldsets = models.ManyToManyField(Fieldset, through=DataCheckerFieldset) diff --git a/src/hope_flex_fields/models/definition.py b/src/hope_flex_fields/models/definition.py index a4c0b3a..11edcda 100644 --- a/src/hope_flex_fields/models/definition.py +++ b/src/hope_flex_fields/models/definition.py @@ -20,6 +20,7 @@ class FieldDefinitionManager(models.Manager): + def get_by_natural_key(self, name): return self.get(name=name) diff --git a/src/hope_flex_fields/models/fieldset.py b/src/hope_flex_fields/models/fieldset.py index 34b494c..2c1cb69 100644 --- a/src/hope_flex_fields/models/fieldset.py +++ b/src/hope_flex_fields/models/fieldset.py @@ -18,6 +18,7 @@ def get_by_natural_key(self, name): class Fieldset(ValidatorMixin, models.Model): + last_modified = models.DateTimeField(auto_now=True) name = models.CharField(max_length=255, unique=True) description = models.TextField(blank=True) extends = models.ForeignKey("self", null=True, blank=True, on_delete=models.CASCADE) diff --git a/src/hope_flex_fields/utils.py b/src/hope_flex_fields/utils.py index b2c857b..334a46c 100644 --- a/src/hope_flex_fields/utils.py +++ b/src/hope_flex_fields/utils.py @@ -1,6 +1,11 @@ import inspect +import io +import tempfile +from io import StringIO +from pathlib import Path from django import forms +from django.core.management import call_command from django.forms.fields import DateTimeFormatsIterator from django.utils.text import slugify @@ -43,3 +48,37 @@ def get_kwargs_from_formfield(field: forms.Field): value = [str(v) for v in value] ret[attr_name] = value return ret + + +def dumpdata_to_buffer(): + buf = StringIO() + call_command( + "dumpdata", + "hope_flex_fields", + use_natural_primary_keys=True, + use_natural_foreign_keys=True, + stdout=buf, + ) + buf.seek(0) + return buf.getvalue() + + +def loaddata_from_buffer(buf): + workdir = Path(".").absolute() + kwargs = { + "dir": workdir, + "prefix": "~LOADDATA", + "suffix": ".json", + "delete": False, + } + with tempfile.NamedTemporaryFile(**kwargs) as fdst: + fdst.write(buf.getvalue()) + fixture = (workdir / fdst.name).absolute() + out = io.StringIO() + try: + call_command("loaddata", fixture, stdout=out, verbosity=3) + except Exception: + raise + finally: + fixture.unlink() + return out.getvalue() diff --git a/tests/admin/data.json b/tests/admin/data.json index fef657b..c8f51ba 100644 --- a/tests/admin/data.json +++ b/tests/admin/data.json @@ -2,6 +2,7 @@ { "model": "hope_flex_fields.fielddefinition", "fields": { + "last_modified": "2024-07-08T13:16:07.112Z", "name": "imported-intfield", "description": "", "attrs": { @@ -15,6 +16,7 @@ { "model": "hope_flex_fields.fielddefinition", "fields": { + "last_modified": "2024-07-08T13:16:07.112Z", "name": "imported-floatfield", "description": "", "attrs": { @@ -28,12 +30,14 @@ { "model": "hope_flex_fields.fieldset", "fields": { - "name": "imported-fieldset" + "name": "imported-fieldset", + "last_modified": "2024-07-08T13:16:07.112Z" } }, { "model": "hope_flex_fields.flexfield", "fields": { + "last_modified": "2024-07-08T13:16:07.112Z", "name": "imported-int", "description": "", "attrs": { @@ -54,6 +58,7 @@ { "model": "hope_flex_fields.flexfield", "fields": { + "last_modified": "2024-07-08T13:16:07.112Z", "name": "imported-float", "description": "", "attrs": { diff --git a/tests/extra/demoapp/demo/api.py b/tests/extra/demoapp/demo/api.py deleted file mode 100644 index e69de29..0000000 diff --git a/tests/extra/demoapp/demo/urls.py b/tests/extra/demoapp/demo/urls.py index 39adc2d..ab84d2b 100644 --- a/tests/extra/demoapp/demo/urls.py +++ b/tests/extra/demoapp/demo/urls.py @@ -20,10 +20,20 @@ from rest_framework import routers -from hope_flex_fields.api.views import FieldDefinitionViewSet +from hope_flex_fields.api.views import ( + DataCheckerViewSet, + FieldDefinitionViewSet, + FieldsetViewSet, + FlexFieldViewSet, + SyncViewSet, +) router = routers.DefaultRouter() -router.register(r"fields", FieldDefinitionViewSet) +router.register(r"field", FieldDefinitionViewSet) +router.register(r"fieldset", FieldsetViewSet) +router.register(r"datachecker", DataCheckerViewSet) +router.register(r"flexfield", FlexFieldViewSet) +router.register(r"sync", SyncViewSet, basename="sync") urlpatterns = [ diff --git a/tests/test_api.py b/tests/test_api.py index cd0e32f..c07dede 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -1,10 +1,86 @@ +from pathlib import Path + +from django import forms + +import pytest from rest_framework.test import APIClient -from testutils.factories import FieldDefinitionFactory +from testutils.factories import DataCheckerFactory + +from hope_flex_fields.utils import loaddata_from_buffer + + +@pytest.fixture +def data(db): + from testutils.factories import ( + DataCheckerFieldsetFactory, + FieldDefinitionFactory, + FieldsetFactory, + FlexFieldFactory, + ) + + fd1 = FieldDefinitionFactory( + field_type=forms.IntegerField, attrs={"min_value": 1, "max_value": 100} + ) + fd2 = FieldDefinitionFactory(field_type=forms.IntegerField, attrs={"min_value": 1}) + fd3 = FieldDefinitionFactory( + field_type=forms.IntegerField, attrs={"max_value": 100} + ) + fd4 = FieldDefinitionFactory(field_type=forms.FloatField, attrs={}) + fd5 = FieldDefinitionFactory(field_type=forms.DateField, attrs={}) + fd6 = FieldDefinitionFactory(field_type=forms.BooleanField) + fd7 = FieldDefinitionFactory( + field_type=forms.ChoiceField, + attrs={"choices": [["a", "a"], ["b", "b"], ["c", "c"]]}, + ) + fs = FieldsetFactory() + FlexFieldFactory(name="int1", field=fd1, fieldset=fs, attrs={"required": False}) + FlexFieldFactory(name="int2", field=fd2, fieldset=fs, attrs={"required": False}) + FlexFieldFactory(name="int3", field=fd3, fieldset=fs, attrs={"required": False}) + FlexFieldFactory(name="float", field=fd4, fieldset=fs, attrs={"required": False}) + FlexFieldFactory(name="date", field=fd5, fieldset=fs, attrs={"required": False}) + FlexFieldFactory(name="bool", field=fd6, fieldset=fs, attrs={"required": False}) + FlexFieldFactory(name="choice", field=fd7, fieldset=fs, attrs={"required": False}) + + dc = DataCheckerFactory() + DataCheckerFieldsetFactory(checker=dc, fieldset=fs, prefix="fs") + return dc + + +def test_fields(admin_user, data): + client = APIClient() + client.force_authenticate(user=admin_user) + response = client.get("http://testserver/api/field/") + assert response.json() + + +def test_flexfield(admin_user, data): + client = APIClient() + client.force_authenticate(user=admin_user) + response = client.get("http://testserver/api/flexfield/") + assert response.json() + + +def test_fieldsets(admin_user, data): + client = APIClient() + client.force_authenticate(user=admin_user) + response = client.get("http://testserver/api/fieldset/") + assert response.json() -def test_fields(admin_user): - FieldDefinitionFactory() +def test_datachecker(admin_user, data): client = APIClient() client.force_authenticate(user=admin_user) - response = client.get("http://testserver/api/fields/") + response = client.get("http://testserver/api/datachecker/") assert response.json() + + +def test_sync(admin_user, data, mocked_responses): + data = (Path(__file__).parent / "admin/data.json").read_text() + + client = APIClient() + client.force_authenticate(user=admin_user) + + mocked_responses.get("http://testserver/api/sync/", json=data) + response = client.get("http://testserver/api/sync/") + out = loaddata_from_buffer(response) + assert "Processed 33 object(s).\nInstalled 33 object(s) from 1 fixture(s)" in out