diff --git a/docs/guide/serializers.rst b/docs/guide/serializers.rst index 722da38..9b9a3b3 100644 --- a/docs/guide/serializers.rst +++ b/docs/guide/serializers.rst @@ -110,10 +110,8 @@ All field classes accept this set of arguments: intead of relying on param labels.* * **source** *(str, optional):* name of internal object key/attribute - that will be passed to field's on ``.to_representation(value)`` call. - Special ``'*'`` value is allowed that will pass whole object to - field when making representation. If not set then default source will - be a field name used as a serializer's attribute. + that will be passed to field's on ``.to_representation(value)`` call. If not + set then default source is a field name used as a serializer's attribute. * **validators** *(list, optional):* list of validator callables. @@ -121,14 +119,12 @@ All field classes accept this set of arguments: of given type objects -.. note:: - - ``source='*'`` is in fact a dirty workaround and will not work well - on validation when new object instances needs to be created/updated - using POST/PUT requests. This works quite well with simple retrieve/list - type resources but in more sophisticated cases it is better to use - custom object properties as sources to encapsulate such fields. +.. versionchanged:: 1.0.0 + Fields no no longer have special case treatment for ``source='*'`` argument. + If you want to access multiple object keys and values within single + serializer field please refer to :ref:`guide-field-attribute-access` section + of this document. .. _field-validation: @@ -143,9 +139,9 @@ in order to provide correct HTTP responses each validator shoud raise .. note:: Concept of validation for fields is understood here as a process of checking - if data of valid type (successfully parsed/processed by - ``.from_representation`` handler) does meet some other constraints - (lenght, bounds, unique, etc). + if data of valid type (i.e. data that was successfully parsed/processed by + ``.from_representation()`` handler) does meet some other constraints + (lenght, bounds, uniquess, etc). Example of simple validator usage: @@ -174,31 +170,53 @@ Resource validation ~~~~~~~~~~~~~~~~~~~ In most cases field level validation is all that you need but sometimes you -need to perfom obejct level validation that needs to access multiple fields -that are already deserialized and validated. Suggested way to do this in -graceful is to override serializer's ``.validate()`` method and raise -:class:`graceful.errors.ValidationError` when your validation fails. This -exception will be then automatically translated to HTTP Bad Request response -on resource-level handlers. Here is example: +need to perfom validation on whole resource representation or deserialized +object. It is possible to access multiple fields that were already deserialized +and pre-validated directly from serializer class. + +You can provide your own object-level serialization handler using serializer's +``validate()`` method. This method accepts two arguments: + +* **object_dict** *(dict):* it is deserialized object dictionary that already + passed validation. Field sources instead of their representation names are + used as its keys. + +* **partial** *(bool):* it is set to ``True`` only on partial object updates + (e.g. on ``PATCH`` requests). If you plan to support partial resource + modification you should check this field and verify if you object has + all the existing keys. + +If your validation fails you should raise the +:class:`graceful.errors.ValidationError` exception. Following is the example +of resource serializer with custom object-level validation: .. code-block:: python class DrinkSerializer(): - alcohol = StringField("main ingredient", required=True) - mixed_with = StringField("what makes it tasty", required=True) + alcohol = StringField("main ingredient") + mixed_with = StringField("what makes it tasty") + + def validate(self, object_dict, partial): + # note: always make sure to call super `validate_object()` + # to make sure that per-field validation is enabled. - def validate(self, object_dict, partial=False): - # note: always make sure to call super `validate()` - # so whole validation of fields works as expected - super().validate(object_dict, partial) + if partial and any([ + 'alcohol' in object_dict, + 'mixed_with' in object_dict, + ]): + raise ValidationError( + "bartender refused to change ingredients" + ) # here is a place for your own validation if ( object_dict['alcohol'] == 'whisky' and object_dict['mixed_with'] == 'cola' ): - raise ValidationError("bartender refused!') + raise ValidationError( + "bartender refused to mix whisky with cola!" + ) Custom fields @@ -228,3 +246,88 @@ as a serialized JSON string that we would like to (de)serialize: def to_representation(data): return json.loads(data) +.. _guide-field-attribute-access: + + +Accessing multiple fields at once +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Sometimes you need to access multiple fields of internal object instance at +once in order to properly represent data in your API. This is very common when +interacting with legacy services/components that cannot be changed or when +your storage engine simply does not allow to store nested or structured objects. + +Serializers generally work on per-field basis and allow only to translate field +names between representation and application internal objects. In order to +manipulate multiple representation or internal object instance keys within the +single field you need to create custom field class and override one or more +of following methods: + +* ``read_instance(self, instance, key_or_attribute)``: read value from the + object instance before serialization. The return value will be later passed + as an argument to ``to_representation()`` method. The ``key_or_attribute`` + argument is field's name or source (if ``source`` explicitly specified). + Base implementation defaults to dictionary key lookup or object attribute + lookup. +* ``read_representation(self, representation, key_or_attribute)``: read value + from the object instance before deserialization. The return value will be + later passed as an argument to ``from_representation()`` method. The + ``key_or_attribute`` argument the field's name. Base implementation defaults + to dictionary key lookup or object attribute lookup. +* ``update_instance(self, instance, key_or_attribute, value)``: update the + content of object instance after deserialization. The ``value`` argument is + the return value of ``from_representation()`` method. The + ``key_or_attribute`` argument the field's name or source (if ``source`` + explicitly specified). Base implementation defaults to dictionary key + assignment or object attribute assignment. +* ``update_representation(self, representation, key_or_attribute, value)``: + update the content of representation instance after serialization. + The ``value`` argument is the return value of ``to_representation()`` method. + The ``key_or_attribute`` argument the field's name. Base implementation + defaults to dictionary key assignment or object attribute assignment. + +To better explain how to use these methods let's assume that due to some +storage backend constraints we cannot save nested dictionaries. All of fields +of some nested object will have to be stored under separate keys but we still +want to present this to the user as separate nested dictionary. And of course +we want to support both writes and saves. + +.. code-block:: python + + class OwnerField(RawField): + def from_representation(self, data): + if not isinstance(data, dict): + raise ValueError("expected object") + + return { + 'owner_name': data.get('name'), + 'owner_age': data.get('age'), + } + + def to_representation(self, value): + return { + 'age': value.get('owner_age'), + 'name': value.get('owner_name'), + } + + def validate(self, value): + print(value) + if 'owner_age' not in value or not isinstance(value['owner_age'], int): + raise ValidationError("invalid owner age") + + if 'owner_name' not in value: + raise ValidationError("invalid owner name") + + def update_instance(self, instance, attribute_or_key, value): + # we assume that instance is always a dictionary so we can + # use the .update() method + instance.update(value) + + def read_instance(self, instance, attribute_or_key): + # .to_representation() method requires acces to whole object + # dictionary so we have to return whole object. + return instance + + +Similar approach may be used to flatten nested objects into more compact +representations. diff --git a/src/graceful/errors.py b/src/graceful/errors.py index 110095e..55fb5d7 100644 --- a/src/graceful/errors.py +++ b/src/graceful/errors.py @@ -34,7 +34,7 @@ def _get_description(self): "forbidden: {}".format(self.forbidden) if self.forbidden else "" ), - "invalid: {}:".format(self.invalid) if self.invalid else "", + "invalid: {}".format(self.invalid) if self.invalid else "", ( "failed to parse: {}".format(self.failed) if self.failed else "" diff --git a/src/graceful/fields.py b/src/graceful/fields.py index b306aa2..6a162f2 100644 --- a/src/graceful/fields.py +++ b/src/graceful/fields.py @@ -1,4 +1,5 @@ import inspect +from collections import Mapping, MutableMapping from graceful.validators import min_validator, max_validator @@ -60,6 +61,14 @@ def from_representation(self, data): def to_representation(self, value): return ["True", "False"][value] + .. versionchanged:: 1.0.0 + Field instances no longer support ``source="*"`` to access whole object + for the purpose of representation serialization/deserialization. If you + want to access multiple fields of object instance and/or its + representation you must override ``read_*`` and ``update_*`` methods in + your custom field classes (see :ref:`guide-field-attribute-access` + section in documentation). + """ #: Two-tuple ``(label, url)`` pointing to represented type specification @@ -172,6 +181,66 @@ def validate(self, value): for validator in self.validators: validator(value) + def update_instance(self, instance, attribute_or_key, value): + """Update object instance after deserialization. + + Args: + instance (object): dictionary or object after serialization. + attribute_or_key (str): field's name or source (if ``source`` + explicitly specified). + value (object): return value from ``from_representation`` method. + """ + if isinstance(instance, MutableMapping): + instance[attribute_or_key] = value + else: + setattr(instance, attribute_or_key, value) + + def read_instance(self, instance, attribute_or_key): + """Read value from the object instance before serialization. + + Args: + instance (object): dictionary or object before serialization. + attribute_or_key (str): field's name or source (if ``source`` + explicitly specified). + + Returns: + The value that will be later passed as an argument to + ``to_representation()`` method. + """ + if isinstance(instance, Mapping): + return instance.get(attribute_or_key, None) + + return getattr(instance, attribute_or_key, None) + + def update_representation(self, representation, attribute_or_key, value): + """Update representation after field serialization. + + Args: + instance (object): representation object. + attribute_or_key (str): field's name. + value (object): return value from ``to_representation`` method. + """ + if isinstance(representation, MutableMapping): + representation[attribute_or_key] = value + else: + setattr(representation, attribute_or_key, value) + + def read_representation(self, representation, attribute_or_key): + """Read value from the representation before deserialization. + + Args: + instance (object): dictionary or object before deserialization. + attribute_or_key (str): field's name. + + Returns: + The value that will be later passed as an argument to + ``from_representation()`` method. + """ + if isinstance(representation, Mapping): + return representation.get(attribute_or_key, None) + + return getattr(representation, attribute_or_key, None) + class RawField(BaseField): """Represents raw field subtype. diff --git a/src/graceful/resources/base.py b/src/graceful/resources/base.py index 5364c7a..f60bce7 100644 --- a/src/graceful/resources/base.py +++ b/src/graceful/resources/base.py @@ -424,11 +424,11 @@ def require_validated(self, req, partial=False, bulk=False): try: for representation in representations: - object_dict = self.serializer.from_representation( - representation + object_dicts.append( + self.serializer.from_representation( + representation, partial + ) ) - self.serializer.validate(object_dict, partial) - object_dicts.append(object_dict) except DeserializationError as err: # when working on Resource we know that we can finally raise diff --git a/src/graceful/serializers.py b/src/graceful/serializers.py index e83a76a..3754bdf 100644 --- a/src/graceful/serializers.py +++ b/src/graceful/serializers.py @@ -1,22 +1,9 @@ from collections import OrderedDict -from collections.abc import Mapping, MutableMapping from graceful.errors import DeserializationError from graceful.fields import BaseField -def _source(name, field): - """Translate field name to instance source name with respect to source=*. - - .. deprecated:: - This function will be removed in 1.0.0. - """ - if field.source == '*': - return name - else: - return field.source or name - - class MetaSerializer(type): """Metaclass for handling serialization with field objects.""" @@ -93,12 +80,15 @@ class CatSerializer(BaseSerializer): """ + instance_factory = dict + representation_factory = dict + @property def fields(self): """Return dictionary of field definition objects of this serializer.""" return getattr(self, self.__class__._fields_storage_key) - def to_representation(self, obj): + def to_representation(self, instance): """Convert given internal object instance into representation dict. Representation dict may be later serialized to the content-type @@ -109,223 +99,164 @@ def to_representation(self, obj): one using ``field.to_representation()`` method. Args: - obj (object): internal object that needs to be represented + instance (object): internal object that needs to be represented Returns: dict: representation dictionary """ - representation = {} + representation = self.representation_factory() for name, field in self.fields.items(): # note fields do not know their names in source representation # but may know what attribute they target from source object - attribute = self.get_attribute(obj, field.source or name) + attribute = field.read_instance(instance, field.source or name) if attribute is None: # Skip none attributes so fields do not have to deal with them - representation[name] = [] if field.many else None + field.update_representation( + representation, name, [] if field.many else None + ) + elif field.many: - representation[name] = [ - field.to_representation(item) for item in attribute - ] + field.update_representation( + representation, name, [ + field.to_representation(item) for item in attribute + ] + ) else: - representation[name] = field.to_representation(attribute) + field.update_representation( + representation, name, field.to_representation(attribute) + ) return representation - def from_representation(self, representation): + def from_representation(self, representation, partial=False): """Convert given representation dict into internal object. Internal object is simply a dictionary of values with respect to field - sources. - - This does not check if all required fields exist or values are - valid in terms of value validation - (see: :meth:`BaseField.validate()`) but still requires all of passed - representation values to be well formed representation (success call - to ``field.from_representation``). - - In case of malformed representation it will run additional validation - only to provide a full detailed exception about all that might be - wrong with provided representation. + sources. This method does not quit on first failure to make sure that + as many as possible issues will be presented to the client. Args: representation (dict): dictionary with field representation values Raises: - DeserializationError: when at least one representation field - is not formed as expected by field object. Information - about additional forbidden/missing/invalid fields is provided - as well. + DeserializationError: when at least of these issues occurs: + + * if at least one of representation field is not formed as + expected by the field object (``ValueError`` raised by + field's ``from_representation()`` method). + * if ``partial=False`` and at least one representation fields + is missing. + * if any non-existing or non-writable field is provided in + representation. + * if any custom field validator fails (raises + ``ValidationError`` or ``ValueError`` exception) + + ValidationError: on custom user validation checks implemented with + ``validate()`` handler. """ - object_dict = {} + instance = self.instance_factory() + failed = {} + invalid = {} + + # note: we need to perform validation on whole representation before + # validation because there is no + missing, forbidden = self._validate_representation( + representation, partial + ) for name, field in self.fields.items(): if name not in representation: continue try: + raw_entry = field.read_representation(representation, name) if ( # note: we cannot check for any sequence or iterable # because of strings and nested dicts. - not isinstance(representation[name], (list, tuple)) and + not isinstance(raw_entry, (list, tuple)) and field.many ): raise ValueError("field should be sequence") - source = _source(name, field) - value = representation[name] + field_values = [ + field.from_representation(item) + for item in ([raw_entry] if not field.many else raw_entry) + ] - if field.many: - object_dict[source] = [ - field.from_representation(single_value) - for single_value in value - ] - else: - object_dict[source] = field.from_representation(value) + for value in field_values: + field.validate(value) + + field.update_instance( + instance, + # If field does not have explicit source string then use + # its name. + field.source or name, + # many=True fields require special care + field_values if field.many else field_values[0] + ) except ValueError as err: failed[name] = str(err) - if failed: - # if failed to parse we eagerly perform validation so full - # information about what is wrong will be returned - try: - self.validate(object_dict) - # note: this exception can be reached with partial==True - # since do not support partial updates yet this has 'no cover' - raise DeserializationError() # pragma: no cover - except DeserializationError as err: - err.failed = failed - raise + if any([missing, forbidden, invalid, failed]): + raise DeserializationError(missing, forbidden, invalid, failed) - return object_dict + # note: expected to raise ValidationError. It is extra feature handle + # so we dont try hard to merge wit previous errors. + self.validate(instance, partial) - def validate(self, object_dict, partial=False): - """Validate given internal object returned by ``to_representation()``. + return instance - Internal object is validated against missing/forbidden/invalid fields - values using fields definitions defined in serializer. + def _validate_representation(self, representation, partial=False): + """Validate resource representation fieldwise. - Args: - object_dict (dict): internal object dictionart to perform - to validate - partial (bool): if set to True then incomplete object_dict - is accepter and will not raise any exceptions when one - of fields is missing + Check if object has all required fields to support full or partial + object modification/creation and ensure it does not contain any + forbidden fields. - Raises: - DeserializationError: + Returns: + A ``(missing, forbidden)`` tuple with lists indicating fields that + failed validation. """ - # we are working on object_dict not an representation so there - # is a need to annotate sources differently - sources = { - _source(name, field): field - for name, field in self.fields.items() - } - - # note: we are checking for all mising and invalid fields so we can - # return exception with all fields that are missing and should - # exist instead of single one missing = [ - name for name, field in sources.items() - if all((not partial, name not in object_dict, not field.read_only)) + name for name, field in self.fields.items() + if all(( + not partial, + name not in representation, + not field.read_only + )) ] forbidden = [ - name for name in object_dict - if any((name not in sources, sources[name].read_only)) + name for name in representation + if name not in self.fields or self.fields[name].read_only ] - invalid = {} - for name, value in object_dict.items(): - try: - field = sources[name] - - if field.many: - for single_value in value: - field.validate(single_value) - else: - field.validate(value) - - except ValueError as err: - invalid[name] = str(err) - - if any([missing, forbidden, invalid]): - # note: We have validated internal object instance but need to - # inform the user about problems with his representation. - # This is why we have to do this dirty transformation. - # note: This will be removed in 1.0.0 where we change how - # validation works and where we remove star-like fields. - # refs: #42 (https://github.com/swistakm/graceful/issues/42) - sources_to_field_names = { - _source(name, field): name - for name, field in self.fields.items() - } - - def _(names): - if isinstance(names, list): - return [ - sources_to_field_names.get(name, name) - for name in names - ] - elif isinstance(names, dict): - return { - sources_to_field_names.get(name, name): value - for name, value in names.items() - } - else: - return names # pragma: nocover - - raise DeserializationError(_(missing), _(forbidden), _(invalid)) - - def get_attribute(self, obj, attr): - """Get attribute of given object instance. + return missing, forbidden - Reason for existence of this method is the fact that 'attribute' can - be also object's key from if is a dict or any other kind of mapping. + def validate(self, instance, partial=False): + """Validate given internal object. - Note: it will return None if attribute key does not exist + Internal object is a dictionary that have sucesfully passed general + validation against missing/forbidden fields and was checked with + per-field custom validators. Args: - obj (object): internal object to retrieve data from - - Returns: - internal object's key value or attribute - - """ - # '*' is a special wildcard character that means whole object - # is passed - if attr == '*': - return obj - - # if this is any mapping then instead of attributes use keys - if isinstance(obj, Mapping): - return obj.get(attr, None) - - return getattr(obj, attr, None) - - def set_attribute(self, obj, attr, value): - """Set value of attribute in given object instance. - - Reason for existence of this method is the fact that 'attribute' can - be also a object's key if it is a dict or any other kind of mapping. - - Args: - obj (object): object instance to modify - attr (str): attribute (or key) to change - value: value to set + instance (dict): internal object instance to be validated. + partial (bool): if set to True then incomplete instance + is accepted (e.g. on PATCH requests) so it is possible that + not every field is available. + Raises: + ValidationError: raised when deserialized object does not meet some + user-defined contraints. """ - # if this is any mutable mapping then instead of attributes use keys - if isinstance(obj, MutableMapping): - obj[attr] = value - else: - setattr(obj, attr, value) def describe(self): """Describe all serialized fields. diff --git a/tests/test_resources.py b/tests/test_resources.py index 0869a18..4ae492f 100644 --- a/tests/test_resources.py +++ b/tests/test_resources.py @@ -398,10 +398,10 @@ class TestSerializer(BaseSerializer): one = StringField("one different than two") two = StringField("two different than one") - def validate(self, object_dict, partial=False): - super().validate(object_dict, partial) + def validate(self, instance, partial=False): + super().validate(instance, partial) # possible use case: kind of uniqueness relationship - if object_dict['one'] == object_dict['two']: + if instance['one'] == instance['two']: raise ValidationError("one must be different than two") class TestResource(Resource): diff --git a/tests/test_serializers.py b/tests/test_serializers.py index 5e7b353..ea91ea4 100644 --- a/tests/test_serializers.py +++ b/tests/test_serializers.py @@ -158,74 +158,6 @@ class SomeConcreteSerializer(BaseSerializer): assert recreated == {"_name": "John", "_address": "US"} -def test_serializer_set_attribute(): - serializer = BaseSerializer() - - # test dict keys are treated as attributes - instance = {} - serializer.set_attribute(instance, 'foo', 'bar') - assert instance == {'foo': 'bar'} - - # test normal objects atrributes are attributes indeed - # in scope of this method - class SomeObject: - def __init__(self): - self.foo = None - - instance = SomeObject() - serializer.set_attribute(instance, 'foo', 'bar') - assert instance.foo == 'bar' - - -def test_serializer_get_attribute(): - serializer = BaseSerializer() - - # test dict keys are treated as attributes - instance = {'foo': 'bar'} - assert serializer.get_attribute(instance, 'foo') == 'bar' - - # test normal objects atrributes are attributes indeed - # in scope of this method - class SomeObject: - def __init__(self): - self.foo = 'bar' - - instance = SomeObject() - assert serializer.get_attribute(instance, 'foo') == 'bar' - - # test that getting non existent attribute returns None - assert serializer.get_attribute(instance, 'nonexistens') is None - - -def test_serializer_source_wildcard(): - """ - Test that '*' wildcard causes whole instance is returned on get attribute - """ - serializer = BaseSerializer() - - instance = {"foo", "bar"} - assert serializer.get_attribute(instance, '*') == instance - - -def test_serializer_source_field_with_wildcard(): - class ExampleSerializer(BaseSerializer): - starfield = ExampleField( - details='whole object instance goes here', - source='*', - ) - - serializer = ExampleSerializer() - instance = {'foo': 'bar'} - representation = {"starfield": "bizbaz"} - - assert serializer.to_representation( - instance - )['starfield'] == instance - assert serializer.from_representation( - representation - )['starfield'] == representation["starfield"] - - def test_serializer_describe(): """ Test that serializers are self-describing """ @@ -287,6 +219,6 @@ class ExampleSerializer(BaseSerializer): serializer = ExampleSerializer() with pytest.raises(ValueError): - serializer.validate(invalid) + serializer.from_representation(invalid) - serializer.validate(valid) + serializer.from_representation(valid)