Update serializer with recursive cleaning and removing null values

CHANGELOG.md

@@ -1,3 +1,7 @@
+## 2.0.10
+
+* Update serializer with recursive cleaning and removing null values
+
 ## 2.0.9
 
 * Update serializer with removing empty dicts/lists and transforming empty dicts into nulls in lists

fhirpy/__init__.py

@@ -1,7 +1,7 @@
 from .lib import AsyncFHIRClient, SyncFHIRClient
 
 __title__ = "fhir-py"
-__version__ = "2.0.9"
+__version__ = "2.0.10"
 __author__ = "beda.software"
 __license__ = "None"
 __copyright__ = "Copyright 2024 beda.software"

fhirpy/base/lib_async.py

@@ -111,7 +111,7 @@ async def save(
         # _as_dict is a private api used internally
         _as_dict: bool = False,
     ) -> Union[TResource, Any]:
-        data = serialize(self.dump(resource), remove_nulls=fields is None)
+        data = serialize(self.dump(resource), drop_nulls_from_dicts=fields is None)
         if fields:
             if not resource.id:
                 raise TypeError("Resource `id` is required for update operation")
@@ -171,7 +171,7 @@ async def patch(
         response_data = await self._do_request(
             "patch",
             f"{resource_type}/{resource_id}",
-            data=serialize(self.dump(kwargs), remove_nulls=False),
+            data=serialize(self.dump(kwargs), drop_nulls_from_dicts=False),
         )
 
         if custom_resource_class:
@@ -473,7 +473,7 @@ async def patch(self, _resource: Any = None, **kwargs) -> TResource:
         )
         data = serialize(
             self.client.dump(_resource if _resource is not None else kwargs),
-            remove_nulls=False,
+            drop_nulls_from_dicts=False,
         )
         response_data = await self.client._do_request(
             "PATCH", self.resource_type, data, self.params

fhirpy/base/lib_sync.py

@@ -111,7 +111,7 @@ def save(
         # _as_dict is a private api used internally
         _as_dict: bool = False,
     ) -> Union[TResource, Any]:
-        data = serialize(self.dump(resource), remove_nulls=fields is None)
+        data = serialize(self.dump(resource), drop_nulls_from_dicts=fields is None)
         if fields:
             if not resource.id:
                 raise TypeError("Resource `id` is required for update operation")
@@ -167,7 +167,7 @@ def patch(
         response_data = self._do_request(
             "patch",
             f"{resource_type}/{resource_id}",
-            data=serialize(self.dump(kwargs), remove_nulls=False),
+            data=serialize(self.dump(kwargs), drop_nulls_from_dicts=False),
         )
 
         if custom_resource_class:
@@ -473,7 +473,7 @@ def patch(self, _resource: Any = None, **kwargs) -> TResource:
 
         data = serialize(
             self.client.dump(_resource if _resource is not None else kwargs),
-            remove_nulls=False,
+            drop_nulls_from_dicts=False,
         )
         response_data = self.client._do_request("patch", self.resource_type, data, self.params)
         return self._dict_to_resource(response_data)

fhirpy/base/resource.py

@@ -5,7 +5,13 @@
 from fhirpy.base.client import TClient
 from fhirpy.base.exceptions import ResourceNotFound
 from fhirpy.base.resource_protocol import TReference, TResource, get_resource_path
-from fhirpy.base.utils import convert_values, get_by_path, parse_path
+from fhirpy.base.utils import (
+    clean_empty_values,
+    convert_values,
+    get_by_path,
+    parse_path,
+    remove_nulls_from_dicts,
+)
 
 
 class AbstractResource(Generic[TClient], dict, ABC):
@@ -251,13 +257,7 @@ def is_local(self):
         pass
 
 
-def serialize(resource: Any, remove_nulls=True) -> dict:
-    """
-    * empty dicts/lists are always removed
-    * nulls are removed only for dicts if `remove_nulls` is set
-    * in lists empty dicts are transformed into nulls because nulls are used for alignment
-    """
-
+def serialize(resource: Any, drop_nulls_from_dicts=True) -> dict:
     def convert_fn(item):
         if isinstance(item, BaseResource):
             return serialize(item.to_reference()), True
@@ -267,40 +267,16 @@ def convert_fn(item):
 
         if _is_serializable_dict_like(item):
             # Handle dict-serializable structures like pydantic Model
-            item = _remove_dict_empty_values(dict(item))
-
-            if remove_nulls:
-                return _remove_nulls(item), False
-            return item, False
-
-        if isinstance(item, list):
-            return _transform_list_empty_values_to_null(item), False
+            return dict(item), False
 
         return item, False
 
-    return convert_values(dict(resource), convert_fn)
-
-
-def _remove_dict_empty_values(d: dict):
-    return {key: value for key, value in d.items() if not _is_empty(value)}
-
-
-def _transform_list_empty_values_to_null(d: list):
-    return [None if _is_empty(value) else value for value in d]
-
-
-def _remove_nulls(d: dict):
-    return {key: value for key, value in d.items() if not _is_null(value)}
-
-
-def _is_empty(d: Any):
-    if isinstance(d, (dict, list)):
-        return not d
-    return False
+    converted_values = convert_values(dict(resource), convert_fn)
 
+    if drop_nulls_from_dicts:
+        converted_values = remove_nulls_from_dicts(converted_values)
 
-def _is_null(d: Any):
-    return d is None
+    return clean_empty_values(converted_values)
 
 
 def _is_serializable_dict_like(item):

fhirpy/base/utils.py

@@ -1,4 +1,5 @@
 import reprlib
+from typing import Any
 from urllib.parse import parse_qs, quote, urlencode, urlparse
 
 from yarl import URL
@@ -218,3 +219,34 @@ def set_by_path(obj, path, value):
 
 def remove_prefix(s, prefix):
     return s[len(prefix) :] if s.startswith(prefix) else s
+
+
+def clean_empty_values(data: Any):
+    if isinstance(data, dict):
+        cleaned_dict = {k: clean_empty_values(v) for k, v in data.items()}
+        return {k: v for k, v in cleaned_dict.items() if not _is_empty(v)}
+
+    if isinstance(data, list):
+        return [clean_empty_values(item) if not _is_empty(item) else None for item in data]
+
+    return data
+
+
+def _is_empty(d: Any):
+    if isinstance(d, (dict, list)):
+        return not d
+    return False
+
+
+def remove_nulls_from_dicts(data: Any):
+    if isinstance(data, dict):
+        return {k: remove_nulls_from_dicts(v) for k, v in data.items() if not _is_null(v)}
+
+    if isinstance(data, list):
+        return [remove_nulls_from_dicts(item) for item in data]
+
+    return data
+
+
+def _is_null(d: Any):
+    return d is None

tests/test_utils.py

@@ -3,6 +3,7 @@
 from pydantic import BaseModel
 
 from fhirpy.base.resource_protocol import get_resource_type_from_class
+from fhirpy.base.utils import clean_empty_values, remove_nulls_from_dicts
 
 
 def test_get_resource_type_from_class_for_pydantic_model_value():
@@ -17,3 +18,25 @@ class Patient(BaseModel):
         resourceType: Literal["Patient"]  # noqa: N815
 
     assert get_resource_type_from_class(Patient) == "Patient"
+
+
+def test_remove_nulls_from_dicts():
+    assert remove_nulls_from_dicts({}) == {}
+    assert remove_nulls_from_dicts({"item": []}) == {"item": []}
+    assert remove_nulls_from_dicts({"item": [None]}) == {"item": [None]}
+    assert remove_nulls_from_dicts({"item": [None, {"item": None}]}) == {"item": [None, {}]}
+    assert remove_nulls_from_dicts({"item": [None, {"item": None}, {}]}) == {"item": [None, {}, {}]}
+
+
+def test_clean_empty_values():
+    assert clean_empty_values({}) == {}
+    assert clean_empty_values({"str": ""}) == {"str": ""}
+    assert clean_empty_values({"nested": {"nested2": [{}]}}) == {"nested": {"nested2": [None]}}
+    assert clean_empty_values({"nested": {"nested2": {}}}) == {}
+    assert clean_empty_values({"item": []}) == {}
+    assert clean_empty_values({"item": []}) == {}
+    assert clean_empty_values({"item": [None]}) == {"item": [None]}
+    assert clean_empty_values({"item": [None, {"item": None}]}) == {"item": [None, {"item": None}]}
+    assert clean_empty_values({"item": [None, {"item": None}, {}]}) == {
+        "item": [None, {"item": None}, None]
+    }