Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

json: Improve tolerance to uncovered C members during JSON encoding #53333

Closed
wants to merge 4 commits into from
Closed

json: Improve tolerance to uncovered C members during JSON encoding #53333

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
0c3bb8f
sys: Introduce SIZEOF_FIELD helper macro in util.h
lucasdietrich Jun 27, 2023
e3963a3
json: Introduce struct_size descriptor member
lucasdietrich Jun 30, 2023
0b5133e
json: Improve tolerance to uncovered C members during JSON encoding
lucasdietrich Jul 26, 2023
5b088a1
test: Add unused members to test structs in JSON test suite
lucasdietrich Jun 27, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
47 changes: 21 additions & 26 deletions include/zephyr/data/json.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -79,27 +79,23 @@ struct json_obj_token {
struct json_obj_descr {
const char *field_name;

/* Alignment can be 1, 2, 4, or 8. The macros to create
* a struct json_obj_descr will store the alignment's
* power of 2 in order to keep this value in the 0-3 range
* and thus use only 2 bits.
*/
uint32_t align_shift : 2;

/* 127 characters is more than enough for a field name. */
uint32_t field_name_len : 7;
uint32_t field_name_len : 8;

/* Valid values here (enum json_tokens): JSON_TOK_STRING,
* JSON_TOK_NUMBER, JSON_TOK_TRUE, JSON_TOK_FALSE,
* JSON_TOK_OBJECT_START, JSON_TOK_ARRAY_START. (All others
* ignored.) Maximum value is '}' (125), so this has to be 7 bits
* long.
*/
uint32_t type : 7;
uint32_t type : 8;

/* 65535 bytes is more than enough for many JSON payloads. */
uint32_t offset : 16;

/* Size of the structure being described by the "object" descriptor */
size_t struct_size;
pdgendt marked this conversation as resolved.
Show resolved Hide resolved

union {
struct {
const struct json_obj_descr *sub_descr;
Expand Down Expand Up @@ -127,10 +123,6 @@ struct json_obj_descr {
typedef int (*json_append_bytes_t)(const char *bytes, size_t len,
void *data);

#define Z_ALIGN_SHIFT(type) (__alignof__(type) == 1 ? 0 : \
__alignof__(type) == 2 ? 1 : \
__alignof__(type) == 4 ? 2 : 3)

/**
* @brief Helper macro to declare a descriptor for supported primitive
* values.
Expand All @@ -154,10 +146,10 @@ typedef int (*json_append_bytes_t)(const char *bytes, size_t len,
#define JSON_OBJ_DESCR_PRIM(struct_, field_name_, type_) \
{ \
.field_name = (#field_name_), \
.align_shift = Z_ALIGN_SHIFT(struct_), \
.field_name_len = sizeof(#field_name_) - 1, \
.type = type_, \
.offset = offsetof(struct_, field_name_), \
.struct_size = 0u, \
}

/**
Expand Down Expand Up @@ -187,10 +179,10 @@ typedef int (*json_append_bytes_t)(const char *bytes, size_t len,
#define JSON_OBJ_DESCR_OBJECT(struct_, field_name_, sub_descr_) \
{ \
.field_name = (#field_name_), \
.align_shift = Z_ALIGN_SHIFT(struct_), \
.field_name_len = (sizeof(#field_name_) - 1), \
.type = JSON_TOK_OBJECT_START, \
.offset = offsetof(struct_, field_name_), \
.struct_size = sizeof(struct_), \
{ \
.object = { \
.sub_descr = sub_descr_, \
Expand All @@ -212,9 +204,9 @@ typedef int (*json_append_bytes_t)(const char *bytes, size_t len,
(const struct json_obj_descr[]) \
{ \
{ \
.align_shift = Z_ALIGN_SHIFT(struct_), \
.type = elem_type_, \
.offset = offsetof(struct_, len_field_), \
.struct_size = 0u, \
union_ \
} \
}
Expand All @@ -238,8 +230,10 @@ typedef int (*json_append_bytes_t)(const char *bytes, size_t len,
*
* @param elem_descr_ Element descriptor, pointer to a descriptor array
* @param elem_descr_len_ Number of elements in elem_descr_
* @param struct_size_ Size of the struct
*/
#define Z_JSON_DESCR_OBJ(elem_descr_, elem_descr_len_) \
#define Z_JSON_DESCR_OBJ(elem_descr_, elem_descr_len_, struct_size_) \
.struct_size = struct_size_, \
{ \
.object = { \
.sub_descr = elem_descr_, \
Expand Down Expand Up @@ -273,10 +267,10 @@ typedef int (*json_append_bytes_t)(const char *bytes, size_t len,
len_field_, elem_type_) \
{ \
.field_name = (#field_name_), \
.align_shift = Z_ALIGN_SHIFT(struct_), \
.field_name_len = sizeof(#field_name_) - 1, \
.type = JSON_TOK_ARRAY_START, \
.offset = offsetof(struct_, field_name_), \
.struct_size = sizeof(struct_), \
{ \
.array = { \
.element_descr = Z_JSON_ELEMENT_DESCR(struct_, len_field_, \
Expand Down Expand Up @@ -324,15 +318,16 @@ typedef int (*json_append_bytes_t)(const char *bytes, size_t len,
len_field_, elem_descr_, elem_descr_len_) \
{ \
.field_name = (#field_name_), \
.align_shift = Z_ALIGN_SHIFT(struct_), \
.field_name_len = sizeof(#field_name_) - 1, \
.type = JSON_TOK_ARRAY_START, \
.offset = offsetof(struct_, field_name_), \
.struct_size = sizeof(struct_), \
{ \
.array = { \
.element_descr = Z_JSON_ELEMENT_DESCR(struct_, len_field_, \
JSON_TOK_OBJECT_START, \
Z_JSON_DESCR_OBJ(elem_descr_, elem_descr_len_)), \
Z_JSON_DESCR_OBJ(elem_descr_, elem_descr_len_, \
SIZEOF_FIELD(struct_, field_name_[0]))), \
.n_elements = (max_len_), \
}, \
}, \
Expand Down Expand Up @@ -385,10 +380,10 @@ typedef int (*json_append_bytes_t)(const char *bytes, size_t len,
elem_descr_, elem_descr_len_) \
{ \
.field_name = (#field_name_), \
.align_shift = Z_ALIGN_SHIFT(struct_), \
.field_name_len = sizeof(#field_name_) - 1, \
.type = JSON_TOK_ARRAY_START, \
.offset = offsetof(struct_, field_name_), \
.struct_size = sizeof(struct_), \
{ \
.array = { \
.element_descr = Z_JSON_ELEMENT_DESCR( \
Expand Down Expand Up @@ -456,7 +451,6 @@ typedef int (*json_append_bytes_t)(const char *bytes, size_t len,
struct_field_name_, type_) \
{ \
.field_name = (json_field_name_), \
.align_shift = Z_ALIGN_SHIFT(struct_), \
.field_name_len = sizeof(json_field_name_) - 1, \
.type = type_, \
.offset = offsetof(struct_, struct_field_name_), \
Expand All @@ -479,10 +473,10 @@ typedef int (*json_append_bytes_t)(const char *bytes, size_t len,
struct_field_name_, sub_descr_) \
{ \
.field_name = (json_field_name_), \
.align_shift = Z_ALIGN_SHIFT(struct_), \
.field_name_len = (sizeof(json_field_name_) - 1), \
.type = JSON_TOK_OBJECT_START, \
.offset = offsetof(struct_, struct_field_name_), \
.struct_size = sizeof(struct_), \
{ \
.object = { \
.sub_descr = sub_descr_, \
Expand Down Expand Up @@ -512,10 +506,10 @@ typedef int (*json_append_bytes_t)(const char *bytes, size_t len,
elem_type_) \
{ \
.field_name = (json_field_name_), \
.align_shift = Z_ALIGN_SHIFT(struct_), \
.field_name_len = sizeof(json_field_name_) - 1, \
.type = JSON_TOK_ARRAY_START, \
.offset = offsetof(struct_, struct_field_name_), \
.struct_size = 0u, \
{ \
.array = { \
.element_descr = \
Expand Down Expand Up @@ -571,15 +565,16 @@ typedef int (*json_append_bytes_t)(const char *bytes, size_t len,
elem_descr_len_) \
{ \
.field_name = json_field_name_, \
.align_shift = Z_ALIGN_SHIFT(struct_), \
.field_name_len = sizeof(json_field_name_) - 1, \
.type = JSON_TOK_ARRAY_START, \
.offset = offsetof(struct_, struct_field_name_), \
.struct_size = 0u, \
{ \
.array = { \
.element_descr = Z_JSON_ELEMENT_DESCR(struct_, len_field_, \
JSON_TOK_OBJECT_START, \
Z_JSON_DESCR_OBJ(elem_descr_, elem_descr_len_)), \
Z_JSON_DESCR_OBJ(elem_descr_, elem_descr_len_, \
SIZEOF_FIELD(struct_, field_name_[0])), \
.n_elements = (max_len_), \
}, \
}, \
Expand Down
20 changes: 20 additions & 0 deletions include/zephyr/sys/util.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -224,6 +224,26 @@ extern "C" {
#define CONTAINER_OF(ptr, type, field) \
((type *)(((char *)(ptr)) - offsetof(type, field)))

/**
* @brief Get the size of a field in a structure type
*
* Example:
*
* struct foo {
* int bar;
* char baz;
* };
*
* size_t size_of_baz = SIZEOF_FIELD(struct foo, baz);
*
* Above, @p size_of_baz is equal to sizeof(char).
*
* @param type the type of the structure containing the field
* @param member the name of the field in the structure
* @return the size of the field in bytes
*/
#define SIZEOF_FIELD(type, member) sizeof(((type *)0)->member)
mrfuchs marked this conversation as resolved.
Show resolved Hide resolved
pdgendt marked this conversation as resolved.
Show resolved Hide resolved

/**
* @brief Value of @p x rounded up to the next multiple of @p align.
*/
Expand Down
41 changes: 17 additions & 24 deletions lib/os/json.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -489,6 +489,16 @@ static int64_t decode_value(struct json_obj *obj,
descr->object.sub_descr_len,
field);
case JSON_TOK_ARRAY_START:
/* For nested arrays, update value to current field,
* so it matches descriptor's offset to length field
*/
if (val == NULL) {
val = field;

if (descr->array.element_descr->type == JSON_TOK_OBJECT_START) {
field = (char *)field + descr->offset;
}
}
return arr_parse(obj, descr->array.element_descr,
descr->array.n_elements, field, val);
case JSON_TOK_OBJ_ARRAY: {
Expand Down Expand Up @@ -546,27 +556,9 @@ static ptrdiff_t get_elem_size(const struct json_obj_descr *descr)
case JSON_TOK_TRUE:
case JSON_TOK_FALSE:
return sizeof(bool);
case JSON_TOK_ARRAY_START: {
ptrdiff_t size;

size = descr->array.n_elements * get_elem_size(descr->array.element_descr);
/* Consider additional item count field for array objects */
if (descr->field_name_len > 0) {
size = size + sizeof(size_t);
}

return size;
}
case JSON_TOK_OBJECT_START: {
ptrdiff_t total = 0;
size_t i;

for (i = 0; i < descr->object.sub_descr_len; i++) {
total += get_elem_size(&descr->object.sub_descr[i]);
}

return ROUND_UP(total, 1 << descr->align_shift);
}
case JSON_TOK_ARRAY_START:
case JSON_TOK_OBJECT_START:
return descr->struct_size;
default:
return -EINVAL;
}
Expand Down Expand Up @@ -602,11 +594,11 @@ static int arr_parse(struct json_obj *obj,
return -ENOSPC;
}

/* For nested arrays, update value to current field,
* so it matches descriptor's offset to length field
/* Set value to NULL to make decode_value know that
* array is nested.
*/
if (elem_descr->type == JSON_TOK_ARRAY_START) {
value = field;
value = NULL;
}

if (decode_value(obj, elem_descr, &tok, field, value) < 0) {
Expand Down Expand Up @@ -903,6 +895,7 @@ static int arr_encode(const struct json_obj_descr *elem_descr,
/* For nested arrays, skip parent descriptor to get elements */
if (elem_descr->type == JSON_TOK_ARRAY_START) {
elem_descr = elem_descr->array.element_descr;
field = (char *)field + elem_descr->offset;
}

elem_size = get_elem_size(elem_descr);
Expand Down
25 changes: 19 additions & 6 deletions tests/lib/json/src/main.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@

struct test_nested {
int nested_int;
uint32_t _unused_member;
pdgendt marked this conversation as resolved.
Show resolved Hide resolved
bool nested_bool;
const char *nested_string;
};
Expand All @@ -19,6 +20,7 @@ struct test_struct {
const char *some_string;
int some_int;
bool some_bool;
uint32_t _unused_member;
struct test_nested some_nested_struct;
int some_array[16];
size_t some_array_len;
Expand All @@ -32,17 +34,21 @@ struct test_struct {
};

struct elt {
uint32_t _unused_member;
const char *name;
int height;
};

struct obj_array {
uint32_t _unused_member1;
struct elt elements[10];
uint32_t _unused_member2;
size_t num_elements;
};

struct test_int_limits {
int int_max;
uint32_t _unused_member;
int int_cero;
int int_min;
};
Expand Down Expand Up @@ -92,11 +98,15 @@ static const struct json_obj_descr obj_limits_descr[] = {
};

struct array {
/* uint32_t _unused_member1; // TODO test fails if this is uncommented */
struct elt objects;
uint32_t _unused_member2;
};

struct obj_array_array {
uint32_t _unused_member1;
struct array objects_array[4];
uint32_t _unused_member2;
size_t objects_array_len;
};

Expand All @@ -111,7 +121,9 @@ static const struct json_obj_descr array_array_descr[] = {
};

struct obj_array_2dim {
uint32_t _unused_member1;
struct obj_array objects_array_array[3];
uint32_t _unused_member2;
size_t objects_array_array_len;
};

Expand Down Expand Up @@ -173,8 +185,8 @@ ZTEST(lib_json_test, test_json_encoding)
.nested_string = "no escape necessary",
},
.nested_obj_array = {
{1, true, "true"},
{0, false, "false"}
{.nested_int = 1, .nested_bool = true, .nested_string = "true"},
{.nested_int = 0, .nested_bool = false, .nested_string = "false"}
},
.obj_array_len = 2
};
Expand Down Expand Up @@ -324,10 +336,11 @@ ZTEST(lib_json_test, test_json_encoding_array_array)
{
struct obj_array_array obj_array_array_ts = {
.objects_array = {
[0] = { { .name = "Sim\303\263n Bol\303\255var", .height = 168 } },
[1] = { { .name = "Pel\303\251", .height = 173 } },
[2] = { { .name = "Usain Bolt", .height = 195 } },
},
[0] = {.objects = {.name = "Sim\303\263n Bol\303\255var",
.height = 168}},
[1] = {.objects = {.name = "Pel\303\251", .height = 173}},
[2] = {.objects = {.name = "Usain Bolt", .height = 195}},
},
.objects_array_len = 3,
};
char encoded[] = "{\"objects_array\":["
Expand Down
Loading