Add check for decoded datatype precision overflow

Adds a check for the case where a decoded datatype's precision could overflow SIZE_MAX due to the size of a datatype being larger than SIZE_MAX / 8
jhendersonHDF · Apr 3, 2024 · d9d25d6 · d9d25d6
1 parent c6e26a1
commit d9d25d6
Show file tree

Hide file tree

Showing 3 changed files with 12 additions and 7 deletions.
diff --git a/src/H5Odtype.c b/src/H5Odtype.c
@@ -162,6 +162,8 @@ H5O__dtype_decode_helper(unsigned *ioflags /*in,out*/, const uint8_t **pp, H5T_t
     /* Check for invalid datatype size */
     if (dt->shared->size == 0)
         HGOTO_ERROR(H5E_OHDR, H5E_BADVALUE, FAIL, "invalid datatype size");
+    if (H5T_IS_ATOMIC(dt->shared) && (dt->shared->size > SIZE_MAX / 8))
+        HGOTO_ERROR(H5E_OHDR, H5E_BADVALUE, FAIL, "number of bits in atomic datatype would overflow size_t");
 
     switch (dt->shared->type) {
         case H5T_INTEGER:

diff --git a/src/H5T.c b/src/H5T.c
@@ -6261,10 +6261,10 @@ H5T_set_loc(H5T_t *dt, H5VL_object_t *file, H5T_loc_t loc)
         /* Check the datatype of this element */
         switch (dt->shared->type) {
             case H5T_ARRAY: /* Recurse on VL, compound and array base element type */
-                /* Recurse if it's VL, compound, enum or array */
+                /* Recurse if it's VL, compound, enum, array or reference */
                 /* (If the force_conv flag is _not_ set, the type cannot change in size, so don't recurse) */
                 if (dt->shared->parent->shared->force_conv &&
-                    H5T_IS_COMPLEX(dt->shared->parent->shared->type)) {
+                    (H5T_IS_COMPLEX(dt->shared->parent->shared->type) || H5T_IS_REF(dt->shared->parent->shared))) {
                     /* Keep the old base element size for later */
                     old_size = dt->shared->parent->shared->size;
 
@@ -6302,10 +6302,11 @@ H5T_set_loc(H5T_t *dt, H5VL_object_t *file, H5T_loc_t loc)
                     /* Set the member type pointer (for convenience) */
                     memb_type = dt->shared->u.compnd.memb[i].type;
 
-                    /* Recurse if it's VL, compound, enum or array */
+                    /* Recurse if it's VL, compound, enum, array or reference */
                     /* (If the force_conv flag is _not_ set, the type cannot change in size, so don't recurse)
                      */
-                    if (memb_type->shared->force_conv && H5T_IS_COMPLEX(memb_type->shared->type)) {
+                    if (memb_type->shared->force_conv &&
+                            (H5T_IS_COMPLEX(memb_type->shared->type) || H5T_IS_REF(memb_type->shared))) {
                         /* Keep the old field size for later */
                         old_size = memb_type->shared->size;
 
@@ -6347,8 +6348,7 @@ H5T_set_loc(H5T_t *dt, H5VL_object_t *file, H5T_loc_t loc)
                  * them as part of the same blob)*/
                 /* (If the force_conv flag is _not_ set, the type cannot change in size, so don't recurse) */
                 if (dt->shared->parent->shared->force_conv &&
-                    H5T_IS_COMPLEX(dt->shared->parent->shared->type) &&
-                    (dt->shared->parent->shared->type != H5T_REFERENCE)) {
+                    H5T_IS_COMPLEX(dt->shared->parent->shared->type)) {
                     if ((changed = H5T_set_loc(dt->shared->parent, file, loc)) < 0)
                         HGOTO_ERROR(H5E_DATATYPE, H5E_CANTINIT, FAIL, "Unable to set VL location");
                     if (changed > 0)

diff --git a/src/H5Tpkg.h b/src/H5Tpkg.h
@@ -46,7 +46,7 @@
 
 /* Macro to ease detecting "complex" datatypes (i.e. those with base types or fields) */
 #define H5T_IS_COMPLEX(t)                                                                                    \
-    ((t) == H5T_COMPOUND || (t) == H5T_ENUM || (t) == H5T_VLEN || (t) == H5T_ARRAY || (t) == H5T_REFERENCE)
+    ((t) == H5T_COMPOUND || (t) == H5T_ENUM || (t) == H5T_VLEN || (t) == H5T_ARRAY)
 
 /* Macro to ease detecting fixed "string" datatypes */
 #define H5T_IS_FIXED_STRING(dt) (H5T_STRING == (dt)->type)
@@ -57,6 +57,9 @@
 /* Macro to ease detecting fixed or variable-length "string" datatypes */
 #define H5T_IS_STRING(dt) (H5T_IS_FIXED_STRING(dt) || H5T_IS_VL_STRING(dt))
 
+/* Macro to ease detecting reference datatypes */
+#define H5T_IS_REF(dt) (H5T_REFERENCE == (dt)->type)
+
 /* Macro to ease detecting atomic datatypes */
 #define H5T_IS_ATOMIC(dt) (!(H5T_IS_COMPLEX((dt)->type) || (dt)->type == H5T_OPAQUE))