Fix CVE-2016-4332 (#3451)

release_docs/RELEASE.txt

@@ -135,6 +135,18 @@ Bug Fixes since HDF5-1.10.10 release
 ===================================
     Library
     -------
+    - Fixed an assertion in a previous fix for CVE-2016-4332
+
+      An assert could fail when processing corrupt files that have invalid
+      shared message flags (as in CVE-2016-4332).
+
+      The assert statement in question has been replaced with pointer checks
+      that don't raise errors. Since the function is in cleanup code, we do
+      our best to close and free things, even when presented with partially
+      initialized structs.
+
+      Fixes CVE-2016-4332 and HDFFV-9950 (confirmed via the cve_hdf5 repo)
+
     - Seg fault on file close
 
       h5debug fails at file close with core dump on a file that has an

src/H5Omessage.c

@@ -675,12 +675,11 @@ H5O__msg_free_real(const H5O_msg_class_t *type, void *msg_native)
 {
     FUNC_ENTER_PACKAGE_NOERR
 
-    /* check args */
-    HDassert(type);
+    /* Don't assert on args since this could be called in cleanup code */
 
     if (msg_native) {
         H5O__msg_reset_real(type, msg_native);
-        if (NULL != (type->free))
+        if (type && type->free)
             (type->free)(msg_native);
         else
             H5MM_xfree(msg_native);