Merge hdf5_1_14 changes 07-29 to 08-04

.clang-format

@@ -85,7 +85,6 @@ StatementMacros:
   - H5_GCC_DIAG_ON
   - H5_LEAVE
   - HGOTO_DONE
-  - HGOTO_ERROR
   - HMPI_DONE_ERROR
   - HMPI_ERROR
   - HMPI_GOTO_ERROR

bin/make_err

@@ -243,9 +243,9 @@ sub create_init ($) {
     foreach $name (keys %major) {
         print HEADER " "x(0*$indent),"assert(${name}_g==(-1));\n";
         print HEADER " "x(0*$indent),"if((msg = H5E__create_msg(cls, H5E_MAJOR, \"${major{$name}}\"))==NULL)\n";
-        print HEADER " "x(1*$indent),"HGOTO_ERROR(H5E_ERROR, H5E_CANTINIT, FAIL, \"error message initialization failed\")\n";
+        print HEADER " "x(1*$indent),"HGOTO_ERROR(H5E_ERROR, H5E_CANTINIT, FAIL, \"error message initialization failed\");\n";
         print HEADER " "x(0*$indent),"if((${name}_g = H5I_register(H5I_ERROR_MSG, msg, FALSE))<0)\n";
-        print HEADER " "x(1*$indent),"HGOTO_ERROR(H5E_ERROR, H5E_CANTREGISTER, FAIL, \"can't register error message\")\n";
+        print HEADER " "x(1*$indent),"HGOTO_ERROR(H5E_ERROR, H5E_CANTREGISTER, FAIL, \"can't register error message\");\n";
     }
 
     # Iterate over all the minor error sections
@@ -259,9 +259,9 @@ sub create_init ($) {
         for $name ( @{$section_list{$sect_name}}) {
             print HEADER " "x(0*$indent),"assert(${name}_g==(-1));\n";
             print HEADER " "x(0*$indent),"if((msg = H5E__create_msg(cls, H5E_MINOR, \"${minor{$name}}\"))==NULL)\n";
-            print HEADER " "x(1*$indent),"HGOTO_ERROR(H5E_ERROR, H5E_CANTINIT, FAIL, \"error message initialization failed\")\n";
+            print HEADER " "x(1*$indent),"HGOTO_ERROR(H5E_ERROR, H5E_CANTINIT, FAIL, \"error message initialization failed\");\n";
             print HEADER " "x(0*$indent),"if((${name}_g = H5I_register(H5I_ERROR_MSG, msg, FALSE))<0)\n";
-            print HEADER " "x(1*$indent),"HGOTO_ERROR(H5E_ERROR, H5E_CANTREGISTER, FAIL, \"can't register error message\")\n";
+            print HEADER " "x(1*$indent),"HGOTO_ERROR(H5E_ERROR, H5E_CANTREGISTER, FAIL, \"can't register error message\");\n";
         }
     }
 

bin/release

@@ -206,7 +206,7 @@ tar2cmakezip()
     cp /mnt/scr1/pre-release/hdf5/CMake/LIBAEC.tar.gz $cmziptmpsubdir
     cp /mnt/scr1/pre-release/hdf5/CMake/ZLib.tar.gz $cmziptmpsubdir
     cp /mnt/scr1/pre-release/hdf5/CMake/HDF5Examples-2.0.3-Source.zip $cmziptmpsubdir
-    cp /mnt/scr1/pre-release/hdf5/CMake/hdf5_plugins-1_14_1.zip $cmziptmpsubdir
+    cp /mnt/scr1/pre-release/hdf5/CMake/hdf5_plugins-1_14_2.zip $cmziptmpsubdir
     cp $cmziptmpsubdir/$version/config/cmake/scripts/CTestScript.cmake $cmziptmpsubdir
     cp $cmziptmpsubdir/$version/config/cmake/scripts/HDF5config.cmake $cmziptmpsubdir
     cp $cmziptmpsubdir/$version/config/cmake/scripts/HDF5options.cmake $cmziptmpsubdir
@@ -298,7 +298,7 @@ tar2cmaketgz()
     cp /mnt/scr1/pre-release/hdf5/CMake/LIBAEC.tar.gz $cmgztmpsubdir
     cp /mnt/scr1/pre-release/hdf5/CMake/ZLib.tar.gz $cmgztmpsubdir
     cp /mnt/scr1/pre-release/hdf5/CMake/HDF5Examples-2.0.3-Source.tar.gz $cmgztmpsubdir
-    cp /mnt/scr1/pre-release/hdf5/CMake/hdf5_plugins-1_14_1.tar.gz $cmgztmpsubdir
+    cp /mnt/scr1/pre-release/hdf5/CMake/hdf5_plugins-1_14_2.tar.gz $cmgztmpsubdir
     cp $cmgztmpsubdir/$version/config/cmake/scripts/CTestScript.cmake $cmgztmpsubdir
     cp $cmgztmpsubdir/$version/config/cmake/scripts/HDF5config.cmake $cmgztmpsubdir
     cp $cmgztmpsubdir/$version/config/cmake/scripts/HDF5options.cmake $cmgztmpsubdir
@@ -377,7 +377,7 @@ tar2hpccmaketgz()
     cp /mnt/scr1/pre-release/hdf5/CMake/LIBAEC.tar.gz $cmgztmpsubdir
     cp /mnt/scr1/pre-release/hdf5/CMake/ZLib.tar.gz $cmgztmpsubdir
     cp /mnt/scr1/pre-release/hdf5/CMake/HDF5Examples-2.0.3-Source.tar.gz $cmgztmpsubdir
-    cp /mnt/scr1/pre-release/hdf5/CMake/hdf5_plugins-1_14_1.tar.gz $cmgztmpsubdir
+    cp /mnt/scr1/pre-release/hdf5/CMake/hdf5_plugins-1_14_2.tar.gz $cmgztmpsubdir
     cp $cmgztmpsubdir/$version/config/cmake/scripts/CTestScript.cmake $cmgztmpsubdir
     cp $cmgztmpsubdir/$version/config/cmake/scripts/HDF5config.cmake $cmgztmpsubdir
 

configure

@@ -4615,14 +4615,21 @@ fi
 ## This allows multiple src-dir builds within one host.
 
 
-## Run post processing on files created by configure.
+## Run post processing on files created by configure
+##
 ## src/H5pubconf.h:
 ## Generate src/H5pubconf.h from src/H5config.h by prepending H5_ to all
 ## macro names. This avoid name conflict between HDF5 macro names and those
 ## generated by another software package that uses the HDF5 library.
+##
 ## src/libhdf5.settings:
-## Remove all lines begun with "#" which are generated by CONDITIONAL's of
-## configure.
+## Remove all lines beginning with "#" which are generated by CONDITIONAL's
+## of configure.
+##
+## src/H5build_settings.c
+## Remove all lines beginning with "#" which are generated by CONDITIONAL's
+## of configure. This uses a check for whitespace after the pound sign
+## to avoid clobbering include statements.
 ac_config_commands="$ac_config_commands pubconf"
 
 
@@ -10364,7 +10371,7 @@ else $as_nop
 JAVA_TEST=Test.java
 CLASS_TEST=Test.class
 cat << \EOF > $JAVA_TEST
-/* #line 10367 "configure" */
+/* #line 10374 "configure" */
 public class Test {
 }
 EOF
@@ -10603,7 +10610,7 @@ else $as_nop
 JAVA_TEST=Test.java
 CLASS_TEST=Test.class
 cat << \EOF > $JAVA_TEST
-/* #line 10606 "configure" */
+/* #line 10613 "configure" */
 public class Test {
 }
 EOF
@@ -10638,7 +10645,7 @@ JAVA_TEST=Test.java
 CLASS_TEST=Test.class
 TEST=Test
 cat << \EOF > $JAVA_TEST
-/* [#]line 10641 "configure" */
+/* [#]line 10648 "configure" */
 public class Test {
 public static void main (String args[]) {
         System.exit (0);
@@ -11063,7 +11070,7 @@ else $as_nop
 
 
 cat << \EOF > Test.java
-/* #line 11066 "configure" */
+/* #line 11073 "configure" */
 import junit.textui.TestRunner;
 public class Test {
 

configure.ac

@@ -81,14 +81,21 @@ AM_MAINTAINER_MODE([disable])
 ## This allows multiple src-dir builds within one host.
 AC_PREFIX_DEFAULT([`pwd`/hdf5])
 
-## Run post processing on files created by configure.
+## Run post processing on files created by configure
+##
 ## src/H5pubconf.h:
 ## Generate src/H5pubconf.h from src/H5config.h by prepending H5_ to all
 ## macro names. This avoid name conflict between HDF5 macro names and those
 ## generated by another software package that uses the HDF5 library.
+##
 ## src/libhdf5.settings:
-## Remove all lines begun with "#" which are generated by CONDITIONAL's of
-## configure.
+## Remove all lines beginning with "#" which are generated by CONDITIONAL's
+## of configure.
+##
+## src/H5build_settings.c
+## Remove all lines beginning with "#" which are generated by CONDITIONAL's
+## of configure. This uses a check for whitespace after the pound sign
+## to avoid clobbering include statements.
 AC_CONFIG_COMMANDS([pubconf], [
   echo "creating src/H5pubconf.h"
   sed 's/#define /#define H5_/' <src/H5config.h |\

doc/getting-started-with-hdf5-development.md

@@ -270,7 +270,7 @@ H5X_do_stuff(/*parameters*/)
 	HDassert(/*parameter check*/);
 
 	if (H5X_other_call() < 0)
-		HGOTO_ERROR(H5E_MAJ, H5E_MIN, FAIL, "badness")
+		HGOTO_ERROR(H5E_MAJ, H5E_MIN, FAIL, "badness");
 
 done:
 	if (ret_value < 0)
@@ -329,12 +329,12 @@ H5Xdo_api_stuff(/*parameters*/)
 	H5TRACE3(/*stuff*/)
 
 	if (/*parameter check*/)
-		HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "badness")
+		HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "badness");
 
 	/* VOL setup */
 
 	if (H5VL_call() < 0)
-		HGOTO_ERROR(H5E_FOO, H5E_BAR, FAIL, "badness")
+		HGOTO_ERROR(H5E_FOO, H5E_BAR, FAIL, "badness");
 
 done:
 	if (ret_value < 0)
@@ -483,7 +483,7 @@ fail. The general format is this:
 
 ```c
 if (function_that_could_fail(foo, bar) < 0)
-    HGOTO_ERROR(H5E_<major>, H5E_<minor>, <bad value>, "tell me about badness")
+    HGOTO_ERROR(H5E_<major>, H5E_<minor>, <bad value>, "tell me about badness");
 ```
 
 `HGOTO_ERROR` is one of a set of macros defined in `H5Eprivate.h`. This macro
@@ -713,7 +713,7 @@ error:
     {
         H5Fclose(fid);
     }
-    H5E_END_TRY;
+    H5E_END_TRY
 
     return FAIL;
 }

java/src/jni/CMakeLists.txt

@@ -106,7 +106,7 @@ if (HDF5_EXPORTED_TARGETS)
           ${HDF5_EXPORTED_TARGETS}
       LIBRARY DESTINATION ${HDF5_INSTALL_LIB_DIR} COMPONENT libraries
       ARCHIVE DESTINATION ${HDF5_INSTALL_LIB_DIR} COMPONENT libraries
-      RUNTIME DESTINATION ${HDF5_INSTALL_LIB_DIR} COMPONENT libraries
+      RUNTIME DESTINATION ${HDF5_INSTALL_BIN_DIR} COMPONENT libraries
       FRAMEWORK DESTINATION ${HDF5_INSTALL_FWRK_DIR} COMPONENT libraries
       INCLUDES DESTINATION include
   )

release_docs/RELEASE.txt

@@ -59,7 +59,12 @@ New Features
 
     Library:
     --------
-    -
+    - Change the error handling for a not found path in the find plugin process.
+
+      While attempting to load a plugin the HDF5 library will fail if one of the
+      directories in the plugin paths does not exist, even if there are more paths
+      to check. Instead of exiting the function with an error, just logged the error
+      and continue processing the list of paths to check.
 
 
     Parallel Library:
@@ -116,6 +121,37 @@ Bug Fixes since HDF5-1.14.1 release
 ===================================
     Library
     -------
+    - Fixed CVE-2018-13867
+
+      A corrupt file containing an invalid local heap datablock address
+      could trigger an assert failure when the metadata cache attempted
+      to load the datablock from storage.
+
+      The local heap now verifies that the datablock address is valid
+      when the local heap header information is parsed.
+
+    - Fixed CVE-2018-11202
+
+      A malformed file could result in chunk index memory leaks. Under most
+      conditions (i.e., when the --enable-using-memchecker option is NOT
+      used), this would result in a small memory leak and and infinite loop
+      and abort when shutting down the library. The infinite loop would be
+      due to the "free list" package not being able to clear its resources
+      so the library couldn't shut down. When the "using a memory checker"
+      option is used, the free lists are disabled so there is just a memory
+      leak with no abort on library shutdown.
+
+      The chunk index resources are now correctly cleaned up when reading
+      misparsed files and valgrind confirms no memory leaks.
+
+    - Fixed an issue where an assert statement was converted to an
+      incorrect error check statement
+
+      An assert statement in the library dealing with undefined dataset data
+      fill values was converted to an improper error check that would always
+      trigger when a dataset's fill value was set to NULL (undefined). This
+      has now been fixed.
+
     - Fixed an assertion failure when attempting to use the Subfiling IOC
       VFD directly
 
@@ -171,7 +207,21 @@ Bug Fixes since HDF5-1.14.1 release
 
     Tools
     -----
-    -
+    - Fixed an issue in h5repack for variable-length typed datasets
+
+      When repacking datasets into a new file, h5repack tries to determines whether
+      it can use H5Ocopy to copy each dataset into the new file, or if it needs to
+      manually re-create the dataset, then read data from the old dataset and write
+      it to the new dataset. H5repack was previously using H5Ocopy for datasets with
+      variable-length datatypes, but this can be problematic if the global heap
+      addresses involved do not match exactly between the old and new files. These
+      addresses could change for a variety of reasons, such as the command-line options
+      provided to h5repack, how h5repack allocate space in the repacked file, etc.      
+      Since H5Ocopy does not currently perform any translation when these addresses
+      change, datasets that were repacked with H5Ocopy could become unreadable in the
+      new file. H5repack has been fixed to repack variable-length typed datasets without
+      using H5Ocopy to ensure that the new datasets always have the correct global heap
+      addresses.
 
 
     Performance

src/H5.c

@@ -115,7 +115,7 @@ H5_default_vfd_init(void)
      * it has of initializing the default VFD.
      */
     if (H5FD_sec2_init() == H5I_INVALID_HID) {
-        HGOTO_ERROR(H5E_FUNC, H5E_CANTINIT, FAIL, "unable to load default VFD ID")
+        HGOTO_ERROR(H5E_FUNC, H5E_CANTINIT, FAIL, "unable to load default VFD ID");
     }
 done:
     FUNC_LEAVE_NOAPI(ret_value)
@@ -267,7 +267,7 @@ H5_init_library(void)
         for (i = 0; i < NELMTS(initializer); i++) {
             if (initializer[i].func() < 0) {
                 HGOTO_ERROR(H5E_FUNC, H5E_CANTINIT, FAIL,
-                    "unable to initialize %s interface", initializer[i].descr)
+                    "unable to initialize %s interface", initializer[i].descr);
             }
         }
         /* clang-format on */
@@ -576,7 +576,7 @@ H5garbage_collect(void)
 
     /* Call the garbage collection routines in the library */
     if (H5FL_garbage_coll() < 0)
-        HGOTO_ERROR(H5E_RESOURCE, H5E_CANTGC, FAIL, "can't garbage collect objects")
+        HGOTO_ERROR(H5E_RESOURCE, H5E_CANTGC, FAIL, "can't garbage collect objects");
 
 done:
     FUNC_LEAVE_API(ret_value)
@@ -622,7 +622,7 @@ H5set_free_list_limits(int reg_global_lim, int reg_list_lim, int arr_global_lim,
     /* Call the free list function to actually set the limits */
     if (H5FL_set_free_list_limits(reg_global_lim, reg_list_lim, arr_global_lim, arr_list_lim, blk_global_lim,
                                   blk_list_lim, blk_global_lim, blk_list_lim) < 0)
-        HGOTO_ERROR(H5E_RESOURCE, H5E_CANTSET, FAIL, "can't set garbage collection limits")
+        HGOTO_ERROR(H5E_RESOURCE, H5E_CANTSET, FAIL, "can't set garbage collection limits");
 
 done:
     FUNC_LEAVE_API(ret_value)
@@ -658,7 +658,7 @@ H5get_free_list_sizes(size_t *reg_size /*out*/, size_t *arr_size /*out*/, size_t
 
     /* Call the free list function to actually get the sizes */
     if (H5FL_get_free_list_sizes(reg_size, arr_size, blk_size, fac_size) < 0)
-        HGOTO_ERROR(H5E_RESOURCE, H5E_CANTGET, FAIL, "can't get garbage collection sizes")
+        HGOTO_ERROR(H5E_RESOURCE, H5E_CANTGET, FAIL, "can't get garbage collection sizes");
 
 done:
     FUNC_LEAVE_API(ret_value)
@@ -1052,11 +1052,11 @@ H5atclose(H5_atclose_func_t func, void *ctx)
 
     /* Check arguments */
     if (NULL == func)
-        HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "NULL func pointer")
+        HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "NULL func pointer");
 
     /* Allocate space for the 'atclose' node */
     if (NULL == (new_atclose = H5FL_MALLOC(H5_atclose_node_t)))
-        HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, FAIL, "can't allocate 'atclose' node")
+        HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, FAIL, "can't allocate 'atclose' node");
 
     /* Set up 'atclose' node */
     new_atclose->func = func;