Merges from 1.14 Aug 4-9 (#3366)

* Make h5dump spacing consistent when printing VLEN datatype (#3351) (#3352) * Add Fortran H5ES module to deploy list (#3342) * Fix for the bug exposed from running test/set_extent.c when selection I/O is enabled. (#3319) The test/set_extent.c is modified to test for selection I/O enabled. * Merge Implementation of the mpio driver with selection I/O. (#3360) * Work around a testphdf5 failure on Cray MPICH machines (#3361) (#3362) * set H5_PAC_C_MAX_REAL_PRECISION default to 0 when cross compiling (#3365) with CMake to fix Fortran build failures.
HDFGroup · Aug 10, 2023 · 0c1ad6c · 0c1ad6c
1 parent f3de9ee
commit 0c1ad6c
Show file tree

Hide file tree

Showing 36 changed files with 4,585 additions and 541 deletions.
diff --git a/config/cmake/ConfigureChecks.cmake b/config/cmake/ConfigureChecks.cmake
@@ -858,21 +858,23 @@ if (HDF5_BUILD_FORTRAN)
 
     list (GET PROG_OUTPUT4 0 H5_LDBL_DIG)
     list (GET PROG_OUTPUT4 1 H5_FLT128_DIG)
-  endif ()
 
-  if (${HDF_PREFIX}_SIZEOF___FLOAT128 EQUAL "0" OR FLT128_DIG EQUAL "0")
-    set (${HDF_PREFIX}_HAVE_FLOAT128 0)
-    set (${HDF_PREFIX}_SIZEOF___FLOAT128 0)
-    set (_PAC_C_MAX_REAL_PRECISION ${H5_LDBL_DIG})
+    if (${HDF_PREFIX}_SIZEOF___FLOAT128 EQUAL "0" OR FLT128_DIG EQUAL "0")
+      set (${HDF_PREFIX}_HAVE_FLOAT128 0)
+      set (${HDF_PREFIX}_SIZEOF___FLOAT128 0)
+      set (_PAC_C_MAX_REAL_PRECISION ${H5_LDBL_DIG})
+    else ()
+      set (_PAC_C_MAX_REAL_PRECISION ${H5_FLT128_DIG})
+    endif ()
+    if (NOT ${_PAC_C_MAX_REAL_PRECISION})
+      set (${HDF_PREFIX}_PAC_C_MAX_REAL_PRECISION 0)
+    else ()
+      set (${HDF_PREFIX}_PAC_C_MAX_REAL_PRECISION ${_PAC_C_MAX_REAL_PRECISION})
+    endif ()
+    message (STATUS "maximum decimal precision for C var - ${${HDF_PREFIX}_PAC_C_MAX_REAL_PRECISION}")
   else ()
-    set (_PAC_C_MAX_REAL_PRECISION ${H5_FLT128_DIG})
-  endif ()
-  if (NOT ${_PAC_C_MAX_REAL_PRECISION})
     set (${HDF_PREFIX}_PAC_C_MAX_REAL_PRECISION 0)
-  else ()
-    set (${HDF_PREFIX}_PAC_C_MAX_REAL_PRECISION ${_PAC_C_MAX_REAL_PRECISION})
   endif ()
-  message (STATUS "maximum decimal precision for C var - ${${HDF_PREFIX}_PAC_C_MAX_REAL_PRECISION}")
 
 endif()
 

diff --git a/fortran/src/CMakeLists.txt b/fortran/src/CMakeLists.txt
@@ -427,29 +427,34 @@ else ()
   )
 endif ()
 
+set (mod_export_files
+    h5fortran_types.mod
+    hdf5.mod
+    h5fortkit.mod
+    h5global.mod
+    h5a.mod
+    h5d.mod
+    h5e.mod
+    h5es.mod
+    h5f.mod
+    h5g.mod
+    h5i.mod
+    h5l.mod
+    h5lib.mod
+    h5o.mod
+    h5p.mod
+    h5r.mod
+    h5s.mod
+    h5t.mod
+    h5vl.mod
+    h5z.mod
+    h5_gen.mod
+)
+
 if (BUILD_STATIC_LIBS)
-  set (mod_files
-      ${MOD_BUILD_DIR}/h5fortran_types.mod
-      ${MOD_BUILD_DIR}/hdf5.mod
-      ${MOD_BUILD_DIR}/h5fortkit.mod
-      ${MOD_BUILD_DIR}/h5global.mod
-      ${MOD_BUILD_DIR}/h5a.mod
-      ${MOD_BUILD_DIR}/h5d.mod
-      ${MOD_BUILD_DIR}/h5e.mod
-      ${MOD_BUILD_DIR}/h5f.mod
-      ${MOD_BUILD_DIR}/h5g.mod
-      ${MOD_BUILD_DIR}/h5i.mod
-      ${MOD_BUILD_DIR}/h5l.mod
-      ${MOD_BUILD_DIR}/h5lib.mod
-      ${MOD_BUILD_DIR}/h5o.mod
-      ${MOD_BUILD_DIR}/h5p.mod
-      ${MOD_BUILD_DIR}/h5r.mod
-      ${MOD_BUILD_DIR}/h5s.mod
-      ${MOD_BUILD_DIR}/h5t.mod
-      ${MOD_BUILD_DIR}/h5vl.mod
-      ${MOD_BUILD_DIR}/h5z.mod
-      ${MOD_BUILD_DIR}/h5_gen.mod
-  )
+  foreach (mod_file ${mod_export_files})
+    set (mod_files ${mod_files} ${MOD_BUILD_DIR}/${mod_file})
+  endforeach ()
   install (
       FILES
           ${mod_files}
@@ -471,28 +476,9 @@ if (BUILD_STATIC_LIBS)
 endif ()
 
 if (BUILD_SHARED_LIBS)
-  set (modsh_files
-      ${MODSH_BUILD_DIR}/h5fortran_types.mod
-      ${MODSH_BUILD_DIR}/hdf5.mod
-      ${MODSH_BUILD_DIR}/h5fortkit.mod
-      ${MODSH_BUILD_DIR}/h5global.mod
-      ${MODSH_BUILD_DIR}/h5a.mod
-      ${MODSH_BUILD_DIR}/h5d.mod
-      ${MODSH_BUILD_DIR}/h5e.mod
-      ${MODSH_BUILD_DIR}/h5f.mod
-      ${MODSH_BUILD_DIR}/h5g.mod
-      ${MODSH_BUILD_DIR}/h5i.mod
-      ${MODSH_BUILD_DIR}/h5l.mod
-      ${MODSH_BUILD_DIR}/h5lib.mod
-      ${MODSH_BUILD_DIR}/h5o.mod
-      ${MODSH_BUILD_DIR}/h5p.mod
-      ${MODSH_BUILD_DIR}/h5r.mod
-      ${MODSH_BUILD_DIR}/h5s.mod
-      ${MODSH_BUILD_DIR}/h5t.mod
-      ${MODSH_BUILD_DIR}/h5vl.mod
-      ${MODSH_BUILD_DIR}/h5z.mod
-      ${MODSH_BUILD_DIR}/h5_gen.mod
-  )
+  foreach (mod_file ${mod_export_files})
+    set (modsh_files ${modsh_files} ${MODSH_BUILD_DIR}/${mod_file})
+  endforeach ()
   install (
       FILES
           ${modsh_files}

diff --git a/hl/fortran/src/CMakeLists.txt b/hl/fortran/src/CMakeLists.txt
@@ -235,16 +235,19 @@ endif ()
 # Add file(s) to CMake Install
 #-----------------------------------------------------------------------------
 
+set (mod_export_files
+    h5ds.mod
+    h5tb.mod
+    h5tb_const.mod
+    h5lt.mod
+    h5lt_const.mod
+    h5im.mod
+)
 
 if (BUILD_STATIC_LIBS)
-  set (mod_files
-      ${MOD_BUILD_DIR}/h5ds.mod
-      ${MOD_BUILD_DIR}/h5tb.mod
-      ${MOD_BUILD_DIR}/h5tb_const.mod
-      ${MOD_BUILD_DIR}/h5lt.mod
-      ${MOD_BUILD_DIR}/h5lt_const.mod
-      ${MOD_BUILD_DIR}/h5im.mod
-  )
+  foreach (mod_file ${mod_export_files})
+    set (mod_files ${mod_files} ${MOD_BUILD_DIR}/${mod_file})
+  endforeach ()
   install (
       FILES
           ${mod_files}
@@ -265,14 +268,9 @@ if (BUILD_STATIC_LIBS)
   endif ()
 endif ()
 if (BUILD_SHARED_LIBS)
-  set (modsh_files
-      ${MODSH_BUILD_DIR}/h5ds.mod
-      ${MODSH_BUILD_DIR}/h5tb.mod
-      ${MODSH_BUILD_DIR}/h5tb_const.mod
-      ${MODSH_BUILD_DIR}/h5lt.mod
-      ${MODSH_BUILD_DIR}/h5lt_const.mod
-      ${MODSH_BUILD_DIR}/h5im.mod
-  )
+  foreach (mod_file ${mod_export_files})
+    set (modsh_files ${modsh_files} ${MODSH_BUILD_DIR}/${mod_file})
+  endforeach ()
   install (
       FILES
           ${modsh_files}

diff --git a/release_docs/RELEASE.txt b/release_docs/RELEASE.txt
@@ -260,6 +260,17 @@ Bug Fixes since HDF5-1.14.1 release
 
     Testing
     -------
+    - Fixed a testing failure in testphdf5 on Cray machines
+
+      On some Cray machines, what appears to be a bug in Cray MPICH was causing
+      calls to H5Fis_accessible to create a 0-byte file with strange Unix
+      permissions. This was causing an H5Fdelete file deletion test in the
+      testphdf5 program to fail due to a just-deleted HDF5 file appearing to
+      still be accessible on the file system. The issue in Cray MPICH has been
+      worked around for the time being by resetting the MPI_Info object on the
+      File Access Property List used to MPI_INFO_NULL before passing it to the
+      H5Fis_accessible call.
+
     - A bug was fixed in the HDF5 API test random datatype generation code
 
       A bug in the random datatype generation code could cause test failures

diff --git a/release_docs/USING_HDF5_VS.txt b/release_docs/USING_HDF5_VS.txt
@@ -38,7 +38,10 @@ Using Visual Studio 2010 and above with HDF5 Libraries built with Visual Studio
           library, and then optionally the HDF5 High Level, Fortran or C++
           libraries. For example, to compile a C++ application, enter:
 
-          szip.lib zlib.lib hdf5.lib hdf5_cpp.lib
+          hdf5.lib hdf5_cpp.lib
+
+          For static linking:
+          libszaec.lib libaec.lib libzlib.lib libhdf5.lib libhdf5_cpp.lib
 
 
 ==========================================================================

diff --git a/src/H5Dchunk.c b/src/H5Dchunk.c
@@ -2731,11 +2731,20 @@ H5D__chunk_read(H5D_io_info_t *io_info, H5D_dset_io_info_t *dset_info)
                 }
             } /* end if */
             else if (!skip_missing_chunks) {
+                /* Set up nonexistent dataset info for (fill value) read from nonexistent chunk */
+                nonexistent_dset_info.layout_io_info.contig_piece_info = chunk_info;
+                nonexistent_dset_info.file_space                       = chunk_info->fspace;
+                nonexistent_dset_info.mem_space                        = chunk_info->mspace;
+                nonexistent_dset_info.nelmts                           = chunk_info->piece_points;
+
+                /* Set request_nelmts.  This is not normally set by the upper layers because selection I/O
+                 * usually does not use strip mining (H5D__scatgath_write), and instead allocates buffers
+                 * large enough for the entire I/O.  Set request_nelmts to be large enough for all selected
+                 * elements in this chunk because it must be at least that large */
+                nonexistent_dset_info.type_info.request_nelmts = nonexistent_dset_info.nelmts;
+
                 /* Perform the actual read operation from the nonexistent chunk
                  */
-                nonexistent_dset_info.file_space = chunk_info->fspace;
-                nonexistent_dset_info.mem_space  = chunk_info->mspace;
-                nonexistent_dset_info.nelmts     = chunk_info->piece_points;
                 if ((dset_info->io_ops.single_read)(&nonexistent_io_info, &nonexistent_dset_info) < 0)
                     HGOTO_ERROR(H5E_DATASET, H5E_READERROR, FAIL, "chunked read failed");
             } /* end if */
@@ -2866,9 +2875,10 @@ H5D__chunk_read(H5D_io_info_t *io_info, H5D_dset_io_info_t *dset_info)
 
                 /* Perform the actual read operation */
                 assert(chk_io_info->count == 1);
-                chk_io_info->dsets_info[0].file_space = chunk_info->fspace;
-                chk_io_info->dsets_info[0].mem_space  = chunk_info->mspace;
-                chk_io_info->dsets_info[0].nelmts     = chunk_info->piece_points;
+                chk_io_info->dsets_info[0].layout_io_info.contig_piece_info = chunk_info;
+                chk_io_info->dsets_info[0].file_space                       = chunk_info->fspace;
+                chk_io_info->dsets_info[0].mem_space                        = chunk_info->mspace;
+                chk_io_info->dsets_info[0].nelmts                           = chunk_info->piece_points;
                 if ((dset_info->io_ops.single_read)(chk_io_info, &chk_io_info->dsets_info[0]) < 0)
                     HGOTO_ERROR(H5E_DATASET, H5E_READERROR, FAIL, "chunked read failed");
 
@@ -3055,10 +3065,19 @@ H5D__chunk_write(H5D_io_info_t *io_info, H5D_dset_io_info_t *dset_info)
                 /* Set up the storage buffer information for this chunk */
                 cpt_store.compact.buf = chunk;
 
+                /* Set up compact dataset info for write to cached chunk */
+                cpt_dset_info.layout_io_info.contig_piece_info = chunk_info;
+                cpt_dset_info.file_space                       = chunk_info->fspace;
+                cpt_dset_info.mem_space                        = chunk_info->mspace;
+                cpt_dset_info.nelmts                           = chunk_info->piece_points;
+
+                /* Set request_nelmts.  This is not normally set by the upper layers because selection I/O
+                 * usually does not use strip mining (H5D__scatgath_write), and instead allocates buffers
+                 * large enough for the entire I/O.  Set request_nelmts to be large enough for all selected
+                 * elements in this chunk because it must be at least that large */
+                cpt_dset_info.type_info.request_nelmts = cpt_dset_info.nelmts;
+
                 /* Perform the actual write operation */
-                cpt_dset_info.file_space = chunk_info->fspace;
-                cpt_dset_info.mem_space  = chunk_info->mspace;
-                cpt_dset_info.nelmts     = chunk_info->piece_points;
                 if ((dset_info->io_ops.single_write)(&cpt_io_info, &cpt_dset_info) < 0)
                     HGOTO_ERROR(H5E_DATASET, H5E_READERROR, FAIL, "chunked write failed");
 
@@ -3253,9 +3272,10 @@ H5D__chunk_write(H5D_io_info_t *io_info, H5D_dset_io_info_t *dset_info)
 
             /* Perform the actual write operation */
             assert(chk_io_info->count == 1);
-            chk_io_info->dsets_info[0].file_space = chunk_info->fspace;
-            chk_io_info->dsets_info[0].mem_space  = chunk_info->mspace;
-            chk_io_info->dsets_info[0].nelmts     = chunk_info->piece_points;
+            chk_io_info->dsets_info[0].layout_io_info.contig_piece_info = chunk_info;
+            chk_io_info->dsets_info[0].file_space                       = chunk_info->fspace;
+            chk_io_info->dsets_info[0].mem_space                        = chunk_info->mspace;
+            chk_io_info->dsets_info[0].nelmts                           = chunk_info->piece_points;
             if ((dset_info->io_ops.single_write)(chk_io_info, &chk_io_info->dsets_info[0]) < 0)
                 HGOTO_ERROR(H5E_DATASET, H5E_READERROR, FAIL, "chunked write failed");
 

diff --git a/src/H5Dcompact.c b/src/H5Dcompact.c
@@ -233,8 +233,9 @@ H5D__compact_io_init(H5D_io_info_t *io_info, H5D_dset_io_info_t *dinfo)
 {
     FUNC_ENTER_PACKAGE_NOERR
 
-    dinfo->store->compact.buf   = dinfo->dset->shared->layout.storage.u.compact.buf;
-    dinfo->store->compact.dirty = &dinfo->dset->shared->layout.storage.u.compact.dirty;
+    dinfo->store->compact.buf               = dinfo->dset->shared->layout.storage.u.compact.buf;
+    dinfo->store->compact.dirty             = &dinfo->dset->shared->layout.storage.u.compact.dirty;
+    dinfo->layout_io_info.contig_piece_info = NULL;
 
     /* Disable selection I/O */
     io_info->use_select_io = H5D_SELECTION_IO_MODE_OFF;

diff --git a/src/H5Defl.c b/src/H5Defl.c
@@ -198,6 +198,9 @@ H5D__efl_io_init(H5D_io_info_t *io_info, H5D_dset_io_info_t *dinfo)
 
     H5MM_memcpy(&dinfo->store->efl, &(dinfo->dset->shared->dcpl_cache.efl), sizeof(H5O_efl_t));
 
+    /* No "pieces" selected */
+    dinfo->layout_io_info.contig_piece_info = NULL;
+
     /* Disable selection I/O */
     io_info->use_select_io = H5D_SELECTION_IO_MODE_OFF;
     io_info->no_selection_io_cause |= H5D_SEL_IO_NOT_CONTIGUOUS_OR_CHUNKED_DATASET;

diff --git a/src/H5Dio.c b/src/H5Dio.c
@@ -415,8 +415,14 @@ H5D__read(size_t count, H5D_dset_io_info_t *dset_info)
                 HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "can't get MPI-I/O transfer mode");
 
             /* Only report the collective I/O mode if we're actually performing collective I/O */
-            if (xfer_mode == H5FD_MPIO_COLLECTIVE)
+            if (xfer_mode == H5FD_MPIO_COLLECTIVE) {
                 H5CX_set_mpio_actual_io_mode(io_info.actual_io_mode);
+
+                /* If we did selection I/O, report that we used "link chunk" mode, since that's the most
+                 * analogous to what selection I/O does */
+                if (io_info.use_select_io == H5D_SELECTION_IO_MODE_ON)
+                    H5CX_set_mpio_actual_chunk_opt(H5D_MPIO_LINK_CHUNK);
+            }
         }
 #endif /* H5_HAVE_PARALLEL */
     }
@@ -1131,20 +1137,16 @@ H5D__typeinfo_init_phase2(H5D_io_info_t *io_info)
     assert(io_info);
 
     /* If selection I/O mode is default (auto), enable it here if the VFD supports it (it will be turned off
-     * later if something else conflicts), otherwise disable it.  If we're using the MPIO VFD, the automatic
-     * selection will happen in H5D__mpio_opt_possible() inside H5D__ioinfo_adjust(). */
-#ifdef H5_HAVE_PARALLEL
-    if (!io_info->using_mpi_vfd)
-#endif /* H5_HAVE_PARALLEL */
-        if (io_info->use_select_io == H5D_SELECTION_IO_MODE_DEFAULT) {
-            if (H5F_has_vector_select_io(io_info->dsets_info[0].dset->oloc.file,
-                                         io_info->op_type == H5D_IO_OP_WRITE))
-                io_info->use_select_io = H5D_SELECTION_IO_MODE_ON;
-            else {
-                io_info->use_select_io = H5D_SELECTION_IO_MODE_OFF;
-                io_info->no_selection_io_cause |= H5D_SEL_IO_DEFAULT_OFF;
-            }
+     * later if something else conflicts), otherwise disable it */
+    if (io_info->use_select_io == H5D_SELECTION_IO_MODE_DEFAULT) {
+        if (H5F_has_vector_select_io(io_info->dsets_info[0].dset->oloc.file,
+                                     io_info->op_type == H5D_IO_OP_WRITE))
+            io_info->use_select_io = H5D_SELECTION_IO_MODE_ON;
+        else {
+            io_info->use_select_io = H5D_SELECTION_IO_MODE_OFF;
+            io_info->no_selection_io_cause |= H5D_SEL_IO_DEFAULT_OFF;
         }
+    }
 
     /* If we're doing type conversion and we might be doing selection I/O, check if the buffers are large
      * enough to handle the whole I/O */

diff --git a/src/H5Dmpio.c b/src/H5Dmpio.c
@@ -622,6 +622,9 @@ H5D__mpio_opt_possible(H5D_io_info_t *io_info)
         if (!H5FD_mpi_opt_types_g)
             local_cause[0] |= H5D_MPIO_MPI_OPT_TYPES_ENV_VAR_DISABLED;
 
+        /* Decision on whether to use selection I/O should have been made by now */
+        assert(io_info->use_select_io != H5D_SELECTION_IO_MODE_DEFAULT);
+
         /* Datatype conversions and transformations are allowed with selection I/O.  If the selection I/O mode
          * is auto (default), disable collective for now and re-enable later if we can */
         if (io_info->use_select_io != H5D_SELECTION_IO_MODE_ON) {
@@ -731,25 +734,6 @@ H5D__mpio_opt_possible(H5D_io_info_t *io_info)
             HMPI_GOTO_ERROR(FAIL, "MPI_Allreduce failed", mpi_code)
     } /* end else */
 
-    /* If the selection I/O mode is default (auto), decide here whether it should be on or off */
-    if (io_info->use_select_io == H5D_SELECTION_IO_MODE_DEFAULT) {
-        /* If the only reason(s) we've disabled collective are type conversions and/or transforms, enable
-         * selection I/O and re-enable collective I/O since it's supported by selection I/O */
-        if (global_cause[0] && !(global_cause[0] & ~((unsigned)H5D_MPIO_DATATYPE_CONVERSION |
-                                                     (unsigned)H5D_MPIO_DATA_TRANSFORMS))) {
-            assert(!(local_cause[0] &
-                     ~((unsigned)H5D_MPIO_DATATYPE_CONVERSION | (unsigned)H5D_MPIO_DATA_TRANSFORMS)));
-            local_cause[0]         = 0;
-            global_cause[0]        = 0;
-            io_info->use_select_io = H5D_SELECTION_IO_MODE_ON;
-        }
-        else {
-            /* Otherwise, there's currently no benefit to selection I/O, so leave it off */
-            io_info->use_select_io = H5D_SELECTION_IO_MODE_OFF;
-            io_info->no_selection_io_cause |= H5D_SEL_IO_DEFAULT_OFF;
-        }
-    }
-
     /* Set the local & global values of no-collective-cause in the API context */
     H5CX_set_mpio_local_no_coll_cause(local_cause[0]);
     H5CX_set_mpio_global_no_coll_cause(global_cause[0]);