Merge pull request #138 from Parallel-NetCDF/remove_hint_nc_header_co…

…llective

Remove hint nc_header_collective but still allow collective I/O for header

benchmarks/C/parallel_run.sh

@@ -30,7 +30,7 @@ unset PNETCDF_HINTS
 for i in ${check_PROGRAMS} ; do
     for j in ${safe_modes} ; do
         if test "$j" = 1 ; then # test only in safe mode
-           export PNETCDF_HINTS="nc_header_collective=true"
+           export PNETCDF_HINTS="romio_no_indep_rw=true"
         fi
         export PNETCDF_SAFE_MODE=$j
         # echo "set PNETCDF_SAFE_MODE ${PNETCDF_SAFE_MODE}"

benchmarks/FLASH-IO/parallel_run.sh

@@ -30,7 +30,7 @@ unset PNETCDF_HINTS
 for i in ${check_PROGRAMS} ; do
     for j in ${safe_modes} ; do
         if test "$j" = 1 ; then # test only in safe mode
-           export PNETCDF_HINTS="nc_header_collective=true"
+           export PNETCDF_HINTS="romio_no_indep_rw=true"
         fi
         export PNETCDF_SAFE_MODE=$j
         # echo "set PNETCDF_SAFE_MODE ${PNETCDF_SAFE_MODE}"

examples/C/get_info.c

@@ -15,7 +15,7 @@
  *
  *    Example standard output:
 
-    MPI File Info: nkeys = 27
+    MPI File Info: nkeys = 35
     MPI File Info: [ 0] key =            cb_buffer_size, value = 16777216
     MPI File Info: [ 1] key =             romio_cb_read, value = automatic
     MPI File Info: [ 2] key =            romio_cb_write, value = automatic
@@ -31,18 +31,26 @@
     MPI File Info: [12] key =             romio_ds_read, value = automatic
     MPI File Info: [13] key =            romio_ds_write, value = automatic
     MPI File Info: [14] key =  romio_synchronized_flush, value = disabled
-    MPI File Info: [15] key =            cb_config_list, value = *:1
-    MPI File Info: [16] key =     romio_filesystem_type, value = UFS: Generic ROMIO driver for all UNIX-like file systems
-    MPI File Info: [17] key =     romio_aggregator_list, value = 0
-    MPI File Info: [18] key =      nc_header_align_size, value = 512
-    MPI File Info: [19] key =         nc_var_align_size, value = 4
-    MPI File Info: [20] key =      nc_record_align_size, value = 4
-    MPI File Info: [21] key = nc_header_read_chunk_size, value = 262144
-    MPI File Info: [22] key =          nc_in_place_swap, value = auto
-    MPI File Info: [23] key =              nc_ibuf_size, value = 16777216
-    MPI File Info: [24] key =         pnetcdf_subfiling, value = disable
-    MPI File Info: [25] key =           nc_num_subfiles, value = 0
-    MPI File Info: [26] key =      nc_header_collective, value = false
+    MPI File Info: [15] key = romio_visibility_immediate, value = true
+    MPI File Info: [16] key =            cb_config_list, value = *:1
+    MPI File Info: [17] key =     romio_filesystem_type, value = UFS: Generic ROMIO driver for all UNIX-like file systems
+    MPI File Info: [18] key =    mpi_memory_alloc_kinds, value = mpi,system
+    MPI File Info: [19] key =     romio_aggregator_list, value = 0
+    MPI File Info: [20] key =             striping_unit, value = 0
+    MPI File Info: [21] key =           striping_factor, value = 0
+    MPI File Info: [22] key =            start_iodevice, value = 0
+    MPI File Info: [23] key =      nc_header_align_size, value = 512
+    MPI File Info: [24] key =         nc_var_align_size, value = 4
+    MPI File Info: [25] key =      nc_record_align_size, value = 4
+    MPI File Info: [26] key = nc_header_read_chunk_size, value = 262144
+    MPI File Info: [27] key =          nc_in_place_swap, value = auto
+    MPI File Info: [28] key =              nc_ibuf_size, value = 16777216
+    MPI File Info: [29] key =         pnetcdf_subfiling, value = disable
+    MPI File Info: [30] key =           nc_num_subfiles, value = 0
+    MPI File Info: [31] key =          nc_hash_size_dim, value = 256
+    MPI File Info: [32] key =          nc_hash_size_var, value = 256
+    MPI File Info: [33] key =        nc_hash_size_gattr, value = 64
+    MPI File Info: [34] key =        nc_hash_size_vattr, value = 8
  */
 
 #include <stdio.h>

examples/C/parallel_run.sh

@@ -33,7 +33,7 @@ unset PNETCDF_HINTS
 for i in ${check_PROGRAMS} ; do
     for j in ${safe_modes} ; do
         if test "$j" = 1 ; then # test only in safe mode
-           export PNETCDF_HINTS="nc_header_collective=true"
+           export PNETCDF_HINTS="romio_no_indep_rw=true"
         fi
         export PNETCDF_SAFE_MODE=$j
         # echo "set PNETCDF_SAFE_MODE ${PNETCDF_SAFE_MODE}"

examples/CXX/parallel_run.sh

@@ -33,7 +33,7 @@ unset PNETCDF_HINTS
 for i in ${check_PROGRAMS} ; do
     for j in ${safe_modes} ; do
         if test "$j" = 1 ; then # test only in safe mode
-           export PNETCDF_HINTS="nc_header_collective=true"
+           export PNETCDF_HINTS="romio_no_indep_rw=true"
         fi
         export PNETCDF_SAFE_MODE=$j
         # echo "set PNETCDF_SAFE_MODE ${PNETCDF_SAFE_MODE}"

examples/F77/parallel_run.sh

@@ -33,7 +33,7 @@ unset PNETCDF_HINTS
 for i in ${check_PROGRAMS} ; do
     for j in ${safe_modes} ; do
         if test "$j" = 1 ; then # test only in safe mode
-           export PNETCDF_HINTS="nc_header_collective=true"
+           export PNETCDF_HINTS="romio_no_indep_rw=true"
         fi
         export PNETCDF_SAFE_MODE=$j
         # echo "set PNETCDF_SAFE_MODE ${PNETCDF_SAFE_MODE}"

examples/F90/parallel_run.sh

@@ -33,7 +33,7 @@ unset PNETCDF_HINTS
 for i in ${check_PROGRAMS} ; do
     for j in ${safe_modes} ; do
         if test "$j" = 1 ; then # test only in safe mode
-           export PNETCDF_HINTS="nc_header_collective=true"
+           export PNETCDF_HINTS="romio_no_indep_rw=true"
         fi
         export PNETCDF_SAFE_MODE=$j
         # echo "set PNETCDF_SAFE_MODE ${PNETCDF_SAFE_MODE}"

examples/adios/parallel_run.sh

@@ -29,7 +29,7 @@ unset PNETCDF_HINTS
 for i in ${check_PROGRAMS} ; do
     for j in ${safe_modes} ; do
         if test "$j" = 1 ; then # test only in safe mode
-           export PNETCDF_HINTS="nc_header_collective=true"
+           export PNETCDF_HINTS="romio_no_indep_rw=true"
         fi
         export PNETCDF_SAFE_MODE=$j
         # echo "set PNETCDF_SAFE_MODE ${PNETCDF_SAFE_MODE}"

examples/burst_buffer/parallel_run.sh

@@ -34,7 +34,7 @@ for i in ${check_PROGRAMS} ; do
     # echo "---- exec=$i"
     for j in ${safe_modes} ; do
         if test "$j" = 1 ; then # test only in safe mode
-           export PNETCDF_HINTS="nc_header_collective=true"
+           export PNETCDF_HINTS="romio_no_indep_rw=true"
         fi
         export PNETCDF_SAFE_MODE=$j
         # echo "set PNETCDF_SAFE_MODE ${PNETCDF_SAFE_MODE}"

examples/tutorial/parallel_run.sh

@@ -33,7 +33,7 @@ unset PNETCDF_HINTS
 for i in ${check_PROGRAMS} ; do
     for j in ${safe_modes} ; do
         if test "$j" = 1 ; then # test only in safe mode
-           export PNETCDF_HINTS="nc_header_collective=true"
+           export PNETCDF_HINTS="romio_no_indep_rw=true"
         fi
         export PNETCDF_SAFE_MODE=$j
         # echo "set PNETCDF_SAFE_MODE ${PNETCDF_SAFE_MODE}"

sneak_peek.md

@@ -16,11 +16,11 @@ This is essentially a placeholder for the next release note ...
     ```
   + PnetCDF now allows a single read/write request from a process of size
     larger than 2 GiB. Such large requests are now passed down to the MP-IO
-    library. This change is because MPI 3.0 introduces the large count feature,
+    library. This change is because MPI 4.0 introduces the large count feature,
     including MPI_Count data type, MPI_XXX_c and MPI_XXX_x APIs that use 8-byte
-    integer type to enable large MPI operations. As many MPI libraries today
+    integer type to enable large MPI operations. As some MPI libraries today
     have implemented this feature, PnetCDF can now take advantage of it to
-    support large single requests. Because of this change configure option
+    support large single requests. Because of this change, configure option
     `--enable-large-single-req` is thus deprecated. See
     See [PR #131](https://github.com/Parallel-NetCDF/PnetCDF/pull/131)
 
@@ -95,24 +95,18 @@ This is essentially a placeholder for the next release note ...
   + none
 
 * New PnetCDF hint
-  + `nc_hash_size_dim:   Set hash table size for dimension names. Default: 256
-  + `nc_hash_size_var:   Set hash table size for variable names. Default: 256
-  + `nc_hash_size_gattr: Set hash table size for global attribute names.
+  + `nc_hash_size_dim`:   Set hash table size for dimension names. Default: 256
+  + `nc_hash_size_var`:   Set hash table size for variable names. Default: 256
+  + `nc_hash_size_gattr`: Set hash table size for global attribute names.
     Default: 64
-  + `nc_hash_size_vattr: Set hash table size for variable attribute names.
+  + `nc_hash_size_vattr`: Set hash table size for variable attribute names.
     Default: 8
   + The above 4 new hints allow users to set different hash table sizes for
     different objects.  For instance, when the number of variables to be
     defined is large and the number of attributes per variable is small,
     increasing `nc_hash_size_var` can speed up the definition time, and
     reducing `nc_hash_size_vattr` can reduce the memory footprint. See
     [PR #132](https://github.com/Parallel-NetCDF/PnetCDF/pull/132).
-  + `nc_header_collective` -- to instruct PnetCDF to call MPI collective APIs to
-    read and write the file header. The default is "false", meaning the file
-    header is only read/written by rank 0, using MPI independent read and write
-    APIs. When set to "true", collective APIs will be used with all other
-    processes making zero-size requests. See
-    [PR #104](https://github.com/Parallel-NetCDF/PnetCDF/pull/104).
 
 * New run-time environment variables
   + none
@@ -133,9 +127,8 @@ This is essentially a placeholder for the next release note ...
       variables. The file is still a valid netCDF file.
     * In the former case, PR #99 changes `ncvalidator` to report a warning,
       rather than an error.
-  + `ncvalidator` - Add printing of the dimension size of a variable when its
-    size is larger than the limitation allowed by the file format. See commit
-    5584d44.
+    * Add printing of the dimension size of a variable when its size is larger
+      than the limitation allowed by the file format. See commit 5584d44.
   + Add file src/utils/README.md which gives short descriptions of the utility
     programs and collapsible bullets to display their manual pages.
 
@@ -158,6 +151,15 @@ This is essentially a placeholder for the next release note ...
   + Silence Intel icc compilation warnings: when CFLAGS contains
     "-Wimplicit-const-int-float-conversion" and "-Wstringop-overread".
     See [PR #110](https://github.com/Parallel-NetCDF/PnetCDF/pull/110).
+  + In all previous PnetCDF's implementations, file header is always
+    written/read by rank 0 using MPI independent APIs. This can nullify ROMIO
+    hint `romio_no_indep_rw` if set by the user. To warrant no independent
+    read/write, PnetCDF now first checks hint `romio_no_indep_rw` and if set to
+    `true`, then all file header I/Os are done using MPI collective I/O calls,
+    where only rank 0 makes non-zero length requests while all others zero
+    length (in order to participate the collective calls). See
+    [PR #104](https://github.com/Parallel-NetCDF/PnetCDF/pull/104) and
+    [PR #138](https://github.com/Parallel-NetCDF/PnetCDF/pull/138).
   + In all prior versions, the file name was checked whether it contains
     character ':'. The prefix name ending with ':' is considered by ROMIO as
     the file system type name. The prefix name, if found, is then stripped, so

src/drivers/ncmpio/ncmpio_enddef.c

@@ -1083,11 +1083,6 @@ ncmpio__enddef(void       *ncdp,
     sprintf(value, "%lld", ncp->r_align);
     MPI_Info_set(ncp->mpiinfo, "nc_record_align_size", value);
 
-    if (fIsSet(ncp->flags, NC_HCOLL))
-        MPI_Info_set(ncp->mpiinfo, "nc_header_collective", "true");
-    else
-        MPI_Info_set(ncp->mpiinfo, "nc_header_collective", "false");
-
 #ifdef ENABLE_SUBFILING
     sprintf(value, "%d", ncp->num_subfiles);
     MPI_Info_set(ncp->mpiinfo, "nc_num_subfiles", value);

src/drivers/ncmpio/ncmpio_util.c

@@ -180,16 +180,17 @@ void ncmpio_set_pnetcdf_hints(NC *ncp,
 #endif
 
     if (user_info != MPI_INFO_NULL) {
-        /* read/write file header using MPI collective APIs */
-        MPI_Info_get(user_info, "nc_header_collective", MPI_MAX_INFO_VAL-1,
+        /* If romio_no_indep_rw is set to true, let all processes participate
+         * the read/write file header using MPI collective APIs, where only
+         * rank 0 has non-zero request count.
+         */
+        MPI_Info_get(user_info, "romio_no_indep_rw", MPI_MAX_INFO_VAL-1,
                      value, &flag);
         if (flag) {
             if (strcasecmp(value, "true") == 0)
                 fSet((ncp)->flags, NC_HCOLL);
         }
     }
-    if (!flag) strcpy(value, "false");
-    MPI_Info_set(info_used, "nc_header_collective", value);
 
     ncp->dims.hash_size = PNC_HSIZE_DIM;
     if (user_info != MPI_INFO_NULL) {

test/C/parallel_run.sh

@@ -28,7 +28,7 @@ unset PNETCDF_HINTS
 
 for j in ${safe_modes} ; do
     if test "$j" = 1 ; then # test only in safe mode
-       export PNETCDF_HINTS="nc_header_collective=true"
+       export PNETCDF_HINTS="romio_no_indep_rw=true"
     fi
     export PNETCDF_SAFE_MODE=$j
     # echo "set PNETCDF_SAFE_MODE ${PNETCDF_SAFE_MODE}"

test/CXX/parallel_run.sh

@@ -30,7 +30,7 @@ unset PNETCDF_HINTS
 for i in ${TESTPROGRAMS} ; do
     for j in ${safe_modes} ; do
         if test "$j" = 1 ; then # test only in safe mode
-           export PNETCDF_HINTS="nc_header_collective=true"
+           export PNETCDF_HINTS="romio_no_indep_rw=true"
         fi
         export PNETCDF_SAFE_MODE=$j
         # echo "set PNETCDF_SAFE_MODE ${PNETCDF_SAFE_MODE}"

test/F90/parallel_run.sh

@@ -30,7 +30,7 @@ unset PNETCDF_HINTS
 for i in ${PARALLEL_PROGS} ; do
     for j in ${safe_modes} ; do
         if test "$j" = 1 ; then # test only in safe mode
-           export PNETCDF_HINTS="nc_header_collective=true"
+           export PNETCDF_HINTS="romio_no_indep_rw=true"
         fi
         export PNETCDF_SAFE_MODE=$j
         # echo "set PNETCDF_SAFE_MODE ${PNETCDF_SAFE_MODE}"

test/adios/parallel_run.sh

@@ -27,7 +27,7 @@ unset PNETCDF_HINTS
 for i in ${check_PROGRAMS} ; do
     for j in ${safe_modes} ; do
         if test "$j" = 1 ; then # test only in safe mode
-           export PNETCDF_HINTS="nc_header_collective=true"
+           export PNETCDF_HINTS="romio_no_indep_rw=true"
         fi
         export PNETCDF_SAFE_MODE=$j
         # echo "set PNETCDF_SAFE_MODE ${PNETCDF_SAFE_MODE}"

test/burst_buffer/parallel_run.sh

@@ -30,7 +30,7 @@ unset PNETCDF_HINTS
 for i in ${TESTPROGRAMS} ; do
     for j in ${safe_modes} ; do
         if test "$j" = 1 ; then # test only in safe mode
-           export PNETCDF_HINTS="nc_header_collective=true"
+           export PNETCDF_HINTS="romio_no_indep_rw=true"
         fi
         export PNETCDF_SAFE_MODE=$j
         # echo "set PNETCDF_SAFE_MODE ${PNETCDF_SAFE_MODE}"

test/cdf_format/parallel_run.sh

@@ -30,7 +30,7 @@ unset PNETCDF_HINTS
 
 for j in ${safe_modes} ; do
     if test "$j" = 1 ; then # test only in safe mode
-       export PNETCDF_HINTS="nc_header_collective=true"
+       export PNETCDF_HINTS="romio_no_indep_rw=true"
     fi
     export PNETCDF_SAFE_MODE=$j
     # echo "set PNETCDF_SAFE_MODE ${PNETCDF_SAFE_MODE}"

test/header/parallel_run.sh

@@ -30,7 +30,7 @@ unset PNETCDF_HINTS
 for i in ${check_PROGRAMS} ; do
     for j in ${safe_modes} ; do
         if test "$j" = 1 ; then # test only in safe mode
-           export PNETCDF_HINTS="nc_header_collective=true"
+           export PNETCDF_HINTS="romio_no_indep_rw=true"
         fi
         export PNETCDF_SAFE_MODE=$j
         # echo "set PNETCDF_SAFE_MODE ${PNETCDF_SAFE_MODE}"

test/nc4/interoperability_rd.m4

@@ -43,7 +43,7 @@ int nc_wr_$2(int rank, int ncid, int* vid, int *did){
     start[1] = 1;
     err = nc_put_var1_$2(ncid, *vid, start, buf + 4); CHECK_ERR
 
-    return err;
+    return nerrs;
 }
 ')dnl
 dnl
@@ -79,18 +79,18 @@ int pnc_rd_$2(int rank, int ncid, int vid, int *did){
         }
     }
 
-    return err;
+    return nerrs;
 }
 ')dnl
 dnl
 define(`CALL_NC_WR',dnl
 `dnl
-    err = nc_wr_$2(rank, ncid, vid + $1, did); CHECK_ERR
+    nerrs += nc_wr_$2(rank, ncid, vid + $1, did); CHECK_NERRS
 ')dnl
 dnl
 define(`CALL_PNC_RD',dnl
 `dnl
-    err = pnc_rd_$2(rank, ncid, vid[$1], did); CHECK_ERR
+    nerrs += pnc_rd_$2(rank, ncid, vid[$1], did); CHECK_NERRS
 ')dnl
 dnl
 /*
@@ -139,6 +139,8 @@ foreach(`dt', (`(`c', `schar', `signed char')', dnl
                ), `PNC_RD(translit(dt, `()'))')dnl
 
 
+#define CHECK_NERRS if (nerrs > 0) goto err_out;
+
 int main(int argc, char **argv) {
     int err, nerrs = 0;
     int rank, np;
@@ -211,6 +213,7 @@ foreach(`dt', (`(`0', `schar', `char')', dnl
                `(`9', `ulonglong', `unsigned long long')', dnl
                ), `CALL_PNC_RD(translit(dt, `()'))')dnl
 
+err_out:
     /* Close file */
     ncmpi_close(ncid);
 

test/nc4/interoperability_wr.m4

@@ -44,7 +44,7 @@ int pnc_wr_$2(int rank, int ncid, int* vid, int *did){
     start[1] = 1;
     err = ncmpi_put_var1_$2_all(ncid, *vid, start, buf + 4); CHECK_ERR
 
-    return err;
+    return nerrs;
 }
 ')dnl
 dnl
@@ -81,18 +81,18 @@ int nc_rd_$2(int rank, int ncid, int vid, int *did){
         }
     }
 
-    return err;
+    return nerrs;
 }
 ')dnl
 dnl
 define(`CALL_PNC_WR',dnl
 `dnl
-    err = pnc_wr_$2(rank, ncid, vid + 0, did); CHECK_ERR
+    nerrs += pnc_wr_$2(rank, ncid, vid + 0, did); CHECK_NERRS
 ')dnl
 dnl
 define(`CALL_NC_RD',dnl
 `dnl
-    err = nc_rd_$2(rank, ncid, vid[0], did); CHECK_ERR
+    nerrs += nc_rd_$2(rank, ncid, vid[0], did); CHECK_NERRS
 ')dnl
 dnl
 /*
@@ -140,6 +140,8 @@ foreach(`dt', (`(`c', `schar', `signed char')', dnl
                ), `NC_RD(translit(dt, `()'))')dnl
 
 
+#define CHECK_NERRS if (nerrs > 0) goto err_out;
+
 int main(int argc, char **argv) {
     int err, nerrs = 0;
     int rank, np;
@@ -191,6 +193,7 @@ foreach(`dt', (`(`0', `schar', `char')', dnl
                `(`9', `ulonglong', `unsigned long long')', dnl
                ), `CALL_PNC_WR(translit(dt, `()'))')dnl
 
+err_out:
     /* Close file */
     ncmpi_close(ncid);