Skip to content

Commit

Permalink
Check MPI_Get_count_c, use it if available
Browse files Browse the repository at this point in the history
  • Loading branch information
wkliao committed Mar 17, 2024
1 parent 0cd64d9 commit 87a20ce
Show file tree
Hide file tree
Showing 5 changed files with 88 additions and 36 deletions.
1 change: 1 addition & 0 deletions configure.ac
Original file line number Diff line number Diff line change
Expand Up @@ -1326,6 +1326,7 @@ AC_CHECK_FUNCS([MPI_Type_create_subarray_c \
MPI_Type_vector_c \
MPI_Type_size_c \
MPI_Type_get_true_extent_c \
MPI_Get_count_c \
MPI_Pack_c \
MPI_Unpack_c], [], [have_mpi_large_count_apis=no])
# If one of the above APIs is not available, have_mpi_large_count_apis will be
Expand Down
35 changes: 23 additions & 12 deletions src/drivers/ncmpio/ncmpio_enddef.c
Original file line number Diff line number Diff line change
Expand Up @@ -133,14 +133,14 @@ move_file_block(NC *ncp,
* bufcount for write. Note that the latter will write the variables
* that have not been written before. Below uses the former option.
*/
#ifdef _USE_MPI_GET_COUNT

/* explicitly initialize mpistatus object to 0. For zero-length read,
* MPI_Get_count may report incorrect result for some MPICH version,
* due to the uninitialized MPI_Status object passed to MPI-IO calls.
* Thus we initialize it above to work around.
*/
memset(&mpistatus, 0, sizeof(MPI_Status));
#endif

TRACE_IO(MPI_File_write_at_all)(ncp->collective_fh,
to+nbytes+rank*chunk_size,
buf, get_size /* bufcount */,
Expand All @@ -150,12 +150,18 @@ move_file_block(NC *ncp,
if (err == NC_EFILE) DEBUG_ASSIGN_ERROR(status, NC_EWRITE)
}
else {
#ifdef _USE_MPI_GET_COUNT
int put_size;
MPI_Get_count(&mpistatus, MPI_BYTE, &put_size);
/* update the number of bytes written since file open */
#ifdef HAVE_MPI_GET_COUNT_C
MPI_Count put_size;
MPI_Get_count_c(&mpistatus, MPI_BYTE, &put_size);
ncp->put_size += put_size;
#else
ncp->put_size += get_size; /* or bufcount */
int put_size;
mpireturn = MPI_Get_count(&mpistatus, MPI_BYTE, &put_size);
if (mpireturn != MPI_SUCCESS || put_size == MPI_UNDEFINED)
ncp->put_size += get_size; /* or bufcount */
else
ncp->put_size += put_size;
#endif
}
TRACE_COMM(MPI_Allreduce)(&status, &min_st, 1, MPI_INT, MPI_MIN, ncp->comm);
Expand Down Expand Up @@ -520,14 +526,13 @@ write_NC(NC *ncp)

/* rank 0's fileview already includes the file header */

#ifdef _USE_MPI_GET_COUNT
/* explicitly initialize mpistatus object to 0. For zero-length read,
* MPI_Get_count may report incorrect result for some MPICH version,
* due to the uninitialized MPI_Status object passed to MPI-IO calls.
* Thus we initialize it above to work around.
*/
memset(&mpistatus, 0, sizeof(MPI_Status));
#endif

/* write the header in chunks */
remain = header_wlen;
for (i=0; i<ntimes; i++) {
Expand All @@ -544,12 +549,18 @@ write_NC(NC *ncp)
if (err == NC_EFILE) DEBUG_ASSIGN_ERROR(status, NC_EWRITE)
}
else {
#ifdef _USE_MPI_GET_COUNT
int put_size;
MPI_Get_count(&mpistatus, MPI_BYTE, &put_size);
/* update the number of bytes written since file open */
#ifdef HAVE_MPI_GET_COUNT_C
MPI_Count put_size;
MPI_Get_count_c(&mpistatus, MPI_BYTE, &put_size);
ncp->put_size += put_size;
#else
ncp->put_size += header_wlen;
int put_size;
mpireturn = MPI_Get_count(&mpistatus, MPI_BYTE, &put_size);
if (mpireturn != MPI_SUCCESS || put_size == MPI_UNDEFINED)
ncp->put_size += header_wlen;
else
ncp->put_size += put_size;
#endif
}
remain -= NC_MAX_INT;
Expand Down
48 changes: 38 additions & 10 deletions src/drivers/ncmpio/ncmpio_file_io.c
Original file line number Diff line number Diff line change
Expand Up @@ -66,14 +66,12 @@ ncmpio_read_write(NC *ncp,
/* request size in bytes, may be > NC_MAX_INT */
req_size = (MPI_Offset)btype_size * buf_count;

#ifdef _USE_MPI_GET_COUNT
/* explicitly initialize mpistatus object to 0. For zero-length read,
* MPI_Get_count may report incorrect result for some MPICH version,
* due to the uninitialized MPI_Status object passed to MPI-IO calls.
* Thus we initialize it above to work around.
*/
memset(&mpistatus, 0, sizeof(MPI_Status));
#endif

if (coll_indep == NC_REQ_COLL)
fh = ncp->collective_fh;
Expand Down Expand Up @@ -141,12 +139,27 @@ ncmpio_read_write(NC *ncp,
}
if (mpireturn == MPI_SUCCESS) {
/* update the number of bytes read since file open */
#ifdef _USE_MPI_GET_COUNT
int get_size;
MPI_Get_count(&mpistatus, MPI_BYTE, &get_size);
#ifdef HAVE_MPI_GET_COUNT_C
MPI_Count get_size;
MPI_Get_count_c(&mpistatus, MPI_BYTE, &get_size);
ncp->get_size += get_size;
#else
ncp->get_size += req_size;
int get_size;
mpireturn = MPI_Get_count(&mpistatus, xbuf_type, &get_size);
if (mpireturn != MPI_SUCCESS || get_size == MPI_UNDEFINED)
ncp->get_size += req_size;
else {
#ifdef HAVE_MPI_TYPE_SIZE_X
/* MPI_Type_size_x is introduced in MPI 3.0 */
mpireturn = MPI_Type_size_x(xbuf_type, &btype_size);
#else
mpireturn = MPI_Type_size(xbuf_type, &btype_size);
#endif
if (mpireturn != MPI_SUCCESS || get_size == MPI_UNDEFINED)
ncp->get_size += req_size;
else
ncp->get_size += btype_size * get_size;
}
#endif
}
if (xbuf != buf) { /* unpack contiguous xbuf to noncontiguous buf */
Expand Down Expand Up @@ -237,12 +250,27 @@ ncmpio_read_write(NC *ncp,
}
if (mpireturn == MPI_SUCCESS) {
/* update the number of bytes written since file open */
#ifdef _USE_MPI_GET_COUNT
int put_size;
MPI_Get_count(&mpistatus, MPI_BYTE, &put_size);
#ifdef HAVE_MPI_GET_COUNT_C
MPI_Count put_size;
MPI_Get_count_c(&mpistatus, MPI_BYTE, &put_size);
ncp->put_size += put_size;
#else
ncp->put_size += req_size;
int put_size;
mpireturn = MPI_Get_count(&mpistatus, xbuf_type, &put_size);
if (mpireturn != MPI_SUCCESS || put_size == MPI_UNDEFINED)
ncp->put_size += req_size;
else {
#ifdef HAVE_MPI_TYPE_SIZE_X
/* MPI_Type_size_x is introduced in MPI 3.0 */
mpireturn = MPI_Type_size_x(xbuf_type, &btype_size);
#else
mpireturn = MPI_Type_size(xbuf_type, &btype_size);
#endif
if (mpireturn != MPI_SUCCESS || put_size == MPI_UNDEFINED)
ncp->put_size += req_size;
else
ncp->put_size += btype_size * put_size;
}
#endif
}
if (xbuf != buf) NCI_Free(xbuf);
Expand Down
23 changes: 15 additions & 8 deletions src/drivers/ncmpio/ncmpio_header_put.c
Original file line number Diff line number Diff line change
Expand Up @@ -544,18 +544,19 @@ int ncmpio_write_header(NC *ncp)
DEBUG_RETURN_ERROR(NC_EINTOVERFLOW)
}

#ifdef _USE_MPI_GET_COUNT
/* explicitly initialize mpistatus object to 0. For zero-length read,
* MPI_Get_count may report incorrect result for some MPICH version,
* due to the uninitialized MPI_Status object passed to MPI-IO calls.
* Thus we initialize it above to work around.
*/
memset(&mpistatus, 0, sizeof(MPI_Status));
#endif

if (fIsSet(ncp->flags, NC_HCOLL)) /* collective write */
TRACE_IO(MPI_File_write_at_all)(fh, 0, buf, (int)ncp->xsz, MPI_BYTE, &mpistatus);
TRACE_IO(MPI_File_write_at_all)(fh, 0, buf, (int)ncp->xsz,
MPI_BYTE, &mpistatus);
else
TRACE_IO(MPI_File_write_at)(fh, 0, buf, (int)ncp->xsz, MPI_BYTE, &mpistatus);
TRACE_IO(MPI_File_write_at)(fh, 0, buf, (int)ncp->xsz,
MPI_BYTE, &mpistatus);

if (mpireturn != MPI_SUCCESS) {
err = ncmpii_error_mpi2nc(mpireturn, "MPI_File_write_at");
Expand All @@ -565,12 +566,18 @@ int ncmpio_write_header(NC *ncp)
}
}
else {
#ifdef _USE_MPI_GET_COUNT
int put_size;
MPI_Get_count(&mpistatus, MPI_BYTE, &put_size);
/* update the number of bytes written since file open */
#ifdef HAVE_MPI_GET_COUNT_C
MPI_Count put_size;
MPI_Get_count_c(&mpistatus, MPI_BYTE, &put_size);
ncp->put_size += put_size;
#else
ncp->put_size += ncp->xsz;
int put_size;
mpireturn = MPI_Get_count(&mpistatus, MPI_BYTE, &put_size);
if (mpireturn != MPI_SUCCESS || put_size == MPI_UNDEFINED)
ncp->put_size += ncp->xsz;
else
ncp->put_size += put_size;
#endif
}
NCI_Free(buf);
Expand Down
17 changes: 11 additions & 6 deletions src/drivers/ncmpio/ncmpio_sync.c
Original file line number Diff line number Diff line change
Expand Up @@ -107,14 +107,13 @@ ncmpio_write_numrecs(NC *ncp,
}
/* ncmpix_put_xxx advances the 1st argument with size len */

#ifdef _USE_MPI_GET_COUNT
/* explicitly initialize mpistatus object to 0. For zero-length read,
* MPI_Get_count may report incorrect result for some MPICH version,
* due to the uninitialized MPI_Status object passed to MPI-IO calls.
* Thus we initialize it above to work around.
*/
memset(&mpistatus, 0, sizeof(MPI_Status));
#endif

/* root's file view always includes the entire file header */
if (fIsSet(ncp->flags, NC_HCOLL))
TRACE_IO(MPI_File_write_at_all)(fh, NC_NUMRECS_OFFSET, (void*)pos,
Expand All @@ -127,12 +126,18 @@ ncmpio_write_numrecs(NC *ncp,
if (err == NC_EFILE) DEBUG_RETURN_ERROR(NC_EWRITE)
}
else {
#ifdef _USE_MPI_GET_COUNT
int put_size;
MPI_Get_count(&mpistatus, MPI_BYTE, &put_size);
/* update the number of bytes written since file open */
#ifdef HAVE_MPI_GET_COUNT_C
MPI_Count put_size;
MPI_Get_count_c(&mpistatus, MPI_BYTE, &put_size);
ncp->put_size += put_size;
#else
ncp->put_size += len;
int put_size;
mpireturn = MPI_Get_count(&mpistatus, MPI_BYTE, &put_size);
if (mpireturn != MPI_SUCCESS || put_size == MPI_UNDEFINED)
ncp->put_size += len;
else
ncp->put_size += put_size;
#endif
}
}
Expand Down

0 comments on commit 87a20ce

Please sign in to comment.