From 20174543ad517320229944058fb2d1e03a90075f Mon Sep 17 00:00:00 2001 From: wkliao Date: Wed, 13 Mar 2024 19:52:42 -0500 Subject: [PATCH] Check availability individual large count MPI functions PnetCDF uses the following MPI datatype construction functions for large requests. MPI_Type_create_subarray_c MPI_Type_contiguous_c MPI_Type_create_hvector_c MPI_Type_create_struct_c MPI_Type_create_hindexed_c MPI_Type_vector_c MPI_Pack_c MPI_Unpack_c MPI_Type_get_true_extent_x MPI_Type_size_x If all are available, then define HAVE_MPI_LARGE_COUNT and proceed with constructing large datatypes for fileview and user buffer. Checking MPI_VERSION against 3 is not sufficient, as some MPI (e.g. OpenMPI 5.0.2) does not support MPI_XXX_c functions. --- configure.ac | 21 ++++++++++++ src/drivers/common/create_imaptype.c | 4 +-- src/drivers/common/pack_unpack.c | 24 ++++++++------ src/drivers/ncbbio/ncbbio_log.c | 4 +-- src/drivers/ncbbio/ncbbio_var.c | 2 +- src/drivers/ncmpio/ncmpio_file_io.c | 48 +++++++++++++++++++++------- src/drivers/ncmpio/ncmpio_filetype.c | 38 ++++++++++++++-------- src/drivers/ncmpio/ncmpio_fill.c | 16 +++++----- src/drivers/ncmpio/ncmpio_subfile.c | 2 +- src/drivers/ncmpio/ncmpio_util.c | 8 ++--- src/drivers/ncmpio/ncmpio_vard.c | 24 +++++++------- src/drivers/ncmpio/ncmpio_wait.c | 47 +++++++++++++-------------- test/largefile/large_header.c | 9 ++++-- 13 files changed, 156 insertions(+), 91 deletions(-) diff --git a/configure.ac b/configure.ac index be0d823a3..d2c4409ed 100644 --- a/configure.ac +++ b/configure.ac @@ -145,6 +145,7 @@ AH_TEMPLATE([PNETCDF_PROFILING], [Define if to enable PnetCDF internal pe AH_TEMPLATE([ENABLE_THREAD_SAFE], [Define if to enable thread-safe capability]) AH_TEMPLATE([ENABLE_REQ_AGGREGATION], [Define if able to support request aggregation in nonblocking routines]) dnl AH_TEMPLATE([HAVE_MPI_COUNT], [Define if type MPI_Count is defined]) +AH_TEMPLATE([HAVE_MPI_LARGE_COUNT], [Define if required MPI APIs have arguments of type MPI_Count]) AH_TOP([#ifndef _CONFIG_H #define _CONFIG_H]) @@ -1314,6 +1315,26 @@ dnl MPI_Type_create_struct \ dnl MPI_Type_create_resized \ dnl MPI_Type_get_extent]) +dnl MPI_count was first introduced in MPI 3.0. Check MPI functions that make +dnl use of MPI_Count. +have_mpi_large_count_apis=yes +AC_CHECK_FUNCS([MPI_Type_create_subarray_c \ + MPI_Type_contiguous_c \ + MPI_Type_create_hvector_c \ + MPI_Type_create_struct_c \ + MPI_Type_create_hindexed_c \ + MPI_Type_vector_c \ + MPI_Pack_c \ + MPI_Unpack_c \ + MPI_Type_get_true_extent_x \ + MPI_Type_size_x], [], [have_mpi_large_count_apis=no]) +# If one of the above APIs is not available, have_mpi_large_count_apis will be +# set to no +UD_MSG_DEBUG([have_mpi_large_count_apis=$have_mpi_large_count_apis]) +if test "x$have_mpi_large_count_apis" = "xyes" ; then + AC_DEFINE(HAVE_MPI_LARGE_COUNT, 1) +fi + dnl Check presence of MPI COMBINERS. These are of type int. dnl These are introduced in MPI 2.0. As PnetCDF requires an MPI library that dnl supports MPI-IO and MPI-IO was first introduced in MPI 2.0, checking these diff --git a/src/drivers/common/create_imaptype.c b/src/drivers/common/create_imaptype.c index 03169d402..5f219702a 100644 --- a/src/drivers/common/create_imaptype.c +++ b/src/drivers/common/create_imaptype.c @@ -69,7 +69,7 @@ ncmpii_create_imaptype(int ndims, */ if (imap_contig_blocklen > NC_MAX_INT || count[dim] > NC_MAX_INT || imap[dim] > NC_MAX_INT) { -#if MPI_VERSION >= 3 +#ifdef HAVE_MPI_LARGE_COUNT mpireturn = MPI_Type_vector_c(count[dim], imap_contig_blocklen, imap[dim], itype, imaptype); if (mpireturn != MPI_SUCCESS) { @@ -99,7 +99,7 @@ ncmpii_create_imaptype(int ndims, MPI_Datatype tmptype; if (count[dim] > NC_MAX_INT) { -#if MPI_VERSION >= 3 +#ifdef HAVE_MPI_LARGE_COUNT mpireturn = MPI_Type_create_hvector_c(count[dim], 1, imap[dim]*el_size, *imaptype, &tmptype); if (mpireturn != MPI_SUCCESS) { diff --git a/src/drivers/common/pack_unpack.c b/src/drivers/common/pack_unpack.c index b6c502184..053eb3c13 100644 --- a/src/drivers/common/pack_unpack.c +++ b/src/drivers/common/pack_unpack.c @@ -42,20 +42,22 @@ ncmpii_pack(int ndims, MPI_Offset buf_size, nelems; MPI_Datatype etype, imaptype=MPI_DATATYPE_NULL; -#if MPI_VERSION >= 3 - MPI_Count position, type_size; - mpireturn = MPI_Type_size_c(buftype, &type_size); +#ifdef HAVE_MPI_TYPE_SIZE_X + MPI_Count type_size; + mpireturn = MPI_Type_size_x(buftype, &type_size); if (mpireturn != MPI_SUCCESS) { - err = ncmpii_error_mpi2nc(mpireturn, "MPI_Type_size_c"); + err = ncmpii_error_mpi2nc(mpireturn, "MPI_Type_size_x"); DEBUG_RETURN_ERROR(err) } #else - int position, type_size; + int type_size; mpireturn = MPI_Type_size(buftype, &type_size); if (mpireturn != MPI_SUCCESS) { err = ncmpii_error_mpi2nc(mpireturn, "MPI_Type_size"); DEBUG_RETURN_ERROR(err) } + else if (type_size == MPI_UNDEFINED) + DEBUG_RETURN_ERROR(NC_EINTOVERFLOW) #endif buf_size = type_size; @@ -107,11 +109,12 @@ ncmpii_pack(int ndims, /* allocate lbuf and pack buf into lbuf */ lbuf = NCI_Malloc((size_t)buf_size); if (lbuf == NULL) DEBUG_RETURN_ERROR(NC_ENOMEM) - position = 0; -#if MPI_VERSION >= 3 +#ifdef HAVE_MPI_LARGE_COUNT + MPI_Count position=0; MPI_Pack_c(buf, (MPI_Count)bufcount, buftype, lbuf, (MPI_Count)buf_size, &position, MPI_COMM_SELF); #else + int position=0; if (buf_size > NC_MAX_INT) DEBUG_RETURN_ERROR(NC_EINTOVERFLOW) @@ -128,13 +131,14 @@ ncmpii_pack(int ndims, /* Step 2: pack lbuf to cbuf if imap is non-contiguous */ if (imaptype != MPI_DATATYPE_NULL) { /* true varm */ /* pack lbuf to cbuf, a contiguous buffer, using imaptype */ - position = 0; -#if MPI_VERSION >= 3 +#ifdef HAVE_MPI_LARGE_COUNT + MPI_Count position=0; *cbuf = NCI_Malloc((size_t)buf_size); MPI_Pack_c(lbuf, 1, imaptype, *cbuf, (MPI_Count)buf_size, &position, MPI_COMM_SELF); #else - if (buf_size > NC_MAX_INT) { + int position=0; + if (buf_size > NC_MAX_INT) DEBUG_RETURN_ERROR(NC_EINTOVERFLOW) *cbuf = NCI_Malloc((size_t)buf_size); diff --git a/src/drivers/ncbbio/ncbbio_log.c b/src/drivers/ncbbio/ncbbio_log.c index 4bb4e2178..27de3fbd5 100644 --- a/src/drivers/ncbbio/ncbbio_log.c +++ b/src/drivers/ncbbio/ncbbio_log.c @@ -362,7 +362,7 @@ int ncbbio_log_enddef(NC_bb *ncbbp) int ncbbio_log_close(NC_bb *ncbbp, int replay) { - int err; + int err, status=NC_NOERR; NC_bb_metadataheader* headerp; #ifdef PNETCDF_PROFILING @@ -484,7 +484,7 @@ int ncbbio_log_close(NC_bb *ncbbp, #endif #endif - return NC_NOERR; + return status; } /* diff --git a/src/drivers/ncbbio/ncbbio_var.c b/src/drivers/ncbbio/ncbbio_var.c index 9bce692a8..74e3dd274 100644 --- a/src/drivers/ncbbio/ncbbio_var.c +++ b/src/drivers/ncbbio/ncbbio_var.c @@ -417,7 +417,7 @@ ncbbio_put_varn(void *ncdp, bnelems *= elsize; if (bnelems > NC_MAX_INT) { -#if MPI_VERSION >= 3 +#ifdef HAVE_MPI_LARGE_COUNT MPI_Count position=0; cbuf = NCI_Malloc(bnelems); mpireturn = MPI_Pack_c((void*)buf, bufcount, buftype, cbuf, diff --git a/src/drivers/ncmpio/ncmpio_file_io.c b/src/drivers/ncmpio/ncmpio_file_io.c index d17dfb023..711d91554 100644 --- a/src/drivers/ncmpio/ncmpio_file_io.c +++ b/src/drivers/ncmpio/ncmpio_file_io.c @@ -28,18 +28,36 @@ ncmpio_read_write(NC *ncp, void *buf, int buftype_is_contig) { - int status=NC_NOERR, mpireturn, err; + int status=NC_NOERR, err=NC_NOERR, mpireturn; MPI_Status mpistatus; MPI_File fh; MPI_Offset req_size; -#if MPI_VERSION >= 3 + +#ifdef HAVE_MPI_TYPE_SIZE_X MPI_Count btype_size; /* MPI_Type_size_x is introduced in MPI 3.0 */ - MPI_Type_size_x(buf_type, &btype_size); + mpireturn = MPI_Type_size_x(buf_type, &btype_size); #else int btype_size; - MPI_Type_size(buf_type, &btype_size); + mpireturn = MPI_Type_size(buf_type, &btype_size); #endif + if (mpireturn != MPI_SUCCESS) { + err = ncmpii_error_mpi2nc(mpireturn, "MPI_Type_size"); + /* return the first encountered error if there is any */ + err = (err == NC_EFILE) ? NC_EREAD : err; + } + else if (btype_size == MPI_UNDEFINED) + err = NC_EINTOVERFLOW; + + if (err != NC_NOERR) { + if (coll_indep == NC_REQ_COLL) { + DEBUG_ASSIGN_ERROR(status, err) + /* write nothing, but participate the collective call */ + buf_count = 0; + } + else + DEBUG_RETURN_ERROR(err) + } /* request size in bytes, may be > NC_MAX_INT */ req_size = (MPI_Offset)btype_size * buf_count; @@ -64,12 +82,15 @@ ncmpio_read_write(NC *ncp, MPI_Datatype xbuf_type=buf_type; if (buf_count > NC_MAX_INT) { -#if MPI_VERSION >= 3 +#ifdef HAVE_MPI_LARGE_COUNT MPI_Type_contiguous_c((MPI_Count)buf_count, buf_type, &xbuf_type); MPI_Type_commit(&xbuf_type); xlen = 1; #else - DEBUG_RETURN_ERROR(NC_EINTOVERFLOW) + if (coll_indep == NC_REQ_COLL) + DEBUG_ASSIGN_ERROR(status, NC_EINTOVERFLOW) + else + DEBUG_RETURN_ERROR(NC_EINTOVERFLOW) #endif } else if (buf_count > 0 && !buftype_is_contig && @@ -110,7 +131,7 @@ ncmpio_read_write(NC *ncp, /* return the first encountered error if there is any */ if (status == NC_NOERR) { err = (err == NC_EFILE) ? NC_EREAD : err; - DEBUG_ASSIGN_ERROR(status, err) + DEBUG_RETURN_ERROR(err) } } } @@ -125,7 +146,7 @@ ncmpio_read_write(NC *ncp, #endif } if (xbuf != buf) { /* unpack contiguous xbuf to noncontiguous buf */ -#if MPI_VERSION >= 3 +#ifdef HAVE_MPI_LARGE_COUNT MPI_Count pos=0; MPI_Unpack_c(xbuf, xlen, &pos, buf, (MPI_Count)buf_count, buf_type, MPI_COMM_SELF); @@ -144,12 +165,15 @@ ncmpio_read_write(NC *ncp, MPI_Datatype xbuf_type=buf_type; if (buf_count > NC_MAX_INT) { -#if MPI_VERSION >= 3 +#ifdef HAVE_MPI_LARGE_COUNT MPI_Type_contiguous_c((MPI_Count)buf_count, buf_type, &xbuf_type); MPI_Type_commit(&xbuf_type); xlen = 1; #else - DEBUG_RETURN_ERROR(NC_EINTOVERFLOW) + if (coll_indep == NC_REQ_COLL) + DEBUG_ASSIGN_ERROR(status, NC_EINTOVERFLOW) + else + DEBUG_RETURN_ERROR(NC_EINTOVERFLOW) #endif } else if (buf_count > 0 && !buftype_is_contig && @@ -160,7 +184,7 @@ ncmpio_read_write(NC *ncp, * noncontiguous. */ if (req_size > NC_MAX_INT) { -#if MPI_VERSION >= 3 +#ifdef HAVE_MPI_LARGE_COUNT MPI_Count pos=0; xbuf = NCI_Malloc(req_size); MPI_Pack_c(buf, (MPI_Count)buf_count, buf_type, xbuf, @@ -203,7 +227,7 @@ ncmpio_read_write(NC *ncp, /* return the first encountered error if there is any */ if (status == NC_NOERR) { err = (err == NC_EFILE) ? NC_EWRITE : err; - DEBUG_ASSIGN_ERROR(status, err) + DEBUG_RETURN_ERROR(err) } } } diff --git a/src/drivers/ncmpio/ncmpio_filetype.c b/src/drivers/ncmpio/ncmpio_filetype.c index 4939bc28d..d2422da87 100644 --- a/src/drivers/ncmpio/ncmpio_filetype.c +++ b/src/drivers/ncmpio/ncmpio_filetype.c @@ -123,7 +123,7 @@ type_create_subarray64(int ndims, if (ndims == 0) DEBUG_RETURN_ERROR(NC_EDIMMETA) -#if MPI_VERSION >= 3 +#ifdef HAVE_MPI_LARGE_COUNT MPI_Count *sizes, *subsizes, *starts; sizes = (MPI_Count*) NCI_Malloc((size_t)ndims * 3 * sizeof(MPI_Count)); @@ -329,7 +329,7 @@ filetype_create_vara(const NC *ncp, /* previously, request size has been checked and it must > 0 */ if (IS_RECVAR(varp)) { -#if MPI_VERSION >= 3 +#ifdef HAVE_MPI_LARGE_COUNT MPI_Count blocklength; #else int blocklength; @@ -358,7 +358,7 @@ filetype_create_vara(const NC *ncp, blocklength = varp->xsz; } -#if MPI_VERSION >= 3 +#ifdef HAVE_MPI_LARGE_COUNT /* concatenate number of count[0] subarray types into filetype */ err = MPI_Type_create_hvector_c(count[0], blocklength, ncp->recsize, rectype, &filetype); @@ -576,8 +576,8 @@ ncmpio_file_set_view(const NC *ncp, MPI_Comm_rank(ncp->comm, &rank); if (rank == 0) { /* prepend the whole file header to filetype */ - MPI_Datatype root_filetype, ftypes[2]; -#if MPI_VERSION >= 3 + MPI_Datatype root_filetype=MPI_BYTE, ftypes[2]; +#ifdef HAVE_MPI_LARGE_COUNT MPI_Count blocklens[2]; MPI_Count disps[2]; #else @@ -585,8 +585,10 @@ ncmpio_file_set_view(const NC *ncp, MPI_Aint disps[2]; /* check if header size > 2^31 */ - if (ncp->begin_var > NC_MAX_INT) - DEBUG_ASSIGN_ERROR(status, NC_EINTOVERFLOW) + if (ncp->begin_var > NC_MAX_INT) { + status = NC_EINTOVERFLOW; + goto err_out; + } #endif /* first block is the header extent */ @@ -599,23 +601,31 @@ ncmpio_file_set_view(const NC *ncp, disps[1] = *offset; ftypes[1] = filetype; -#if (MPI_VERSION < 3) && (SIZEOF_MPI_AINT != SIZEOF_MPI_OFFSET) +#if !defined(HAVE_MPI_LARGE_COUNT) && (SIZEOF_MPI_AINT != SIZEOF_MPI_OFFSET) if (*offset > NC_MAX_INT) { - blocklens[1] = 0; - DEBUG_ASSIGN_ERROR(status, NC_EAINT_TOO_SMALL) + status = NC_EINTOVERFLOW; + goto err_out; } #endif -#if MPI_VERSION >= 3 - MPI_Type_create_struct_c(2, blocklens, disps, ftypes, &root_filetype); +#ifdef HAVE_MPI_LARGE_COUNT + mpireturn = MPI_Type_create_struct_c(2, blocklens, disps, ftypes, + &root_filetype); #else - MPI_Type_create_struct(2, blocklens, disps, ftypes, &root_filetype); + mpireturn = MPI_Type_create_struct(2, blocklens, disps, ftypes, + &root_filetype); #endif + if (mpireturn != MPI_SUCCESS) { + err = ncmpii_error_mpi2nc(mpireturn, "MPI_Type_create_struct"); + if (status == NC_NOERR) status = err; + } MPI_Type_commit(&root_filetype); +err_out: TRACE_IO(MPI_File_set_view)(fh, 0, MPI_BYTE, root_filetype, "native", MPI_INFO_NULL); - MPI_Type_free(&root_filetype); + if (root_filetype != MPI_BYTE) + MPI_Type_free(&root_filetype); /* now update the explicit offset to be used in MPI-IO call later */ *offset = ncp->begin_var; diff --git a/src/drivers/ncmpio/ncmpio_fill.c b/src/drivers/ncmpio/ncmpio_fill.c index 49fbc902b..437070dd8 100644 --- a/src/drivers/ncmpio/ncmpio_fill.c +++ b/src/drivers/ncmpio/ncmpio_fill.c @@ -211,7 +211,7 @@ fill_var_rec(NC *ncp, bufType = MPI_BYTE; if (count > NC_MAX_INT) { -#if MPI_VERSION >= 3 +#ifdef HAVE_MPI_LARGE_COUNT mpireturn = MPI_Type_contiguous_c((MPI_Count)count, MPI_BYTE, &bufType); if (mpireturn != MPI_SUCCESS) { err = ncmpii_error_mpi2nc(mpireturn, "MPI_Type_contiguous_c"); @@ -374,7 +374,7 @@ fillerup_aggregate(NC *ncp, NC *old_ncp) MPI_Status mpistatus; NC_var *varp; -#if MPI_VERSION >= 3 +#ifdef HAVE_MPI_LARGE_COUNT MPI_Count *blocklengths, *offset; #else int *blocklengths; @@ -423,7 +423,7 @@ fillerup_aggregate(NC *ncp, NC *old_ncp) /* find the number of write segments (upper bound) */ nsegs = (size_t)(ncp->vars.ndefined + ncp->vars.num_rec_vars * nrecs); count = (MPI_Offset*) NCI_Malloc(nsegs * SIZEOF_MPI_OFFSET); -#if MPI_VERSION >= 3 +#ifdef HAVE_MPI_LARGE_COUNT offset = (MPI_Count*) NCI_Malloc(nsegs * sizeof(MPI_Count)); #else offset = (MPI_Aint*) NCI_Malloc(nsegs * SIZEOF_MPI_AINT); @@ -515,7 +515,7 @@ fillerup_aggregate(NC *ncp, NC *old_ncp) } /* allocate one contiguous buffer space for all writes */ -#if MPI_VERSION >= 3 +#ifdef HAVE_MPI_LARGE_COUNT blocklengths = (MPI_Count*) NCI_Malloc((size_t)j * sizeof(MPI_Count)); #else blocklengths = (int*) NCI_Malloc((size_t)j * SIZEOF_INT); @@ -543,7 +543,7 @@ fillerup_aggregate(NC *ncp, NC *old_ncp) } count[k] *= varp->xsz; -#if MPI_VERSION >= 3 +#ifdef HAVE_MPI_LARGE_COUNT blocklengths[k] = (MPI_Count)count[k]; #else if (count[k] != (int)count[k]) { @@ -578,7 +578,7 @@ fillerup_aggregate(NC *ncp, NC *old_ncp) } count[k] *= varp->xsz; -#if MPI_VERSION >= 3 +#ifdef HAVE_MPI_LARGE_COUNT blocklengths[k] = (MPI_Count)count[k]; #else if (count[k] != (int)count[k]) { @@ -600,7 +600,7 @@ fillerup_aggregate(NC *ncp, NC *old_ncp) } else { /* create fileview: a list of contiguous segment for each variable */ -#if MPI_VERSION >= 3 +#ifdef HAVE_MPI_LARGE_COUNT mpireturn = MPI_Type_create_hindexed_c(k, blocklengths, offset, MPI_BYTE, &filetype); #else @@ -628,7 +628,7 @@ fillerup_aggregate(NC *ncp, NC *old_ncp) bufType = MPI_BYTE; if (buf_len > NC_MAX_INT) { -#if MPI_VERSION >= 3 +#ifdef HAVE_MPI_LARGE_COUNT mpireturn = MPI_Type_contiguous_c((MPI_Count)buf_len, MPI_BYTE, &bufType); if (mpireturn != MPI_SUCCESS) { diff --git a/src/drivers/ncmpio/ncmpio_subfile.c b/src/drivers/ncmpio/ncmpio_subfile.c index 0586c9a42..eef114dbd 100644 --- a/src/drivers/ncmpio/ncmpio_subfile.c +++ b/src/drivers/ncmpio/ncmpio_subfile.c @@ -785,7 +785,7 @@ ncmpio_subfile_getput_vars(NC *ncp, MPI_Offset outsize = bnelems * bufcount * el_size; cbuf = NCI_Malloc((size_t)outsize); if (fIsSet(reqMode, NC_REQ_WR)) { -#if MPI_VERSION >= 3 +#ifdef HAVE_MPI_LARGE_COUNT MPI_Count position=0; MPI_Pack_c(buf, (MPI_Count)bufcount, buftype, cbuf, (MPI_Count)outsize, &position, MPI_COMM_SELF); diff --git a/src/drivers/ncmpio/ncmpio_util.c b/src/drivers/ncmpio/ncmpio_util.c index 6579c9425..3940feee7 100644 --- a/src/drivers/ncmpio/ncmpio_util.c +++ b/src/drivers/ncmpio/ncmpio_util.c @@ -402,7 +402,7 @@ ncmpio_pack_xbuf(int fmt, /* NC_FORMAT_CDF2 NC_FORMAT_CDF5 etc. */ if (buf != lbuf) { /* pack buf into lbuf based on buftype */ -#if MPI_VERSION >= 3 +#ifdef HAVE_MPI_LARGE_COUNT MPI_Count position = 0; MPI_Pack_c(buf, (MPI_Count)bufcount, buftype, lbuf, (MPI_Count)ibuf_size, &position, MPI_COMM_SELF); @@ -437,7 +437,7 @@ ncmpio_pack_xbuf(int fmt, /* NC_FORMAT_CDF2 NC_FORMAT_CDF5 etc. */ } /* pack lbuf to cbuf based on imaptype */ -#if MPI_VERSION >= 3 +#ifdef HAVE_MPI_LARGE_COUNT MPI_Count position = 0; MPI_Pack_c(lbuf, 1, imaptype, cbuf, (MPI_Count)ibuf_size, &position, MPI_COMM_SELF); @@ -670,7 +670,7 @@ ncmpio_unpack_xbuf(int fmt, /* NC_FORMAT_CDF2 NC_FORMAT_CDF5 etc. */ /* unpacked cbuf into lbuf based on imap -------------------------------*/ if (imaptype != MPI_DATATYPE_NULL) { /* unpack cbuf to lbuf based on imaptype */ -#if MPI_VERSION >= 3 +#ifdef HAVE_MPI_LARGE_COUNT MPI_Count position = 0; MPI_Unpack_c(cbuf, (MPI_Count)ibuf_size, &position, lbuf, 1, imaptype, MPI_COMM_SELF); @@ -687,7 +687,7 @@ ncmpio_unpack_xbuf(int fmt, /* NC_FORMAT_CDF2 NC_FORMAT_CDF5 etc. */ /* unpacked lbuf into buf based on buftype -----------------------------*/ if (!buftype_is_contig && lbuf != buf) { /* no need unpack when buftype is used in MPI_File_read (lbuf == buf) */ -#if MPI_VERSION >= 3 +#ifdef HAVE_MPI_LARGE_COUNT MPI_Count position = 0; MPI_Unpack_c(lbuf, (MPI_Count)ibuf_size, &position, buf, (MPI_Count)bufcount, buftype, MPI_COMM_SELF); diff --git a/src/drivers/ncmpio/ncmpio_vard.c b/src/drivers/ncmpio/ncmpio_vard.c index be958ee59..5fab2b27f 100644 --- a/src/drivers/ncmpio/ncmpio_vard.c +++ b/src/drivers/ncmpio/ncmpio_vard.c @@ -59,13 +59,7 @@ getput_vard(NC *ncp, MPI_File fh; MPI_Offset nelems=0, fnelems=0, bnelems=0, offset=0; MPI_Datatype etype=MPI_DATATYPE_NULL, xtype=MPI_BYTE; -#if MPI_VERSION >= 3 - MPI_Count filetype_size=0; - MPI_Count true_lb=0, true_ub=0, true_extent=0; -#else - int filetype_size=0; - MPI_Aint true_lb=0, true_ub=0, true_extent=0; -#endif + MPI_Offset filetype_size; #ifdef ENABLE_SUBFILING /* call a separate routine if variable is stored in subfiles */ @@ -90,14 +84,18 @@ getput_vard(NC *ncp, * MPI_Type_create_hindexed), we need to find the true last byte accessed * by this request, true_ub, in order to calculate new_numrecs. */ -#if MPI_VERSION >= 3 +#ifdef HAVE_MPI_TYPE_SIZE_X /* MPI_Type_size_x is introduced in MPI 3.0 */ - mpireturn = MPI_Type_size_x(filetype, &filetype_size); + MPI_Count true_lb=0, true_ub=0, true_extent=0; + MPI_Count type_size; + + mpireturn = MPI_Type_size_x(filetype, &type_size); if (mpireturn != MPI_SUCCESS) { err = ncmpii_error_mpi2nc(mpireturn, "MPI_Type_size_x"); xtype = MPI_BYTE; goto err_check; } + filetype_size = type_size; /* MPI_Type_get_true_extent_x is introduced in MPI 3.0 */ MPI_Type_get_true_extent_x(filetype, &true_lb, &true_extent); true_ub = true_lb + true_extent; @@ -106,16 +104,20 @@ getput_vard(NC *ncp, * cannot be used for large filetypes. Prior to MPI 3.0 standard, argument * "size" of MPI_Type_size is of type int. When int overflows, the returned * value in argument "size" may be a negative. */ - mpireturn = MPI_Type_size(filetype, &filetype_size); + MPI_Aint true_lb=0, true_ub=0, true_extent=0; + int type_size; + + mpireturn = MPI_Type_size(filetype, &type_size); if (mpireturn != MPI_SUCCESS) { err = ncmpii_error_mpi2nc(mpireturn, "MPI_Type_size"); xtype = MPI_BYTE; goto err_check; } - if (filetype_size < 0) { /* int overflow */ + if (type_size == MPI_UNDEFINED) { /* int overflow */ DEBUG_ASSIGN_ERROR(err, NC_EINTOVERFLOW) goto err_check; } + filetype_size = type_size; MPI_Type_get_true_extent(filetype, &true_lb, &true_extent); true_ub = true_lb + true_extent; #endif diff --git a/src/drivers/ncmpio/ncmpio_wait.c b/src/drivers/ncmpio/ncmpio_wait.c index d90c8d759..587c2f83b 100644 --- a/src/drivers/ncmpio/ncmpio_wait.c +++ b/src/drivers/ncmpio/ncmpio_wait.c @@ -327,7 +327,7 @@ static int construct_filetypes(NC *ncp, NC_lead_req *lead_list, /* NC_REQ_WR or NC_REQ_RD */ int num_reqs, -#if MPI_VERSION >= 3 +#ifdef HAVE_MPI_LARGE_COUNT MPI_Count *blocklens, /* [num_reqs] temp buffer */ MPI_Count *disps, /* [num_reqs] temp buffer */ #else @@ -413,7 +413,7 @@ construct_filetypes(NC *ncp, coalesced_len = blocklens[j]; if (last_contig_req >= 0) coalesced_len += blocklens[last_contig_req]; -#if MPI_VERSION >= 3 +#ifdef HAVE_MPI_LARGE_COUNT if (last_contig_req >= 0 && disps[j] - disps[last_contig_req] == blocklens[last_contig_req]) { @@ -458,7 +458,7 @@ construct_filetypes(NC *ncp, int mpireturn; if (all_ftype_contig) { -#if MPI_VERSION >= 3 +#ifdef HAVE_MPI_LARGE_COUNT mpireturn = MPI_Type_create_hindexed_c(num_reqs, blocklens, disps, MPI_BYTE, filetype); #else @@ -473,7 +473,7 @@ construct_filetypes(NC *ncp, } } else { -#if MPI_VERSION >= 3 +#ifdef HAVE_MPI_LARGE_COUNT mpireturn = MPI_Type_create_struct_c(num_reqs, blocklens, disps, ftypes, filetype); #else @@ -506,7 +506,7 @@ construct_filetypes(NC *ncp, static int construct_buffertypes(NC_lead_req *lead_list, int num_reqs, -#if MPI_VERSION >= 3 +#ifdef HAVE_MPI_LARGE_COUNT MPI_Count *blocklens, /* [num_reqs] temp buffer */ MPI_Count *disps, /* [num_reqs] temp buffer */ #else @@ -540,7 +540,7 @@ construct_buffertypes(NC_lead_req *lead_list, for (k=1; kvarp->ndims; k++) req_size *= count[k]; } -#if MPI_VERSION >= 3 +#ifdef HAVE_MPI_LARGE_COUNT blocklens[j] = req_size; #else /* check int overflow */ @@ -562,7 +562,7 @@ construct_buffertypes(NC_lead_req *lead_list, if (num_reqs > 0) { /* concatenate buffer addresses into a single buffer type */ -#if MPI_VERSION >= 3 +#ifdef HAVE_MPI_LARGE_COUNT mpireturn = MPI_Type_create_hindexed_c(num_reqs, blocklens, disps, MPI_BYTE, buf_type); #else @@ -1487,7 +1487,7 @@ type_create_off_len(MPI_Offset nsegs, /* no. off-len pairs */ { int i, j, mpireturn; MPI_Offset next_off, next_len, true_nsegs; -#if MPI_VERSION >= 3 +#ifdef HAVE_MPI_LARGE_COUNT MPI_Count *blocklens; MPI_Count *disps; #else @@ -1519,7 +1519,7 @@ type_create_off_len(MPI_Offset nsegs, /* no. off-len pairs */ } /* j+1 is the coalesced length */ true_nsegs = j + 1; -#if MPI_VERSION >= 3 +#ifdef HAVE_MPI_LARGE_COUNT blocklens = (MPI_Count*) NCI_Malloc(sizeof(MPI_Count) * true_nsegs); disps = (MPI_Count*) NCI_Malloc(sizeof(MPI_Count) * true_nsegs); @@ -1601,7 +1601,7 @@ type_create_off_len(MPI_Offset nsegs, /* no. off-len pairs */ } /* j+1 is the coalesced length */ -#if MPI_VERSION >= 3 +#ifdef HAVE_MPI_LARGE_COUNT if (true_nsegs < j + 1) { blocklens = (MPI_Count*) NCI_Realloc(blocklens, (j+1) * sizeof(MPI_Count)); disps = (MPI_Count*) NCI_Realloc(disps, (j+1) * sizeof(MPI_Count)); @@ -1620,6 +1620,10 @@ type_create_off_len(MPI_Offset nsegs, /* no. off-len pairs */ blocklens[j] = segs[i].len; } } + /* j+1 is the coalesced length */ + + mpireturn = MPI_Type_create_hindexed_c(j+1, blocklens, disps, MPI_BYTE, + buf_type); #else if (true_nsegs < j + 1) { blocklens = (int*) NCI_Realloc(blocklens, (j+1) * SIZEOF_INT); @@ -1649,13 +1653,8 @@ type_create_off_len(MPI_Offset nsegs, /* no. off-len pairs */ blocklens[j] = (int)segs[i].len; } } -#endif /* j+1 is the coalesced length */ -#if MPI_VERSION >= 3 - mpireturn = MPI_Type_create_hindexed_c(j+1, blocklens, disps, MPI_BYTE, - buf_type); -#else mpireturn = MPI_Type_create_hindexed(j+1, blocklens, disps, MPI_BYTE, buf_type); #endif @@ -1696,7 +1695,7 @@ req_aggregation(NC *ncp, { int i, gtype, err, status=NC_NOERR, ngroups, mpireturn, buf_len; int *group_index, *group_type, numLeadReqs; -#if MPI_VERSION >= 3 +#ifdef HAVE_MPI_LARGE_COUNT MPI_Count *blocklens, *f_blocklens, *b_blocklens; MPI_Count *disps, *f_disps, *b_disps; #else @@ -1827,7 +1826,7 @@ req_aggregation(NC *ncp, btypes = ftypes + ngroups; /* temp buffers, used by multiple calls to construct_filetypes() */ -#if MPI_VERSION >= 3 +#ifdef HAVE_MPI_LARGE_COUNT blocklens = (MPI_Count*) NCI_Malloc(sizeof(MPI_Count) * num_reqs); disps = (MPI_Count*) NCI_Malloc(sizeof(MPI_Count) * num_reqs); f_blocklens = (MPI_Count*) NCI_Malloc(sizeof(MPI_Count) * ngroups); @@ -1953,7 +1952,7 @@ req_aggregation(NC *ncp, } else { /* concatenate all ftypes[] to filetype */ -#if MPI_VERSION >= 3 +#ifdef HAVE_MPI_LARGE_COUNT mpireturn = MPI_Type_create_struct_c(ngroups, f_blocklens, f_disps, ftypes, &filetype); #else @@ -1976,7 +1975,7 @@ req_aggregation(NC *ncp, } /* concatenate all btypes[] to buf_type */ -#if MPI_VERSION >= 3 +#ifdef HAVE_MPI_LARGE_COUNT mpireturn = MPI_Type_create_struct_c(ngroups, b_blocklens, b_disps, btypes, &buf_type); #else @@ -2260,7 +2259,7 @@ mgetput(NC *ncp, MPI_Offset offset=0, buf_count=0; MPI_File fh; -#if MPI_VERSION >= 3 +#ifdef HAVE_MPI_LARGE_COUNT MPI_Count *blocklens; MPI_Count *disps; blocklens = (MPI_Count*) NCI_Malloc(sizeof(MPI_Count) * num_reqs); @@ -2302,7 +2301,7 @@ mgetput(NC *ncp, if (fIsSet(lead->flag, NC_REQ_SKIP)) buf_count = 0; else { -#if MPI_VERSION >= 3 +#ifdef HAVE_MPI_LARGE_COUNT buf_count = reqs[0].nelems * lead->varp->xsz; #else MPI_Offset req_size = reqs[0].nelems * lead->varp->xsz; @@ -2333,7 +2332,7 @@ mgetput(NC *ncp, req_size = reqs[i].nelems * lead->varp->xsz; -#if MPI_VERSION >= 3 +#ifdef HAVE_MPI_LARGE_COUNT blocklens[j] = req_size; #else /* check int overflow */ @@ -2355,7 +2354,7 @@ mgetput(NC *ncp, req_size = blocklens[last_contig_req]; req_size += blocklens[j]; -#if MPI_VERSION >= 3 +#ifdef HAVE_MPI_LARGE_COUNT if (ai - a_last_contig == blocklens[last_contig_req]) { /* user buffer of request j is contiguous from j-1 * we coalesce j to j-1 */ @@ -2392,7 +2391,7 @@ mgetput(NC *ncp, int num_contig_reqs = last_contig_req+1; /* concatenate buffer addresses into a single buffer type */ -#if MPI_VERSION >= 3 +#ifdef HAVE_MPI_LARGE_COUNT mpireturn = MPI_Type_create_hindexed_c(num_contig_reqs, blocklens, disps, MPI_BYTE, &buf_type); #else diff --git a/test/largefile/large_header.c b/test/largefile/large_header.c index c6c2f7f0e..65c20d6f9 100644 --- a/test/largefile/large_header.c +++ b/test/largefile/large_header.c @@ -60,7 +60,9 @@ int main(int argc, char** argv) CHECK_ERR /* define a variable */ - err = ncmpi_def_var(ncid, "var", NC_INT, 1, &dimid, &varid); + err = ncmpi_def_var(ncid, "var0", NC_INT, 1, &dimid, &varid); + CHECK_ERR + err = ncmpi_def_var(ncid, "var1", NC_INT, 1, &dimid, &varid); CHECK_ERR /* make file header extent > 4 GiB */ @@ -77,11 +79,13 @@ int main(int argc, char** argv) err = ncmpi_close(ncid); CHECK_ERR + if (err != NC_NOERR) goto err_out; + err = ncmpi_open(MPI_COMM_WORLD, filename, NC_NOWRITE, MPI_INFO_NULL, &ncid); CHECK_ERR /* inquire ID of the variable */ - err = ncmpi_inq_varid(ncid, "var", &varid); + err = ncmpi_inq_varid(ncid, "var1", &varid); CHECK_ERR /* read from the variable */ @@ -109,6 +113,7 @@ int main(int argc, char** argv) } } +err_out: MPI_Allreduce(MPI_IN_PLACE, &nerrs, 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD); if (rank == 0) { if (nerrs) printf(FAIL_STR,nerrs);