diff --git a/examples/C/Makefile.am b/examples/C/Makefile.am index 42c1d213c..7221eb247 100644 --- a/examples/C/Makefile.am +++ b/examples/C/Makefile.am @@ -18,6 +18,7 @@ check_PROGRAMS = collective_write \ column_wise \ block_cyclic \ flexible_api \ + flexible_bottom \ get_info \ hints \ mput \ @@ -30,6 +31,7 @@ check_PROGRAMS = collective_write \ transpose \ transpose2D \ vard_int \ + vard_bottom \ i_varn_int64 \ bput_varn_uint \ bput_varn_int64 \ diff --git a/examples/C/flexible_bottom.c b/examples/C/flexible_bottom.c new file mode 100644 index 000000000..a41208abe --- /dev/null +++ b/examples/C/flexible_bottom.c @@ -0,0 +1,215 @@ +/********************************************************************* + * + * Copyright (C) 2024, Northwestern University and Argonne National Laboratory + * See COPYRIGHT notice in top-level directory. + * + *********************************************************************/ + +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * + * This example shows how to use PnetCDF blocking and nonblocking flexible APIs + * by supplying an MPI derived datatype and MPI_BOTTOM in the buffer argument. + * The MPI derived datatype is constructed through MPI_Type_create_hindexed(), + * which defines a noncontiguous memory space consisting of two separate blocks. + * + * A 2D global variable of size NY * NX * nprocs of type float is defined in + * the file. Each process allocates two write buffers and constructs an MPI + * derived datatype for the two buffers. + * + * The compile and run commands are given below. + * + * % mpicc -O2 -o flexible_bottom flexible_bottom.c -lpnetcdf + * + * % mpiexec -l -n 4 ./flexible_bottom ./testfile.nc + * + * % ncmpidump ./testfile.nc + * netcdf testfile { + * // file format: CDF-5 (big variables) + * dimensions: + * Y = 4 ; + * X = 16 ; + * variables: + * float var(Y, X) ; + * data: + * + * var = + * 10, 10, 10, 10, 11, 11, 11, 11, 12, 12, 12, 12, 13, 13, 13, 13, + * 10, 10, 10, 10, 11, 11, 11, 11, 12, 12, 12, 12, 13, 13, 13, 13, + * 50, 50, 50, 50, 51, 51, 51, 51, 52, 52, 52, 52, 53, 53, 53, 53, + * 50, 50, 50, 50, 51, 51, 51, 51, 52, 52, 52, 52, 53, 53, 53, 53 ; + * } + * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +#include +#include +#include /* strcpy(), strncpy() */ +#include /* getopt() */ +#include +#include + +#define NY 4 +#define NX 4 + +static int verbose; + +#define ERR {if(err!=NC_NOERR){printf("Error at %s:%d : %s\n", __FILE__,__LINE__, ncmpi_strerror(err));nerrs++;}} + +#define CHECK_MPI_ERR \ + if (err != MPI_SUCCESS) { \ + int errorStringLen; \ + char errorString[MPI_MAX_ERROR_STRING]; \ + MPI_Error_string(err, errorString, &errorStringLen); \ + printf("Error at line %d: %s\n",__LINE__, errorString); \ + } + +static void +usage(char *argv0) +{ + char *help = + "Usage: %s [-h] | [-q] [file_name]\n" + " [-h] Print help\n" + " [-q] Quiet mode (reports when fail)\n" + " [filename] output netCDF file name\n"; + fprintf(stderr, help, argv0); +} + +/*----< pnetcdf_check_mem_usage() >------------------------------------------*/ +/* check PnetCDF library internal memory usage */ +static int +pnetcdf_check_mem_usage(MPI_Comm comm) +{ + int err, nerrs=0, rank; + MPI_Offset malloc_size, sum_size; + + MPI_Comm_rank(comm, &rank); + + /* print info about PnetCDF internal malloc usage */ + err = ncmpi_inq_malloc_max_size(&malloc_size); + if (err == NC_NOERR) { + MPI_Reduce(&malloc_size, &sum_size, 1, MPI_OFFSET, MPI_SUM, 0, MPI_COMM_WORLD); + if (rank == 0 && verbose) + printf("maximum heap memory allocated by PnetCDF internally is %lld bytes\n", + sum_size); + + /* check if there is any PnetCDF internal malloc residue */ + err = ncmpi_inq_malloc_size(&malloc_size); + MPI_Reduce(&malloc_size, &sum_size, 1, MPI_OFFSET, MPI_SUM, 0, MPI_COMM_WORLD); + if (rank == 0 && sum_size > 0) + printf("heap memory allocated by PnetCDF internally has %lld bytes yet to be freed\n", + sum_size); + } + else if (err != NC_ENOTENABLED) { + printf("Error at %s:%d: %s\n", __FILE__,__LINE__,ncmpi_strerror(err)); + nerrs++; + } + return nerrs; +} + +int main(int argc, char** argv) +{ + extern int optind; + char filename[256]; + int i, rank, nprocs, err, nerrs=0, req, status; + int ncid, cmode, varid, dimid[2]; + float *buf[2]; + MPI_Offset start[2], count[2]; + MPI_Datatype btype; + int array_of_blocklengths[2]; + MPI_Aint array_of_displacements[2]; + + MPI_Init(&argc, &argv); + MPI_Comm_rank(MPI_COMM_WORLD, &rank); + MPI_Comm_size(MPI_COMM_WORLD, &nprocs); + + verbose = 1; + + /* get command-line arguments */ + while ((i = getopt(argc, argv, "hq")) != EOF) + switch(i) { + case 'q': verbose = 0; + break; + case 'h': + default: if (rank==0) usage(argv[0]); + MPI_Finalize(); + return 1; + } + if (argv[optind] == NULL) strcpy(filename, "testfile.nc"); + else snprintf(filename, 256, "%s", argv[optind]); + + if (verbose && rank == 0) printf("%s: example of using MPI_BOTTOM in flexible APIs\n",__FILE__); + + /* create a new file for writing ----------------------------------------*/ + cmode = NC_CLOBBER | NC_64BIT_DATA; + err = ncmpi_create(MPI_COMM_WORLD, filename, cmode, MPI_INFO_NULL, &ncid); + ERR + + /* define dimensions */ + err = ncmpi_def_dim(ncid, "Y", NY, &dimid[0]); ERR + err = ncmpi_def_dim(ncid, "X", NX*nprocs, &dimid[1]); ERR + + /* define a variable of size NY * (NX * nprocs) */ + err = ncmpi_def_var(ncid, "var", NC_FLOAT, 2, dimid, &varid); ERR + + /* exit define mode */ + err = ncmpi_enddef(ncid); ERR + + /* allocate two buffers */ + buf[0] = (float*) malloc(sizeof(float) * NY * NX / 2); + buf[1] = (float*) malloc(sizeof(float) * NY * NX / 2); + for (i=0; i 0); +} + diff --git a/examples/C/vard_bottom.c b/examples/C/vard_bottom.c new file mode 100644 index 000000000..bbeda3a58 --- /dev/null +++ b/examples/C/vard_bottom.c @@ -0,0 +1,339 @@ +/********************************************************************* + * + * Copyright (C) 2024, Northwestern University and Argonne National Laboratory + * See COPYRIGHT notice in top-level directory. + * + *********************************************************************/ + +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * This program is basically the same as example program vard_mvars.c, but + * instead of using user buffer pointer when calling the vard APIs, it + * constructs an MPI derived data type using absolute memory addresses and + * uses MPI_BOTTOM in the call to vard APIs. + * + * It shows how to use a single vard API call to write or read two + * consecutive variables. This example uses two MPI datatype constructors, + * MPI_Type_create_subarray and MPI_Type_create_hindexed, to create the same + * subarray view for two variables, but with different lower and upper bound + * MPI type maps. The two datatypes are then concatenated into a single + * filetype. + * + * To compile: + * mpicc -O2 vard_mvars.c -o vard_mvars -lpnetcdf + * + * Example commands for MPI run and outputs from running ncmpidump on the + * NetCDF file produced by this example program: + * + * % mpiexec -n 4 ./vard_mvars /pvfs2/wkliao/testfile.nc + * + * The expected results from the output file contents are: + * + * % ncmpidump testfile.nc + * netcdf testfile { + * // file format: CDF-1 + * dimensions: + * REC_DIM = UNLIMITED ; // (2 currently) + * Y = 2 ; + * X = 20 ; + * variables: + * int fix_var0(Y, X) ; + * int fix_var1(Y, X) ; + * int rec_var2(REC_DIM, Y, X) ; + * int rec_var3(REC_DIM, Y, X) ; + * data: + * + * fix_var0 = + * 0, 1, 2, 3, 4, 100, 101, 102, 103, 104, 200, 201, 202, 203, 204, 300, 301, + * 302, 303, 304, + * 10, 11, 12, 13, 14, 110, 111, 112, 113, 114, 210, 211, 212, 213, 214, 310, + * 311, 312, 313, 314 ; + * + * fix_var1 = + * 1000, 1001, 1002, 1003, 1004, 1100, 1101, 1102, 1103, 1104, 1200, 1201, + * 1202, 1203, 1204, 1300, 1301, 1302, 1303, 1304, + * 1010, 1011, 1012, 1013, 1014, 1110, 1111, 1112, 1113, 1114, 1210, 1211, + * 1212, 1213, 1214, 1310, 1311, 1312, 1313, 1314 ; + * + * rec_var2 = + * _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, + * _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, + * 2000, 2001, 2002, 2003, 2004, 2100, 2101, 2102, 2103, 2104, 2200, 2201, + * 2202, 2203, 2204, 2300, 2301, 2302, 2303, 2304, + * 2010, 2011, 2012, 2013, 2014, 2110, 2111, 2112, 2113, 2114, 2210, 2211, + * 2212, 2213, 2214, 2310, 2311, 2312, 2313, 2314 ; + * + * rec_var3 = + * _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, + * _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, + * 3000, 3001, 3002, 3003, 3004, 3100, 3101, 3102, 3103, 3104, 3200, 3201, + * 3202, 3203, 3204, 3300, 3301, 3302, 3303, 3304, + * 3010, 3011, 3012, 3013, 3014, 3110, 3111, 3112, 3113, 3114, 3210, 3211, + * 3212, 3213, 3214, 3310, 3311, 3312, 3313, 3314 ; + * } + */ + +#include +#include +#include +#include /* getopt() */ + +#include +#include + +#define NY 2 +#define NX 5 + +static int verbose; + +#define ERR {if(err!=NC_NOERR){printf("Error at %s:%d : %s\n", __FILE__,__LINE__, ncmpi_strerror(err));nerrs++;}} + +#define CHECK_VALUE(buf,base) { \ + for (j=0; j------------------------------------------*/ +/* check PnetCDF library internal memory usage */ +static int +pnetcdf_check_mem_usage(MPI_Comm comm) +{ + int err, nerrs=0, rank; + MPI_Offset malloc_size, sum_size; + + MPI_Comm_rank(comm, &rank); + + /* print info about PnetCDF internal malloc usage */ + err = ncmpi_inq_malloc_max_size(&malloc_size); + if (err == NC_NOERR) { + MPI_Reduce(&malloc_size, &sum_size, 1, MPI_OFFSET, MPI_SUM, 0, MPI_COMM_WORLD); + if (rank == 0 && verbose) + printf("maximum heap memory allocated by PnetCDF internally is %lld bytes\n", + sum_size); + + /* check if there is any PnetCDF internal malloc residue */ + err = ncmpi_inq_malloc_size(&malloc_size); + MPI_Reduce(&malloc_size, &sum_size, 1, MPI_OFFSET, MPI_SUM, 0, MPI_COMM_WORLD); + if (rank == 0 && sum_size > 0) + printf("heap memory allocated by PnetCDF internally has %lld bytes yet to be freed\n", + sum_size); + } + else if (err != NC_ENOTENABLED) { + printf("Error at %s:%d: %s\n", __FILE__,__LINE__,ncmpi_strerror(err)); + nerrs++; + } + return nerrs; +} + +/*----< main() >------------------------------------------------------------*/ +int main(int argc, char **argv) +{ + extern int optind; + char filename[256]; + int i, j, err, ncid, varid[4], dimids[3], nerrs=0; + int rank, nprocs, *buf[2]; + int array_of_sizes[2], array_of_subsizes[2], array_of_starts[2]; + int array_of_blocklengths[NY]; + MPI_Offset recsize, offset[2]; + MPI_Aint array_of_displacements[NY]; + MPI_Datatype buftype, vtype[2], filetype; + + MPI_Init(&argc, &argv); + MPI_Comm_size(MPI_COMM_WORLD, &nprocs); + MPI_Comm_rank(MPI_COMM_WORLD, &rank); + + verbose = 1; + + /* get command-line arguments */ + while ((i = getopt(argc, argv, "hq")) != EOF) + switch(i) { + case 'q': verbose = 0; + break; + case 'h': + default: if (rank==0) usage(argv[0]); + MPI_Finalize(); + return 1; + } + if (argv[optind] == NULL) strcpy(filename, "testfile.nc"); + else snprintf(filename, 256, "%s", argv[optind]); + + MPI_Bcast(filename, 256, MPI_CHAR, 0, MPI_COMM_WORLD); + + if (verbose && rank == 0) printf("%s: example of using vard APIs to write/read two variables\n",__FILE__); + + buf[0] = (int*)malloc(NY * NX * sizeof(int)); + for (j=0; j 0); +} diff --git a/examples/C/vard_mvars.c b/examples/C/vard_mvars.c index ccb915580..caca1995d 100644 --- a/examples/C/vard_mvars.c +++ b/examples/C/vard_mvars.c @@ -187,7 +187,7 @@ int main(int argc, char **argv) array_of_displacements[0] = 0; MPI_Get_address(buf[0], &a0); MPI_Get_address(buf[1], &a1); - array_of_displacements[1] = a1 - a0; + array_of_displacements[1] = MPI_Aint_diff(a1, a0); vtype[0] = vtype[1] = MPI_INT; MPI_Type_create_struct(2, array_of_blocklengths, array_of_displacements, vtype, &buftype); diff --git a/examples/README.md b/examples/README.md index 9f4b45904..bd1e09c89 100644 --- a/examples/README.md +++ b/examples/README.md @@ -70,7 +70,7 @@ with C, C++, F77, and F90 versions. ``` * flexible_api - + ./C/flexible_api.c + + ./C/flexible_api.c ./C/flexible_bottom.c ./C/vard_bottom.c + ./CXX/flexible_api.cpp + ./F77/flexible_api.f + ./F90/flexible_api.f90 diff --git a/sneak_peek.md b/sneak_peek.md index 65031f55b..812e7515d 100644 --- a/sneak_peek.md +++ b/sneak_peek.md @@ -60,7 +60,9 @@ This is essentially a placeholder for the next release note ... + none * New example programs - + none + + C/flexible_bottom.c and C/vard_bottom.c - These two examples construct MPI + derived data types using absolute memory addresses first and use MPI_BOTTOM + when calling the PnetCDF flexible APIs. * New programs for I/O benchmarks + none diff --git a/src/drivers/ncmpio/ncmpio_wait.c b/src/drivers/ncmpio/ncmpio_wait.c index 50f3b5c49..676e7ee51 100644 --- a/src/drivers/ncmpio/ncmpio_wait.c +++ b/src/drivers/ncmpio/ncmpio_wait.c @@ -569,7 +569,7 @@ construct_buffertypes(NC_lead_req *lead_list, MPI_Get_address(reqs[i].xbuf, &ai); if (j == 0) a0 = ai; - disps[j] = ai - a0; + disps[j] = MPI_Aint_diff(ai, a0); j++; } /* update num_reqs to number of valid requests */ @@ -1421,7 +1421,7 @@ merge_requests(NC *ncp, /* buf_addr is the buffer address of the first valid request */ MPI_Get_address(reqs[i].xbuf, &addr); - addr -= buf_addr, /* distance to the buf of first req */ + addr = MPI_Aint_diff(addr, buf_addr); /* distance to the buf of first req */ ndims = lead->varp->ndims; if (ndims > 0) { @@ -1478,15 +1478,15 @@ merge_requests(NC *ncp, MPI_Offset gap = (*segs)[i].off + (*segs)[i].len - (*segs)[j].off; if (gap >= 0) { /* segments i and j overlaps */ - if ((*segs)[i].buf_addr + (*segs)[i].len == - (*segs)[j].buf_addr + gap) { + if (MPI_Aint_add((*segs)[i].buf_addr, (*segs)[i].len) == + MPI_Aint_add((*segs)[j].buf_addr, gap)) { /* buffers i and j are contiguous, merge j to i */ - (*segs)[i].len += (*segs)[j].len - gap; + (*segs)[i].len = MPI_Aint_add((*segs)[i].len, (*segs)[j].len - gap); } else { /* buffers are not contiguous, reduce j's len */ (*segs)[i+1].off = (*segs)[j].off + gap; (*segs)[i+1].len = (*segs)[j].len - gap; - (*segs)[i+1].buf_addr = (*segs)[j].buf_addr + gap; + (*segs)[i+1].buf_addr = MPI_Aint_add((*segs)[j].buf_addr, gap); i++; } } @@ -1959,7 +1959,7 @@ req_aggregation(NC *ncp, if (i > 0) { /* get the buffer address of the first request in this group */ MPI_Get_address(g_reqs[0].xbuf, &b_addr); - b_disps[i] = b_addr - b_begin; /* to 1st buffer of 1st group*/ + b_disps[i] = MPI_Aint_diff(b_addr, b_begin); /* to 1st buffer of 1st group*/ } b_blocklens[i] = 1; } @@ -2374,12 +2374,12 @@ mgetput(NC *ncp, a_last_contig = a0 = ai; buf = reqs[i].xbuf; } - disps[j] = ai - a0; + disps[j] = MPI_Aint_diff(ai, a0); req_size = blocklens[last_contig_req]; req_size += blocklens[j]; #ifdef HAVE_MPI_LARGE_COUNT - if (ai - a_last_contig == blocklens[last_contig_req]) { + if (MPI_Aint_diff(ai, a_last_contig) == blocklens[last_contig_req]) { /* user buffer of request j is contiguous from j-1 * we coalesce j to j-1 */ blocklens[last_contig_req] += blocklens[j]; @@ -2387,7 +2387,7 @@ mgetput(NC *ncp, #else /* if req_size overflows 4-byte int, then skip coalescing */ if (req_size <= NC_MAX_INT && - ai - a_last_contig == blocklens[last_contig_req]) { + MPI_Aint_diff(ai,- a_last_contig) == blocklens[last_contig_req]) { /* user buffer of request j is contiguous from j-1 * we coalesce j to j-1 */ blocklens[last_contig_req] += blocklens[j]; @@ -2397,7 +2397,7 @@ mgetput(NC *ncp, /* not contiguous from request last_contig_req */ last_contig_req++; a_last_contig = ai; - disps[last_contig_req] = ai - a0; + disps[last_contig_req] = MPI_Aint_diff(ai, a0); blocklens[last_contig_req] = blocklens[i]; } j++; diff --git a/src/utils/ncvalidator/ncvalidator.c b/src/utils/ncvalidator/ncvalidator.c index ce8c942e3..9ecb86167 100644 --- a/src/utils/ncvalidator/ncvalidator.c +++ b/src/utils/ncvalidator/ncvalidator.c @@ -2446,6 +2446,8 @@ val_get_NC(int fd, NC *ncp) /* End Of get NC */ +#ifndef BUILD_CDFDIFF + /* File system types recognized by ROMIO in MPICH 4.0.0 */ static const char* fstypes[] = {"ufs", "nfs", "xfs", "pvfs2", "gpfs", "panfs", "lustre", "daos", "testfs", "ime", "quobyte", NULL}; @@ -2543,7 +2545,6 @@ check_signature(char *filename) return NC_FORMAT_UNKNOWN; } -#ifndef BUILD_CDFDIFF static void usage(char *argv0) {