From 064b17bf071bcf96dbbfa91a406c8a3bf68e7c94 Mon Sep 17 00:00:00 2001 From: wkliao Date: Fri, 23 Feb 2024 21:56:38 -0600 Subject: [PATCH 1/3] Save the alignment hints and reuse them in each ncmpi__enddef ncmpi_endf and ncmpi__enddef may be called multiple times. We need to save the alignment hints, h_align, v_align, and r_align as they maybe changed in each call to ncmpi__enddef. Reset them to the original hints allows users to use different alignments in each call to ncmpi__enddef. --- src/drivers/ncmpio/ncmpio_NC.h | 4 ++++ src/drivers/ncmpio/ncmpio_enddef.c | 6 ++++++ src/drivers/ncmpio/ncmpio_util.c | 21 ++++++++++++--------- 3 files changed, 22 insertions(+), 9 deletions(-) diff --git a/src/drivers/ncmpio/ncmpio_NC.h b/src/drivers/ncmpio/ncmpio_NC.h index e4078c384..2d1a15889 100644 --- a/src/drivers/ncmpio/ncmpio_NC.h +++ b/src/drivers/ncmpio/ncmpio_NC.h @@ -341,6 +341,7 @@ typedef struct NC_buf { #define NC_NDIRTY 0x400000 /* numrecs has changed */ #define NC_HDIRTY 0x800000 /* header info has changed */ #define NC_HCOLL 0x000001 /* write header collectively */ + struct NC { int ncid; /* file ID */ int flags; /* various modes, i.e. define/data, fill, @@ -361,6 +362,9 @@ struct NC { MPI_Offset h_align; /* file alignment for header size */ MPI_Offset v_align; /* alignment of the beginning of fixed-size variables */ MPI_Offset r_align; /* file alignment for record variable section */ + MPI_Offset env_h_align; /* h_align set in environment variable */ + MPI_Offset env_v_align; /* v_align set in environment variable */ + MPI_Offset env_r_align; /* r_align set in environment variable */ MPI_Offset h_minfree; /* pad at the end of the header section */ MPI_Offset v_minfree; /* pad at the end of the data section for fixed-size variables */ MPI_Offset ibuf_size; /* packing buffer size for flushing noncontig diff --git a/src/drivers/ncmpio/ncmpio_enddef.c b/src/drivers/ncmpio/ncmpio_enddef.c index a870654e3..cd62593ce 100644 --- a/src/drivers/ncmpio/ncmpio_enddef.c +++ b/src/drivers/ncmpio/ncmpio_enddef.c @@ -973,6 +973,11 @@ ncmpio__enddef(void *ncdp, num_fix_vars = ncp->vars.ndefined - ncp->vars.num_rec_vars; + /* reset to hints set at file create/open time */ + ncp->h_align = ncp->env_h_align; + ncp->v_align = ncp->env_v_align; + ncp->r_align = ncp->env_r_align; + if (ncp->h_align == 0) { /* hint nc_header_align_size is not set */ if (ncp->v_align > 0) /* hint nc_var_align_size is set */ ncp->h_align = ncp->v_align; @@ -986,6 +991,7 @@ ncmpio__enddef(void *ncdp, else if (r_align > 0) /* r_align is passed from ncmpi__enddef */ ncp->h_align = r_align; } + if (ncp->h_align == 0 && ncp->old == NULL) /* h_align is still not set. Set h_align only when creating a new * file. When opening an existing file file, setting h_align here diff --git a/src/drivers/ncmpio/ncmpio_util.c b/src/drivers/ncmpio/ncmpio_util.c index ecf65deb3..cd87bf7ab 100644 --- a/src/drivers/ncmpio/ncmpio_util.c +++ b/src/drivers/ncmpio/ncmpio_util.c @@ -43,42 +43,45 @@ void ncmpio_set_pnetcdf_hints(NC *ncp, * a file is created, or opened and later adding more metadata or variable * data */ + ncp->env_h_align = 0; if (user_info != MPI_INFO_NULL) { /* aligns the size of header extent of a newly created file */ MPI_Info_get(user_info, "nc_header_align_size", MPI_MAX_INFO_VAL-1, value, &flag); if (flag) { errno = 0; /* errno must set to zero before calling strtoll */ - ncp->h_align = strtoll(value, NULL, 10); - if (errno != 0) ncp->h_align = 0; - else if (ncp->h_align < 0) ncp->h_align = 0; + ncp->env_h_align = strtoll(value, NULL, 10); + if (errno != 0) ncp->env_h_align = 0; + else if (ncp->env_h_align < 0) ncp->env_h_align = 0; } } if (!flag) sprintf(value, "%d", FILE_ALIGNMENT_DEFAULT); MPI_Info_set(info_used, "nc_header_align_size", value); + ncp->env_v_align = 0; if (user_info != MPI_INFO_NULL) { /* aligns starting file offsets of individual fixed-size variables */ MPI_Info_get(user_info, "nc_var_align_size", MPI_MAX_INFO_VAL-1, value, &flag); if (flag) { errno = 0; /* errno must set to zero before calling strtoll */ - ncp->v_align = strtoll(value, NULL, 10); - if (errno != 0) ncp->v_align = 0; - else if (ncp->v_align < 0) ncp->v_align = 0; + ncp->env_v_align = strtoll(value, NULL, 10); + if (errno != 0) ncp->env_v_align = 0; + else if (ncp->env_v_align < 0) ncp->env_v_align = 0; } } if (!flag) sprintf(value, "%d", FILE_ALIGNMENT_DEFAULT); MPI_Info_set(info_used, "nc_var_align_size", value); + ncp->env_r_align = 0; if (user_info != MPI_INFO_NULL) { /* aligns starting file offset of the record variable section */ MPI_Info_get(user_info, "nc_record_align_size", MPI_MAX_INFO_VAL-1, value, &flag); if (flag) { errno = 0; /* errno must set to zero before calling strtoll */ - ncp->r_align = strtoll(value, NULL, 10); - if (errno != 0) ncp->r_align = 0; - else if (ncp->r_align < 0) ncp->r_align = 0; + ncp->env_r_align = strtoll(value, NULL, 10); + if (errno != 0) ncp->env_r_align = 0; + else if (ncp->env_r_align < 0) ncp->env_r_align = 0; } } if (!flag) sprintf(value, "%d", FILE_ALIGNMENT_DEFAULT); From 95ec93d5467027359ecf20918d652e83333e5b98 Mon Sep 17 00:00:00 2001 From: wkliao Date: Fri, 23 Feb 2024 22:01:18 -0600 Subject: [PATCH 2/3] test program tst_redefine.c to test multiple entries of ncmpi__enddef --- test/testcases/Makefile.am | 3 +- test/testcases/tst_redefine.c | 367 ++++++++++++++++++++++++++++++++++ 2 files changed, 369 insertions(+), 1 deletion(-) create mode 100644 test/testcases/tst_redefine.c diff --git a/test/testcases/Makefile.am b/test/testcases/Makefile.am index 5a078efdc..e16da7230 100644 --- a/test/testcases/Makefile.am +++ b/test/testcases/Makefile.am @@ -95,7 +95,8 @@ TESTPROGRAMS = ncmpi_vars_null_stride \ tst_free_comm \ flexible_var \ test_get_varn \ - tst_del_attr + tst_del_attr \ + tst_redefine M4_SRCS = put_all_kinds.m4 \ erange_fill.m4 \ diff --git a/test/testcases/tst_redefine.c b/test/testcases/tst_redefine.c new file mode 100644 index 000000000..04e2f07e2 --- /dev/null +++ b/test/testcases/tst_redefine.c @@ -0,0 +1,367 @@ +/* + * Copyright (C) 2024, Northwestern University and Argonne National Laboratory + * See COPYRIGHT notice in top-level directory. + * + */ + +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * + * This program tests header extent alignment if entering the redefine mode. + * + * The compile and run commands are given below. + * + * % mpicc -g -o tst_redefine tst_redefine.c -lpnetcdf + * + * % mpiexec -l -n 4 ./tst_redefine testfile.nc + * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +#include +#include +#include +#include /* strcasecmp() */ +#include /* basename() */ +#include + +#include + +static int verbose; + +static int +tst_fmt(char *filename, int cmode) +{ + int rank, ncid, err, nerrs=0; + int dimid[3], varid[4], minfree=100; + MPI_Offset old_hsize, hsize; + MPI_Offset old_extent, extent; + MPI_Offset old_var_off, var_off; + MPI_Offset v_align, r_align; + MPI_Comm comm = MPI_COMM_WORLD; + + MPI_Comm_rank(comm, &rank); + + unsetenv("PNETCDF_HINTS"); + + /* create a new file */ + cmode |= NC_CLOBBER; + err = ncmpi_create(comm, filename, cmode, MPI_INFO_NULL, &ncid); CHECK_ERR + + err = ncmpi_def_dim(ncid, "time", NC_UNLIMITED, &dimid[0]); CHECK_ERR + err = ncmpi_def_dim(ncid, "dim", 25, &dimid[1]); CHECK_ERR + err = ncmpi_def_var(ncid, "fa", NC_INT, 1, dimid+1, &varid[0]); CHECK_ERR + err = ncmpi_def_var(ncid, "fb", NC_INT, 1, dimid+1, &varid[1]); CHECK_ERR + err = ncmpi_def_var(ncid, "ta", NC_INT, 2, dimid, &varid[2]); CHECK_ERR + err = ncmpi_def_var(ncid, "tb", NC_INT, 2, dimid, &varid[3]); CHECK_ERR + + /* disable header alignment */ + err = ncmpi__enddef(ncid, 0, 4, 0, 4); CHECK_ERR + + err = ncmpi_close(ncid); CHECK_ERR + + /* reopen the file and check file header size and extent */ + err = ncmpi_open(comm, filename, NC_WRITE, MPI_INFO_NULL, &ncid); CHECK_ERR + + err = ncmpi_inq_header_size(ncid, &hsize); CHECK_ERR + err = ncmpi_inq_header_extent(ncid, &extent); CHECK_ERR + if (verbose) + printf("File header size = %lld extent = %lld\n", hsize, extent); + + /* make sure header size == extent */ + if (hsize != extent) { + nerrs++; + printf("Error at line %d in %s: File header size %lld != extent %lld\n", + __LINE__,__FILE__, hsize, extent); + } + + /* enter redefine mode and add a dimension to grow the header */ + err = ncmpi_redef(ncid); CHECK_ERR + + err = ncmpi_def_dim(ncid, "dim_x", 25, &dimid[2]); CHECK_ERR + + /* disable header alignment */ + err = ncmpi__enddef(ncid, 0, 4, 0, 4); CHECK_ERR + + old_hsize = hsize; + old_extent = extent; + err = ncmpi_inq_header_size(ncid, &hsize); CHECK_ERR + err = ncmpi_inq_header_extent(ncid, &extent); CHECK_ERR + if (verbose) { + printf("File old header size = %6lld old extent = %6lld\n", old_hsize, old_extent); + printf("File new header size = %6lld new extent = %6lld\n", hsize, extent); + } + + /* make sure header size grows */ + if (hsize <= old_hsize) { + nerrs++; + printf("Error at line %d in %s: File header size %lld fails to grow from %lld\n", + __LINE__,__FILE__, hsize, old_hsize); + } + /* make sure header extent grows */ + if (extent <= old_extent) { + nerrs++; + printf("Error at line %d in %s: File header extent %lld fails to grow from %lld\n", + __LINE__,__FILE__, extent, old_extent); + } + + /* enter redefine mode and add nothing */ + err = ncmpi_redef(ncid); CHECK_ERR + + /* add free space into header extent */ + err = ncmpi__enddef(ncid, minfree, 0, 0, 0); CHECK_ERR + + old_hsize = hsize; + old_extent = extent; + err = ncmpi_inq_header_size(ncid, &hsize); CHECK_ERR + err = ncmpi_inq_header_extent(ncid, &extent); CHECK_ERR + if (verbose) { + printf("File old header size = %6lld old extent = %6lld\n", old_hsize, old_extent); + printf("File new header size = %6lld new extent = %6lld\n", hsize, extent); + } + + /* make sure header size does not change */ + if (hsize != old_hsize) { + nerrs++; + printf("Error at line %d in %s: File header size %lld changed from %lld\n", + __LINE__,__FILE__, hsize, old_hsize); + } + /* make sure header extent grows minfree bytes */ + if (extent != old_extent + minfree) { + nerrs++; + printf("Error at line %d in %s: File header extent %lld fails to grow into %lld\n", + __LINE__,__FILE__, extent, old_extent + minfree); + } + + /* enter redefine mode and add nothing */ + err = ncmpi_redef(ncid); CHECK_ERR + + /* align header extent to 512 bytes */ + err = ncmpi__enddef(ncid, 0, 512, 0, 0); CHECK_ERR + + old_hsize = hsize; + old_extent = extent; + err = ncmpi_inq_header_size(ncid, &hsize); CHECK_ERR + err = ncmpi_inq_header_extent(ncid, &extent); CHECK_ERR + if (verbose) { + printf("File old header size = %6lld old extent = %6lld\n", old_hsize, old_extent); + printf("File new header size = %6lld new extent = %6lld\n", hsize, extent); + } + + /* make sure header size does not change */ + if (hsize != old_hsize) { + nerrs++; + printf("Error at line %d in %s: File header size %lld changed from %lld\n", + __LINE__,__FILE__, hsize, old_hsize); + } + /* make sure header extent grows to 512 bytes */ + if (extent != 512) { + nerrs++; + printf("Error at line %d in %s: File header extent %lld fails to grow into 512\n", + __LINE__,__FILE__, extent); + } + + err = ncmpi_close(ncid); CHECK_ERR + + + setenv("PNETCDF_HINTS", "nc_header_align_size=700", 1); + + /* reopen the file and check file header size and extent */ + err = ncmpi_open(comm, filename, NC_WRITE, MPI_INFO_NULL, &ncid); CHECK_ERR + + /* enter redefine mode and add nothing */ + err = ncmpi_redef(ncid); CHECK_ERR + + /* align header extent to 800 bytes, this should take no effect, as + * run-time hints precedes function calls. */ + err = ncmpi__enddef(ncid, 0, 800, 0, 0); CHECK_ERR + + old_hsize = hsize; + old_extent = extent; + err = ncmpi_inq_header_size(ncid, &hsize); CHECK_ERR + err = ncmpi_inq_header_extent(ncid, &extent); CHECK_ERR + if (verbose) { + printf("File old header size = %6lld old extent = %6lld\n", old_hsize, old_extent); + printf("File new header size = %6lld new extent = %6lld\n", hsize, extent); + } + + /* make sure header size does not change */ + if (hsize != old_hsize) { + nerrs++; + printf("Error at line %d in %s: File header size %lld changed from %lld\n", + __LINE__,__FILE__, hsize, old_hsize); + } + /* make sure header extent grows to 700 bytes */ + if (extent != 700) { + nerrs++; + printf("Error at line %d in %s: File header extent %lld fails to grow into 700\n", + __LINE__,__FILE__, extent); + } + + unsetenv("PNETCDF_HINTS"); + + /* enter redefine mode and add nothing */ + err = ncmpi_redef(ncid); CHECK_ERR + + /* set minfree to the free space, this should not grow the extent */ + err = ncmpi__enddef(ncid, extent-hsize, 0, 0, 0); CHECK_ERR + + old_hsize = hsize; + old_extent = extent; + err = ncmpi_inq_header_size(ncid, &hsize); CHECK_ERR + err = ncmpi_inq_header_extent(ncid, &extent); CHECK_ERR + if (verbose) { + printf("File old header size = %6lld old extent = %6lld\n", old_hsize, old_extent); + printf("File new header size = %6lld new extent = %6lld\n", hsize, extent); + } + + /* make sure header size does not change */ + if (hsize != old_hsize) { + nerrs++; + printf("Error at line %d in %s: File header size %lld changed from %lld\n", + __LINE__,__FILE__, hsize, old_hsize); + } + /* make sure header extent remain the same */ + if (extent != old_extent) { + nerrs++; + printf("Error at line %d in %s: File header extent %lld changed from %lld\n", + __LINE__,__FILE__, extent, old_extent); + } + + /* obtain 1st record variable's file offset */ + err = ncmpi_inq_varoffset(ncid, varid[2], &old_var_off); CHECK_ERR + + /* enter redefine mode and add nothing */ + err = ncmpi_redef(ncid); CHECK_ERR + + /* set v_minfree of size 400 */ + err = ncmpi__enddef(ncid, 0, 0, 400, 0); CHECK_ERR + + /* obtain 1st record variable's file offset */ + err = ncmpi_inq_varoffset(ncid, varid[2], &var_off); CHECK_ERR + + /* var_off should grows 400 bytes */ + if (var_off != old_var_off + 400) { + nerrs++; + printf("Error at line %d in %s: 1st record variable offset %lld (expecting %lld)\n", + __LINE__,__FILE__, var_off, old_var_off+400); + } + +#if 0 + err = ncmpi_close(ncid); CHECK_ERR + + /* reopen the file and set r_align */ + err = ncmpi_open(comm, filename, NC_WRITE, MPI_INFO_NULL, &ncid); CHECK_ERR +#endif + /* enter redefine mode and add nothing */ + err = ncmpi_redef(ncid); CHECK_ERR + + /* set r_align to 1500 */ + r_align = 1500; + err = ncmpi__enddef(ncid, 0, 0, 0, r_align); CHECK_ERR + + /* obtain 1st record variable's file offset */ + err = ncmpi_inq_varoffset(ncid, varid[2], &var_off); CHECK_ERR + + /* var_off should grows into 1500 bytes */ + if (var_off != r_align) { + nerrs++; + printf("Error at line %d in %s: 1st record variable offset %lld (expecting %lld)\n", + __LINE__,__FILE__, var_off, r_align); + } + + err = ncmpi_close(ncid); CHECK_ERR + + unsetenv("PNETCDF_HINTS"); + + /* re-create a new file */ + cmode |= NC_CLOBBER; + err = ncmpi_create(comm, filename, cmode, MPI_INFO_NULL, &ncid); CHECK_ERR + + /* define only record variables */ + err = ncmpi_def_dim(ncid, "time", NC_UNLIMITED, &dimid[0]); CHECK_ERR + err = ncmpi_def_dim(ncid, "dim", 25, &dimid[1]); CHECK_ERR + err = ncmpi_def_var(ncid, "ta", NC_INT, 2, dimid, &varid[2]); CHECK_ERR + err = ncmpi_def_var(ncid, "tb", NC_INT, 2, dimid, &varid[3]); CHECK_ERR + + /* set v_align and r_align. Without record variables, r_align will take + * effect over v_align + */ + v_align = 100; + r_align = 512; + err = ncmpi__enddef(ncid, 0, v_align, 0, r_align); CHECK_ERR + + err = ncmpi_inq_header_size(ncid, &hsize); CHECK_ERR + err = ncmpi_inq_header_extent(ncid, &extent); CHECK_ERR + if (verbose) + printf("File header size = %lld extent = %lld\n", hsize, extent); + + /* obtain 1st record variable's file offset */ + err = ncmpi_inq_varoffset(ncid, varid[2], &var_off); CHECK_ERR + if (var_off != extent) { + nerrs++; + printf("Error at line %d in %s: 1st record variable offset %lld (expecting %lld)\n", + __LINE__,__FILE__, var_off, extent); + } + + /* extent should be r_align */ + if (extent != r_align) { + nerrs++; + printf("Error at line %d in %s: file extent %lld (expecting %lld)\n", + __LINE__,__FILE__, extent, r_align); + } + + err = ncmpi_close(ncid); CHECK_ERR + + return nerrs; +} + +int main(int argc, char** argv) +{ + char filename[256]; + int commsize, rank, err, nerrs=0; + MPI_Comm comm = MPI_COMM_WORLD; + + MPI_Init(&argc, &argv); + MPI_Comm_size(comm, &commsize); + MPI_Comm_rank(comm, &rank); + + verbose = 0; + + if (argc > 2) { + if (!rank) printf("Usage: %s [filename]\n",argv[0]); + MPI_Finalize(); + return 1; + } + if (argc == 2) snprintf(filename, 256, "%s", argv[1]); + else strcpy(filename, "tst_redefine.nc"); + + if (rank == 0) { + char *cmd_str = (char*)malloc(strlen(argv[0]) + 256); + sprintf(cmd_str, "*** TESTING C %s for header alignment ", basename(argv[0])); + printf("%-66s ------ ", cmd_str); fflush(stdout); + free(cmd_str); + } + + nerrs += tst_fmt(filename, 0); + nerrs += tst_fmt(filename, NC_64BIT_OFFSET); + nerrs += tst_fmt(filename, NC_64BIT_DATA); + + /* check if PnetCDF freed all internal malloc */ + MPI_Offset malloc_size, sum_size; + err = ncmpi_inq_malloc_size(&malloc_size); + if (err == NC_NOERR) { + MPI_Reduce(&malloc_size, &sum_size, 1, MPI_OFFSET, MPI_SUM, 0, MPI_COMM_WORLD); + if (rank == 0 && sum_size > 0) + printf("heap memory allocated by PnetCDF internally has %lld bytes yet to be freed\n", + sum_size); + if (malloc_size > 0) ncmpi_inq_malloc_list(); + } + + MPI_Allreduce(MPI_IN_PLACE, &nerrs, 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD); + if (rank == 0) { + if (nerrs) printf(FAIL_STR,nerrs); + else printf(PASS_STR); + } + + MPI_Finalize(); + return (nerrs > 0); +} From fd97bca7a79554a382f2abd746b783cc3259eeff Mon Sep 17 00:00:00 2001 From: wkliao Date: Fri, 23 Feb 2024 22:07:49 -0600 Subject: [PATCH 3/3] add note of new test program tst_redefine.c --- sneak_peek.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/sneak_peek.md b/sneak_peek.md index 382f5315c..f9f7a9067 100644 --- a/sneak_peek.md +++ b/sneak_peek.md @@ -159,6 +159,8 @@ This is essentially a placeholder for the next release note ... + none * New test program + + test/testcases/tst_redefine.c - test multiple entries of ncmpi__enddef + [PR #126](https://github.com/Parallel-NetCDF/PnetCDF/pull/126). + test/testcases/tst_symlink.c - test NC_CLOBBER on a symbolic link. + test/testcases/tst_del_attr.c - test delete attributes. See [PR #99](https://github.com/Parallel-NetCDF/PnetCDF/pull/99).