Skip to content

Commit

Permalink
calculate record variable offset dynamically
Browse files Browse the repository at this point in the history
  • Loading branch information
@wkliao
wkliao committed Mar 20, 2024
1 parent da11a84 commit 8cc0138
Show file tree
Hide file tree
Showing 3 changed files with 91 additions and 83 deletions.
32 changes: 16 additions & 16 deletions test/largefile/large_reqs.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -117,8 +117,8 @@ int tst_one_var(char *filename, MPI_Comm comm)

gsize[0] = NY;
gsize[1] = NX;
lsize[0] = count[1];
lsize[1] = count[2];
lsize[0] = (int)count[1];
lsize[1] = (int)count[2];
lstart[0] = 0;
lstart[1] = 0;
MPI_Type_create_subarray(2, gsize, lsize, lstart, MPI_ORDER_C,
Expand Down Expand Up @@ -203,32 +203,32 @@ int tst_vars(char *filename, MPI_Comm comm)
buf = (int*) malloc(buf_len * sizeof(int));
for (i=0; i<buf_len; i++) buf[i] = (i + rank) % 128;

/* set subarray offset and length */
start[0] = 0;
start[1] = LEN * (rank / psize[1]);
start[2] = LEN * (rank % psize[1]);
count[0] = 1;
count[1] = LEN - gap;
count[2] = LEN - gap;

if (verbose)
printf("rank %d start=%lld %lld count=%lld %lld\n",
rank, start[1],start[2], count[1],count[2]);

/* create a subarray datatype for user buffer */
int gsize[2], lsize[2], lstart[2];
MPI_Datatype buftype;

gsize[0] = LEN;
gsize[1] = LEN;
lsize[0] = count[1];
lsize[1] = count[2];
lsize[0] = LEN - gap;
lsize[1] = LEN - gap;
lstart[0] = 0;
lstart[1] = 0;
MPI_Type_create_subarray(2, gsize, lsize, lstart, MPI_ORDER_C,
MPI_INT, &buftype);
MPI_Type_commit(&buftype);

/* set subarray offset and length */
start[0] = 0;
start[1] = LEN * (rank / psize[1]);
start[2] = LEN * (rank % psize[1]);
count[0] = 1;
count[1] = lsize[0];
count[2] = lsize[1];

if (verbose)
printf("rank %d start=%lld %lld count=%lld %lld\n",
rank, start[1],start[2], count[1],count[2]);

if (verbose)
printf("%d: nonblocking write total amount = %.1f GiB\n",
rank, (float)count[1]*count[2]*NVARS*sizeof(int)/1073741824);
Expand Down
11 changes: 6 additions & 5 deletions test/largefile/large_var.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -61,7 +61,8 @@ swapn(void *buf,
int main(int argc, char** argv)
{
char filename[256];
int i, j, rank, nprocs, err, nerrs=0, bufsize, expected;
size_t bufsize;
int i, j, rank, nprocs, err, nerrs=0, expected;
int ncid, cmode, varid, dimid[3], req[3], st[3], *buf, *buf_ptr;
MPI_Offset offset, var_offset, start[3], count[3];
MPI_File fh;
Expand Down Expand Up @@ -436,7 +437,7 @@ int main(int argc, char** argv)
for (i=0; i<count[0]; i++) {
for (j=0; j<count[1]; j++) {
offset = var_offset + ((start[0] + i) * NY * NX + (start[1] + j) * NX + start[2]) * sizeof(int);
MPI_File_read_at(fh, offset, buf_ptr, count[2], MPI_INT, &status);
MPI_File_read_at(fh, offset, buf_ptr, (int)count[2], MPI_INT, &status);
#ifndef WORDS_BIGENDIAN
swapn(buf_ptr, count[2]);
#endif
Expand Down Expand Up @@ -466,7 +467,7 @@ int main(int argc, char** argv)
for (i=0; i<count[0]; i++) {
for (j=0; j<count[1]; j++) {
offset = var_offset + ((start[0] + i) * NY * NX + (start[1] + j) * NX + start[2]) * sizeof(int);
MPI_File_read_at(fh, offset, buf_ptr, count[2], MPI_INT, &status);
MPI_File_read_at(fh, offset, buf_ptr, (int)count[2], MPI_INT, &status);
#ifndef WORDS_BIGENDIAN
swapn(buf_ptr, count[2]);
#endif
Expand Down Expand Up @@ -495,7 +496,7 @@ int main(int argc, char** argv)
for (i=0; i<count[0]; i++) {
for (j=0; j<count[1]; j++) {
offset = var_offset + ((start[0] + i) * NY * NX + (start[1] + j) * NX + start[2]) * sizeof(int);
MPI_File_read_at(fh, offset, buf_ptr, count[2], MPI_INT, &status);
MPI_File_read_at(fh, offset, buf_ptr, (int)count[2], MPI_INT, &status);
#ifndef WORDS_BIGENDIAN
swapn(buf_ptr, count[2]);
#endif
Expand Down Expand Up @@ -523,7 +524,7 @@ int main(int argc, char** argv)
for (i=0; i<count[0]; i++) {
for (j=0; j<count[1]; j++) {
offset = var_offset + ((start[0] + i) * NY * NX + (start[1] + j) * NX + start[2]) * sizeof(int);
MPI_File_read_at(fh, offset, buf_ptr, count[2], MPI_INT, &status);
MPI_File_read_at(fh, offset, buf_ptr, (int)count[2], MPI_INT, &status);
#ifndef WORDS_BIGENDIAN
swapn(buf_ptr, count[2]);
#endif
Expand Down
131 changes: 69 additions & 62 deletions test/testcases/tst_redefine.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,8 @@

static int verbose;

#define NUM_RNDUP(x, unit) ((((x) + (unit) - 1) / (unit)) * (unit))

#define LEN 16

#define CHECK_VAL(ncid, varid, ii, val, expect) { \
Expand All @@ -44,25 +46,24 @@ static int verbose;
static int
check_fix_vars(MPI_Comm comm, int ncid, int *varid)
{
int i, nerrs=0, err, rank, nprocs, *buf;
int i, nerrs=0, err, rank, *buf;
MPI_Offset start[2], count[2];

MPI_Comm_rank(comm, &rank);
MPI_Comm_size(comm, &nprocs);

start[0] = 0; start[1] = rank * (LEN / nprocs);
count[0] = 1; count[1] = LEN / nprocs;
start[0] = 0; start[1] = rank * LEN;
count[0] = 1; count[1] = LEN;

buf = (int*) malloc(sizeof(int) * count[0] * count[1]);

for (i=0; i<count[0]*count[1]; i++) buf[i] = -1;
err = ncmpi_get_vara_int_all(ncid, varid[0], start+1, count+1, buf); CHECK_ERROUT
for (i=0; i<LEN/nprocs; i++)
for (i=0; i<LEN; i++)
CHECK_VAL(ncid, varid[0], i, buf[i], rank+i)

for (i=0; i<count[0]*count[1]; i++) buf[i] = -1;
err = ncmpi_get_vara_int_all(ncid, varid[1], start+1, count+1, buf); CHECK_ERROUT
for (i=0; i<LEN/nprocs; i++)
for (i=0; i<LEN; i++)
CHECK_VAL(ncid, varid[1], i, buf[i], rank+i+100)

err_out:
Expand All @@ -73,25 +74,24 @@ check_fix_vars(MPI_Comm comm, int ncid, int *varid)
static int
check_rec_vars(MPI_Comm comm, int ncid, int *varid)
{
int i, nerrs=0, err, rank, nprocs, *buf;
int i, nerrs=0, err, rank, *buf;
MPI_Offset start[2], count[2];

MPI_Comm_rank(comm, &rank);
MPI_Comm_size(comm, &nprocs);

start[0] = 0; start[1] = rank * (LEN / nprocs);
count[0] = 2; count[1] = LEN / nprocs;
start[0] = 0; start[1] = rank * LEN;
count[0] = 2; count[1] = LEN;

buf = (int*) malloc(sizeof(int) * count[0] * count[1]);

for (i=0; i<count[0]*count[1]; i++) buf[i] = -1;
err = ncmpi_get_vara_int_all(ncid, varid[0], start, count, buf); CHECK_ERROUT
for (i=0; i<2*LEN/nprocs; i++)
for (i=0; i<2*LEN; i++)
CHECK_VAL(ncid, varid[0], i, buf[i], rank+i+1000)

for (i=0; i<count[0]*count[1]; i++) buf[i] = -1;
err = ncmpi_get_vara_int_all(ncid, varid[1], start, count, buf); CHECK_ERROUT
for (i=0; i<2*LEN/nprocs; i++)
for (i=0; i<2*LEN; i++)
CHECK_VAL(ncid, varid[1], i, buf[i], rank+i+10000)

err_out:
Expand All @@ -108,7 +108,7 @@ tst_fmt(char *filename, int cmode)
MPI_Offset old_hsize, hsize;
MPI_Offset old_extent, extent;
MPI_Offset old_var_off, var_off;
MPI_Offset v_align, r_align;
MPI_Offset v_align, r_align, exp_var_off;
MPI_Comm comm = MPI_COMM_WORLD;

MPI_Comm_rank(comm, &rank);
Expand All @@ -121,7 +121,7 @@ tst_fmt(char *filename, int cmode)
err = ncmpi_create(comm, filename, cmode, MPI_INFO_NULL, &ncid); CHECK_ERR

err = ncmpi_def_dim(ncid, "time", NC_UNLIMITED, &dimid[0]); CHECK_ERR
err = ncmpi_def_dim(ncid, "dim", LEN, &dimid[1]); CHECK_ERR
err = ncmpi_def_dim(ncid, "dim", LEN*nprocs, &dimid[1]); CHECK_ERR
err = ncmpi_def_var(ncid, "fa", NC_INT, 1, dimid+1, &varid[0]); CHECK_ERR
err = ncmpi_def_var(ncid, "fb", NC_INT, 1, dimid+1, &varid[1]); CHECK_ERR
err = ncmpi_def_var(ncid, "ta", NC_INT, 2, dimid, &varid[2]); CHECK_ERR
Expand All @@ -130,24 +130,24 @@ tst_fmt(char *filename, int cmode)
/* disable header alignment */
err = ncmpi__enddef(ncid, 0, 4, 0, 4); CHECK_ERR

start[0] = 0; start[1] = rank * (LEN / nprocs);
count[0] = 2; count[1] = LEN / nprocs;
start[0] = 0; start[1] = rank * LEN;
count[0] = 2; count[1] = LEN;

buf = (int*) malloc(sizeof(int) * count[0] * count[1]);

for (i=0; i<LEN/nprocs; i++) buf[i] = rank + i;
for (i=0; i<LEN; i++) buf[i] = rank + i;
err = ncmpi_put_vara_int_all(ncid, varid[0], start+1, count+1, buf); CHECK_ERR
for (i=0; i<LEN/nprocs; i++) buf[i] = rank + i + 100;
for (i=0; i<LEN; i++) buf[i] = rank + i + 100;
err = ncmpi_put_vara_int_all(ncid, varid[1], start+1, count+1, buf); CHECK_ERR
for (i=0; i<2*LEN/nprocs; i++) buf[i] = rank + i + 1000;
for (i=0; i<2*LEN; i++) buf[i] = rank + i + 1000;
err = ncmpi_put_vara_int_all(ncid, varid[2], start, count, buf); CHECK_ERR
for (i=0; i<2*LEN/nprocs; i++) buf[i] = rank + i + 10000;
for (i=0; i<2*LEN; i++) buf[i] = rank + i + 10000;
err = ncmpi_put_vara_int_all(ncid, varid[3], start, count, buf); CHECK_ERR

err = check_fix_vars(comm, ncid, varid);
if (err > 0) goto err_out;
err = check_rec_vars(comm, ncid, varid+2);
if (err > 0) goto err_out;
nerrs += check_fix_vars(comm, ncid, varid);
if (nerrs > 0) goto err_out;
nerrs += check_rec_vars(comm, ncid, varid+2);
if (nerrs > 0) goto err_out;

err = ncmpi_close(ncid); CHECK_ERR

Expand Down Expand Up @@ -201,10 +201,10 @@ tst_fmt(char *filename, int cmode)
__LINE__,__FILE__, extent, old_extent);
}

err = check_fix_vars(comm, ncid, varid);
if (err > 0) goto err_out;
err = check_rec_vars(comm, ncid, varid+2);
if (err > 0) goto err_out;
nerrs += check_fix_vars(comm, ncid, varid);
if (nerrs > 0) goto err_out;
nerrs += check_rec_vars(comm, ncid, varid+2);
if (nerrs > 0) goto err_out;

/* enter redefine mode and add nothing */
err = ncmpi_redef(ncid); CHECK_ERR
Expand Down Expand Up @@ -234,10 +234,10 @@ tst_fmt(char *filename, int cmode)
__LINE__,__FILE__, extent, old_extent + minfree);
}

err = check_fix_vars(comm, ncid, varid);
if (err > 0) goto err_out;
err = check_rec_vars(comm, ncid, varid+2);
if (err > 0) goto err_out;
nerrs += check_fix_vars(comm, ncid, varid);
if (nerrs > 0) goto err_out;
nerrs += check_rec_vars(comm, ncid, varid+2);
if (nerrs > 0) goto err_out;

/* enter redefine mode and add nothing */
err = ncmpi_redef(ncid); CHECK_ERR
Expand Down Expand Up @@ -267,10 +267,10 @@ tst_fmt(char *filename, int cmode)
__LINE__,__FILE__, extent);
}

err = check_fix_vars(comm, ncid, varid);
if (err > 0) goto err_out;
err = check_rec_vars(comm, ncid, varid+2);
if (err > 0) goto err_out;
nerrs += check_fix_vars(comm, ncid, varid);
if (nerrs > 0) goto err_out;
nerrs += check_rec_vars(comm, ncid, varid+2);
if (nerrs > 0) goto err_out;

err = ncmpi_close(ncid); CHECK_ERR

Expand Down Expand Up @@ -316,10 +316,10 @@ tst_fmt(char *filename, int cmode)

unsetenv("PNETCDF_HINTS");

err = check_fix_vars(comm, ncid, varid);
if (err > 0) goto err_out;
err = check_rec_vars(comm, ncid, varid+2);
if (err > 0) goto err_out;
nerrs += check_fix_vars(comm, ncid, varid);
if (nerrs > 0) goto err_out;
nerrs += check_rec_vars(comm, ncid, varid+2);
if (nerrs > 0) goto err_out;

/* enter redefine mode and add nothing */
err = ncmpi_redef(ncid); CHECK_ERR
Expand Down Expand Up @@ -352,10 +352,10 @@ tst_fmt(char *filename, int cmode)
/* obtain 1st record variable's file offset */
err = ncmpi_inq_varoffset(ncid, varid[2], &old_var_off); CHECK_ERR

err = check_fix_vars(comm, ncid, varid);
if (err > 0) goto err_out;
err = check_rec_vars(comm, ncid, varid+2);
if (err > 0) goto err_out;
nerrs += check_fix_vars(comm, ncid, varid);
if (nerrs > 0) goto err_out;
nerrs += check_rec_vars(comm, ncid, varid+2);
if (nerrs > 0) goto err_out;

/* enter redefine mode and add nothing */
err = ncmpi_redef(ncid); CHECK_ERR
Expand All @@ -373,17 +373,21 @@ tst_fmt(char *filename, int cmode)
__LINE__,__FILE__, var_off, old_var_off+400);
}

err = check_fix_vars(comm, ncid, varid);
if (err > 0) goto err_out;
err = check_rec_vars(comm, ncid, varid+2);
if (err > 0) goto err_out;
nerrs += check_fix_vars(comm, ncid, varid);
if (nerrs > 0) goto err_out;
nerrs += check_rec_vars(comm, ncid, varid+2);
if (nerrs > 0) goto err_out;

#if 0
err = ncmpi_close(ncid); CHECK_ERR

/* reopen the file and set r_align */
err = ncmpi_open(comm, filename, NC_WRITE, MPI_INFO_NULL, &ncid); CHECK_ERR
#endif

/* obtained the old offset of 1st record variable */
err = ncmpi_inq_varoffset(ncid, varid[2], &old_var_off); CHECK_ERR

/* enter redefine mode and add nothing */
err = ncmpi_redef(ncid); CHECK_ERR

Expand All @@ -394,17 +398,20 @@ tst_fmt(char *filename, int cmode)
/* obtain 1st record variable's file offset */
err = ncmpi_inq_varoffset(ncid, varid[2], &var_off); CHECK_ERR

/* round up to r_align */
exp_var_off = NUM_RNDUP(old_var_off, r_align);

/* var_off should grows into 1500 bytes */
if (var_off != r_align) {
if (var_off != exp_var_off) {
nerrs++;
printf("Error at line %d in %s: 1st record variable offset %lld (expecting %lld)\n",
__LINE__,__FILE__, var_off, r_align);
__LINE__,__FILE__, var_off, exp_var_off);
}

err = check_fix_vars(comm, ncid, varid);
if (err > 0) goto err_out;
err = check_rec_vars(comm, ncid, varid+2);
if (err > 0) goto err_out;
nerrs += check_fix_vars(comm, ncid, varid);
if (nerrs > 0) goto err_out;
nerrs += check_rec_vars(comm, ncid, varid+2);
if (nerrs > 0) goto err_out;

err = ncmpi_close(ncid); CHECK_ERR

Expand All @@ -416,7 +423,7 @@ tst_fmt(char *filename, int cmode)

/* define only record variables */
err = ncmpi_def_dim(ncid, "time", NC_UNLIMITED, &dimid[0]); CHECK_ERR
err = ncmpi_def_dim(ncid, "dim", LEN, &dimid[1]); CHECK_ERR
err = ncmpi_def_dim(ncid, "dim", LEN*nprocs, &dimid[1]); CHECK_ERR
err = ncmpi_def_var(ncid, "ta", NC_INT, 2, dimid, &varid[2]); CHECK_ERR
err = ncmpi_def_var(ncid, "tb", NC_INT, 2, dimid, &varid[3]); CHECK_ERR

Expand All @@ -427,12 +434,12 @@ tst_fmt(char *filename, int cmode)
r_align = 512;
err = ncmpi__enddef(ncid, 0, v_align, 0, r_align); CHECK_ERR

start[0] = 0; start[1] = rank * (LEN / nprocs);
count[0] = 2; count[1] = LEN / nprocs;
start[0] = 0; start[1] = rank * LEN;
count[0] = 2; count[1] = LEN;

for (i=0; i<2*LEN/nprocs; i++) buf[i] = rank + i + 1000;
for (i=0; i<2*LEN; i++) buf[i] = rank + i + 1000;
err = ncmpi_put_vara_int_all(ncid, varid[2], start, count, buf); CHECK_ERR
for (i=0; i<2*LEN/nprocs; i++) buf[i] = rank + i + 10000;
for (i=0; i<2*LEN; i++) buf[i] = rank + i + 10000;
err = ncmpi_put_vara_int_all(ncid, varid[3], start, count, buf); CHECK_ERR

err = ncmpi_inq_header_size(ncid, &hsize); CHECK_ERR
Expand All @@ -455,13 +462,13 @@ tst_fmt(char *filename, int cmode)
__LINE__,__FILE__, extent, r_align);
}

err = check_rec_vars(comm, ncid, varid+2);
if (err > 0) goto err_out;
nerrs += check_rec_vars(comm, ncid, varid+2);
if (nerrs > 0) goto err_out;

err_out:
err = ncmpi_close(ncid); CHECK_ERR
free(buf);

err_out:
return nerrs;
}

Expand Down

0 comments on commit 8cc0138

Please sign in to comment.