From ae502a9847a548173706698f88d4607fd15e11b9 Mon Sep 17 00:00:00 2001
From: Wei-keng Liao <wkliao@users.noreply.github.com>
Date: Sun, 11 Feb 2024 14:38:33 -0600
Subject: [PATCH] UnifyFS requires a stronger file consistency

---
 test/F90/f90tst_parallel2.f90     |  5 ++++
 test/F90/f90tst_parallel3.f90     |  5 ++++
 test/largefile/high_dim_var.c     | 18 ++++++++++++
 test/nonblocking/i_varn_indef.c   | 48 +++++++++++++++++++++++++++++++
 test/nonblocking/mcoll_perf.c     | 12 ++++++++
 test/testcases/flexible.c         | 12 ++++++++
 test/testcases/ivarn.c            | 12 ++++++++
 test/testcases/mix_collectives.c  |  6 ++++
 test/testcases/test_vardf.F       | 12 ++++++++
 test/testcases/test_vardf90.f90   | 10 +++++++
 test/testcases/tst_def_var_fill.c |  6 ++++
 test/testcases/varn_real.f90      | 10 +++++++
 12 files changed, 156 insertions(+)

diff --git a/test/F90/f90tst_parallel2.f90 b/test/F90/f90tst_parallel2.f90
index e00399951..d173027c2 100644
--- a/test/F90/f90tst_parallel2.f90
+++ b/test/F90/f90tst_parallel2.f90
@@ -109,6 +109,11 @@ program f90tst_parallel
   ! With classic model netCDF-4 file, enddef must be called.
   call handle_err(nf90mpi_enddef(ncid))
 
+  ! UnifyFS requires a stronger file consistency
+  call handle_err(nf90mpi_sync(ncid))
+  call MPI_Barrier(MPI_COMM_WORLD, ierr)
+  call handle_err(nf90mpi_sync(ncid))
+
   ! Determine what part of the variable will be written for this
   ! processor. It's a checkerboard decomposition.
   count = (/ NX / 4, NY / 4 /)
diff --git a/test/F90/f90tst_parallel3.f90 b/test/F90/f90tst_parallel3.f90
index 18d3ee9ba..08bd615a0 100644
--- a/test/F90/f90tst_parallel3.f90
+++ b/test/F90/f90tst_parallel3.f90
@@ -113,6 +113,11 @@ program f90tst_parallel3
   ! This will be the last collective operation.
   call check(nf90mpi_enddef(ncid))
 
+  ! UnifyFS requires a stronger file consistency
+  call check(nf90mpi_sync(ncid))
+  call MPI_Barrier(MPI_COMM_WORLD, ierr)
+  call check(nf90mpi_sync(ncid))
+
   ! Determine what part of the variable will be written/read for this
   ! processor. It's a checkerboard decomposition.
   count = (/ HALF_NX, HALF_NY /)
diff --git a/test/largefile/high_dim_var.c b/test/largefile/high_dim_var.c
index d06c95293..127c2405d 100644
--- a/test/largefile/high_dim_var.c
+++ b/test/largefile/high_dim_var.c
@@ -78,6 +78,12 @@ int main(int argc, char** argv) {
     err = ncmpi_enddef(ncid); CHECK_ERR
     if (err != NC_NOERR) goto fn_exit;
 
+#ifdef STRONGER_CONSISTENCY
+    ncmpi_sync(ncid);
+    MPI_Barrier(MPI_COMM_WORLD);
+    ncmpi_sync(ncid);
+#endif
+
     nelms = (NRECS > DIMLEN) ? NRECS : DIMLEN;
     for (i=1; i<NDIMS; i++) nelms *= DIMLEN;
     buffer = (short*) malloc(nelms * sizeof(short));
@@ -96,6 +102,12 @@ int main(int argc, char** argv) {
         }
     }
 
+#ifdef STRONGER_CONSISTENCY
+    ncmpi_sync(ncid);
+    MPI_Barrier(MPI_COMM_WORLD);
+    ncmpi_sync(ncid);
+#endif
+
     for (i=0; i<nelms; i++) buffer[i] = i % 32768;
 
     for (i=0; i<NDIMS; i++) {
@@ -115,6 +127,12 @@ int main(int argc, char** argv) {
                                        buffer); CHECK_ERR
     }
 
+#ifdef STRONGER_CONSISTENCY
+    ncmpi_sync(ncid);
+    MPI_Barrier(MPI_COMM_WORLD);
+    ncmpi_sync(ncid);
+#endif
+
     /* all processes read and verify */
     if (rank > 0) for (i=0; i<NDIMS; i++) count[i]  = 2;
     for (nelms=1,i=0; i<NDIMS; i++) nelms *= count[i];
diff --git a/test/nonblocking/i_varn_indef.c b/test/nonblocking/i_varn_indef.c
index 786b5e0ea..9ea1098b4 100644
--- a/test/nonblocking/i_varn_indef.c
+++ b/test/nonblocking/i_varn_indef.c
@@ -339,12 +339,24 @@ int main(int argc, char** argv)
 
     err = ncmpi_enddef(ncid); CHECK_ERR
 
+#ifdef STRONGER_CONSISTENCY
+    ncmpi_sync(ncid);
+    MPI_Barrier(MPI_COMM_WORLD);
+    ncmpi_sync(ncid);
+#endif
+
     /* clear the file contents using a blocking API, before commit the
      * nonblocking requests posted in define mode */
     if (!bb_enabled) {
         nerrs += clear_file_contents(ncid, varid);
     }
 
+#ifdef STRONGER_CONSISTENCY
+    ncmpi_sync(ncid);
+    MPI_Barrier(MPI_COMM_WORLD);
+    ncmpi_sync(ncid);
+#endif
+
     nerrs += check_num_pending_reqs(ncid, nreqs, __LINE__);
     err = ncmpi_wait_all(ncid, nreqs, reqs, sts);
     CHECK_ERR
@@ -393,12 +405,24 @@ int main(int argc, char** argv)
     }
     err = ncmpi_enddef(ncid); CHECK_ERR
 
+#ifdef STRONGER_CONSISTENCY
+    ncmpi_sync(ncid);
+    MPI_Barrier(MPI_COMM_WORLD);
+    ncmpi_sync(ncid);
+#endif
+
     /* clear the file contents using a blocking API, before commit the
      * nonblocking requests posted in define mode */
     if (!bb_enabled) {
         nerrs += clear_file_contents(ncid, varid);
     }
 
+#ifdef STRONGER_CONSISTENCY
+    ncmpi_sync(ncid);
+    MPI_Barrier(MPI_COMM_WORLD);
+    ncmpi_sync(ncid);
+#endif
+
     nerrs += check_num_pending_reqs(ncid, nreqs, __LINE__);
     err = ncmpi_wait_all(ncid, nreqs, reqs, sts);
     CHECK_ERR
@@ -457,12 +481,24 @@ int main(int argc, char** argv)
 
     err = ncmpi_enddef(ncid); CHECK_ERR
 
+#ifdef STRONGER_CONSISTENCY
+    ncmpi_sync(ncid);
+    MPI_Barrier(MPI_COMM_WORLD);
+    ncmpi_sync(ncid);
+#endif
+
     /* clear the file contents using a blocking API, before commit the
      * nonblocking requests posted in define mode */
     if (!bb_enabled) {
         nerrs += clear_file_contents(ncid, varid);
     }
 
+#ifdef STRONGER_CONSISTENCY
+    ncmpi_sync(ncid);
+    MPI_Barrier(MPI_COMM_WORLD);
+    ncmpi_sync(ncid);
+#endif
+
     nerrs += check_num_pending_reqs(ncid, nreqs*2, __LINE__);
     err = ncmpi_wait_all(ncid, nreqs, reqs, sts);
     CHECK_ERR
@@ -555,12 +591,24 @@ int main(int argc, char** argv)
 
     err = ncmpi_enddef(ncid); CHECK_ERR
 
+#ifdef STRONGER_CONSISTENCY
+    ncmpi_sync(ncid);
+    MPI_Barrier(MPI_COMM_WORLD);
+    ncmpi_sync(ncid);
+#endif
+
     /* clear the file contents using a blocking API, before commit the
      * nonblocking requests posted in define mode */
     if (!bb_enabled) {
         nerrs += clear_file_contents(ncid, varid);
     }
 
+#ifdef STRONGER_CONSISTENCY
+    ncmpi_sync(ncid);
+    MPI_Barrier(MPI_COMM_WORLD);
+    ncmpi_sync(ncid);
+#endif
+
     nerrs += check_num_pending_reqs(ncid, nreqs*3, __LINE__);
 
     /* flush nonblocking write requests */
diff --git a/test/nonblocking/mcoll_perf.c b/test/nonblocking/mcoll_perf.c
index 6e9d7934f..8df476518 100644
--- a/test/nonblocking/mcoll_perf.c
+++ b/test/nonblocking/mcoll_perf.c
@@ -512,6 +512,12 @@ int main(int argc, char **argv)
         err = ncmpi_enddef(ncid);
         CHECK_ERR
 
+#ifdef STRONGER_CONSISTENCY
+        ncmpi_sync(ncid);
+        MPI_Barrier(MPI_COMM_WORLD);
+        ncmpi_sync(ncid);
+#endif
+
         if (k == 0) {
             if (rank == 0 && verbose)
                 printf("*** Testing to write 2 non-record variables and 2 record variables by using ncmpi_put_vara_all() ...");
@@ -522,6 +528,12 @@ int main(int argc, char **argv)
                     CHECK_ERR
                 }
             }
+#ifdef STRONGER_CONSISTENCY
+            ncmpi_sync(ncid);
+            MPI_Barrier(MPI_COMM_WORLD);
+            ncmpi_sync(ncid);
+#endif
+
             for (i=0; i<nvars; i++){
                 err = ncmpi_put_vara_all(ncid, varid[i], starts[i], counts[i], buf[i], bufcounts[i], MPI_INT);
                 CHECK_ERR
diff --git a/test/testcases/flexible.c b/test/testcases/flexible.c
index cfa856e23..d43dd4a8d 100644
--- a/test/testcases/flexible.c
+++ b/test/testcases/flexible.c
@@ -88,6 +88,12 @@ int main(int argc, char **argv) {
     err = ncmpi_set_fill(ncid, NC_FILL, NULL); CHECK_ERR /* enable fill mode */
     err = ncmpi_enddef(ncid); CHECK_ERR
 
+#ifdef STRONGER_CONSISTENCY
+    ncmpi_sync(ncid);
+    MPI_Barrier(MPI_COMM_WORLD);
+    ncmpi_sync(ncid);
+#endif
+
     /* fill 2 records with default fill values */
     err = ncmpi_fill_var_rec(ncid, varid1, 0); CHECK_ERR
     err = ncmpi_fill_var_rec(ncid, varid1, 1); CHECK_ERR
@@ -96,6 +102,12 @@ int main(int argc, char **argv) {
     err = ncmpi_fill_var_rec(ncid, varid3, 0); CHECK_ERR
     err = ncmpi_fill_var_rec(ncid, varid3, 1); CHECK_ERR
 
+#ifdef STRONGER_CONSISTENCY
+    ncmpi_sync(ncid);
+    MPI_Barrier(MPI_COMM_WORLD);
+    ncmpi_sync(ncid);
+#endif
+
     /* initialize the contents of the array */
     for (j=0; j<NY; j++) for (i=0; i<NX; i++) buf[j][i] = j+10;
 
diff --git a/test/testcases/ivarn.c b/test/testcases/ivarn.c
index 0dc412f8c..a12898482 100644
--- a/test/testcases/ivarn.c
+++ b/test/testcases/ivarn.c
@@ -208,6 +208,12 @@ int main(int argc, char** argv)
     }
     err = ncmpi_enddef(ncid); CHECK_ERR
 
+#ifdef STRONGER_CONSISTENCY
+    ncmpi_sync(ncid);
+    MPI_Barrier(MPI_COMM_WORLD);
+    ncmpi_sync(ncid);
+#endif
+
     if (nprocs < 4) { /* need 4 processes to fill the variables */
         err = ncmpi_fill_var_rec(ncid, vari0001, 0); CHECK_ERR
         err = ncmpi_fill_var_rec(ncid, varr0001, 0); CHECK_ERR
@@ -217,6 +223,12 @@ int main(int argc, char** argv)
         err = ncmpi_fill_var_rec(ncid, vard0002, 0); CHECK_ERR
     }
 
+#ifdef STRONGER_CONSISTENCY
+    ncmpi_sync(ncid);
+    MPI_Barrier(MPI_COMM_WORLD);
+    ncmpi_sync(ncid);
+#endif
+
     starts    = (MPI_Offset**) malloc(2 *    sizeof(MPI_Offset*));
     counts    = (MPI_Offset**) malloc(2 *    sizeof(MPI_Offset*));
     starts[0] = (MPI_Offset*)  calloc(2 * 2, sizeof(MPI_Offset));
diff --git a/test/testcases/mix_collectives.c b/test/testcases/mix_collectives.c
index c07ff07c3..a0570ee38 100644
--- a/test/testcases/mix_collectives.c
+++ b/test/testcases/mix_collectives.c
@@ -69,6 +69,12 @@ int main(int argc, char **argv)
     err = ncmpi_set_fill(ncid, NC_FILL, NULL); CHECK_ERR
     err = ncmpi_enddef(ncid); CHECK_ERR
 
+#ifdef STRONGER_CONSISTENCY
+    ncmpi_sync(ncid);
+    MPI_Barrier(MPI_COMM_WORLD);
+    ncmpi_sync(ncid);
+#endif
+
     for (j=0; j<6; j++) for (i=0; i<4; i++) buf[j][i] = j*4+i + rank*100;
 
     /* now, each of 4 processes makes a call to different kinds of put APIs */
diff --git a/test/testcases/test_vardf.F b/test/testcases/test_vardf.F
index d2956ef10..80d43875b 100644
--- a/test/testcases/test_vardf.F
+++ b/test/testcases/test_vardf.F
@@ -308,6 +308,12 @@ program main
           err = nfmpi_enddef(ncid)
           call check(err, 'In nfmpi_enddef: ')
 
+#ifdef STRONGER_CONSISTENCY
+          err = nfmpi_sync(ncid)
+          call MPI_Barrier(MPI_COMM_WORLD, ierr)
+          err = nfmpi_sync(ncid)
+#endif
+
           ! now we are in data mode
 
           ! fill 2 records with default fill values
@@ -324,6 +330,12 @@ program main
           err = nfmpi_fill_var_rec(ncid, varid2, recno)
           call check(err, 'In nfmpi_fill_var_rec: varid2, 2 ')
 
+#ifdef STRONGER_CONSISTENCY
+          err = nfmpi_sync(ncid)
+          call MPI_Barrier(MPI_COMM_WORLD, ierr)
+          err = nfmpi_sync(ncid)
+#endif
+
           ! create a file type for the record variable */
           err = nfmpi_inq_recsize(ncid, recsize)
           call check(err, 'In nfmpi_inq_recsize: ')
diff --git a/test/testcases/test_vardf90.f90 b/test/testcases/test_vardf90.f90
index b32f8f2c3..efc13e5cc 100644
--- a/test/testcases/test_vardf90.f90
+++ b/test/testcases/test_vardf90.f90
@@ -290,6 +290,11 @@ program main
           err = nf90mpi_enddef(ncid)
           call check(err, 'In nf90mpi_enddef: ')
 
+          ! UnifyFS requires a stronger file consistency
+          err = nf90mpi_sync(ncid)
+          call MPI_Barrier(MPI_COMM_WORLD, err)
+          err = nf90mpi_sync(ncid)
+
           ! now we are in data mode
 
           ! fill 2 records with default fill values
@@ -306,6 +311,11 @@ program main
           err = nf90mpi_fill_var_rec(ncid, varid2, recno)
           call check(err, 'In nf90mpi_fill_var_rec: varid2, 2 ')
 
+          ! UnifyFS requires a stronger file consistency
+          err = nf90mpi_sync(ncid)
+          call MPI_Barrier(MPI_COMM_WORLD, err)
+          err = nf90mpi_sync(ncid)
+
           ! create a file type for the record variable */
           err = nf90mpi_inq_recsize(ncid, recsize)
           call check(err, 'In nf90mpi_inq_recsize: ')
diff --git a/test/testcases/tst_def_var_fill.c b/test/testcases/tst_def_var_fill.c
index 873f1229a..00b07c720 100644
--- a/test/testcases/tst_def_var_fill.c
+++ b/test/testcases/tst_def_var_fill.c
@@ -61,6 +61,12 @@ tst_fmt(char *filename, int cmode)
 
     err = ncmpi_enddef(ncid); CHECK_ERR
 
+#ifdef STRONGER_CONSISTENCY
+    err = ncmpi_sync(ncid); CHECK_ERR
+    MPI_Barrier(MPI_COMM_WORLD);
+    err = ncmpi_sync(ncid); CHECK_ERR
+#endif
+
     /* initialize I/O buffer */
     for (i=0; i<NY*NX; i++) buf[i] = rank+5;
 
diff --git a/test/testcases/varn_real.f90 b/test/testcases/varn_real.f90
index 2648f1b7f..7ec3a2eb4 100644
--- a/test/testcases/varn_real.f90
+++ b/test/testcases/varn_real.f90
@@ -113,6 +113,11 @@ program main
           err = nf90mpi_enddef(ncid)
           call check(err, 'In nf90mpi_enddef: ')
 
+          ! UnifyFS requires a stronger file consistency
+          err = nf90mpi_sync(ncid)
+          call MPI_Barrier(MPI_COMM_WORLD, err)
+          err = nf90mpi_sync(ncid)
+
           ! pick arbitrary numbers of requests for 4 processes
           num_reqs = 0
           if (rank .EQ.  0) then
@@ -231,6 +236,11 @@ program main
 
           if (nprocs .GT. 4) call MPI_Barrier(MPI_COMM_WORLD, err)
 
+          ! UnifyFS requires a stronger file consistency
+          err = nf90mpi_sync(ncid)
+          call MPI_Barrier(MPI_COMM_WORLD, err)
+          err = nf90mpi_sync(ncid)
+
           ! read a scalar back and check the content
           oneReal = -1.0  ! a scalar
           if (rank .GE. 4) starts = 1_MPI_OFFSET_KIND