Merge pull request #130 from Parallel-NetCDF/check_2GiB

change checking for size of dim/attr/header limit

src/dispatchers/dimension.c

@@ -57,9 +57,8 @@ ncmpi_def_dim(int         ncid,    /* IN:  file ID */
     /* MPI_Offset is usually a signed value, but serial netcdf uses size_t.
      * In 1999 ISO C standard, size_t is an unsigned integer type of at least
      * 16 bit. */
-    if (pncp->format == NC_FORMAT_CDF2) { /* CDF-2 format, max is 2^32-4 */
-        if (size > NC_MAX_UINT - 3 || (size < 0))
-            /* "-3" handles rounded-up size */
+    if (pncp->format == NC_FORMAT_CDF2) { /* CDF-2 format, max is NC_MAX_INT */
+        if (size > NC_MAX_INT || (size < 0))
             err = NC_EDIMSIZE;
     } else if (pncp->format == NC_FORMAT_CDF5) { /* CDF-5 format */
         if (size < 0)
@@ -68,9 +67,8 @@ ncmpi_def_dim(int         ncid,    /* IN:  file ID */
                pncp->format == NC_FORMAT_NETCDF4_CLASSIC) { /* NetCDF-4 format */
         if (size < 0)
             err = NC_EDIMSIZE;
-    } else { /* CDF-1 format, max is 2^31-4 */
-        if (size > NC_MAX_INT - 3 || (size < 0))
-            /* "-3" handles rounded-up size */
+    } else { /* CDF-1 format, max is NC_MAX_INT */
+        if (size > NC_MAX_INT || (size < 0))
             err = NC_EDIMSIZE;
     }
     if (err != NC_NOERR) {

src/drivers/ncmpio/ncmpio_header_put.c

@@ -80,7 +80,8 @@ hdr_put_NC_dim(bufferinfo   *pbp,
     /* copy dim_length */
     if (pbp->version < 5) {
         /* TODO: Isn't checking dimension size already done in def_dim()? */
-        if (dimp->size != (uint)dimp->size) DEBUG_RETURN_ERROR(NC_EINTOVERFLOW)
+        if (dimp->size > NC_MAX_INT)
+            DEBUG_RETURN_ERROR(NC_EINTOVERFLOW)
         err = ncmpix_put_uint32((void**)(&pbp->pos), (uint)dimp->size);
     }
     else
@@ -174,7 +175,8 @@ hdr_put_NC_attrV(bufferinfo    *pbp,
     sz = attrp->nelems * xsz;
     padding = attrp->xsz - sz;
 
-    if (sz != (size_t) sz) DEBUG_RETURN_ERROR(NC_EINTOVERFLOW)
+    if (sz > NC_MAX_INT)
+        DEBUG_RETURN_ERROR(NC_EINTOVERFLOW)
     memcpy(pbp->pos, attrp->xvalue, (size_t)sz);
     pbp->pos = (void *)((char *)pbp->pos + sz);
 
@@ -212,7 +214,7 @@ hdr_put_NC_attr(bufferinfo    *pbp,
 
     /* copy nelems */
     if (pbp->version < 5) {
-        if (attrp->nelems != (uint)attrp->nelems)
+        if (attrp->nelems  > NC_MAX_INT)
             DEBUG_RETURN_ERROR(NC_EINTOVERFLOW)
         status = ncmpix_put_uint32((void**)(&pbp->pos), (uint)attrp->nelems);
     }
@@ -365,7 +367,8 @@ hdr_put_NC_var(bufferinfo   *pbp,
      * in CDF-2 and CDF-5, it is a 64-bit integer
      */
     if (pbp->version == 1) {
-        if (varp->begin != (uint)varp->begin) DEBUG_RETURN_ERROR(NC_EINTOVERFLOW)
+        if (varp->begin  > NC_MAX_INT)
+            DEBUG_RETURN_ERROR(NC_EINTOVERFLOW)
         status = ncmpix_put_uint32((void**)(&pbp->pos), (uint)varp->begin);
     }
     else
@@ -473,7 +476,8 @@ ncmpio_hdr_put_NC(NC *ncp, void *buf)
     /* copy numrecs, number of records */
     nrecs = ncp->numrecs;
     if (ncp->format < 5) {
-        if (nrecs != (uint)nrecs) DEBUG_RETURN_ERROR(NC_EINTOVERFLOW)
+        if (nrecs  > NC_MAX_INT)
+            DEBUG_RETURN_ERROR(NC_EINTOVERFLOW)
         status = ncmpix_put_uint32((void**)(&putbuf.pos), (uint)nrecs);
     }
     else {
@@ -535,7 +539,7 @@ int ncmpio_write_header(NC *ncp)
         /* copy header object to write buffer */
         status = ncmpio_hdr_put_NC(ncp, buf);
 
-        if (ncp->xsz != (int)ncp->xsz) {
+        if (ncp->xsz  > NC_MAX_INT) {
             NCI_Free(buf);
             DEBUG_RETURN_ERROR(NC_EINTOVERFLOW)
         }

test/cdf_format/dim_cdf12.c

@@ -55,7 +55,6 @@
 #include <stdlib.h>
 #include <string.h> /* strcpy() */
 #include <libgen.h> /* basename() */
-#include <limits.h>
 #include <mpi.h>
 #include <pnetcdf.h>
 #include <testutils.h>
@@ -98,27 +97,27 @@ int main(int argc, char** argv)
     /* create a new CDF-1 file ----------------------------------------------*/
     cmode = NC_CLOBBER;
 
-    /* max dimension size for CDF-1 file is 2^31-3 = 2147483647 - 3 */
+    /* max dimension size for CDF-1 file is NC_MAX_INT */
     err = ncmpi_create(MPI_COMM_WORLD, filename, cmode, info, &ncid); CHECK_ERR
-    err = ncmpi_def_dim(ncid, "Y", INT_MAX, &dimid[0]);
+    err = ncmpi_def_dim(ncid, "Y", (MPI_Offset)1+NC_MAX_INT, &dimid[0]);
     EXP_ERR(NC_EDIMSIZE)
-    err = ncmpi_def_dim(ncid, "Y", INT_MAX-3, &dimid[0]); CHECK_ERR
+    err = ncmpi_def_dim(ncid, "Y", NC_MAX_INT, &dimid[0]); CHECK_ERR
     err = ncmpi_close(ncid); CHECK_ERR
     err = ncmpi_open(MPI_COMM_WORLD, filename, NC_NOWRITE, info, &ncid); CHECK_ERR
     err = ncmpi_close(ncid); CHECK_ERR
 
     /* use the max dimension size to define a 1D variable */
     err = ncmpi_create(MPI_COMM_WORLD, filename, cmode, info, &ncid); CHECK_ERR
-    err = ncmpi_def_dim(ncid, "Y", INT_MAX-3, &dimid[0]); CHECK_ERR
+    err = ncmpi_def_dim(ncid, "Y", NC_MAX_INT, &dimid[0]); CHECK_ERR
     err = ncmpi_def_var(ncid, "var0", NC_CHAR, 1, dimid, &varid); CHECK_ERR
     err = ncmpi_close(ncid); CHECK_ERR
     err = ncmpi_open(MPI_COMM_WORLD, filename, NC_NOWRITE, info, &ncid); CHECK_ERR
     err = ncmpi_close(ncid); CHECK_ERR
 
     /* use the max dimension size to define a 1D variable, followed by
-     * another variable to make the file size > 2147483647 */
+     * another variable to make the file size > NC_MAX_INT */
     err = ncmpi_create(MPI_COMM_WORLD, filename, cmode, info, &ncid); CHECK_ERR
-    err = ncmpi_def_dim(ncid, "Y", INT_MAX-3,  &dimid[0]); CHECK_ERR
+    err = ncmpi_def_dim(ncid, "Y", NC_MAX_INT, &dimid[0]); CHECK_ERR
     err = ncmpi_def_dim(ncid, "X", 2,          &dimid[1]); CHECK_ERR
     err = ncmpi_def_var(ncid, "var0", NC_CHAR, 1, &dimid[0], &varid); CHECK_ERR
     err = ncmpi_def_var(ncid, "var1", NC_INT,  1, &dimid[1], &varid); CHECK_ERR
@@ -129,37 +128,37 @@ int main(int argc, char** argv)
     /* use the max dimension size - 1024 to define a 1D variable, followed
      * by another variable to make the file size < 2147483647 */
     err = ncmpi_create(MPI_COMM_WORLD, filename, cmode, info, &ncid); CHECK_ERR
-    err = ncmpi_def_dim(ncid, "Y", INT_MAX-1024, &dimid[0]); CHECK_ERR
-    err = ncmpi_def_dim(ncid, "X", 2,            &dimid[1]); CHECK_ERR
+    err = ncmpi_def_dim(ncid, "Y", NC_MAX_INT-1024, &dimid[0]); CHECK_ERR
+    err = ncmpi_def_dim(ncid, "X", 2,               &dimid[1]); CHECK_ERR
     err = ncmpi_def_var(ncid, "var0", NC_CHAR, 1, &dimid[0], &varid); CHECK_ERR
     err = ncmpi_def_var(ncid, "var1", NC_INT,  1, &dimid[1], &varid); CHECK_ERR
     err = ncmpi_close(ncid); CHECK_ERR
     err = ncmpi_open(MPI_COMM_WORLD, filename, NC_NOWRITE, info, &ncid); CHECK_ERR
     err = ncmpi_close(ncid); CHECK_ERR
 
     /* define the first variable of type short that makes the file size >
-     * 2147483647. error should be reported in ncmpi_enddef() or
+     * NC_MAX_INT. error should be reported in ncmpi_enddef() or
      * ncmpi_close() */
     err = ncmpi_create(MPI_COMM_WORLD, filename, cmode, info, &ncid); CHECK_ERR
-    err = ncmpi_def_dim(ncid, "Y", INT_MAX-3, &dimid[0]); CHECK_ERR
-    err = ncmpi_def_dim(ncid, "X", 2,         &dimid[1]); CHECK_ERR
+    err = ncmpi_def_dim(ncid, "Y", NC_MAX_INT, &dimid[0]); CHECK_ERR
+    err = ncmpi_def_dim(ncid, "X", 2,          &dimid[1]); CHECK_ERR
     err = ncmpi_def_var(ncid, "var0", NC_SHORT, 1, &dimid[0], &varid); CHECK_ERR
     err = ncmpi_def_var(ncid, "var1", NC_CHAR,  1, &dimid[1], &varid); CHECK_ERR
     err = ncmpi_close(ncid);
     EXP_ERR(NC_EVARSIZE)
 
     /* define two variables to make the file size just < 2147483647 */
     err = ncmpi_create(MPI_COMM_WORLD, filename, cmode, info, &ncid); CHECK_ERR
-    err = ncmpi_def_dim(ncid, "Y", INT_MAX-3-512-8, &dimid[0]); CHECK_ERR
-    err = ncmpi_def_dim(ncid, "X", 2,       &dimid[1]); CHECK_ERR
+    err = ncmpi_def_dim(ncid, "Y", NC_MAX_INT-512-8, &dimid[0]); CHECK_ERR
+    err = ncmpi_def_dim(ncid, "X", 2,                &dimid[1]); CHECK_ERR
     err = ncmpi_def_var(ncid, "var0", NC_CHAR, 1, &dimid[0], &varid); CHECK_ERR
     err = ncmpi_def_var(ncid, "var1", NC_INT,  1, &dimid[1], &varid); CHECK_ERR
     err = ncmpi_close(ncid); CHECK_ERR
 
-    /* define two variables to make the file size just > 2147483647 */
+    /* define two variables to make the file size just > NC_MAX_INT */
     err = ncmpi_create(MPI_COMM_WORLD, filename, cmode, info, &ncid); CHECK_ERR
-    err = ncmpi_def_dim(ncid, "Y", INT_MAX/2+1, &dimid[0]); CHECK_ERR
-    err = ncmpi_def_dim(ncid, "X", 2,           &dimid[1]); CHECK_ERR
+    err = ncmpi_def_dim(ncid, "Y", NC_MAX_INT/2+1, &dimid[0]); CHECK_ERR
+    err = ncmpi_def_dim(ncid, "X", 2,              &dimid[1]); CHECK_ERR
     err = ncmpi_def_var(ncid, "var0", NC_INT, 1, &dimid[0], &varid); CHECK_ERR
     err = ncmpi_def_var(ncid, "var1", NC_INT, 1, &dimid[1], &varid); CHECK_ERR
     err = ncmpi_close(ncid);
@@ -168,27 +167,27 @@ int main(int argc, char** argv)
     /* create a new CDF-2 file ----------------------------------------------*/
     cmode = NC_CLOBBER | NC_64BIT_OFFSET;
 
-    /* max dimension size for CDF-2 file is 2^32-3 = 4294967295 - 3 */
+    /* max dimension size for CDF-2 file is NC_MAX_INT */
     err = ncmpi_create(MPI_COMM_WORLD, filename, cmode, info, &ncid); CHECK_ERR
-    err = ncmpi_def_dim(ncid, "Y", UINT_MAX, &dimid[0]);
+    err = ncmpi_def_dim(ncid, "Y", (MPI_Offset)1+NC_MAX_INT, &dimid[0]);
     EXP_ERR(NC_EDIMSIZE)
-    err = ncmpi_def_dim(ncid, "Y", UINT_MAX-3, &dimid[0]); CHECK_ERR
+    err = ncmpi_def_dim(ncid, "Y", NC_MAX_INT, &dimid[0]); CHECK_ERR
     err = ncmpi_close(ncid); CHECK_ERR
     err = ncmpi_open(MPI_COMM_WORLD, filename, NC_NOWRITE, info, &ncid); CHECK_ERR
     err = ncmpi_close(ncid); CHECK_ERR
 
     /* use the max dimension size to define a 1D variable */
     err = ncmpi_create(MPI_COMM_WORLD, filename, cmode, info, &ncid); CHECK_ERR
-    err = ncmpi_def_dim(ncid, "Y", UINT_MAX-3, &dimid[0]); CHECK_ERR
+    err = ncmpi_def_dim(ncid, "Y", NC_MAX_INT, &dimid[0]); CHECK_ERR
     err = ncmpi_def_var(ncid, "var0", NC_CHAR, 1, dimid, &varid); CHECK_ERR
     err = ncmpi_close(ncid); CHECK_ERR
     err = ncmpi_open(MPI_COMM_WORLD, filename, NC_NOWRITE, info, &ncid); CHECK_ERR
     err = ncmpi_close(ncid); CHECK_ERR
 
     /* use the max dimension size to define a 1D variable, followed by
-     * another variable to make the file size > 4294967295 */
+     * another variable to make the file size > 2 * NC_MAX_INT */
     err = ncmpi_create(MPI_COMM_WORLD, filename, cmode, info, &ncid); CHECK_ERR
-    err = ncmpi_def_dim(ncid, "Y", UINT_MAX-3, &dimid[0]); CHECK_ERR
+    err = ncmpi_def_dim(ncid, "Y", NC_MAX_INT, &dimid[0]); CHECK_ERR
     err = ncmpi_def_dim(ncid, "X", 2,          &dimid[1]); CHECK_ERR
     err = ncmpi_def_var(ncid, "var0", NC_CHAR, 1, &dimid[0], &varid); CHECK_ERR
     err = ncmpi_def_var(ncid, "var1", NC_INT,  1, &dimid[1], &varid); CHECK_ERR
@@ -200,25 +199,25 @@ int main(int argc, char** argv)
      * 4294967295. error should be reported in ncmpi_enddef() or
      * ncmpi_close() */
     err = ncmpi_create(MPI_COMM_WORLD, filename, cmode, info, &ncid); CHECK_ERR
-    err = ncmpi_def_dim(ncid, "Y", UINT_MAX-3, &dimid[0]); CHECK_ERR
-    err = ncmpi_def_dim(ncid, "X", 2,          &dimid[1]); CHECK_ERR
+    err = ncmpi_def_dim(ncid, "Y", NC_MAX_INT, &dimid[0]); CHECK_ERR
+    err = ncmpi_def_dim(ncid, "X", 2,       &dimid[1]); CHECK_ERR
     err = ncmpi_def_var(ncid, "var0", NC_SHORT, 1, &dimid[0], &varid); CHECK_ERR
     err = ncmpi_def_var(ncid, "var1", NC_INT,   1, &dimid[1], &varid); CHECK_ERR
     err = ncmpi_close(ncid);
     EXP_ERR(NC_EVARSIZE)
 
-    /* define 2 1D int variables of dimension size > max */
+    /* define 2 1D int variables of dimension size > NC_MAX_INT */
     err = ncmpi_create(MPI_COMM_WORLD, filename, cmode, info, &ncid); CHECK_ERR
-    err = ncmpi_def_dim(ncid, "Y", INT_MAX, &dimid[0]); CHECK_ERR
-    err = ncmpi_def_dim(ncid, "X", 2,       &dimid[1]); CHECK_ERR
+    err = ncmpi_def_dim(ncid, "Y", NC_MAX_INT, &dimid[0]); CHECK_ERR
+    err = ncmpi_def_dim(ncid, "X", 2,          &dimid[1]); CHECK_ERR
     err = ncmpi_def_var(ncid, "var0", NC_INT, 1, &dimid[0], &varid); CHECK_ERR
     err = ncmpi_def_var(ncid, "var1", NC_INT, 1, &dimid[1], &varid); CHECK_ERR
     err = ncmpi_close(ncid);
     EXP_ERR(NC_EVARSIZE)
 
     err = ncmpi_create(MPI_COMM_WORLD, filename, cmode, info, &ncid); CHECK_ERR
-    err = ncmpi_def_dim(ncid, "Y", INT_MAX/2+1, &dimid[0]); CHECK_ERR
-    err = ncmpi_def_dim(ncid, "X", 2,           &dimid[1]); CHECK_ERR
+    err = ncmpi_def_dim(ncid, "Y", NC_MAX_INT/2+1, &dimid[0]); CHECK_ERR
+    err = ncmpi_def_dim(ncid, "X", 2,              &dimid[1]); CHECK_ERR
     err = ncmpi_def_var(ncid, "var0", NC_INT, 1, &dimid[0], &varid); CHECK_ERR
     err = ncmpi_def_var(ncid, "var1", NC_INT, 1, &dimid[1], &varid); CHECK_ERR
     err = ncmpi_close(ncid);
@@ -229,8 +228,8 @@ int main(int argc, char** argv)
      */
     err = ncmpi_create(MPI_COMM_WORLD, filename, cmode, info, &ncid); CHECK_ERR
     err = ncmpi_def_dim(ncid, "Z", NC_UNLIMITED, &dimid[0]); CHECK_ERR
-    err = ncmpi_def_dim(ncid, "Y", INT_MAX/64,   &dimid[1]); CHECK_ERR
-    err = ncmpi_def_dim(ncid, "X", 64,           &dimid[2]); CHECK_ERR
+    err = ncmpi_def_dim(ncid, "Y", NC_MAX_INT/64,   &dimid[1]); CHECK_ERR
+    err = ncmpi_def_dim(ncid, "X", 64,              &dimid[2]); CHECK_ERR
     err = ncmpi_def_var(ncid, "var0", NC_INT, 3, dimid, &varid); CHECK_ERR
     err = ncmpi_def_var(ncid, "var1", NC_INT, 3, dimid, &varid); CHECK_ERR
     err = ncmpi_close(ncid);
@@ -239,8 +238,8 @@ int main(int argc, char** argv)
     /* test large record variable that is not defined last */
     err = ncmpi_create(MPI_COMM_WORLD, filename, cmode, info, &ncid); CHECK_ERR
     err = ncmpi_def_dim(ncid, "Z", NC_UNLIMITED, &dimid[0]); CHECK_ERR
-    err = ncmpi_def_dim(ncid, "Y", INT_MAX/64,   &dimid[1]); CHECK_ERR
-    err = ncmpi_def_dim(ncid, "X", 64,           &dimid[2]); CHECK_ERR
+    err = ncmpi_def_dim(ncid, "Y", NC_MAX_INT/64,   &dimid[1]); CHECK_ERR
+    err = ncmpi_def_dim(ncid, "X", 64,              &dimid[2]); CHECK_ERR
     err = ncmpi_def_var(ncid, "var0", NC_INT, 3, dimid, &varid); CHECK_ERR
     err = ncmpi_def_var(ncid, "var1", NC_INT, 2, dimid, &varid); CHECK_ERR
     err = ncmpi_close(ncid);
@@ -250,8 +249,8 @@ int main(int argc, char** argv)
      * output file can be tested by ncvalidator in wrap_runs.sh
      */
     err = ncmpi_create(MPI_COMM_WORLD, filename, cmode, info, &ncid); CHECK_ERR
-    err = ncmpi_def_dim(ncid, "Y", INT_MAX/2, &dimid[0]); CHECK_ERR
-    err = ncmpi_def_dim(ncid, "X", 2,         &dimid[1]); CHECK_ERR
+    err = ncmpi_def_dim(ncid, "Y", NC_MAX_INT/2, &dimid[0]); CHECK_ERR
+    err = ncmpi_def_dim(ncid, "X", 2,            &dimid[1]); CHECK_ERR
     err = ncmpi_def_var(ncid, "var0", NC_INT, 1, &dimid[0], &varid); CHECK_ERR
     err = ncmpi_def_var(ncid, "var1", NC_INT, 1, &dimid[1], &varid); CHECK_ERR
     err = ncmpi_close(ncid); CHECK_ERR

test/testcases/tst_dimsizes.c

@@ -25,14 +25,24 @@
 
 #include <testutils.h>
 
-#define DIMMAXCLASSIC (NC_MAX_INT - 3)
-#define DIMMAX64OFFSET (NC_MAX_UINT - 3)
+#define DIMMAXCLASSIC NC_MAX_INT
+#define DIMMAX64OFFSET NC_MAX_INT
 #define DIMMAX64DATA NC_MAX_INT64
 
 /*
- * NC_CLASSIC => NC_INT_MAX - 3
- * NC_64BIT_OFFSET => NC_UINT_MAX - 3
- * NC_64BIT_DATA => NC_INT64_MAX
+ * NetCDF file format specification:
+ *   netcdf_file  = header data
+ *   header       = magic numrecs dim_list gatt_list var_list
+ *   dim_list     = ABSENT | NC_DIMENSION nelems [dim ...]
+ *   dim          = name dim_length
+ *   dim_length   = NON_NEG
+ *   NON_NEG      = <non-negative INT>
+ *   INT          = <32-bit signed integer, Bigendian, two's complement>
+ *
+ * Therefore, the max dimension size are:
+ * NC_CLASSIC       Max dimension size is  NC_INT_MAX
+ * NC_64BIT_OFFSET  Max dimension size is  NC_INT_MAX
+ * NC_64BIT_DATA    Max dimension size is  NC_INT64_MAX
  * Note that for NC_64BIT_DATA, the max dimension size is different from netCDF
  * library. This is because PnetCDF uses MPI_Offset for dimension size and
  * MPI_Offset is a signed long long.
@@ -72,7 +82,7 @@ main(int argc, char **argv)
     err = ncmpi_def_dim(ncid, "testdim", dimsize, &dimid); CHECK_ERR
     dimsize = -1;
     err = ncmpi_def_dim(ncid, "testdim1", dimsize, &dimid); EXP_ERR(NC_EDIMSIZE)
-    dimsize = DIMMAXCLASSIC+1;
+    dimsize = (MPI_Offset)DIMMAXCLASSIC+1;
     err = ncmpi_def_dim(ncid, "testdim1", dimsize, &dimid); EXP_ERR(NC_EDIMSIZE)
     err = ncmpi_close(ncid); CHECK_ERR
 
@@ -92,7 +102,7 @@ main(int argc, char **argv)
     err = ncmpi_def_dim(ncid, "testdim", dimsize, &dimid); CHECK_ERR
     dimsize = -1;
     err = ncmpi_def_dim(ncid, "testdim1", dimsize, &dimid); EXP_ERR(NC_EDIMSIZE)
-    dimsize = DIMMAX64OFFSET+1;
+    dimsize = (MPI_Offset)DIMMAX64OFFSET+1;
     err = ncmpi_def_dim(ncid, "testdim1", dimsize, &dimid); EXP_ERR(NC_EDIMSIZE)
     err = ncmpi_close(ncid); CHECK_ERR