Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

change checking for size of dim/attr/header limit #130

Merged
merged 3 commits into from
Mar 13, 2024
Merged

change checking for size of dim/attr/header limit #130

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
121ff46
check header size against INT_MAX
wkliao Mar 12, 2024
830295c
Dimension size limit should be INT_MAX
wkliao Mar 12, 2024
117d109
test dimension size limits
wkliao Mar 12, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 4 additions & 6 deletions src/dispatchers/dimension.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -57,9 +57,8 @@ ncmpi_def_dim(int ncid, /* IN: file ID */
/* MPI_Offset is usually a signed value, but serial netcdf uses size_t.
* In 1999 ISO C standard, size_t is an unsigned integer type of at least
* 16 bit. */
if (pncp->format == NC_FORMAT_CDF2) { /* CDF-2 format, max is 2^32-4 */
if (size > NC_MAX_UINT - 3 || (size < 0))
/* "-3" handles rounded-up size */
if (pncp->format == NC_FORMAT_CDF2) { /* CDF-2 format, max is NC_MAX_INT */
if (size > NC_MAX_INT || (size < 0))
err = NC_EDIMSIZE;
} else if (pncp->format == NC_FORMAT_CDF5) { /* CDF-5 format */
if (size < 0)
Expand All @@ -68,9 +67,8 @@ ncmpi_def_dim(int ncid, /* IN: file ID */
pncp->format == NC_FORMAT_NETCDF4_CLASSIC) { /* NetCDF-4 format */
if (size < 0)
err = NC_EDIMSIZE;
} else { /* CDF-1 format, max is 2^31-4 */
if (size > NC_MAX_INT - 3 || (size < 0))
/* "-3" handles rounded-up size */
} else { /* CDF-1 format, max is NC_MAX_INT */
if (size > NC_MAX_INT || (size < 0))
err = NC_EDIMSIZE;
}
if (err != NC_NOERR) {
Expand Down
16 changes: 10 additions & 6 deletions src/drivers/ncmpio/ncmpio_header_put.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -80,7 +80,8 @@ hdr_put_NC_dim(bufferinfo *pbp,
/* copy dim_length */
if (pbp->version < 5) {
/* TODO: Isn't checking dimension size already done in def_dim()? */
if (dimp->size != (uint)dimp->size) DEBUG_RETURN_ERROR(NC_EINTOVERFLOW)
if (dimp->size > NC_MAX_INT)
DEBUG_RETURN_ERROR(NC_EINTOVERFLOW)
err = ncmpix_put_uint32((void**)(&pbp->pos), (uint)dimp->size);
}
else
Expand Down Expand Up @@ -174,7 +175,8 @@ hdr_put_NC_attrV(bufferinfo *pbp,
sz = attrp->nelems * xsz;
padding = attrp->xsz - sz;

if (sz != (size_t) sz) DEBUG_RETURN_ERROR(NC_EINTOVERFLOW)
if (sz > NC_MAX_INT)
DEBUG_RETURN_ERROR(NC_EINTOVERFLOW)
memcpy(pbp->pos, attrp->xvalue, (size_t)sz);
pbp->pos = (void *)((char *)pbp->pos + sz);

Expand Down Expand Up @@ -212,7 +214,7 @@ hdr_put_NC_attr(bufferinfo *pbp,

/* copy nelems */
if (pbp->version < 5) {
if (attrp->nelems != (uint)attrp->nelems)
if (attrp->nelems > NC_MAX_INT)
DEBUG_RETURN_ERROR(NC_EINTOVERFLOW)
status = ncmpix_put_uint32((void**)(&pbp->pos), (uint)attrp->nelems);
}
Expand Down Expand Up @@ -365,7 +367,8 @@ hdr_put_NC_var(bufferinfo *pbp,
* in CDF-2 and CDF-5, it is a 64-bit integer
*/
if (pbp->version == 1) {
if (varp->begin != (uint)varp->begin) DEBUG_RETURN_ERROR(NC_EINTOVERFLOW)
if (varp->begin > NC_MAX_INT)
DEBUG_RETURN_ERROR(NC_EINTOVERFLOW)
status = ncmpix_put_uint32((void**)(&pbp->pos), (uint)varp->begin);
}
else
Expand Down Expand Up @@ -473,7 +476,8 @@ ncmpio_hdr_put_NC(NC *ncp, void *buf)
/* copy numrecs, number of records */
nrecs = ncp->numrecs;
if (ncp->format < 5) {
if (nrecs != (uint)nrecs) DEBUG_RETURN_ERROR(NC_EINTOVERFLOW)
if (nrecs > NC_MAX_INT)
DEBUG_RETURN_ERROR(NC_EINTOVERFLOW)
status = ncmpix_put_uint32((void**)(&putbuf.pos), (uint)nrecs);
}
else {
Expand Down Expand Up @@ -535,7 +539,7 @@ int ncmpio_write_header(NC *ncp)
/* copy header object to write buffer */
status = ncmpio_hdr_put_NC(ncp, buf);

if (ncp->xsz != (int)ncp->xsz) {
if (ncp->xsz > NC_MAX_INT) {
NCI_Free(buf);
DEBUG_RETURN_ERROR(NC_EINTOVERFLOW)
}
Expand Down
71 changes: 35 additions & 36 deletions test/cdf_format/dim_cdf12.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -55,7 +55,6 @@
#include <stdlib.h>
#include <string.h> /* strcpy() */
#include <libgen.h> /* basename() */
#include <limits.h>
#include <mpi.h>
#include <pnetcdf.h>
#include <testutils.h>
Expand Down Expand Up @@ -98,27 +97,27 @@ int main(int argc, char** argv)
/* create a new CDF-1 file ----------------------------------------------*/
cmode = NC_CLOBBER;

/* max dimension size for CDF-1 file is 2^31-3 = 2147483647 - 3 */
/* max dimension size for CDF-1 file is NC_MAX_INT */
err = ncmpi_create(MPI_COMM_WORLD, filename, cmode, info, &ncid); CHECK_ERR
err = ncmpi_def_dim(ncid, "Y", INT_MAX, &dimid[0]);
err = ncmpi_def_dim(ncid, "Y", (MPI_Offset)1+NC_MAX_INT, &dimid[0]);
EXP_ERR(NC_EDIMSIZE)
err = ncmpi_def_dim(ncid, "Y", INT_MAX-3, &dimid[0]); CHECK_ERR
err = ncmpi_def_dim(ncid, "Y", NC_MAX_INT, &dimid[0]); CHECK_ERR
err = ncmpi_close(ncid); CHECK_ERR
err = ncmpi_open(MPI_COMM_WORLD, filename, NC_NOWRITE, info, &ncid); CHECK_ERR
err = ncmpi_close(ncid); CHECK_ERR

/* use the max dimension size to define a 1D variable */
err = ncmpi_create(MPI_COMM_WORLD, filename, cmode, info, &ncid); CHECK_ERR
err = ncmpi_def_dim(ncid, "Y", INT_MAX-3, &dimid[0]); CHECK_ERR
err = ncmpi_def_dim(ncid, "Y", NC_MAX_INT, &dimid[0]); CHECK_ERR
err = ncmpi_def_var(ncid, "var0", NC_CHAR, 1, dimid, &varid); CHECK_ERR
err = ncmpi_close(ncid); CHECK_ERR
err = ncmpi_open(MPI_COMM_WORLD, filename, NC_NOWRITE, info, &ncid); CHECK_ERR
err = ncmpi_close(ncid); CHECK_ERR

/* use the max dimension size to define a 1D variable, followed by
* another variable to make the file size > 2147483647 */
* another variable to make the file size > NC_MAX_INT */
err = ncmpi_create(MPI_COMM_WORLD, filename, cmode, info, &ncid); CHECK_ERR
err = ncmpi_def_dim(ncid, "Y", INT_MAX-3, &dimid[0]); CHECK_ERR
err = ncmpi_def_dim(ncid, "Y", NC_MAX_INT, &dimid[0]); CHECK_ERR
err = ncmpi_def_dim(ncid, "X", 2, &dimid[1]); CHECK_ERR
err = ncmpi_def_var(ncid, "var0", NC_CHAR, 1, &dimid[0], &varid); CHECK_ERR
err = ncmpi_def_var(ncid, "var1", NC_INT, 1, &dimid[1], &varid); CHECK_ERR
Expand All @@ -129,37 +128,37 @@ int main(int argc, char** argv)
/* use the max dimension size - 1024 to define a 1D variable, followed
* by another variable to make the file size < 2147483647 */
err = ncmpi_create(MPI_COMM_WORLD, filename, cmode, info, &ncid); CHECK_ERR
err = ncmpi_def_dim(ncid, "Y", INT_MAX-1024, &dimid[0]); CHECK_ERR
err = ncmpi_def_dim(ncid, "X", 2, &dimid[1]); CHECK_ERR
err = ncmpi_def_dim(ncid, "Y", NC_MAX_INT-1024, &dimid[0]); CHECK_ERR
err = ncmpi_def_dim(ncid, "X", 2, &dimid[1]); CHECK_ERR
err = ncmpi_def_var(ncid, "var0", NC_CHAR, 1, &dimid[0], &varid); CHECK_ERR
err = ncmpi_def_var(ncid, "var1", NC_INT, 1, &dimid[1], &varid); CHECK_ERR
err = ncmpi_close(ncid); CHECK_ERR
err = ncmpi_open(MPI_COMM_WORLD, filename, NC_NOWRITE, info, &ncid); CHECK_ERR
err = ncmpi_close(ncid); CHECK_ERR

/* define the first variable of type short that makes the file size >
* 2147483647. error should be reported in ncmpi_enddef() or
* NC_MAX_INT. error should be reported in ncmpi_enddef() or
* ncmpi_close() */
err = ncmpi_create(MPI_COMM_WORLD, filename, cmode, info, &ncid); CHECK_ERR
err = ncmpi_def_dim(ncid, "Y", INT_MAX-3, &dimid[0]); CHECK_ERR
err = ncmpi_def_dim(ncid, "X", 2, &dimid[1]); CHECK_ERR
err = ncmpi_def_dim(ncid, "Y", NC_MAX_INT, &dimid[0]); CHECK_ERR
err = ncmpi_def_dim(ncid, "X", 2, &dimid[1]); CHECK_ERR
err = ncmpi_def_var(ncid, "var0", NC_SHORT, 1, &dimid[0], &varid); CHECK_ERR
err = ncmpi_def_var(ncid, "var1", NC_CHAR, 1, &dimid[1], &varid); CHECK_ERR
err = ncmpi_close(ncid);
EXP_ERR(NC_EVARSIZE)

/* define two variables to make the file size just < 2147483647 */
err = ncmpi_create(MPI_COMM_WORLD, filename, cmode, info, &ncid); CHECK_ERR
err = ncmpi_def_dim(ncid, "Y", INT_MAX-3-512-8, &dimid[0]); CHECK_ERR
err = ncmpi_def_dim(ncid, "X", 2, &dimid[1]); CHECK_ERR
err = ncmpi_def_dim(ncid, "Y", NC_MAX_INT-512-8, &dimid[0]); CHECK_ERR
err = ncmpi_def_dim(ncid, "X", 2, &dimid[1]); CHECK_ERR
err = ncmpi_def_var(ncid, "var0", NC_CHAR, 1, &dimid[0], &varid); CHECK_ERR
err = ncmpi_def_var(ncid, "var1", NC_INT, 1, &dimid[1], &varid); CHECK_ERR
err = ncmpi_close(ncid); CHECK_ERR

/* define two variables to make the file size just > 2147483647 */
/* define two variables to make the file size just > NC_MAX_INT */
err = ncmpi_create(MPI_COMM_WORLD, filename, cmode, info, &ncid); CHECK_ERR
err = ncmpi_def_dim(ncid, "Y", INT_MAX/2+1, &dimid[0]); CHECK_ERR
err = ncmpi_def_dim(ncid, "X", 2, &dimid[1]); CHECK_ERR
err = ncmpi_def_dim(ncid, "Y", NC_MAX_INT/2+1, &dimid[0]); CHECK_ERR
err = ncmpi_def_dim(ncid, "X", 2, &dimid[1]); CHECK_ERR
err = ncmpi_def_var(ncid, "var0", NC_INT, 1, &dimid[0], &varid); CHECK_ERR
err = ncmpi_def_var(ncid, "var1", NC_INT, 1, &dimid[1], &varid); CHECK_ERR
err = ncmpi_close(ncid);
Expand All @@ -168,27 +167,27 @@ int main(int argc, char** argv)
/* create a new CDF-2 file ----------------------------------------------*/
cmode = NC_CLOBBER | NC_64BIT_OFFSET;

/* max dimension size for CDF-2 file is 2^32-3 = 4294967295 - 3 */
/* max dimension size for CDF-2 file is NC_MAX_INT */
err = ncmpi_create(MPI_COMM_WORLD, filename, cmode, info, &ncid); CHECK_ERR
err = ncmpi_def_dim(ncid, "Y", UINT_MAX, &dimid[0]);
err = ncmpi_def_dim(ncid, "Y", (MPI_Offset)1+NC_MAX_INT, &dimid[0]);
EXP_ERR(NC_EDIMSIZE)
err = ncmpi_def_dim(ncid, "Y", UINT_MAX-3, &dimid[0]); CHECK_ERR
err = ncmpi_def_dim(ncid, "Y", NC_MAX_INT, &dimid[0]); CHECK_ERR
err = ncmpi_close(ncid); CHECK_ERR
err = ncmpi_open(MPI_COMM_WORLD, filename, NC_NOWRITE, info, &ncid); CHECK_ERR
err = ncmpi_close(ncid); CHECK_ERR

/* use the max dimension size to define a 1D variable */
err = ncmpi_create(MPI_COMM_WORLD, filename, cmode, info, &ncid); CHECK_ERR
err = ncmpi_def_dim(ncid, "Y", UINT_MAX-3, &dimid[0]); CHECK_ERR
err = ncmpi_def_dim(ncid, "Y", NC_MAX_INT, &dimid[0]); CHECK_ERR
err = ncmpi_def_var(ncid, "var0", NC_CHAR, 1, dimid, &varid); CHECK_ERR
err = ncmpi_close(ncid); CHECK_ERR
err = ncmpi_open(MPI_COMM_WORLD, filename, NC_NOWRITE, info, &ncid); CHECK_ERR
err = ncmpi_close(ncid); CHECK_ERR

/* use the max dimension size to define a 1D variable, followed by
* another variable to make the file size > 4294967295 */
* another variable to make the file size > 2 * NC_MAX_INT */
err = ncmpi_create(MPI_COMM_WORLD, filename, cmode, info, &ncid); CHECK_ERR
err = ncmpi_def_dim(ncid, "Y", UINT_MAX-3, &dimid[0]); CHECK_ERR
err = ncmpi_def_dim(ncid, "Y", NC_MAX_INT, &dimid[0]); CHECK_ERR
err = ncmpi_def_dim(ncid, "X", 2, &dimid[1]); CHECK_ERR
err = ncmpi_def_var(ncid, "var0", NC_CHAR, 1, &dimid[0], &varid); CHECK_ERR
err = ncmpi_def_var(ncid, "var1", NC_INT, 1, &dimid[1], &varid); CHECK_ERR
Expand All @@ -200,25 +199,25 @@ int main(int argc, char** argv)
* 4294967295. error should be reported in ncmpi_enddef() or
* ncmpi_close() */
err = ncmpi_create(MPI_COMM_WORLD, filename, cmode, info, &ncid); CHECK_ERR
err = ncmpi_def_dim(ncid, "Y", UINT_MAX-3, &dimid[0]); CHECK_ERR
err = ncmpi_def_dim(ncid, "X", 2, &dimid[1]); CHECK_ERR
err = ncmpi_def_dim(ncid, "Y", NC_MAX_INT, &dimid[0]); CHECK_ERR
err = ncmpi_def_dim(ncid, "X", 2, &dimid[1]); CHECK_ERR
err = ncmpi_def_var(ncid, "var0", NC_SHORT, 1, &dimid[0], &varid); CHECK_ERR
err = ncmpi_def_var(ncid, "var1", NC_INT, 1, &dimid[1], &varid); CHECK_ERR
err = ncmpi_close(ncid);
EXP_ERR(NC_EVARSIZE)

/* define 2 1D int variables of dimension size > max */
/* define 2 1D int variables of dimension size > NC_MAX_INT */
err = ncmpi_create(MPI_COMM_WORLD, filename, cmode, info, &ncid); CHECK_ERR
err = ncmpi_def_dim(ncid, "Y", INT_MAX, &dimid[0]); CHECK_ERR
err = ncmpi_def_dim(ncid, "X", 2, &dimid[1]); CHECK_ERR
err = ncmpi_def_dim(ncid, "Y", NC_MAX_INT, &dimid[0]); CHECK_ERR
err = ncmpi_def_dim(ncid, "X", 2, &dimid[1]); CHECK_ERR
err = ncmpi_def_var(ncid, "var0", NC_INT, 1, &dimid[0], &varid); CHECK_ERR
err = ncmpi_def_var(ncid, "var1", NC_INT, 1, &dimid[1], &varid); CHECK_ERR
err = ncmpi_close(ncid);
EXP_ERR(NC_EVARSIZE)

err = ncmpi_create(MPI_COMM_WORLD, filename, cmode, info, &ncid); CHECK_ERR
err = ncmpi_def_dim(ncid, "Y", INT_MAX/2+1, &dimid[0]); CHECK_ERR
err = ncmpi_def_dim(ncid, "X", 2, &dimid[1]); CHECK_ERR
err = ncmpi_def_dim(ncid, "Y", NC_MAX_INT/2+1, &dimid[0]); CHECK_ERR
err = ncmpi_def_dim(ncid, "X", 2, &dimid[1]); CHECK_ERR
err = ncmpi_def_var(ncid, "var0", NC_INT, 1, &dimid[0], &varid); CHECK_ERR
err = ncmpi_def_var(ncid, "var1", NC_INT, 1, &dimid[1], &varid); CHECK_ERR
err = ncmpi_close(ncid);
Expand All @@ -229,8 +228,8 @@ int main(int argc, char** argv)
*/
err = ncmpi_create(MPI_COMM_WORLD, filename, cmode, info, &ncid); CHECK_ERR
err = ncmpi_def_dim(ncid, "Z", NC_UNLIMITED, &dimid[0]); CHECK_ERR
err = ncmpi_def_dim(ncid, "Y", INT_MAX/64, &dimid[1]); CHECK_ERR
err = ncmpi_def_dim(ncid, "X", 64, &dimid[2]); CHECK_ERR
err = ncmpi_def_dim(ncid, "Y", NC_MAX_INT/64, &dimid[1]); CHECK_ERR
err = ncmpi_def_dim(ncid, "X", 64, &dimid[2]); CHECK_ERR
err = ncmpi_def_var(ncid, "var0", NC_INT, 3, dimid, &varid); CHECK_ERR
err = ncmpi_def_var(ncid, "var1", NC_INT, 3, dimid, &varid); CHECK_ERR
err = ncmpi_close(ncid);
Expand All @@ -239,8 +238,8 @@ int main(int argc, char** argv)
/* test large record variable that is not defined last */
err = ncmpi_create(MPI_COMM_WORLD, filename, cmode, info, &ncid); CHECK_ERR
err = ncmpi_def_dim(ncid, "Z", NC_UNLIMITED, &dimid[0]); CHECK_ERR
err = ncmpi_def_dim(ncid, "Y", INT_MAX/64, &dimid[1]); CHECK_ERR
err = ncmpi_def_dim(ncid, "X", 64, &dimid[2]); CHECK_ERR
err = ncmpi_def_dim(ncid, "Y", NC_MAX_INT/64, &dimid[1]); CHECK_ERR
err = ncmpi_def_dim(ncid, "X", 64, &dimid[2]); CHECK_ERR
err = ncmpi_def_var(ncid, "var0", NC_INT, 3, dimid, &varid); CHECK_ERR
err = ncmpi_def_var(ncid, "var1", NC_INT, 2, dimid, &varid); CHECK_ERR
err = ncmpi_close(ncid);
Expand All @@ -250,8 +249,8 @@ int main(int argc, char** argv)
* output file can be tested by ncvalidator in wrap_runs.sh
*/
err = ncmpi_create(MPI_COMM_WORLD, filename, cmode, info, &ncid); CHECK_ERR
err = ncmpi_def_dim(ncid, "Y", INT_MAX/2, &dimid[0]); CHECK_ERR
err = ncmpi_def_dim(ncid, "X", 2, &dimid[1]); CHECK_ERR
err = ncmpi_def_dim(ncid, "Y", NC_MAX_INT/2, &dimid[0]); CHECK_ERR
err = ncmpi_def_dim(ncid, "X", 2, &dimid[1]); CHECK_ERR
err = ncmpi_def_var(ncid, "var0", NC_INT, 1, &dimid[0], &varid); CHECK_ERR
err = ncmpi_def_var(ncid, "var1", NC_INT, 1, &dimid[1], &varid); CHECK_ERR
err = ncmpi_close(ncid); CHECK_ERR
Expand Down
24 changes: 17 additions & 7 deletions test/testcases/tst_dimsizes.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,14 +25,24 @@

#include <testutils.h>

#define DIMMAXCLASSIC (NC_MAX_INT - 3)
#define DIMMAX64OFFSET (NC_MAX_UINT - 3)
#define DIMMAXCLASSIC NC_MAX_INT
#define DIMMAX64OFFSET NC_MAX_INT
#define DIMMAX64DATA NC_MAX_INT64

/*
* NC_CLASSIC => NC_INT_MAX - 3
* NC_64BIT_OFFSET => NC_UINT_MAX - 3
* NC_64BIT_DATA => NC_INT64_MAX
* NetCDF file format specification:
* netcdf_file = header data
* header = magic numrecs dim_list gatt_list var_list
* dim_list = ABSENT | NC_DIMENSION nelems [dim ...]
* dim = name dim_length
* dim_length = NON_NEG
* NON_NEG = <non-negative INT>
* INT = <32-bit signed integer, Bigendian, two's complement>
*
* Therefore, the max dimension size are:
* NC_CLASSIC Max dimension size is NC_INT_MAX
* NC_64BIT_OFFSET Max dimension size is NC_INT_MAX
* NC_64BIT_DATA Max dimension size is NC_INT64_MAX
* Note that for NC_64BIT_DATA, the max dimension size is different from netCDF
* library. This is because PnetCDF uses MPI_Offset for dimension size and
* MPI_Offset is a signed long long.
Expand Down Expand Up @@ -72,7 +82,7 @@ main(int argc, char **argv)
err = ncmpi_def_dim(ncid, "testdim", dimsize, &dimid); CHECK_ERR
dimsize = -1;
err = ncmpi_def_dim(ncid, "testdim1", dimsize, &dimid); EXP_ERR(NC_EDIMSIZE)
dimsize = DIMMAXCLASSIC+1;
dimsize = (MPI_Offset)DIMMAXCLASSIC+1;
err = ncmpi_def_dim(ncid, "testdim1", dimsize, &dimid); EXP_ERR(NC_EDIMSIZE)
err = ncmpi_close(ncid); CHECK_ERR

Expand All @@ -92,7 +102,7 @@ main(int argc, char **argv)
err = ncmpi_def_dim(ncid, "testdim", dimsize, &dimid); CHECK_ERR
dimsize = -1;
err = ncmpi_def_dim(ncid, "testdim1", dimsize, &dimid); EXP_ERR(NC_EDIMSIZE)
dimsize = DIMMAX64OFFSET+1;
dimsize = (MPI_Offset)DIMMAX64OFFSET+1;
err = ncmpi_def_dim(ncid, "testdim1", dimsize, &dimid); EXP_ERR(NC_EDIMSIZE)
err = ncmpi_close(ncid); CHECK_ERR

Expand Down
Loading