Extend NCZarr to support unlimited dimensions.

RELEASE_NOTES.md

@@ -7,6 +7,7 @@ This file contains a high-level description of this package's evolution. Release
 
 ## 4.9.3 - TBD
 
+* Extend NCZarr to support unlimited dimensions. See [Github #2736](https://github.com/Unidata/netcdf-c/pull/2736).
 * Fix default parameters for caching of NCZarr. See [Github #2734](https://github.com/Unidata/netcdf-c/pull/2734).
 * Introducing configure-time options to disable various filters, even if the required libraries are available on the system, in support of [GitHub #2712](https://github.com/Unidata/netcdf-c/pull/2712). 
 * Fix memory leak WRT unreclaimed HDF5 plist. See [Github #2752](https://github.com/Unidata/netcdf-c/pull/2752).

docs/nczarr.md

@@ -53,6 +53,7 @@ are also [supported](./md_filters.html "filters").
 Specifically, the model supports the following.
 - "Atomic" types: char, byte, ubyte, short, ushort, int, uint, int64, uint64, string.
 - Shared (named) dimensions
+- Unlimited dimensions
 - Attributes with specified types -- both global and per-variable
 - Chunking
 - Fill values
@@ -65,7 +66,6 @@ Specifically, the model supports the following.
 With respect to full netCDF-4, the following concepts are
 currently unsupported.
 - User-defined types (enum, opaque, VLEN, and Compound)
-- Unlimited dimensions
 - Contiguous or compact storage
 
 Note that contiguous and compact are not actually supported
@@ -375,7 +375,7 @@ Currently it contains the following key(s):
 _\_nczarr_group\__ -- this key appears in every _.zgroup_ object.
 It contains any netcdf specific group information.
 Specifically it contains the following keys:
-* "dims" -- the name and size of shared dimensions defined in this group.
+* "dims" -- the name and size of shared dimensions defined in this group, as well an optional flag indictating if the dimension is UNLIMITED.
 * "vars" -- the name of variables defined in this group.
 * "groups" -- the name of sub-groups defined in this group.
 These lists allow walking the NCZarr dataset without having to use the potentially costly search operation.

lib_flags.am

@@ -6,14 +6,13 @@
 # libraries for netCDF-4.
 #
 
-AM_CPPFLAGS = -I$(top_builddir)/include -I$(top_srcdir)/include
+AM_CPPFLAGS = -I$(top_builddir)/include -I$(top_srcdir)/include -I$(top_builddir) -I$(top_srcdir)
 AM_LDFLAGS =
 
 if USE_DAP
 AM_CPPFLAGS += -I${top_srcdir}/oc2
 endif
 
-
 if ENABLE_NCZARR
 AM_CPPFLAGS += -I${top_srcdir}/libnczarr
 endif

libnczarr/zarr.h

@@ -89,7 +89,7 @@ EXTERNL int NCZ_copy_fill_value(NC_VAR_INFO_T* var, void** dstp);
 EXTERNL int NCZ_get_maxstrlen(NC_OBJ* obj);
 EXTERNL int NCZ_fixed2char(const void* fixed, char** charp, size_t count, int maxstrlen);
 EXTERNL int NCZ_char2fixed(const char** charp, void* fixed, size_t count, int maxstrlen);
-EXTERNL int NCZ_copy_data(NC_FILE_INFO_T* file, NC_TYPE_INFO_T* xtype, const void* memory, size_t count, int nofill, void* copy);
+EXTERNL int NCZ_copy_data(NC_FILE_INFO_T* file, NC_VAR_INFO_T* var, const void* memory, size_t count, int noclear, void* copy);
 EXTERNL int NCZ_iscomplexjson(NCjson* value, nc_type typehint);
 
 /* zwalk.c */

libnczarr/zcache.h

@@ -67,5 +67,6 @@ extern size64_t NCZ_cache_size(NCZChunkCache* cache);
 extern int NCZ_buildchunkpath(NCZChunkCache* cache, const size64_t* chunkindices, struct ChunkKey* key);
 extern int NCZ_ensure_fill_chunk(NCZChunkCache* cache);
 extern int NCZ_reclaim_fill_chunk(NCZChunkCache* cache);
+extern int NCZ_chunk_cache_modify(NCZChunkCache* cache, const size64_t* indices);
 
 #endif /*ZCACHE_H*/

libnczarr/zchunking.c

@@ -10,7 +10,7 @@
 static int pcounter = 0;
 
 /* Forward */
-static int compute_intersection(const NCZSlice* slice, const size64_t chunklen, NCZChunkRange* range);
+static int compute_intersection(const NCZSlice* slice, const size64_t chunklen, unsigned char isunlimited, NCZChunkRange* range);
 static void skipchunk(const NCZSlice* slice, NCZProjection* projection);
 static int verifyslice(const NCZSlice* slice);
 
@@ -20,30 +20,44 @@ static int verifyslice(const NCZSlice* slice);
    absolute position) of the first chunk that intersects the slice
    and the index of the last chunk that intersects the slice.
    In practice, the count = last - first + 1 is stored instead of the last index.
+   Note that this n-dim array of indices may have holes in it if the slice stride
+   is greater than the chunk length.
+   @param rank variable rank
+   @param slices the complete set of slices |slices| == R
+   @param ncr (out) the vector of computed chunk ranges.
+   @return NC_EXXX error code
 */
 int
 NCZ_compute_chunk_ranges(
-	int rank, /* variable rank */
+	struct Common* common,
         const NCZSlice* slices, /* the complete set of slices |slices| == R*/
-	const size64_t* chunklen, /* the chunk length corresponding to the dimensions */
         NCZChunkRange* ncr)
 {
     int stat = NC_NOERR;
     int i;
+    int rank = common->rank;
 
     for(i=0;i<rank;i++) {
-	if((stat = compute_intersection(&slices[i],chunklen[i],&ncr[i])))
+	if((stat = compute_intersection(&slices[i],common->chunklens[i],common->isunlimited[i],&ncr[i])))
 	    goto done;
     }
 
 done:
     return stat;
 }
 
+/**
+@param Compute chunk range for a single slice.
+@param chunklen size of the chunk
+@param isunlimited if corresponding dim is unlimited
+@param range (out) the range of chunks covered by this slice
+@return NC_EXX error code
+*/
 static int
 compute_intersection(
         const NCZSlice* slice,
-	const size64_t chunklen,
+	size64_t chunklen,
+	unsigned char isunlimited,
         NCZChunkRange* range)
 {
     range->start = floordiv(slice->start, chunklen);
@@ -53,6 +67,9 @@ compute_intersection(
 
 /**
 Compute the projection of a slice as applied to n'th chunk.
+A projection defines the set of grid points touched within a
+chunk by a slice. This set of points is the "projection"
+of the slice onto the chunk.
 This is somewhat complex because:
 1. for the first projection, the start is the slice start,
    but after that, we have to take into account that for
@@ -295,4 +312,3 @@ clearallprojections(NCZAllProjections* nap)
     }
 }
 #endif
-

libnczarr/zchunking.h

@@ -8,8 +8,6 @@
 
 #include "ncexternl.h"
 
-/* Callback functions so we can use with unit tests */
-
 typedef int (*NCZ_reader)(void* source, size64_t* chunkindices, void** chunkdata);
 struct Reader {void* source; NCZ_reader read;};
 
@@ -29,6 +27,10 @@ typedef struct NCZSlice {
     size64_t len; /* full dimension length */
 } NCZSlice;
 
+/* A projection defines the set of grid points
+   for a given set of slices as projected onto
+   a single chunk.
+*/
 typedef struct NCProjection {
     int id;
     int skip; /* Should this projection be skipped? */
@@ -54,30 +56,31 @@ typedef struct NCZSliceProjections {
 				   the chunk */
 } NCZSliceProjections;
 
-/* Combine some values to simplify internal argument lists */
+/* Combine some values to avoid having to pass long argument lists*/
 struct Common {
     NC_FILE_INFO_T* file;
     NC_VAR_INFO_T* var;
     struct NCZChunkCache* cache;
     int reading; /* 1=> read, 0 => write */
     int rank;
     int scalar; /* 1 => scalar variable */
-    size64_t* dimlens;
-    size64_t* chunklens;
-    size64_t* memshape;
+    size64_t dimlens[NC_MAX_VAR_DIMS];
+    unsigned char isunlimited[NC_MAX_VAR_DIMS];
+    size64_t chunklens[NC_MAX_VAR_DIMS];
+    size64_t memshape[NC_MAX_VAR_DIMS];
     void* memory;
     size_t typesize;
     size64_t chunkcount; /* computed product of chunklens; warning indices, not bytes */
     int swap; /* var->format_info_file->native_endianness == var->endianness */
     size64_t shape[NC_MAX_VAR_DIMS]; /* shape of the output hyperslab */
     NCZSliceProjections* allprojections;
-    /* Parametric chunk reader so we can do unittests */
+    /* Parametric chunk reader */
     struct Reader reader;
 };
 
 /**************************************************/
 /* From zchunking.c */
-EXTERNL int NCZ_compute_chunk_ranges(int rank, const NCZSlice*, const size64_t*, NCZChunkRange* ncr);
+EXTERNL int NCZ_compute_chunk_ranges(struct Common*, const NCZSlice*, NCZChunkRange* ncr);
 EXTERNL int NCZ_compute_projections(struct Common*, int r, size64_t chunkindex, const NCZSlice* slice, size_t n, NCZProjection* projections);
 EXTERNL int NCZ_compute_per_slice_projections(struct Common*, int rank, const NCZSlice*, const NCZChunkRange*, NCZSliceProjections* slp);
 EXTERNL int NCZ_compute_all_slice_projections(struct Common*, const NCZSlice* slices, const NCZChunkRange*, NCZSliceProjections*);
@@ -94,10 +97,7 @@ EXTERNL size64_t NCZ_computelinearoffset(size_t, const size64_t*, const size64_t
 /* Special entry points for unit testing */
 struct Common;
 struct NCZOdometer;
-EXTERNL int NCZ_projectslices(size64_t* dimlens,
-		  size64_t* chunklens,
-		  NCZSlice* slices,
-		  struct Common*, struct NCZOdometer**);
+EXTERNL int NCZ_projectslices(struct Common*, NCZSlice* slices, struct NCZOdometer**);
 EXTERNL int NCZ_chunkindexodom(int rank, const NCZChunkRange* ranges, size64_t*, struct NCZOdometer** odom);
 EXTERNL void NCZ_clearsliceprojections(int count, NCZSliceProjections* slpv);
 EXTERNL void NCZ_clearcommon(struct Common* common);

libnczarr/zdim.c

@@ -83,8 +83,8 @@ NCZ_def_dim(int ncid, const char *name, size_t len, int *idp)
     if ((stat = nc4_check_name(name, norm_name)))
         return stat;
 
-    /* Since unlimited is not supported, len > 0 */
-    if(len <= 0)
+    /* Since unlimited is supported, len >= 0 */
+    if(len < 0)
         return NC_EDIMSIZE;
 
     /* For classic model: dim length has to fit in a 32-bit unsigned
@@ -110,10 +110,14 @@ NCZ_def_dim(int ncid, const char *name, size_t len, int *idp)
     if ((stat = nc4_dim_list_add(grp, norm_name, len, -1, &dim)))
         return stat;
 
-    /* Create struct for NCZ-specific dim info. */
-    if (!(dim->format_dim_info = calloc(1, sizeof(NCZ_DIM_INFO_T))))
-        return NC_ENOMEM;
-    ((NCZ_DIM_INFO_T*)dim->format_dim_info)->common.file = h5;
+    {
+        NCZ_DIM_INFO_T* diminfo = NULL;
+        /* Create struct for NCZ-specific dim info. */
+        if (!(diminfo = calloc(1, sizeof(NCZ_DIM_INFO_T))))
+            return NC_ENOMEM;
+        dim->format_dim_info = diminfo;
+        diminfo->common.file = h5;
+    }    
 
     /* Pass back the dimid. */
     if (idp)
@@ -269,10 +273,3 @@ NCZ_rename_dim(int ncid, int dimid, const char *name)
 
     return NC_NOERR;
 }
-
-int
-NCZ_inq_unlimdims(int ncid, int *ndimsp, int *unlimdimidsp)
-{
-    if(ndimsp) *ndimsp = 0;
-    return NC_NOERR;
-}

libnczarr/zdispatch.c

@@ -38,7 +38,7 @@ static const NC_Dispatch NCZ_dispatcher = {
     NCZ_def_dim,
     NCZ_inq_dimid,
     NCZ_inq_dim,
-    NCZ_inq_unlimdim,
+    NC4_inq_unlimdim,
     NCZ_rename_dim,
 
     NCZ_inq_att,
@@ -65,7 +65,7 @@ static const NC_Dispatch NCZ_dispatcher = {
     NCZ_def_var_fill,
 
     NCZ_show_metadata,
-    NCZ_inq_unlimdims,
+    NC4_inq_unlimdims,
 
     NCZ_inq_ncid,
     NCZ_inq_grps,

libnczarr/zodom.c

@@ -25,8 +25,8 @@ nczodom_new(int rank, const size64_t* start, const size64_t* stop, const size64_
     odom->properties.start0 = 1; /* assume */
     for(i=0;i<rank;i++) { 
 	odom->start[i] = (size64_t)start[i];
-	odom->stop[i] = (size64_t)stop[i];
 	odom->stride[i] = (size64_t)stride[i];
+	odom->stop[i] = (size64_t)stop[i];
 	odom->len[i] = (size64_t)len[i];
 	if(odom->start[i] != 0) odom->properties.start0 = 0;
 	if(odom->stride[i] != 1) odom->properties.stride1 = 0;
@@ -131,11 +131,11 @@ buildodom(int rank, NCZOdometer** odomp)
         if((odom = calloc(1,sizeof(NCZOdometer))) == NULL)
 	    goto done;   
         odom->rank = rank;
-        if((odom->start=malloc(sizeof(size64_t)*rank))==NULL) goto nomem;
-        if((odom->stop=malloc(sizeof(size64_t)*rank))==NULL) goto nomem;
-        if((odom->stride=malloc(sizeof(size64_t)*rank))==NULL) goto nomem;
-        if((odom->len=malloc(sizeof(size64_t)*rank))==NULL) goto nomem;
-        if((odom->index=malloc(sizeof(size64_t)*rank))==NULL) goto nomem;
+        if((odom->start=calloc(1,(sizeof(size64_t)*rank)))==NULL) goto nomem;
+        if((odom->stop=calloc(1,(sizeof(size64_t)*rank)))==NULL) goto nomem;
+        if((odom->stride=calloc(1,(sizeof(size64_t)*rank)))==NULL) goto nomem;
+        if((odom->len=calloc(1,(sizeof(size64_t)*rank)))==NULL) goto nomem;
+        if((odom->index=calloc(1,(sizeof(size64_t)*rank)))==NULL) goto nomem;
         *odomp = odom; odom = NULL;
     }
 done:
@@ -168,7 +168,6 @@ nczodom_skipavail(NCZOdometer* odom)
         odom->index[odom->rank-1] = odom->stop[odom->rank-1];
 }
 
-#if 0
 size64_t
 nczodom_laststride(const NCZOdometer* odom)
 {
@@ -182,4 +181,26 @@ nczodom_lastlen(const NCZOdometer* odom)
     assert(odom != NULL && odom->rank > 0);
     return odom->len[odom->rank-1];
 }
-#endif
+
+void
+nczodom_print(const NCZOdometer* odom)
+{
+    size_t i;
+    fprintf(stderr,"odom{rank=%d offset=%llu avail=%llu",odom->rank,nczodom_offset(odom),nczodom_avail(odom));
+    fprintf(stderr," start=(");
+        for(i=0;i<odom->rank;i++) {fprintf(stderr,"%s%llu",(i==0?"":" "),(unsigned long long)odom->start[i]);}
+	fprintf(stderr,")");
+    fprintf(stderr," stride=(");
+        for(i=0;i<odom->rank;i++) {fprintf(stderr,"%s%llu",(i==0?"":" "),(unsigned long long)odom->stride[i]);}
+	fprintf(stderr,")");
+    fprintf(stderr," stop=(");
+        for(i=0;i<odom->rank;i++) {fprintf(stderr,"%s%llu",(i==0?"":" "),(unsigned long long)odom->stop[i]);}
+	fprintf(stderr,")");
+    fprintf(stderr," len=(");
+        for(i=0;i<odom->rank;i++) {fprintf(stderr,"%s%llu",(i==0?"":" "),(unsigned long long)odom->len[i]);}
+	fprintf(stderr,")");
+    fprintf(stderr," index=(");
+        for(i=0;i<odom->rank;i++) {fprintf(stderr,"%s%llu",(i==0?"":" "),(unsigned long long)odom->index[i]);}
+	fprintf(stderr,")");
+    fprintf(stderr,"}\n");
+}

libnczarr/zodom.h

@@ -11,8 +11,8 @@ struct NCZSlice;
 typedef struct NCZOdometer {
     int rank; /*rank */
     size64_t* start;
-    size64_t* stop; /* start + (count*stride) */
     size64_t* stride;
+    size64_t* stop; /* start + (count*stride) */
     size64_t* len; /* for computing offset */
     size64_t* index; /* current value of the odometer*/
     struct NCZOprop {
@@ -34,5 +34,8 @@ extern void nczodom_reset(NCZOdometer* odom);
 extern void nczodom_free(NCZOdometer*);
 extern size64_t nczodom_avail(const NCZOdometer*);
 extern void nczodom_skipavail(NCZOdometer* odom);
+extern size64_t nczodom_laststride(const NCZOdometer* odom);
+extern size64_t nczodom_lastlen(const NCZOdometer* odom);
+extern void nczodom_print(const NCZOdometer* odom);
 
 #endif /*ZODOM_H*/