up chunk sizes and move to paged aggregation

ubarsc · Mar 6, 2024 · 9b3535d · 9b3535d
1 parent c83e4dc
commit 9b3535d
Show file tree

Hide file tree

Showing 5 changed files with 18 additions and 6 deletions.
diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -79,6 +79,9 @@ if(NOT "x$ENV{HDF5_ROOT}" STREQUAL "x")
     message(STATUS "HDF5_ROOT environment variable specified: $ENV{HDF5_ROOT}")
 endif()
 find_package(HDF5 COMPONENTS C CXX REQUIRED)
+IF(${HDF5_VERSION} VERSION_LESS "1.10.1")
+    message(FATAL_ERROR "HDF5 must be at least version 1.10.1")
+endif()
 
 # https://support.hdfgroup.org/HDF5/doc/TechNotes/ThreadSafeLibrary.html
 include(CheckSymbolExists)

diff --git a/gdal/keadataset.cpp b/gdal/keadataset.cpp
@@ -249,6 +249,11 @@ GDALDataset *KEADataset::Create( const char * pszFilename,
     pszValue = CSLFetchNameValue( papszParmList, "ATTBLOCKSIZE" );
     if( pszValue != nullptr )
         nattblockSize = atol( pszValue );
+
+    hsize_t npageSize = kealib::KEA_PAGE_SIZE;
+    pszValue = CSLFetchNameValue( papszParmList, "PAGESIZE" );
+    if( pszValue != nullptr )
+        npageSize = atol( pszValue );
 
     unsigned int nmdcElmts = kealib::KEA_MDC_NELMTS;
     pszValue = CSLFetchNameValue( papszParmList, "MDC_NELMTS" );
@@ -296,7 +301,7 @@ GDALDataset *KEADataset::Create( const char * pszFilename,
         H5::H5File *keaImgH5File = kealib::KEAImageIO::createKEAImage( pszFilename,
                                                     GDAL_to_KEA_Type( eType ),
                                                     nXSize, nYSize, nBands,
-                                                    nullptr, nullptr, nimageblockSize, 
+                                                    nullptr, nullptr, npageSize, nimageblockSize, 
                                                     nattblockSize, nmdcElmts, nrdccNElmts,
                                                     nrdccNBytes, nrdccW0, nsieveBuf, 
                                                     nmetaBlockSize, ndeflate );

diff --git a/include/libkea/KEACommon.h b/include/libkea/KEACommon.h
@@ -135,8 +135,9 @@ namespace kealib{
     static const hsize_t  KEA_SIEVE_BUF( 65536 ); // 65536
     static const hsize_t  KEA_META_BLOCKSIZE( 2048 ); // 2048
     static const unsigned int KEA_DEFLATE( 1 ); // 1
-    static const hsize_t KEA_IMAGE_CHUNK_SIZE( 256 ); // 256
-    static const hsize_t KEA_ATT_CHUNK_SIZE( 1000 ); // 1000
+    static const hsize_t KEA_IMAGE_CHUNK_SIZE( 512 ); // 512
+    static const hsize_t KEA_ATT_CHUNK_SIZE( 10000 ); // 10000
+    static const hsize_t KEA_PAGE_SIZE(8192); // 8192
 
     enum KEADataType
     {

diff --git a/include/libkea/KEAImageIO.h b/include/libkea/KEAImageIO.h
@@ -126,7 +126,7 @@ namespace kealib{
         // remove band from file
         virtual void removeImageBand(const uint32_t bandIndex);
 
-        static H5::H5File* createKEAImage(const std::string &fileName, KEADataType dataType, uint32_t xSize, uint32_t ySize, uint32_t numImgBands, std::vector<std::string> *bandDescrips=NULL, KEAImageSpatialInfo *spatialInfo=NULL, uint32_t imageBlockSize=KEA_IMAGE_CHUNK_SIZE, uint32_t attBlockSize=KEA_ATT_CHUNK_SIZE, int mdcElmts=KEA_MDC_NELMTS, hsize_t rdccNElmts=KEA_RDCC_NELMTS, hsize_t rdccNBytes=KEA_RDCC_NBYTES, double rdccW0=KEA_RDCC_W0, hsize_t sieveBuf=KEA_SIEVE_BUF, hsize_t metaBlockSize=KEA_META_BLOCKSIZE, uint32_t deflate=KEA_DEFLATE);
+        static H5::H5File* createKEAImage(const std::string &fileName, KEADataType dataType, uint32_t xSize, uint32_t ySize, uint32_t numImgBands, std::vector<std::string> *bandDescrips=NULL, KEAImageSpatialInfo *spatialInfo=NULL, uint32_t imageBlockSize=KEA_IMAGE_CHUNK_SIZE, uint32_t attBlockSize=KEA_ATT_CHUNK_SIZE, hsize_t pageSize=KEA_PAGE_SIZE, int mdcElmts=KEA_MDC_NELMTS, hsize_t rdccNElmts=KEA_RDCC_NELMTS, hsize_t rdccNBytes=KEA_RDCC_NBYTES, double rdccW0=KEA_RDCC_W0, hsize_t sieveBuf=KEA_SIEVE_BUF, hsize_t metaBlockSize=KEA_META_BLOCKSIZE, uint32_t deflate=KEA_DEFLATE);
         static bool isKEAImage(const std::string &fileName);
         static H5::H5File* openKeaH5RW(const std::string &fileName, int mdcElmts=KEA_MDC_NELMTS, hsize_t rdccNElmts=KEA_RDCC_NELMTS, hsize_t rdccNBytes=KEA_RDCC_NBYTES, double rdccW0=KEA_RDCC_W0, hsize_t sieveBuf=KEA_SIEVE_BUF, hsize_t metaBlockSize=KEA_META_BLOCKSIZE);
         static H5::H5File* openKeaH5RDOnly(const std::string &fileName, int mdcElmts=KEA_MDC_NELMTS, hsize_t rdccNElmts=KEA_RDCC_NELMTS, hsize_t rdccNBytes=KEA_RDCC_NBYTES, double rdccW0=KEA_RDCC_W0, hsize_t sieveBuf=KEA_SIEVE_BUF, hsize_t metaBlockSize=KEA_META_BLOCKSIZE);

diff --git a/src/libkea/KEAImageIO.cpp b/src/libkea/KEAImageIO.cpp
@@ -2967,7 +2967,7 @@ namespace kealib{
         }
     }
 
-    H5::H5File* KEAImageIO::createKEAImage(const std::string &fileName, KEADataType dataType, uint32_t xSize, uint32_t ySize, uint32_t numImgBands, std::vector<std::string> *bandDescrips, KEAImageSpatialInfo * spatialInfo, uint32_t imageBlockSize, uint32_t attBlockSize, int mdcElmts, hsize_t rdccNElmts, hsize_t rdccNBytes, double rdccW0, hsize_t sieveBuf, hsize_t metaBlockSize, uint32_t deflate)
+    H5::H5File* KEAImageIO::createKEAImage(const std::string &fileName, KEADataType dataType, uint32_t xSize, uint32_t ySize, uint32_t numImgBands, std::vector<std::string> *bandDescrips, KEAImageSpatialInfo * spatialInfo, uint32_t imageBlockSize, uint32_t attBlockSize, hsize_t pageSize, int mdcElmts, hsize_t rdccNElmts, hsize_t rdccNBytes, double rdccW0, hsize_t sieveBuf, hsize_t metaBlockSize, uint32_t deflate)
     {
         KEAStackPrintState printState;
 
@@ -2982,7 +2982,10 @@ namespace kealib{
             keaAccessPlist.setMetaBlockSize(metaBlockSize);
 
             // CREATE THE HDF FILE - EXISTING FILE WILL BE TRUNCATED
-            keaImgH5File = new H5::H5File( fileName, H5F_ACC_TRUNC, H5::FileCreatPropList::DEFAULT, keaAccessPlist);
+            H5::FileCreatPropList keaPlist = H5::FileCreatPropList();
+            keaPlist.setFileSpaceStrategy(H5F_FSPACE_STRATEGY_AGGR, false, 0);
+            keaPlist.setFileSpacePagesize(pageSize);
+            keaImgH5File = new H5::H5File( fileName, H5F_ACC_TRUNC, keaPlist, keaAccessPlist);
 
             //////////// CREATE GLOBAL HEADER ////////////////
             keaImgH5File->createGroup( KEA_DATASETNAME_HEADER );