From a7c888b23618e7940bcccca178ddbfb47627f65f Mon Sep 17 00:00:00 2001 From: Greg Sjaardema Date: Tue, 13 Jun 2023 17:11:53 -0600 Subject: [PATCH 01/11] Add capability to enable/disable compression libraries --- CMakeLists.txt | 22 +++++++++++++++------- 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 9460f47d8d..20b598261e 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1134,20 +1134,28 @@ macro(set_std_filter filter) # Upper case the filter name string(TOUPPER "${filter}" upfilter) string(TOLOWER "${filter}" downfilter) +if(ENABLE_${upfilter}) # Define a test flag for filter -IF(${filter}_FOUND) - INCLUDE_DIRECTORIES(${${filter}_INCLUDE_DIRS}) - SET(ENABLE_${upfilter} TRUE) - SET(HAVE_${upfilter} ON) - SET(STD_FILTERS "${STD_FILTERS} ${downfilter}") - MESSAGE(">>> Standard Filter: ${downfilter}") + IF(${filter}_FOUND) + INCLUDE_DIRECTORIES(${${filter}_INCLUDE_DIRS}) + SET(ENABLE_${upfilter} TRUE) + SET(HAVE_${upfilter} ON) + SET(STD_FILTERS "${STD_FILTERS} ${downfilter}") + MESSAGE(">>> Standard Filter: ${downfilter}") + ELSE() + SET(ENABLE_${upfilter} FALSE) + SET(HAVE_${upfilter} OFF) + ENDIF() ELSE() - SET(ENABLE_${upfilter} FALSE) SET(HAVE_${upfilter} OFF) ENDIF() endmacro(set_std_filter) # Locate some compressors +OPTION(ENABLE_SZIP "Enable use of Szip compression library if it is available." ON) +OPTION(ENABLE_BZ2 "Enable use of Bz2 compression library if it is available." ON) +OPTION(ENABLE_BLOSC "Enable use of blosc compression library if it is available." ON) +OPTION(ENABLE_ZSTD "Enable use of Zstd compression library if it is available." ON) FIND_PACKAGE(Szip) FIND_PACKAGE(Bz2) FIND_PACKAGE(Blosc) From 05d5b3c13018762fdbcbb9e7e329de0438ed3b36 Mon Sep 17 00:00:00 2001 From: Greg Sjaardema Date: Wed, 14 Jun 2023 08:22:01 -0600 Subject: [PATCH 02/11] Don't call find_package if not enabled The `FIND_PACKAGE` should not be called if the filter/compression library is not enabled. It was causing some inconsistencied in link libraries and CMake configure output... --- CMakeLists.txt | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 20b598261e..550e7b42ab 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1156,10 +1156,18 @@ OPTION(ENABLE_SZIP "Enable use of Szip compression library if it is available." OPTION(ENABLE_BZ2 "Enable use of Bz2 compression library if it is available." ON) OPTION(ENABLE_BLOSC "Enable use of blosc compression library if it is available." ON) OPTION(ENABLE_ZSTD "Enable use of Zstd compression library if it is available." ON) -FIND_PACKAGE(Szip) -FIND_PACKAGE(Bz2) -FIND_PACKAGE(Blosc) -FIND_PACKAGE(Zstd) +IF (ENABLE_SZIP) + FIND_PACKAGE(Szip) +ENDIF() +IF (ENABLE_BZ2) + FIND_PACKAGE(Bz2) +ENDIF() +IF (ENABLE_BLOSC) + FIND_PACKAGE(Blosc) +ENDIF() +IF (ENABLE_ZSTD) + FIND_PACKAGE(Zstd) +ENDIF() # Accumulate standard filters set(STD_FILTERS "deflate") # Always have deflate*/ From 401bdd554107b07cdbb848a60302fdfb00cd6c76 Mon Sep 17 00:00:00 2001 From: Ward Fisher Date: Thu, 20 Jul 2023 15:54:56 -0600 Subject: [PATCH 03/11] Parity for enable_bz2. BZ2 cannot be disabled altogether, but can fall back to inbternal implementation. --- CMakeLists.txt | 8 ++++---- configure.ac | 49 +++++++++++++++++++++++++++++++++++-------------- 2 files changed, 39 insertions(+), 18 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index fab49501d9..049d325b4d 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1129,13 +1129,13 @@ endmacro(set_std_filter) # Locate some compressors OPTION(ENABLE_SZIP "Enable use of Szip compression library if it is available." ON) -OPTION(ENABLE_BZ2 "Enable use of Bz2 compression library if it is available." ON) -OPTION(ENABLE_BLOSC "Enable use of blosc compression library if it is available." ON) -OPTION(ENABLE_ZSTD "Enable use of Zstd compression library if it is available." ON) +OPTION(ENABLE_FILTER_BZ2 "Enable use of Bz2 compression library if it is available." ON) +OPTION(ENABLE_FILTER_BLOSC "Enable use of blosc compression library if it is available." ON) +OPTION(ENABLE_FILTER_ZSTD "Enable use of Zstd compression library if it is available." ON) IF (ENABLE_SZIP) FIND_PACKAGE(Szip) ENDIF() -IF (ENABLE_BZ2) +IF (ENABLE_FILTER_BZ2) FIND_PACKAGE(Bz2) ENDIF() IF (ENABLE_BLOSC) diff --git a/configure.ac b/configure.ac index 9acb948729..f1727c05ea 100644 --- a/configure.ac +++ b/configure.ac @@ -740,22 +740,43 @@ if test "x$have_zstd" = "xyes" ; then fi fi -# See if we have libbz2 -AC_CHECK_LIB([bz2],[BZ2_bzCompress],[have_bz2=yes],[have_bz2=no]) -if test "x$have_bz2" = "xyes" ; then - AC_SEARCH_LIBS([BZ2_bzCompress],[bz2 bz2.dll cygbz2.dll], [], []) - AC_DEFINE([HAVE_BZ2], [1], [if true, bz2 library is installed]) -fi -AC_MSG_CHECKING([whether libbz2 library is available]) -AC_MSG_RESULT([${have_bz2}]) - -if test "x$have_bz2" = "xno" ; then - have_local_bz2=yes - AC_MSG_NOTICE([Defaulting to internal libbz2]) -else - have_local_bz2=no +## +# Beging bz2 checks +## + +# See if we want to enable BZ2, and if so, enable it. +# Then see if we have libbz2 +AC_MSG_CHECKING([whether to search for and enable external bz2 support]) +AC_ARG_ENABLE([filter-bz2], + [AS_HELP_STRING([--disable-filter-bz2], + [disable external bz2 filter support. bz2 support defaults to internal implementation if this is disabled or if the external library is not found.])]) +test "x$enable_filter_bz2" = xno || enable_filter_bz2=yes +AC_MSG_RESULT($enable_filter_bz2) + +if test "x$enable_filter_bz2" = "xyes" ; then + + AC_CHECK_LIB([bz2],[BZ2_bzCompress],[have_bz2=yes],[have_bz2=no]) + if test "x$have_bz2" = "xyes" ; then + AC_SEARCH_LIBS([BZ2_bzCompress],[bz2 bz2.dll cygbz2.dll], [], []) + AC_DEFINE([HAVE_BZ2], [1], [if true, bz2 library is installed]) + fi + AC_MSG_CHECKING([whether libbz2 library is available]) + AC_MSG_RESULT([${have_bz2}]) + + + + if test "x$have_bz2" = "xno" ; then + have_local_bz2=yes + AC_MSG_NOTICE([Defaulting to internal libbz2]) + else + have_local_bz2=no + fi fi + AM_CONDITIONAL(HAVE_LOCAL_BZ2, [test "x$have_local_bz2" = xyes]) +## +# End bz2 checks +## # Note that szip management is tricky. # This is because we have three things to consider: From dc7da87e7c6842fad4782dff4e9438cf75160de5 Mon Sep 17 00:00:00 2001 From: Ward Fisher Date: Thu, 20 Jul 2023 15:59:53 -0600 Subject: [PATCH 04/11] Add option for blosc filter. --- configure.ac | 32 ++++++++++++++++++++++++-------- 1 file changed, 24 insertions(+), 8 deletions(-) diff --git a/configure.ac b/configure.ac index f1727c05ea..3033be49d2 100644 --- a/configure.ac +++ b/configure.ac @@ -711,12 +711,29 @@ AM_CONDITIONAL(ENABLE_NCZARR, [test x$enable_nczarr = xyes]) # Look for Standardized libraries ########## -# See if we have libblosc -AC_CHECK_LIB([blosc],[blosc_init],[have_blosc=yes],[have_blosc=no]) -if test "x$have_blosc" = "xyes" ; then - AC_SEARCH_LIBS([blosc_init],[blosc blosc.dll cygblosc.dll], [], []) - AC_DEFINE([HAVE_BLOSC], [1], [if true, blosc library is available]) +## +# blosc checks +## + +# See if we want to enable blosc, and if so, search for the library. +AC_MSG_CHECKING([whether to search for and enable blosc filter support]) +AC_ARG_ENABLE([filter-blosc], + [AS_HELP_STRING([--disable-filter-blosc], + [disable blosc filter support.])]) +test "x$enable_filter_blosc" = xno || enable_filter_blosc=yes +AC_MSG_RESULT($enable_filter_blosc) + +if test "x$enable_filter_blosc" = "xyes" ; then + # See if we have libblosc + AC_CHECK_LIB([blosc],[blosc_init],[have_blosc=yes],[have_blosc=no]) + if test "x$have_blosc" = "xyes" ; then + AC_SEARCH_LIBS([blosc_init],[blosc blosc.dll cygblosc.dll], [], []) + AC_DEFINE([HAVE_BLOSC], [1], [if true, blosc library is available]) + fi fi +## +# End blosc checks +## # See if we have libzstd AC_CHECK_LIB([zstd],[ZSTD_compress],[have_zstd=yes],[have_zstd=no]) @@ -744,9 +761,8 @@ fi # Beging bz2 checks ## -# See if we want to enable BZ2, and if so, enable it. -# Then see if we have libbz2 -AC_MSG_CHECKING([whether to search for and enable external bz2 support]) +# See if we want to enable BZ2, and if so, search for the library. +AC_MSG_CHECKING([whether to search for and enable external bz2 filter support]) AC_ARG_ENABLE([filter-bz2], [AS_HELP_STRING([--disable-filter-bz2], [disable external bz2 filter support. bz2 support defaults to internal implementation if this is disabled or if the external library is not found.])]) From 4a61f4771bd01447dbee5ff4453d527804bd39f5 Mon Sep 17 00:00:00 2001 From: Ward Fisher Date: Thu, 20 Jul 2023 16:08:07 -0600 Subject: [PATCH 05/11] Add autotools option to disable checking for libzstd. --- configure.ac | 58 ++++++++++++++++++++++++++++++++++++---------------- 1 file changed, 40 insertions(+), 18 deletions(-) diff --git a/configure.ac b/configure.ac index 3033be49d2..7975009c46 100644 --- a/configure.ac +++ b/configure.ac @@ -730,35 +730,57 @@ if test "x$enable_filter_blosc" = "xyes" ; then AC_SEARCH_LIBS([blosc_init],[blosc blosc.dll cygblosc.dll], [], []) AC_DEFINE([HAVE_BLOSC], [1], [if true, blosc library is available]) fi +else + have_blosc=no fi ## # End blosc checks ## -# See if we have libzstd -AC_CHECK_LIB([zstd],[ZSTD_compress],[have_zstd=yes],[have_zstd=no]) -if test "x$have_zstd" = "xyes" ; then - AC_SEARCH_LIBS([ZSTD_compress],[zstd zstd.dll cygzstd.dll], [], []) - AC_DEFINE([HAVE_ZSTD], [1], [if true, zstd library is available]) - -fi -AC_MSG_CHECKING([whether libzstd library is available]) -AC_MSG_RESULT([${have_zstd}]) - ## -# Ensure that the zstd.h dev files are also available. +# libzstd checks ## -if test "x$have_zstd" = "xyes" ; then - AC_CHECK_HEADERS([zstd.h], [], [nc_zstd_h_missing=yes]) - if test "x$nc_zstd_h_missing" = xyes; then - AC_MSG_WARN([zstd library detected, but zstd.h development file not found. Ensure that the zstd development files are installed in order to build zstd support.]) - AC_DEFINE([HAVE_ZSTD], [0], [if true, zstd library is available]) - have_zstd=no + +# See if we want to enable libzstd, and if so, search for the library. +AC_MSG_CHECKING([whether to search for and enable libzstd filter support]) +AC_ARG_ENABLE([filter-zstd], + [AS_HELP_STRING([--disable-filter-zstd], + [disable zstd filter support.])]) +test "x$enable_filter_zstd" = xno || enable_filter_zstd=yes +AC_MSG_RESULT($enable_filter_zstd) + +if test "x$enable_filter_zstd" = "xyes" ; then + # See if we have libzstd + AC_CHECK_LIB([zstd],[ZSTD_compress],[have_zstd=yes],[have_zstd=no]) + if test "x$have_zstd" = "xyes" ; then + AC_SEARCH_LIBS([ZSTD_compress],[zstd zstd.dll cygzstd.dll], [], []) + AC_DEFINE([HAVE_ZSTD], [1], [if true, zstd library is available]) + fi + AC_MSG_CHECKING([whether libzstd library is available]) + AC_MSG_RESULT([${have_zstd}]) + + ## + # Ensure that the zstd.h dev files are also available. + ## + if test "x$have_zstd" = "xyes" ; then + AC_CHECK_HEADERS([zstd.h], [], [nc_zstd_h_missing=yes]) + if test "x$nc_zstd_h_missing" = xyes; then + AC_MSG_WARN([zstd library detected, but zstd.h development file not found. Ensure that the zstd development files are installed in order to build zstd support.]) + AC_DEFINE([HAVE_ZSTD], [0], [if true, zstd library is available]) + have_zstd=no + fi + fi + +else + have_zstd=no fi +## +# End zstd checks +## ## -# Beging bz2 checks +# Begin bz2 checks ## # See if we want to enable BZ2, and if so, search for the library. From 9787de121c642819a0cefe8ff6f4589e5c47b895 Mon Sep 17 00:00:00 2001 From: Ward Fisher Date: Fri, 21 Jul 2023 14:02:30 -0600 Subject: [PATCH 06/11] Small change in CMakeLists.txt --- CMakeLists.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 049d325b4d..b1effd4b38 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1138,10 +1138,10 @@ ENDIF() IF (ENABLE_FILTER_BZ2) FIND_PACKAGE(Bz2) ENDIF() -IF (ENABLE_BLOSC) +IF (ENABLE_FILTER_BLOSC) FIND_PACKAGE(Blosc) ENDIF() -IF (ENABLE_ZSTD) +IF (ENABLE_FILTER_ZSTD) FIND_PACKAGE(Zstd) ENDIF() From 890251c6116df007925d0e32a5afe114ee93ba86 Mon Sep 17 00:00:00 2001 From: Ward Fisher Date: Fri, 21 Jul 2023 14:19:50 -0600 Subject: [PATCH 07/11] String handling in CMakeLists.txt --- CMakeLists.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index b1effd4b38..cfc78638f5 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1110,7 +1110,7 @@ macro(set_std_filter filter) # Upper case the filter name string(TOUPPER "${filter}" upfilter) string(TOLOWER "${filter}" downfilter) -if(ENABLE_${upfilter}) +if(ENABLE_FILTER_${upfilter}) # Define a test flag for filter IF(${filter}_FOUND) INCLUDE_DIRECTORIES(${${filter}_INCLUDE_DIRS}) @@ -1148,7 +1148,7 @@ ENDIF() # Accumulate standard filters set(STD_FILTERS "deflate") # Always have deflate*/ set_std_filter(Szip) - SET(HAVE_SZ ${Szip_FOUND}) +SET(HAVE_SZ ${Szip_FOUND}) set_std_filter(Blosc) set_std_filter(Zstd) IF(Bz2_FOUND) From ae28dd36e68a48b8821cd594323c9d77e2db8414 Mon Sep 17 00:00:00 2001 From: Ward Fisher Date: Fri, 21 Jul 2023 14:32:25 -0600 Subject: [PATCH 08/11] Additional tweaking of search logic. --- CMakeLists.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index cfc78638f5..cef6ad4337 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1128,11 +1128,11 @@ ENDIF() endmacro(set_std_filter) # Locate some compressors -OPTION(ENABLE_SZIP "Enable use of Szip compression library if it is available." ON) +OPTION(ENABLE_FILTER_SZIP "Enable use of Szip compression library if it is available." ON) OPTION(ENABLE_FILTER_BZ2 "Enable use of Bz2 compression library if it is available." ON) OPTION(ENABLE_FILTER_BLOSC "Enable use of blosc compression library if it is available." ON) OPTION(ENABLE_FILTER_ZSTD "Enable use of Zstd compression library if it is available." ON) -IF (ENABLE_SZIP) +IF (ENABLE_FILTER_SZIP) FIND_PACKAGE(Szip) ENDIF() IF (ENABLE_FILTER_BZ2) From c6b853a860228aee0b3318793a69ebf4e833785d Mon Sep 17 00:00:00 2001 From: Ward Fisher Date: Fri, 21 Jul 2023 14:46:45 -0600 Subject: [PATCH 09/11] Logic to ensure libsz is searched for if Zarr is enabled but enable_filter_szip is false. --- CMakeLists.txt | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index cef6ad4337..19a5a07be7 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1128,12 +1128,14 @@ ENDIF() endmacro(set_std_filter) # Locate some compressors -OPTION(ENABLE_FILTER_SZIP "Enable use of Szip compression library if it is available." ON) +OPTION(ENABLE_FILTER_SZIP "Enable use of Szip compression library if it is available. Required if ENABLE_NCZARR is true." ON) OPTION(ENABLE_FILTER_BZ2 "Enable use of Bz2 compression library if it is available." ON) OPTION(ENABLE_FILTER_BLOSC "Enable use of blosc compression library if it is available." ON) OPTION(ENABLE_FILTER_ZSTD "Enable use of Zstd compression library if it is available." ON) IF (ENABLE_FILTER_SZIP) FIND_PACKAGE(Szip) +ELSEIF(ENABLE_NCZARR) + FIND_PACKAGE(Szip) ENDIF() IF (ENABLE_FILTER_BZ2) FIND_PACKAGE(Bz2) From b65bba0b79d9a65a92875137f47c913afb55e542 Mon Sep 17 00:00:00 2001 From: Ward Fisher Date: Mon, 24 Jul 2023 09:32:39 -0600 Subject: [PATCH 10/11] Additional cmake-based logic. --- CMakeLists.txt | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index b1effd4b38..19378845e7 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1148,9 +1148,11 @@ ENDIF() # Accumulate standard filters set(STD_FILTERS "deflate") # Always have deflate*/ set_std_filter(Szip) - SET(HAVE_SZ ${Szip_FOUND}) +SET(HAVE_SZ ${Szip_FOUND}) set_std_filter(Blosc) +IF(Zstd_FOUND) set_std_filter(Zstd) +ENDIF() IF(Bz2_FOUND) set_std_filter(Bz2) ELSE() From 4c71b59b523425f0b3297841906135ef23fdd9ad Mon Sep 17 00:00:00 2001 From: Ward Fisher Date: Wed, 26 Jul 2023 14:33:48 -0600 Subject: [PATCH 11/11] Update Release Notes --- RELEASE_NOTES.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/RELEASE_NOTES.md b/RELEASE_NOTES.md index da8d64fc8f..81f0bbcf4e 100644 --- a/RELEASE_NOTES.md +++ b/RELEASE_NOTES.md @@ -7,7 +7,8 @@ This file contains a high-level description of this package's evolution. Release ## 4.9.3 - TBD -* Fix memory leak WRT unreclaimed HDF5 plist. See [Github #????](https://github.com/Unidata/netcdf-c/pull/????). +* Introducing configure-time options to disable various filters, even if the required libraries are available on the system, in support of [GitHub #2712](https://github.com/Unidata/netcdf-c/pull/2712). +* Fix memory leak WRT unreclaimed HDF5 plist. See [Github #2752](https://github.com/Unidata/netcdf-c/pull/2752). * Support HDF5 transient types when reading an HDF5 file. See [Github #2724](https://github.com/Unidata/netcdf-c/pull/2724). * Suppress filters on variables with non-fixed-size types. See [Github #2716](https://github.com/Unidata/netcdf-c/pull/2716). * Provide a single option to disable all network access and testing. See [Github #2708](https://github.com/Unidata/netcdf-c/pull/2708).