Support CMake VOL builds with FetchContent from local directory (HDFG…

…roup#3455) * Update VOL CMake for REST VOL * Prevent linking static libs to VOLs * index on fetch_local: 5c5c3f1 Prevent linking static libs to VOLs * index on (no branch): 9a36d3e On fetch_local: WIP:add source dir fetch option for vols * Allow building of VOL from local source * Move LOCAL_DIR option to HDF5_VOL_ALLOW_EXTERNAL
qkoziol · Sep 30, 2023 · aae68c7 · aae68c7
1 parent 25b5330
commit aae68c7
Show file tree

Hide file tree

Showing 2 changed files with 100 additions and 42 deletions.
diff --git a/CMakeVOL.cmake b/CMakeVOL.cmake
@@ -27,12 +27,12 @@ function (get_generated_cmake_targets out_var dir)
 endfunction ()
 
 # For now, only support building of external VOL connectors with FetchContent
-option (HDF5_VOL_ALLOW_EXTERNAL "Allow building of external HDF5 VOL connectors with FetchContent" OFF)
+option (HDF5_VOL_ALLOW_EXTERNAL "Allow building of external HDF5 VOL connectors with FetchContent" "NO")
 mark_as_advanced (HDF5_VOL_ALLOW_EXTERNAL)
 if (HDF5_VOL_ALLOW_EXTERNAL)
-  if (HDF5_ALLOW_EXTERNAL_SUPPORT MATCHES "NO" OR NOT HDF5_ALLOW_EXTERNAL_SUPPORT MATCHES "GIT")
-    message (FATAL_ERROR "HDF5_ALLOW_EXTERNAL_SUPPORT must be set to 'GIT' to allow building of external HDF5 VOL connectors")
-  endif ()
+  if (HDF5_VOL_ALLOW_EXTERNAL MATCHES "NO" OR (NOT HDF5_VOL_ALLOW_EXTERNAL MATCHES "GIT" AND NOT HDF5_VOL_ALLOW_EXTERNAL MATCHES "LOCAL_DIR"))
+    message (FATAL_ERROR "HDF5_VOL_ALLOW_EXTERNAL must be set to 'GIT' or 'LOCAL_DIR' to allow building of external HDF5 VOL connectors")
+  endif()
 
   # For compatibility, set some variables that projects would
   # typically look for after calling find_package(HDF5)
@@ -51,7 +51,7 @@ if (HDF5_VOL_ALLOW_EXTERNAL)
 
   foreach (vol_idx RANGE 1 ${HDF5_MAX_EXTERNAL_VOLS})
     # Generate fixed-width index number prepended with 0s
-    # so URLs come in order from 1 - HDF5_MAX_EXTERNAL_VOLS
+    # so VOL sources come in order from 1 - HDF5_MAX_EXTERNAL_VOLS
     set (vol_idx_num_digits 2) # Based on HDF5_MAX_EXTERNAL_VOLS
     set (vol_idx_fixed "${vol_idx}")
     string (LENGTH "${vol_idx_fixed}" vol_idx_len)
@@ -60,33 +60,53 @@ if (HDF5_VOL_ALLOW_EXTERNAL)
       math (EXPR vol_idx_len "${vol_idx_len}+1")
     endwhile ()
 
-    set (HDF5_VOL_URL${vol_idx_fixed} "" CACHE STRING "Git repository URL of an external HDF5 VOL connector to build")
-    mark_as_advanced (HDF5_VOL_URL${vol_idx_fixed})
+    if (HDF5_VOL_ALLOW_EXTERNAL MATCHES "GIT")
+      set (HDF5_VOL_URL${vol_idx_fixed} "" CACHE STRING "Git repository URL of an external HDF5 VOL connector to build")
+      mark_as_advanced (HDF5_VOL_URL${vol_idx_fixed})
+      set (HDF5_VOL_SOURCE "${HDF5_VOL_URL${vol_idx_fixed}}")
+    elseif(HDF5_VOL_ALLOW_EXTERNAL MATCHES "LOCAL_DIR")
+      set (HDF5_VOL_PATH${vol_idx_fixed} "" CACHE STRING "Path to the source directory of an external HDF5 VOL connector to build")
+      mark_as_advanced (HDF5_VOL_PATH${vol_idx_fixed})
+      set (HDF5_VOL_SOURCE "${HDF5_VOL_PATH${vol_idx_fixed}}")
+    endif()
+
+    if (NOT "${HDF5_VOL_SOURCE}" STREQUAL "")
+      # Deal with trailing slash in path for LOCAL_DIR case
+      if (HDF5_VOL_ALLOW_EXTERNAL MATCHES "LOCAL_DIR")
+        # Erase trailing slash
+        string (REGEX REPLACE "/$" "" HDF5_VOL_SOURCE ${HDF5_VOL_SOURCE})
+      endif()
 
-    if (NOT "${HDF5_VOL_URL${vol_idx_fixed}}" STREQUAL "")
       # Extract the name of the VOL connector
-      string (FIND "${HDF5_VOL_URL${vol_idx_fixed}}" "/" hdf5_vol_name_pos REVERSE)
+      string (FIND "${HDF5_VOL_SOURCE}" "/" hdf5_vol_name_pos REVERSE)
       if (hdf5_vol_name_pos EQUAL -1)
-        message (SEND_ERROR "Invalid URL '${HDF5_VOL_URL${vol_idx_fixed}}' specified for HDF5_VOL_URL${vol_idx_fixed}")
+        if (HDF5_VOL_ALLOW_EXTERNAL MATCHES "GIT")
+          message (SEND_ERROR "Invalid URL '${HDF5_VOL_SOURCE}' specified for HDF5_VOL_URL${vol_idx_fixed}")
+        elseif (HDF5_VOL_ALLOW_EXTERNAL MATCHES "LOCAL_DIR")
+          message (SEND_ERROR "Invalid source path '${HDF5_VOL_SOURCE}' specified for HDF5_VOL_PATH${vol_idx_fixed}")
+        endif()
       endif ()
 
       math (EXPR hdf5_vol_name_pos "${hdf5_vol_name_pos}+1")
 
-      string (SUBSTRING "${HDF5_VOL_URL${vol_idx_fixed}}" ${hdf5_vol_name_pos} -1 hdf5_vol_name)
+      string (SUBSTRING "${HDF5_VOL_SOURCE}" ${hdf5_vol_name_pos} -1 hdf5_vol_name)
       string (REPLACE ".git" "" hdf5_vol_name "${hdf5_vol_name}")
       string (STRIP "${hdf5_vol_name}" hdf5_vol_name)
       string (TOUPPER "${hdf5_vol_name}" hdf5_vol_name_upper)
       string (TOLOWER "${hdf5_vol_name}" hdf5_vol_name_lower)
 
-      message (VERBOSE "Building VOL connector '${hdf5_vol_name}' with FetchContent")
+      message (VERBOSE "Building VOL connector '${hdf5_vol_name}' with FetchContent from source ${HDF5_VOL_SOURCE}")
 
       # Set some cache variables that can be set by users when building
+      if (HDF5_VOL_ALLOW_EXTERNAL MATCHES "GIT")
+        set ("HDF5_VOL_${hdf5_vol_name_upper}_BRANCH" "main" CACHE STRING "Git branch (or tag) to use when building VOL connector '${hdf5_vol_name}'")
+        mark_as_advanced ("HDF5_VOL_${hdf5_vol_name_upper}_BRANCH")
+      endif()
+
       set ("HDF5_VOL_${hdf5_vol_name_upper}_NAME" "" CACHE STRING "Name of VOL connector to set for the HDF5_VOL_CONNECTOR environment variable")
-      set ("HDF5_VOL_${hdf5_vol_name_upper}_BRANCH" "main" CACHE STRING "Git branch (or tag) to use when building VOL connector '${hdf5_vol_name}'")
       option ("HDF5_VOL_${hdf5_vol_name_upper}_TEST_PARALLEL" "Whether to test VOL connector '${hdf5_vol_name}' against the parallel API tests" OFF)
 
       mark_as_advanced ("HDF5_VOL_${hdf5_vol_name_upper}_NAME")
-      mark_as_advanced ("HDF5_VOL_${hdf5_vol_name_upper}_BRANCH")
       mark_as_advanced ("HDF5_VOL_${hdf5_vol_name_upper}_TEST_PARALLEL")
 
       if (HDF5_TEST_API)
@@ -95,21 +115,36 @@ if (HDF5_VOL_ALLOW_EXTERNAL)
         endif ()
       endif ()
 
-      if ("${HDF5_VOL_${hdf5_vol_name_upper}_BRANCH}" STREQUAL "")
+      if ((HDF5_VOL_ALLOW_EXTERNAL MATCHES "GIT") AND ("${HDF5_VOL_${hdf5_vol_name_upper}_BRANCH}" STREQUAL ""))
         message (SEND_ERROR "HDF5_VOL_${hdf5_vol_name_upper}_BRANCH must be set to a valid git branch name (or git tag) to build VOL connector '${hdf5_vol_name}'")
       endif ()
 
-      FetchContent_Declare (HDF5_VOL_${hdf5_vol_name_lower}
-        GIT_REPOSITORY "${HDF5_VOL_URL${vol_idx_fixed}}"
+      if ((HDF5_VOL_ALLOW_EXTERNAL MATCHES "LOCAL_DIR")
+        AND NOT (EXISTS ${HDF5_VOL_SOURCE} AND IS_DIRECTORY ${HDF5_VOL_SOURCE}))
+          message (FATAL_ERROR "HDF5_VOL_PATH${vol_idx_fixed} must be an absolute path to a valid directory")
+      endif ()
+
+      if (HDF5_VOL_ALLOW_EXTERNAL MATCHES "GIT")
+        FetchContent_Declare (HDF5_VOL_${hdf5_vol_name_lower}
+        GIT_REPOSITORY "${HDF5_VOL_SOURCE}"
         GIT_TAG "${HDF5_VOL_${hdf5_vol_name_upper}_BRANCH}"
-      )
+        )
+      elseif(HDF5_VOL_ALLOW_EXTERNAL MATCHES "LOCAL_DIR")
+        FetchContent_Declare (HDF5_VOL_${hdf5_vol_name_lower}
+        SOURCE_DIR "${HDF5_VOL_SOURCE}"
+        )
+      endif()
 
       FetchContent_GetProperties(HDF5_VOL_${hdf5_vol_name_lower})
       if (NOT hdf5_vol_${hdf5_vol_name_lower}_POPULATED)
         FetchContent_Populate(HDF5_VOL_${hdf5_vol_name_lower})
 
         if (NOT EXISTS "${hdf5_vol_${hdf5_vol_name_lower}_SOURCE_DIR}/CMakeLists.txt")
-          message (SEND_ERROR "The git repository branch '${HDF5_VOL_${hdf5_vol_name_upper}_BRANCH}' for VOL connector '${hdf5_vol_name}' does not appear to contain a CMakeLists.txt file")
+          if (HDF5_VOL_ALLOW_EXTERNAL MATCHES "GIT")
+            message (SEND_ERROR "The git repository branch '${HDF5_VOL_${hdf5_vol_name_upper}_BRANCH}' for VOL connector '${hdf5_vol_name}' does not appear to contain a CMakeLists.txt file")
+          elseif (HDF5_VOL_ALLOW_EXTERNAL MATCHES "LOCAL_DIR")
+            message(SEND_ERROR "The local directory '${HDF5_VOL_SOURCE}' for VOL connector '${hdf5_vol_name}' does not appear to contain a CMakeLists.txt file")
+          endif()
         endif ()
 
         # If there are any calls to find_package(HDF5) in the connector's

diff --git a/doc/cmake-vols-fetchcontent.md b/doc/cmake-vols-fetchcontent.md
@@ -18,19 +18,18 @@ are the following:
 ## Building
 
 To enable building of an HDF5 VOL connector using HDF5's CMake functionality,
-two CMake variables must first be set:
+a CMake variable must first be set:
 
-    HDF5_VOL_ALLOW_EXTERNAL (Default: OFF)
-        This variable determines whether or not building of external HDF5 VOL connectors
-        is enabled.
-
-    HDF5_ALLOW_EXTERNAL_SUPPORT (Default: "NO")
+    HDF5_VOL_ALLOW_EXTERNAL (Default: "NO")
         This variable is a string that specifies the manner in which the source code for
-        an external VOL connector will be retrieved. Currently, this variable must be set
-        to "GIT" for building external VOL connectors.
+        an external VOL connector will be retrieved. This variable must be set
+        to "GIT" for building external VOL connectors from a Github repository, or
+        set to "LOCAL_DIR" to build from a local source directory.
+
+
+### Building
 
-Once the `HDF5_VOL_ALLOW_EXTERNAL` option is set to ON and the `HDF5_ALLOW_EXTERNAL_SUPPORT`
-variable is set to "GIT", the CMake cache will be populated with a predefined
+If the `HDF5_VOL_ALLOW_EXTERNAL` option is set to "GIT", the CMake cache will be populated with a predefined
 (currently 10) amount of new variables, named:
 
     HDF5_VOL_URL01
@@ -49,26 +48,44 @@ For each URL specified, HDF5's CMake code will attempt to use CMake's
 [FetchContent](https://cmake.org/cmake/help/latest/module/FetchContent.html)
 functionality to retrieve the source code for a VOL connector pointed to by
 that URL and will try to build that VOL connector as part of the HDF5 library
-build process. The VOL connector must be able to be built by CMake and currently
+build process. 
+
+If `HDF5_VOL_ALLOW_EXTERNAL` is instead set to "LOCAL_DIR", then the CMake cache 
+will instead be populated with the variables:
+
+    HDF5_VOL_PATH01
+    HDF5_VOL_PATH02
+    HDF5_VOL_PATH03
+    ...
+
+For each of these variables, an absolute path that points to a local 
+directory containing source code for an HDF5 VOL connector
+can be specified. For example, to specify a local clone of the 
+REST VOL connector stored under one's home directory, one can provide 
+the following option to `cmake`:
+
+    -DHDF5_VOL_PATH01=/home/vol-rest
+
+Regardless of the method used to obtain the VOL source code, 
+the VOL connector must be able to be built by CMake and currently
 must have a CMakeLists.txt file in the top level of the source tree in order to
 be buildable by this process. If the source code for a VOL connector is successfully
 retrieved, the HDF5 build's CMake cache will be populated with variables from
 the VOL connector's CMake code, as if one were building the connector by itself.
 This gives one the ability to customize the build of the connector as usual.
 
 The CMake cache will also be populated with a few new variables for each VOL
-connector that was successfully retrieved from a given URL. To generate these
-variables, the CMake code first creates an internal name for the VOL connector
+connector that was successfully retrieved. To generate these
+variables, the CMake code first creates an internal name for the VOL connector.
+If the source was retrieved from a URL, then the name is generated
 by stripping off the last part of the Git repository URL given for the connector,
 removing the ".git" suffix and any whitespace and then upper-casing the result.
 For example, the name of the VOL connector located at the URL
-https://github.com/hpc-io/vol-async.git would become "VOL-ASYNC". Then, the following
-new variables get created:
+https://github.com/hpc-io/vol-async.git would become "VOL-ASYNC". If the source was 
+retrieved from a local directory, then the source directory's name is trimmed of whitespace, 
+upper-cased, and has any trailing slashes removed.
 
-    HDF5_VOL_<VOL name>_BRANCH (Default: "main")
-        This variable specifies the git branch name or tag to use when fetching
-        the source code for the VOL connector with the CMake-internal name
-        '<VOL name>'.
+After the VOL's internal name is generated, the following new variables get created:
 
     HDF5_VOL_<VOL name>_NAME (Default: "")
         This variable specifies the string that should be used when setting the
@@ -84,8 +101,15 @@ new variables get created:
         This variable determines whether the VOL connector with the CMake-internal
         name '<VOL name>' should be tested against HDF5's parallel tests.
 
+If the source was retrieved from a Git URL, then the following variable will additionally be created:
+
+    HDF5_VOL_<VOL name>_BRANCH (Default: "main")
+        This variable specifies the git branch name or tag to use when fetching
+        the source code for the VOL connector with the CMake-internal name
+        '<VOL name>'.
+
 As an example, this would create the following variables for the
-previously-mentioned VOL connector:
+previously-mentioned VOL connector if it is retrieved from a URL:
 
     HDF5_VOL_VOL-ASYNC_BRANCH
     HDF5_VOL_VOL-ASYNC_NAME
@@ -99,7 +123,7 @@ doesn't parse the string as a list. If `cmake` is run from a shell, extra care
 may need to be taken when escaping the semicolons depending on how the
 shell interprets backslashes.
 
-### Example - Build and test HDF5 Asynchronous I/O VOL connector
+### Example - Build and test HDF5 Asynchronous I/O VOL connector from GIT
 
 Assuming that the HDF5 source code has been checked out and a build directory
 has been created, running the following cmake command from that build directory
@@ -113,8 +137,7 @@ would typically be passed when building HDF5, such as `CMAKE_INSTALL_PREFIX`,
       -DHDF5_ENABLE_PARALLEL=ON
       -DALLOW_UNSUPPORTED=ON
       -DHDF5_TEST_API=ON
-      -DHDF5_VOL_ALLOW_EXTERNAL=ON
-      -DHDF5_ALLOW_EXTERNAL_SUPPORT="GIT"
+      -DHDF5_VOL_ALLOW_EXTERNAL="GIT"
       -DHDF5_VOL_URL01=https://github.com/hpc-io/vol-async.git
       -DHDF5_VOL_VOL-ASYNC_BRANCH=develop 
       -DHDF5_VOL_VOL-ASYNC_NAME="async under_vol=0\;under_info={}"