Merge pull request #637 from lukemartinlogan/dev

Begin OpenMPI support

CMakeLists.txt

@@ -12,13 +12,14 @@ option(HERMES_ENABLE_DOXYGEN "Check how well the code is documented" OFF)
 
 option(HERMES_ENABLE_POSIX_ADAPTER "Build the Hermes POSIX adapter." ON)
 option(HERMES_ENABLE_STDIO_ADAPTER "Build the Hermes stdio adapter." ON)
-option(HERMES_ENABLE_MPIIO_ADAPTER "Build the Hermes MPI-IO adapter." ON)
+option(HERMES_ENABLE_MPIIO_ADAPTER "Build the Hermes MPI-IO adapter." OFF)
+option(HERMES_ENABLE_VFD "Build the Hermes HDF5 Virtual File Driver" OFF)
 option(HERMES_ENABLE_PUBSUB_ADAPTER "Build the Hermes pub/sub adapter." OFF)
 option(HERMES_ENABLE_KVSTORE "Build the Hermes KVStore adapter." OFF)
-option(HERMES_ENABLE_VFD "Build the Hermes HDF5 Virtual File Driver" OFF)
 option(HERMES_ENABLE_PYTHON "Build the Hermes Python wrapper" ON)
 
-message("HERMES_ENABLE_PYTHON: ${HERMES_ENABLE_PYTHON}")
+option(HERMES_MPICH "Specify that this a MPICH build" OFF)
+option(HERMES_OPENMPI "Specify that this a OpenMPI build" OFF)
 
 #-----------------------------------------------------------------------------
 # Compiler Optimization
@@ -61,7 +62,7 @@ message(STATUS "found yaml-cpp at ${yaml-cpp_DIR}")
 find_package(Catch2 3.0.1 REQUIRED)
 message(STATUS "found catch2.h at ${Catch2_CXX_INCLUDE_DIRS}")
 
-# MPICH
+# MPI
 if(BUILD_MPI_TESTS)
     find_package(MPI REQUIRED COMPONENTS C CXX)
     message(STATUS "found mpi.h at ${MPI_CXX_INCLUDE_DIRS}")
@@ -105,6 +106,38 @@ endif()
 include_directories(${Boost_INCLUDE_DIRS})
 message("Boost: ${Boost_LIBRARIES}")
 
+# HDF5
+if(HERMES_ENABLE_VFD)
+    set(HERMES_REQUIRED_HDF5_VERSION 1.14.0)
+    set(HERMES_REQUIRED_HDF5_COMPONENTS C)
+    find_package(HDF5 ${HERMES_REQUIRED_HDF5_VERSION} CONFIG NAMES hdf5
+            COMPONENTS ${HERMES_REQUIRED_HDF5_COMPONENTS} shared)
+    if(HDF5_FOUND)
+        message(STATUS "found HDF5 ${HDF5_VERSION} at ${HDF5_INCLUDE_DIR}")
+        set(HDF5_HERMES_VFD_EXT_INCLUDE_DEPENDENCIES
+                ${HDF5_HERMES_VFD_EXT_INCLUDE_DEPENDENCIES}
+                ${HDF5_INCLUDE_DIR})
+        set(HDF5_HERMES_VFD_EXT_LIB_DEPENDENCIES
+                ${HDF5_HERMES_VFD_EXT_LIB_DEPENDENCIES}
+                ${HDF5_C_SHARED_LIBRARY})
+    else()
+        # Allow for HDF5 autotools builds
+        find_package(HDF5 ${HERMES_REQUIRED_HDF5_VERSION} MODULE REQUIRED
+                COMPONENTS ${HERMES_REQUIRED_HDF5_COMPONENTS})
+        if(HDF5_FOUND)
+            set(HDF5_HERMES_VFD_EXT_INCLUDE_DEPENDENCIES
+                    ${HDF5_HERMES_VFD_EXT_INCLUDE_DEPENDENCIES}
+                    ${HDF5_INCLUDE_DIRS})
+            set(HDF5_HERMES_VFD_EXT_LIB_DEPENDENCIES
+                    ${HDF5_HERMES_VFD_EXT_LIB_DEPENDENCIES}
+                    ${HDF5_LIBRARIES})
+        else()
+            message(FATAL_ERROR "Could not find HDF5, please set HDF5_DIR (1.13.0) or HDF5_ROOT (1.13.1).")
+        endif()
+    endif()
+endif()
+
+
 #------------------------------------------------------------------------------
 # Setup CMake Environment
 #------------------------------------------------------------------------------

ci/hermes/packages/hermes/package.py

@@ -8,8 +8,6 @@ class Hermes(CMakePackage):
     version('master',
             branch='master', submodules=True)
     version('dev', branch='dev', submodules=True)
-    version('dev-priv', git='https://github.com/lukemartinlogan/hermes.git',
-            branch='dev', submodules=True)
     version("1.0.5-beta", sha256="1f3ba51a8beda4bc1314d6541b800de1525f5e233a6f498fcde6dc43562ddcb7")
     version("1.0.0-beta", sha256="301084cced32aa00532ab4bebd638c31b0512c881ffab20bf5da4b7739defac2")
     version("0.9.9-beta", sha256="d2e0025a9bd7a3f05d3ab608c727ed15d86ed30cf582549fe996875daf6cb649")

ci/hermes/packages/hermes_shm/package.py

@@ -8,20 +8,22 @@ class HermesShm(CMakePackage):
     version('master', branch='master')
     version("1.0.0", sha256="a79f01d531ce89985ad59a2f62b41d74c2385e48d929e2f4ad895ae34137573b")
 
+    variant('mpiio', default=True, description='Enable MPI I/O adapter')
+    variant('stdio', default=True, description='Enable STDIO adapter')
+    variant('vfd', default=False, description='Enable HDF5 VFD')
     variant('ares', default=False, description='Enable full libfabric install')
     variant('only_verbs', default=False, description='Only verbs')
-    variant('vfd', default=False, description='Enable HDF5 VFD')
     variant('debug', default=False, description='Build shared libraries')
     variant('zmq', default=False, description='Build ZeroMQ tests')
 
     depends_on('[email protected]')
     depends_on('[email protected]')
-    depends_on('[email protected]')
+    depends_on('mpi')
     depends_on('cereal')
     depends_on('yaml-cpp')
     depends_on('[email protected]')
     depends_on('[email protected]: +context +fiber +filesystem +system +atomic +chrono +serialization +signals +pic +regex')
-    depends_on('libfabric fabrics=sockets,tcp,udp,rxm,rxd,verbs',
+    depends_on('libfabric fabrics=sockets,tcp,udp,verbs',
                when='+ares')
     depends_on('libfabric fabrics=verbs',
                when='+only_verbs')
@@ -34,6 +36,16 @@ def cmake_args(self):
             args.append('-DCMAKE_BUILD_TYPE=Debug')
         else:
             args.append('-DCMAKE_BUILD_TYPE=Release')
+        if '+mpiio' in self.spec:
+            args.append('-DHERMES_ENABLE_MPIIO_ADAPTER=ON')
+            if 'openmpi' in self.spec:
+                args.append('-DHERMES_OPENMPI=ON')
+            elif 'mpich' in self.spec:
+                args.append('-DHERMES_MPICH=ON')
+        if '+stdio' in self.spec:
+            args.append('-HERMES_ENABLE_STDIO_ADAPTER=ON')
+        if '+vfd' in self.spec:
+            args.append('-HERMES_ENABLE_VFD=ON')
         return args
 
     def set_include(self, env, path):

hermes_adapters/CMakeLists.txt

@@ -9,7 +9,7 @@ endif()
 if (HERMES_ENABLE_MPIIO_ADAPTER)
     add_subdirectory(mpiio)
 endif()
-if (HERMES_ENABLE_HDF5_ADAPTER)
+if (HERMES_ENABLE_VFD)
     add_subdirectory(vfd)
 endif()
 

hermes_adapters/mpiio/CMakeLists.txt

@@ -11,6 +11,13 @@ add_dependencies(hermes_mpiio_io_client
         hermes hermes_fs_base)
 target_link_libraries(hermes_mpiio_io_client
         hermes hermes_fs_base MPI::MPI_CXX stdc++fs dl)
+if (HERMES_MPICH)
+    message(STATUS "Using HERMES_MPICH")
+    add_definitions(-DHERMES_MPICH)
+elseif(HERMES_OPENMPI)
+    message(STATUS "Using HERMES_OPENMPI")
+    add_definitions(-DHERMES_OPENMPI)
+endif()
 
 # Create the MPIIO interceptor
 set(INTERCEPTOR_DEPS

hermes_adapters/mpiio/mpiio_api.h

@@ -17,7 +17,9 @@
 #include <iostream>
 #include "hermes_shm/util/logging.h"
 #include <mpi.h>
+#ifdef HERMES_MPICH
 #include <mpio.h>
+#endif
 #include "hermes_adapters/real_api.h"
 
 #ifndef MPI_MODE_TRUNCATE

hermes_adapters/mpiio/mpiio_io_client.cc

@@ -214,8 +214,15 @@ void MpiioIoClient::ReadBlob(const std::string &bkt_name,
 
 /** Update the I/O status after a ReadBlob or WriteBlob */
 void MpiioIoClient::UpdateIoStatus(size_t count, IoStatus &status) {
+#ifdef HERMES_OPENMPI
+  status.mpi_status_ptr_->_cancelled = 0;
+  status.mpi_status_ptr_->_ucount = count;
+#elif defined(HERMES_MPICH)
   status.mpi_status_ptr_->count_hi_and_cancelled = 0;
   status.mpi_status_ptr_->count_lo = count;
+#else
+#error "No MPI implementation specified for MPIIO adapter"
+#endif
 }
 
 }  // namespace hermes::adapter::fs

hermes_adapters/vfd/H5FDhermes.cc

@@ -40,7 +40,7 @@
 #include "H5FDhermes.h"     /* Hermes file driver     */
 
 #include "hermes_adapters/posix/posix_io_client.h"
-#include "posix/posix_fs_api.h"
+#include "hermes_adapters/posix/posix_fs_api.h"
 
 /**
  * Make this adapter use Hermes.
@@ -211,7 +211,8 @@ H5FD__hermes_term(void) {
   /* Reset VFL ID */
   H5FD_HERMES_g = H5I_INVALID_HID;
 
-  HERMES->Finalize();
+  // TODO(llogan): Probably should add back at some point.
+  // HERMES->Finalize();
   return ret_value;
 } /* end H5FD__hermes_term() */
 
@@ -515,13 +516,13 @@ static herr_t H5FD__hermes_write(H5FD_t *_file, H5FD_mem_t type,
  */
 H5PL_type_t
 H5PLget_plugin_type(void) {
-  TRANSPARENT_HERMES
+  TRANSPARENT_HERMES();
   return H5PL_TYPE_VFD;
 }
 
 const void*
 H5PLget_plugin_info(void) {
-  TRANSPARENT_HERMES
+  TRANSPARENT_HERMES();
   return &H5FD_hermes_g;
 }
 

tasks/data_stager/src/data_stager.cc

@@ -65,6 +65,7 @@ class Server : public TaskLib {
     std::unordered_map<hermes::BucketId, std::unique_ptr<AbstractStager>>::iterator it =
         url_map_[rctx.lane_id_].find(task->bkt_id_);
     if (it == url_map_[rctx.lane_id_].end()) {
+      //
       // HELOG(kError, "Could not find stager for bucket: {}", task->bkt_id_);
       // TODO(llogan): Probably should add back...
       // task->SetModuleComplete();

test/unit/hermes_adapters/vfd/hermes_vfd_test.cc

@@ -23,12 +23,10 @@
 #include <hdf5.h>
 
 #include "hermes/hermes.h"
-#include "hermes_types.h"
+#include "hermes/hermes_types.h"
 #include "adapter_test_utils.h"
 #include "catch_config.h"
 
-using hermes::f32;
-using hermes::u32;
 namespace stdfs = std::filesystem;
 
 namespace hermes::adapter::vfd::test {
@@ -399,7 +397,7 @@ void RemoveFiles() {
 int init(int* argc, char*** argv) {
 #if HERMES_INTERCEPT == 1
   setenv("HERMES_FLUSH_MODE", "kSync", 1);
-  TRANSPARENT_HERMES
+  TRANSPARENT_HERMES();
   HERMES_CLIENT_CONF.flushing_mode_ = hermes::FlushingMode::kSync;
 #endif
   MPI_Init(argc, argv);

test/unit/pipelines/test_ior.yaml

@@ -0,0 +1,14 @@
+name: hermes_unit_ior
+env: hermes
+pkgs:
+  - pkg_type: asan
+    pkg_name: asan
+  - pkg_type: hermes_run
+    pkg_name: hermes_run
+    sleep: 5
+  - pkg_type: hermes_api
+    pkg_name: hermes_api
+    posix: true
+  - pkg_type: ior
+    pkg_name: ior
+    api: posix