Use hugepages on the hpe-cray-ex platform.

Enable use of hugepages on the hpe-cray-ex platform. Due to limitations in the
hugepage interface, the runtime will not size the heap dynamically based on
available physical memory, but will instead use the default heap size or the
size specified by CHPL_RT_MAX_HEAP_SIZE. The program may fail with an
out-of-memory error if there is insufficient physical memory for the heap.

Signed-off-by: John H. Hartman <[email protected]>

runtime/src/comm/ofi/Makefile.share

@@ -48,9 +48,10 @@ ifneq (, $(findstring pmi2, $(COMM_SRCS)))
 endif
 
 #
-# Use hugepages only on Cray XC systems.
+# Use hugepages on Cray XC and HPE Cray EX systems.
 #
-ifneq (, $(findstring cray-x,$(CHPL_MAKE_TARGET_PLATFORM)))
+
+ifneq (, $(findstring $(CHPL_MAKE_TARGET_PLATFORM), cray-xc hpe-cray-ex))
   COMM_SRCS += comm-ofi-hugepages.c
 else
   COMM_SRCS += comm-ofi-no-hugepages.c

runtime/src/comm/ofi/comm-ofi-launch.c

@@ -84,4 +84,9 @@ void chpl_comm_preLaunch(int32_t numLocales) {
     //
     chpl_env_set("PMI_NO_PREINITIALIZE", "y", 1);
   }
+  //
+  // Don't map virtual hugepages to physical pages if we allocate a
+  // fixed heap.
+  //
+  chpl_env_set("HUGETLB_NO_RESERVE", "yes", 0);
 }

runtime/src/comm/ofi/comm-ofi.c

@@ -3473,50 +3473,60 @@ void init_fixedHeap(void) {
   // them.  But if you're on such a system and don't, we'll emit the
   // message later.
   //
+  void *start;
   size_t page_size;
   chpl_bool have_hugepages;
 
   if ((page_size = get_hugepageSize()) == 0) {
-    page_size = chpl_getSysPageSize();
     have_hugepages = false;
-  } else {
-    have_hugepages = true;
-  }
-
-  //
-  // We'll make a fixed heap, on whole (huge)pages.
-  //
-  size = ALIGN_UP(size, page_size);
+    page_size = chpl_getSysPageSize();
+    //
+    // We'll make a fixed heap, on whole pages.
+    //
+    size = ALIGN_UP(size, page_size);
 
-  //
-  // Work our way down from the starting size in (roughly) 5% steps
-  // until we can actually allocate a heap that size.
-  //
-  size_t decrement;
-  if ((decrement = ALIGN_DN((size_t) (0.05 * size), page_size)) < page_size) {
-    decrement = page_size;
-  }
+    //
+    // Work our way down from the starting size in (roughly) 5% steps
+    // until we can actually allocate a heap that size.
+    //
+    size_t decrement;
+    if ((decrement = ALIGN_DN((size_t) (0.05 * size), page_size)) < page_size) {
+      decrement = page_size;
+    }
 
-  void* start;
-  size += decrement;
-  do {
-    size -= decrement;
+    size += decrement;
+    do {
+      size -= decrement;
 #ifdef CHPL_COMM_DEBUG
-    if (DBG_TEST_MASK(DBG_HEAP)) {
-      char buf[10];
-      DBG_PRINTF(DBG_HEAP, "try allocating fixed heap, size %s (%#zx)",
-                 chpl_snprintf_KMG_z(buf, sizeof(buf), size), size);
-    }
+      if (DBG_TEST_MASK(DBG_HEAP)) {
+        char buf[10];
+        DBG_PRINTF(DBG_HEAP, "try allocating fixed heap, size %s (%#zx)",
+                   chpl_snprintf_KMG_z(buf, sizeof(buf), size), size);
+      }
 #endif
-    if (have_hugepages) {
-      start = chpl_comm_ofi_hp_get_huge_pages(size);
-    } else {
       CHK_SYS_MEMALIGN(start, page_size, size);
-    }
-  } while (start == NULL && size > decrement);
+    } while (start == NULL && size > decrement);
 
-  if (start == NULL)
-    chpl_error("cannot create fixed heap: cannot get memory", 0, 0);
+    if (start == NULL)
+      chpl_error("cannot create fixed heap: cannot get memory", 0, 0);
+  } else {
+    have_hugepages = true;
+    //
+    // We'll make a fixed heap on whole hugepages.
+    //
+    size = ALIGN_UP(size, page_size);
+#ifdef CHPL_COMM_DEBUG
+      if (DBG_TEST_MASK(DBG_HEAP)) {
+        char buf[10];
+        DBG_PRINTF(DBG_HEAP, "try allocating fixed hugepage heap, size %s (%#zx)",
+                   chpl_snprintf_KMG_z(buf, sizeof(buf), size), size);
+      }
+#endif
+    start = chpl_comm_ofi_hp_get_huge_pages(size);
+    if (start == NULL) {
+      chpl_error("cannot create fixed heap: cannot get huge pages", 0, 0);
+    }
+  }
 
   chpl_comm_regMemHeapTouch(start, size);