From 37d458e575ced66356f0a323a5773d5839a08fda Mon Sep 17 00:00:00 2001
From: Ye Luo <yeluo@anl.gov>
Date: Thu, 27 Jul 2023 13:13:34 +0000
Subject: [PATCH 1/3] Free MKL reserved memory before destroy the SYCL queue.

---
 src/QMCWaveFunctions/Fermion/DelayedUpdateSYCL.h | 10 +++++++++-
 1 file changed, 9 insertions(+), 1 deletion(-)

diff --git a/src/QMCWaveFunctions/Fermion/DelayedUpdateSYCL.h b/src/QMCWaveFunctions/Fermion/DelayedUpdateSYCL.h
index 18246ac30f..4a0276ea9d 100644
--- a/src/QMCWaveFunctions/Fermion/DelayedUpdateSYCL.h
+++ b/src/QMCWaveFunctions/Fermion/DelayedUpdateSYCL.h
@@ -22,6 +22,9 @@
 #include "PrefetchedRange.h"
 #include "syclSolverInverter.hpp"
 #include "DeviceManager.h"
+#if defined(HAVE_MKL)
+#include <mkl_service.h>
+#endif
 
 //#define SYCL_BLOCKING
 
@@ -73,7 +76,12 @@ class DelayedUpdateSYCL
   /// default constructor
   DelayedUpdateSYCL() : delay_count(0) { m_queue_ = DeviceManager::getGlobal().getSYCLDM().createQueueDefaultDevice(); }
 
-  ~DelayedUpdateSYCL() {}
+  ~DelayedUpdateSYCL()
+  {
+#if defined(HAVE_MKL)
+    mkl_free_buffers();
+#endif
+  }
 
   /** resize the internal storage
    * @param norb number of electrons/orbitals

From 30e8c18b9276ae703c657c268de0b8501d6acc3e Mon Sep 17 00:00:00 2001
From: Ye Luo <yeluo@anl.gov>
Date: Thu, 27 Jul 2023 13:14:23 +0000
Subject: [PATCH 2/3] Update Sunspot recipe.

---
 config/build_alcf_sunspot_icpx.sh | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/config/build_alcf_sunspot_icpx.sh b/config/build_alcf_sunspot_icpx.sh
index b1b133b1e4..7fd8458f57 100755
--- a/config/build_alcf_sunspot_icpx.sh
+++ b/config/build_alcf_sunspot_icpx.sh
@@ -8,8 +8,8 @@
 # build_alcf_sunspot_icpx.sh <source_dir> <install_dir> # build all the variants with a given source directory <source_dir> and install to <install_dir>
 
 module load spack libxml2 cmake
-module load cray-hdf5/1.12.2.1
-module load oneapi/eng-compiler/2023.05.15.003
+module load cray-hdf5
+module load oneapi/eng-compiler/2023.05.15.007
 
 module list >& module_list.txt
 
@@ -23,7 +23,7 @@ echo "**********************************"
 
 TYPE=Release
 Machine=sunspot
-Compiler=icpx20230510
+Compiler=icpx20230613
 
 if [[ $# -eq 0 ]]; then
   source_folder=`pwd`
@@ -46,6 +46,7 @@ for name in offload_sycl_real_MP offload_sycl_real offload_sycl_cplx_MP offload_
 do
 
 CMAKE_FLAGS="-DCMAKE_BUILD_TYPE=$TYPE -DMPIEXEC_PREFLAGS='--cpu-bind;depth;-d;8'"
+unset CMAKE_CXX_FLAGS
 
 if [[ $name == *"cplx"* ]]; then
   CMAKE_FLAGS="$CMAKE_FLAGS -DQMC_COMPLEX=ON"

From 18457ba6a3898a3543f3973c69b3168bad36e39a Mon Sep 17 00:00:00 2001
From: Ye Luo <yeluo@anl.gov>
Date: Thu, 27 Jul 2023 08:23:57 -0500
Subject: [PATCH 3/3] Add syclSolver::freeBuffer()

---
 src/Platforms/SYCL/syclSolver.hpp                |  7 +++++--
 src/QMCWaveFunctions/Fermion/DelayedUpdateSYCL.h | 10 +---------
 2 files changed, 6 insertions(+), 11 deletions(-)

diff --git a/src/Platforms/SYCL/syclSolver.hpp b/src/Platforms/SYCL/syclSolver.hpp
index fe18f8a7bd..de45e01bfa 100644
--- a/src/Platforms/SYCL/syclSolver.hpp
+++ b/src/Platforms/SYCL/syclSolver.hpp
@@ -12,14 +12,17 @@
 #ifndef QMCPLUSPLUS_SYCL_MKL_SOLVER_H
 #define QMCPLUSPLUS_SYCL_MKL_SOLVER_H
 
-#include "oneapi/mkl/lapack.hpp"
+#include <oneapi/mkl/lapack.hpp>
+#include <mkl_service.h>
 
 namespace qmcplusplus
 {
 namespace syclSolver
 {
 using namespace oneapi::mkl::lapack;
-}
+
+inline void freeBuffer() { mkl_free_buffers(); }
+} // namespace syclSolver
 } // namespace qmcplusplus
 
 #endif
diff --git a/src/QMCWaveFunctions/Fermion/DelayedUpdateSYCL.h b/src/QMCWaveFunctions/Fermion/DelayedUpdateSYCL.h
index 4a0276ea9d..604fff2822 100644
--- a/src/QMCWaveFunctions/Fermion/DelayedUpdateSYCL.h
+++ b/src/QMCWaveFunctions/Fermion/DelayedUpdateSYCL.h
@@ -22,9 +22,6 @@
 #include "PrefetchedRange.h"
 #include "syclSolverInverter.hpp"
 #include "DeviceManager.h"
-#if defined(HAVE_MKL)
-#include <mkl_service.h>
-#endif
 
 //#define SYCL_BLOCKING
 
@@ -76,12 +73,7 @@ class DelayedUpdateSYCL
   /// default constructor
   DelayedUpdateSYCL() : delay_count(0) { m_queue_ = DeviceManager::getGlobal().getSYCLDM().createQueueDefaultDevice(); }
 
-  ~DelayedUpdateSYCL()
-  {
-#if defined(HAVE_MKL)
-    mkl_free_buffers();
-#endif
-  }
+  ~DelayedUpdateSYCL() { syclSolver::freeBuffer(); }
 
   /** resize the internal storage
    * @param norb number of electrons/orbitals