From 3af0aeeda8cd77b2792f74d8cad7da1f8ec8da5a Mon Sep 17 00:00:00 2001
From: Dom Heinzeller <dom.heinzeller@icloud.com>
Date: Fri, 30 Aug 2024 11:21:30 -0600
Subject: [PATCH] Add configs/common/packages_aocc.yaml and preliminary aocc
 support for Nautilus

---
 configs/common/packages_aocc.yaml             | 15 +++++++
 configs/sites/tier1/nautilus/compilers.yaml   |  5 +--
 .../sites/tier1/nautilus/packages_aocc.yaml   | 40 +++++++++++++++++++
 .../sites/tier1/nautilus/packages_gcc.yaml    |  5 +++
 4 files changed, 61 insertions(+), 4 deletions(-)
 create mode 100644 configs/common/packages_aocc.yaml
 create mode 100644 configs/sites/tier1/nautilus/packages_aocc.yaml

diff --git a/configs/common/packages_aocc.yaml b/configs/common/packages_aocc.yaml
new file mode 100644
index 000000000..70012d1a0
--- /dev/null
+++ b/configs/common/packages_aocc.yaml
@@ -0,0 +1,15 @@
+packages:
+  all:
+    providers:
+      blas: [openblas]
+      fftw-api: [fftw]
+      lapack: [openblas]
+  ectrans:
+    require:
+    - '~mkl +fftw'
+  gsibec:
+    require:
+    - '~mkl'
+  py-numpy:
+    require:
+    - '^openblas'
diff --git a/configs/sites/tier1/nautilus/compilers.yaml b/configs/sites/tier1/nautilus/compilers.yaml
index 331f9f425..aff3f05dd 100644
--- a/configs/sites/tier1/nautilus/compilers.yaml
+++ b/configs/sites/tier1/nautilus/compilers.yaml
@@ -6,10 +6,7 @@ compilers:
       cxx: /p/app/compilers/amd/aocc/4.0.0/bin/clang++
       f77: /p/app/compilers/amd/aocc/4.0.0/bin/flang
       fc: /p/app/compilers/amd/aocc/4.0.0/bin/flang
-    flags:
-      cflags: null
-      cxxflags: null
-      fflags: null
+    flags: {}
     operating_system: rhel8
     target: x86_64
     modules:
diff --git a/configs/sites/tier1/nautilus/packages_aocc.yaml b/configs/sites/tier1/nautilus/packages_aocc.yaml
new file mode 100644
index 000000000..6e98e570f
--- /dev/null
+++ b/configs/sites/tier1/nautilus/packages_aocc.yaml
@@ -0,0 +1,40 @@
+packages:
+# On Nautilus, use intel-oneapi-mkl as provider
+# for blas, lapack, fftw-api with AOCC because of
+# problems using openblas in downstream applications
+# (e.g. py-pandas):
+# ELF load command address/offset not properly aligned" when loading libopenblas.so
+# https://github.com/OpenMathLib/OpenBLAS/wiki/Faq#ELFoffset
+  all:
+    compiler:: [aocc@4.0.0]
+    providers:
+      mpi:: [openmpi@4.1.6]
+      blas:: [intel-oneapi-mkl]
+      fftw-api:: [intel-oneapi-mkl]
+      lapack:: [intel-oneapi-mkl]
+  mpi:
+    buildable: False
+  openmpi:
+    externals:
+    - spec: openmpi@4.1.6%aocc@4.0.0~cuda~java~memchecker~static~wrapper-rpath fabrics=ucx schedulers=slurm
+      prefix: /p/app/penguin/openmpi/4.1.6/aocc-4.0.0
+      modules:
+      - penguin/openmpi/4.1.6/aocc-4.0.0
+      - penguin/mpi-vars/aocc
+      - slurm
+  intel-oneapi-mkl:
+    buildable: False
+    externals:
+    - spec: intel-oneapi-mkl@2024.1%oneapi@2024.1.2
+      prefix: /p/app/projects/NEPTUNE/spack-stack/oneapi-2024.1.2
+  openblas:
+    buildable: False
+  ectrans:
+    require::
+    - '@1.2.0 +mkl ~fftw'
+  gsibec:
+    require::
+    - '@1.2.1 +mkl'
+  py-numpy:
+    require::
+    - '@:1.25 ^intel-oneapi-mkl'
diff --git a/configs/sites/tier1/nautilus/packages_gcc.yaml b/configs/sites/tier1/nautilus/packages_gcc.yaml
index 841ff05b1..9603a1082 100644
--- a/configs/sites/tier1/nautilus/packages_gcc.yaml
+++ b/configs/sites/tier1/nautilus/packages_gcc.yaml
@@ -21,6 +21,11 @@ packages:
       modules:
       - penguin/openmpi/5.0.1/gcc-8.5.0
       - slurm
+  intel-oneapi-mkl:
+    buildable: False
+    externals:
+    - spec: intel-oneapi-mkl@2024.1%oneapi@2024.1.2
+      prefix: /p/app/projects/NEPTUNE/spack-stack/oneapi-2024.1.2
   openblas:
     buildable: False
   ectrans: