Enable modules for packages not compiled with preferred compiler (update spack-stack setup-meta-modules); remove need for external ecflow

.github/workflows/ubuntu-ci-x86_64-intel.yaml

@@ -105,14 +105,13 @@ jobs:
           #echo "    - spec: [email protected]%[email protected]" >> ${SPACK_SYSTEM_CONFIG_PATH}/packages.yaml
           #echo "      prefix: /opt/intel/oneapi" >> ${SPACK_SYSTEM_CONFIG_PATH}/packages.yaml
 
-          # Add external ecflow for Intel
+          # Add external qt@5 to build ecflow
           echo "" >> ${SPACK_SYSTEM_CONFIG_PATH}/packages.yaml
-          echo "  ecflow::" >> ${SPACK_SYSTEM_CONFIG_PATH}/packages.yaml
+          echo "  qt::" >> ${SPACK_SYSTEM_CONFIG_PATH}/packages.yaml
           echo "    buildable: False" >> ${SPACK_SYSTEM_CONFIG_PATH}/packages.yaml
           echo "    externals:" >> ${SPACK_SYSTEM_CONFIG_PATH}/packages.yaml
-          echo "    - spec: [email protected]+ui+static_boost" >> ${SPACK_SYSTEM_CONFIG_PATH}/packages.yaml
-          echo "      modules:" >> ${SPACK_SYSTEM_CONFIG_PATH}/packages.yaml
-          echo "      - ecflow/5.11.4" >> ${SPACK_SYSTEM_CONFIG_PATH}/packages.yaml
+          echo "    - spec: [email protected]" >> ${SPACK_SYSTEM_CONFIG_PATH}/packages.yaml
+          echo "      prefix: /usr" >> ${SPACK_SYSTEM_CONFIG_PATH}/packages.yaml
 
           export -n SPACK_SYSTEM_CONFIG_PATH
 
@@ -121,13 +120,13 @@ jobs:
 
           # Set compiler and MPI specs
           spack config add "packages:all:providers:mpi:[[email protected]]"
-          spack config add "packages:all:compiler:[[email protected]]"
+          spack config add "packages:all:compiler:[[email protected], [email protected]]"
 
           # Add additional variants for MET packages, different from config/common/packages.yaml
           spack config add "packages:met:variants:+python +grib2 +graphics +lidar2nc +modis"
 
-          # Don't generate ecflow module when using external package
-          spack config add "modules:default:tcl:exclude:[ecflow]"
+          # Don't generate qt module when using external package
+          spack config add "modules:default:tcl:exclude:[qt]"
 
           # Concretize and check for duplicates
           spack concretize 2>&1 | tee log.concretize.intel-2021.10.0-buildcache

.github/workflows/ubuntu-ci-x86_64-oneapi.yaml

@@ -111,7 +111,7 @@ jobs:
 
           # Set compiler and MPI specs
           spack config add "packages:all:providers:mpi:[[email protected]]"
-          spack config add "packages:all:compiler:[[email protected]]"
+          spack config add "packages:all:compiler:[[email protected],[email protected]]"
 
           # Add additional variants for MET packages, different from config/common/packages.yaml
           spack config add "packages:met:variants:+python +grib2 +graphics +lidar2nc +modis"

configs/common/packages.yaml

@@ -17,7 +17,8 @@ packages:
     require: '@3.4:'
   # 1.85 incompatible with [email protected] - use latest 'good version'
   boost:
-    require: '@1.84 ~atomic +chrono +date_time +exception +filesystem ~graph ~iostreams ~locale ~log ~math ~mpi ~numpy +pic +program_options +python ~random +regex +serialization ~signals +system +test +thread +timer ~wave cxxstd=17 visibility=hidden'
+    require:
+    - '@1.84 ~atomic +chrono +date_time +exception +filesystem ~graph ~iostreams ~locale ~log ~math ~mpi ~numpy +pic +program_options +python ~random +regex +serialization ~signals +system +test +thread +timer ~wave cxxstd=17 visibility=hidden'
   bufr:
     require: '@12.1.0 +python'
   bufr-query:
@@ -37,7 +38,8 @@ packages:
   eccodes:
     require: '@2.33.0 +png +tools'
   ecflow:
-    require: '@5.11.4 +ui'
+    require:
+    - '@5.11.4 +ui'
   eckit:
     require: '@1.27.0 linalg=eigen,lapack compression=lz4,bzip2'
   ecmwf-atlas:

configs/common/packages_intel.yaml

@@ -18,3 +18,9 @@ packages:
   py-numpy:
     require:
     - '^intel-oneapi-mkl'
+  boost:
+    require:
+    - '%gcc'
+  ecflow:
+    require:
+    - '%gcc'

configs/sites/tier1/acorn/compilers.yaml

@@ -70,3 +70,25 @@ compilers:
         # which confuses some packages.
         CONFIG_SITE: ''
     extra_rpaths: []
+# Needed for Intel 19/2022 backend
+- compiler:
+    spec: [email protected]
+    paths:
+      cc: cc
+      cxx: CC
+      f77: ftn
+      fc: ftn
+    flags: {}
+    operating_system: sles15
+    modules:
+    - PrgEnv-gnu/8.3.3
+    - craype/2.7.13
+    - gcc/10.2.0
+    - libfabric
+    environment:
+      set:
+        # OpenSUSE on WCOSS2 machines sets CONFIG_SITE so 
+        # Automake-based builds are installed in lib64 
+        # which confuses some packages.
+        CONFIG_SITE: ''
+    extra_rpaths: []

configs/sites/tier1/acorn/packages_intel.yaml

@@ -1,6 +1,6 @@
   packages:
     all:
-      compiler:: [[email protected],[email protected]]
+      compiler:: [[email protected],[email protected],[email protected]]
       providers:
         mpi:: [[email protected]]
         # Remove the next three lines to switch to intel-oneapi-mkl

configs/sites/tier1/atlantis/compilers.yaml

@@ -16,3 +16,4 @@ compilers:
       prepend_path:
         LD_LIBRARY_PATH: /cm/shared/apps/intel/oneapi/compiler/2022.1.0/linux/compiler/lib/intel64_lin
     extra_rpaths: []
+# Todo: Define gcc compiler (default for Atlantis) for Intel (ecflow)

configs/sites/tier1/atlantis/packages.yaml

@@ -1,6 +1,6 @@
 packages:
   all:
-    compiler:: [[email protected]]
+    compiler:: [[email protected]] # todo: add gcc here
     providers:
       mpi:: [[email protected]]
 

configs/sites/tier1/aws-pcluster/packages_intel.yaml

@@ -1,6 +1,6 @@
 packages:
   all:
-    compiler:: [[email protected]]
+    compiler:: [[email protected],[email protected]]
     providers:
       mpi:: [[email protected]]
       # Remove the next three lines to switch to intel-oneapi-mkl

configs/sites/tier1/derecho/packages_intel.yaml

@@ -1,6 +1,6 @@
 packages:
   all:
-    compiler:: [[email protected]]
+    compiler:: [[email protected],[email protected]]
     providers:
       mpi:: [[email protected]]
       # Remove the next three lines to switch to intel-oneapi-mkl

configs/sites/tier1/discover-scu16/compilers.yaml

@@ -34,3 +34,18 @@ compilers:
     - comp/gcc/12.1.0
     environment: {}
     extra_rpaths: []
+# Needed for Intel backend
+- compiler:
+    spec: gcc@=11.2.0
+    paths:
+      cc: /usr/local/other/gcc/11.2.0/bin/gcc
+      cxx: /usr/local/other/gcc/11.2.0/bin/g++
+      f77: /usr/local/other/gcc/11.2.0/bin/gfortran
+      fc: /usr/local/other/gcc/11.2.0/bin/gfortran
+    flags: {}
+    operating_system: sles12
+    target: x86_64
+    modules:
+    - comp/gcc/11.2.0
+    environment: {}
+    extra_rpaths: []

configs/sites/tier1/discover-scu16/packages_intel.yaml

@@ -1,6 +1,6 @@
 packages:
   all:
-    compiler:: [[email protected]]
+    compiler:: [[email protected],[email protected]]
     providers:
       mpi:: [[email protected]]
   mpi:

configs/sites/tier1/discover-scu17/compilers.yaml

@@ -51,3 +51,18 @@ compilers:
     - comp/gcc/12.3.0
     environment: {}
     extra_rpaths: []
+# Needed for Intel backend
+- compiler:
+    spec: gcc@=11.4.0
+    paths:
+      cc: /usr/local/other/gcc/11.4.0/bin/gcc
+      cxx: /usr/local/other/gcc/11.4.0/bin/g++
+      f77: /usr/local/other/gcc/11.4.0/bin/gfortran
+      fc: /usr/local/other/gcc/11.4.0/bin/gfortran
+    flags: {}
+    operating_system: sles15
+    target: x86_64
+    modules:
+    - comp/gcc/11.4.0
+    environment: {}
+    extra_rpaths: []

configs/sites/tier1/discover-scu17/packages_intel.yaml

@@ -1,6 +1,6 @@
 packages:
   all:
-    compiler:: [[email protected]]
+    compiler:: [[email protected],[email protected]]
     providers:
       mpi:: [[email protected]]
   mpi:

configs/sites/tier1/gaea-c5/compilers.yaml

@@ -23,3 +23,4 @@ compilers:
         # which confuses some packages.
         CONFIG_SITE: ''
     extra_rpaths: []
+# Todo: Define gcc compiler 10.3.0 for Intel backend (ecflow)

configs/sites/tier1/gaea-c5/packages.yaml

@@ -1,6 +1,6 @@
 packages:
   all:
-    compiler:: [[email protected]]
+    compiler:: [[email protected]] # todo: add gcc here
     providers:
       mpi:: [[email protected]]
       # Remove the next three lines to switch to intel-oneapi-mkl

configs/sites/tier1/gaea-c6/compilers.yaml

@@ -23,3 +23,4 @@ compilers:
         # which confuses some packages.
         CONFIG_SITE: ''
     extra_rpaths: []
+# Todo: Define gcc compiler 12.x.y for Intel backend (ecflow)

configs/sites/tier1/gaea-c6/packages.yaml

@@ -1,6 +1,6 @@
 packages:
   all:
-    compiler:: [[email protected]]
+    compiler:: [[email protected]] # todo: add gcc here
     providers:
       mpi:: [[email protected]]
       # Remove the next three lines to switch to intel-oneapi-mkl

configs/sites/tier1/hera/packages_intel.yaml

@@ -1,6 +1,6 @@
 packages:
   all:
-    compiler:: [[email protected]]
+    compiler:: [[email protected],[email protected]]
     providers:
       mpi:: [[email protected]]
       # Remove the next three lines to switch to intel-oneapi-mkl

configs/sites/tier1/hercules/packages_intel.yaml

@@ -1,6 +1,6 @@
 packages:
   all:
-    compiler:: [[email protected]]
+    compiler:: [[email protected],[email protected]]
     providers:
       mpi:: [[email protected]]
       # Remove the next three lines to switch to intel-oneapi-mkl

configs/sites/tier1/jet/packages_intel.yaml

@@ -1,6 +1,6 @@
 packages:
   all:
-    compiler:: [[email protected]]
+    compiler:: [[email protected],[email protected]]
     providers:
       mpi:: [[email protected]]
   mpi:

configs/sites/tier1/narwhal/compilers.yaml

@@ -6,10 +6,7 @@ compilers::
         cxx: CC
         f77: ftn
         fc: ftn
-      flags:
-        cflags: -diag-disable=10441
-        cxxflags: -diag-disable=10441
-        fflags: -diag-disable=10448
+      flags: {}
       operating_system: sles15
       modules:
       - PrgEnv-intel/8.3.3
@@ -27,20 +24,20 @@ compilers::
   - compiler:
       spec: [email protected]
       paths:
-        cc: /opt/cray/pe/gcc/10.3.0/bin/gcc
-        cxx: /opt/cray/pe/gcc/10.3.0/bin/g++
-        f77: /opt/cray/pe/gcc/10.3.0/bin/gfortran
-        fc: /opt/cray/pe/gcc/10.3.0/bin/gfortran
+        cc: cc
+        cxx: CC
+        f77: ftn
+        fc: ftn
       flags: {}
       operating_system: sles15
       modules:
       - PrgEnv-gnu/8.3.3
       - gcc/10.3.0
-      - cray-libsci/22.11.1.2
+      - cray-libsci/23.05.1.4
       - libfabric/1.12.1.2.2.1
       environment:
         prepend_path:
-          LD_LIBRARY_PATH: '/opt/cray/libfabric/1.12.1.2.2.1/lib64:/opt/cray/pe/libsci/22.11.1.2/GNU/9.1/x86_64/lib'
+          LD_LIBRARY_PATH: '/opt/cray/libfabric/1.12.1.2.2.1/lib64:/opt/cray/pe/libsci/23.05.1.4/GNU/10.3/x86_64/lib'
         set:
           CRAYPE_LINK_TYPE: 'dynamic'
       extra_rpaths: []

configs/sites/tier1/narwhal/packages_gcc.yaml

@@ -7,7 +7,7 @@ packages:
     buildable: False
   cray-mpich:
     externals:
-    - spec: [email protected]%[email protected] +wrappers
+    - spec: [email protected]%[email protected] ~wrappers
       modules:
       - cray-mpich-ucx/8.1.21
       - craype-network-ucx

configs/sites/tier1/narwhal/packages_intel.yaml

@@ -1,6 +1,6 @@
 packages:
   all:
-    compiler:: [[email protected]]
+    compiler:: [[email protected],[email protected]]
     providers:
       mpi:: [[email protected]]
   mpi:

configs/sites/tier1/nautilus/compilers.yaml

@@ -35,6 +35,25 @@ compilers:
         CPATH: '/opt/rh/gcc-toolset-11/root/usr/include'
         LD_LIBRARY_PATH: '/opt/scyld/slurm/lib64:/opt/scyld/slurm/lib64/slurm:/p/app/compilers/intel/oneapi/compiler/2022.0.2/linux/compiler/lib/intel64_lin:/opt/rh/gcc-toolset-11/root/usr/lib64:/opt/rh/gcc-toolset-11/root/usr/lib'
     extra_rpaths: []
+- compiler:
+    spec: [email protected]
+    paths:
+      cc: /p/app/projects/NEPTUNE/spack-stack/oneapi-2024.1.2/compiler/2024.1/bin/icx
+      cxx: /p/app/projects/NEPTUNE/spack-stack/oneapi-2024.1.2/compiler/2024.1/bin/icpx
+      f77: /p/app/projects/NEPTUNE/spack-stack/oneapi-2024.1.2/compiler/2024.1/bin/ifort
+      fc: /p/app/projects/NEPTUNE/spack-stack/oneapi-2024.1.2/compiler/2024.1/bin/ifort
+    flags: {}
+    operating_system: rhel8
+    target: x86_64
+    modules: []
+    environment:
+      prepend_path:
+        PATH: '/opt/rh/gcc-toolset-11/root/usr/bin'
+        CPATH: '/opt/rh/gcc-toolset-11/root/usr/include'
+        LD_LIBRARY_PATH: '/opt/scyld/slurm/lib64:/opt/scyld/slurm/lib64/slurm:/opt/rh/gcc-toolset-11/root/usr/lib64:/opt/rh/gcc-toolset-11/root/usr/lib'
+      append_path:
+        CPATH: '/p/app/projects/NEPTUNE/spack-stack/oneapi-2024.1.2/compiler/2024.1/opt/compiler/include/intel64'
+    extra_rpaths: []
 - compiler:
     spec: [email protected]
     paths:
@@ -49,22 +68,18 @@ compilers:
     - scl/gcc-toolset-12
     environment: {}
     extra_rpaths: []
+# Required for Intel backend
 - compiler:
-    spec: [email protected]
+    spec: [email protected]
     paths:
-      cc: /p/app/projects/NEPTUNE/spack-stack/oneapi-2024.1.2/compiler/2024.1/bin/icx
-      cxx: /p/app/projects/NEPTUNE/spack-stack/oneapi-2024.1.2/compiler/2024.1/bin/icpx
-      f77: /p/app/projects/NEPTUNE/spack-stack/oneapi-2024.1.2/compiler/2024.1/bin/ifort
-      fc: /p/app/projects/NEPTUNE/spack-stack/oneapi-2024.1.2/compiler/2024.1/bin/ifort
+      cc: /opt/rh/gcc-toolset-11/root/usr/bin/gcc
+      cxx: /opt/rh/gcc-toolset-11/root/usr/bin/g++
+      f77: /opt/rh/gcc-toolset-11/root/usr/bin/gfortran
+      fc: /opt/rh/gcc-toolset-11/root/usr/bin/gfortran
     flags: {}
     operating_system: rhel8
     target: x86_64
-    modules: []
-    environment:
-      prepend_path:
-        PATH: '/opt/rh/gcc-toolset-11/root/usr/bin'
-        CPATH: '/opt/rh/gcc-toolset-11/root/usr/include'
-        LD_LIBRARY_PATH: '/opt/scyld/slurm/lib64:/opt/scyld/slurm/lib64/slurm:/opt/rh/gcc-toolset-11/root/usr/lib64:/opt/rh/gcc-toolset-11/root/usr/lib'
-      append_path:
-        CPATH: '/p/app/projects/NEPTUNE/spack-stack/oneapi-2024.1.2/compiler/2024.1/opt/compiler/include/intel64'
+    modules:
+    - scl/gcc-toolset-11
+    environment: {}
     extra_rpaths: []

configs/sites/tier1/nautilus/packages_intel.yaml

@@ -1,6 +1,6 @@
 packages:
   all:
-    compiler:: [[email protected]]
+    compiler:: [[email protected],[email protected]]
     providers:
       mpi:: [[email protected]]
   mpi:

configs/sites/tier1/nautilus/packages_oneapi.yaml

@@ -1,6 +1,6 @@
 packages:
   all:
-    compiler:: [[email protected]]
+    compiler:: [[email protected],[email protected]]
     providers:
       mpi:: [[email protected]]
   mpi:

configs/sites/tier1/noaa-aws/packages_intel.yaml

@@ -1,6 +1,6 @@
 packages:
   all:
-    compiler:: [[email protected]]
+    compiler:: [[email protected],[email protected]]
     providers:
       mpi:: [[email protected]]
       # Remove the next three lines to switch to intel-oneapi-mkl

configs/sites/tier1/noaa-azure/packages_intel.yaml

@@ -1,6 +1,6 @@
 packages:
   all:
-    compiler:: [[email protected]]
+    compiler:: [[email protected],[email protected]]
     providers:
       mpi:: [[email protected]]
       # Remove the next three lines to switch to intel-oneapi-mkl

configs/sites/tier1/noaa-gcloud/packages_intel.yaml

@@ -1,6 +1,6 @@
 packages:
   all:
-    compiler:: [[email protected]]
+    compiler:: [[email protected],[email protected]]
     providers:
       mpi:: [[email protected]]
       # Remove the next three lines to switch to intel-oneapi-mkl