Fix container builds, merge/rename spack-stack extension repos, add Intel oneAPI 2024.0.2 compiler/MPI on AWS ParallelCluster

.github/workflows/ubuntu-ci-x86_64.yaml

@@ -116,9 +116,6 @@ jobs:
           # Add additional variants for MET packages, different from config/common/packages.yaml
           spack config add "packages:met:variants:+python +grib2 +graphics +lidar2nc +modis"
 
-          # https://github.com/spack/spack/issues/42137
-          spack config add "packages:pflogger:variants:+mpi"
-
           # Concretize and check for duplicates
           spack concretize 2>&1 | tee log.concretize.intel-2022.1.0
           ${SPACK_STACK_DIR}/util/show_duplicate_packages.py -d log.concretize.intel-2022.1.0 -i fms -i crtm -i esmf -i mapl

.github/workflows/ubuntu-rnd-x86_64.yaml

@@ -64,9 +64,6 @@ jobs:
           # Add additional variants for MET packages, different from config/common/packages.yaml
           spack config add "packages:met:variants:+python +grib2 +graphics +lidar2nc +modis"
 
-          # https://github.com/spack/spack/issues/42137
-          spack config add "packages:pflogger:variants:+mpi"
-
           # Overrides for spack build and staging areas to speed up builds,
           # separate from the default site config locations to avoid conflicts
           spack config add "config:build_stage:/home/ubuntu/spack-stack/CI/tmp/build_stage"
@@ -133,9 +130,6 @@ jobs:
           # Add additional variants for MET packages, different from config/common/packages.yaml
           spack config add "packages:met:variants:+python +grib2 +graphics +lidar2nc +modis"
 
-          # https://github.com/spack/spack/issues/42137
-          spack config add "packages:pflogger:variants:+mpi"
-
           # Overrides for spack build and staging areas to speed up builds,
           # separate from the default site config locations to avoid conflicts
           spack config add "config:build_stage:/home/ubuntu/spack-stack/CI/tmp/build_stage"

configs/common/packages.yaml

@@ -136,11 +136,6 @@
     libyaml:
       version: ['0.2.5']
     mapl:
-      # 2.35.2 goes with [email protected], 2.40.3 goes with [email protected]
-      # turn off ~pflogger and extdata2g to avoid compilation
-      # errors with [email protected]+, see
-      # https://github.com/JCSDA/spack-stack/issues/769
-      # also: ... extdata2g segfault UFS?
       version: ['2.40.3']
       variants: +shared +pflogger ~f2py
     # If making changes here, also check the Discover site config and the CI workflows
@@ -199,6 +194,9 @@
       variants: +pnetcdf
     parallel-netcdf:
       version: ['1.12.2']
+    pflogger:
+      version: ['1.12.0']
+      variants: +mpi
     pixman:
       variants: +pic
     # Do not build pkgconf - https://github.com/jcsda/spack-stack/issues/123

configs/containers/docker-ubuntu-clang-mpich.yaml

@@ -30,7 +30,7 @@ spack:
       require: '%clang'
       target: [x86_64]
       providers:
-        mpi: [[email protected].1]
+        mpi: [[email protected].2]
       compiler: [[email protected]]
     gcc:
       buildable: false
@@ -68,12 +68,6 @@ spack:
       externals:
       - spec: [email protected] +clang
         prefix: /usr
-    mpich:
-      buildable: false
-      externals:
-      - spec: [email protected]
-        prefix: /opt/mpich-4.1.1
-      version: [4.1.1]
     mysql:
       buildable: false
       externals:
@@ -175,47 +169,26 @@ spack:
     # Extra instructions
     extra_instructions:
       pre_build: |
-        #Create symbolic links for clang compilers
+        # Copy the spack-stack extension repo into the spack code
+        COPY spack-ext-SPACK_STACK_HASH/repos/spack-stack $SPACK_ROOT/var/spack/repos/spack-stack
+        # Register spack-stack extension repo
+        RUN spack repo add $SPACK_ROOT/var/spack/repos/spack-stack --scope defaults
+        # Create symbolic links for clang compilers
         RUN cd /usr/bin && \
         ln -svf clang-10 clang && \
         ln -svf clang++-10 clang++ && \
         ln -svf clang-cpp-10 clang-cpp && \
         cd /usr/lib/llvm-10/lib && \
         ln -svf libc++abi.so.1.0 libc++abi.so
-        # Build mpich outside of spack-stack
-        ENV MPICH_VERSION=4.1.1
-        ENV CC=clang
-        ENV CXX=clang++
-        ENV FC=gfortran
-        RUN mkdir -p /opt/mpich-${MPICH_VERSION}/src && \
-        cd /opt/mpich-${MPICH_VERSION}/src && \
-        wget http://www.mpich.org/static/downloads/${MPICH_VERSION}/mpich-${MPICH_VERSION}.tar.gz && \
-        tar -xzf mpich-${MPICH_VERSION}.tar.gz && \
-        cd mpich-${MPICH_VERSION} && \
-        CFLAGS="-fPIC" CXXFLAGS="-fPIC" FCFLAGS="-fPIC" LDFLAGS="-fPIC" \
-        ./configure --prefix=/opt/mpich-${MPICH_VERSION} --enable-fortran --enable-cxx --with-device=ch4:ofi && \
-        make -j2 && \
-        make install && \
-        ls /opt/mpich-${MPICH_VERSION}
-        ENV PATH=/opt/mpich-${MPICH_VERSION}/bin:${PATH}
-        ENV CPATH=/opt/mpich-${MPICH_VERSION}/include:${CPATH}
-        ENV LD_LIBRARY_PATH=/opt/mpich-${MPICH_VERSION}/lib:${LD_LIBRARY_PATH}
       build: |
         # Put output of spack find into a file
         RUN cd /opt/spack-environment && \
         spack env activate -d . && \
         spack find 2>&1 | tee /root/spack_find.out
       pre_final: |
-        #Set environment variables for installing tzdata
+        # Set environment variables for installing tzdata
         ENV DEBIAN_FRONTEND=noninteractive
         ENV TZ=Etc/UTC
-        ENV MPICH_VERSION=4.1.1
-        ENV CC=clang
-        ENV CXX=clang++
-        ENV FC=gfortran
-        ENV PATH=/opt/mpich-${MPICH_VERSION}/bin:${PATH}
-        ENV CPATH=/opt/mpich-${MPICH_VERSION}/include:${CPATH}
-        ENV LD_LIBRARY_PATH=/opt/mpich-${MPICH_VERSION}/lib:${LD_LIBRARY_PATH}
       final: |
         # Create symbolic links for clang compilers
         RUN cd /usr/bin && \
@@ -224,11 +197,7 @@ spack:
         ln -svf clang-cpp-10 clang-cpp && \
         cd /usr/lib/llvm-10/lib && \
         ln -svf libc++abi.so.1.0 libc++abi.so
-        # Copy spack find output from builder
-        ENV MPICH_VERSION=4.1.1
         COPY --from=builder /root/spack_find.out /root/spack_find.out
-        # Copy mpich-${MPICH_VERSION} installation from builder
-        COPY --from=builder /opt/mpich-${MPICH_VERSION} /opt/mpich-${MPICH_VERSION}
         # Make a non-root user:nonroot / group:nonroot for running MPI
         RUN useradd -U -k /etc/skel -s /bin/bash -d /home/nonroot -m nonroot --uid 43891 && \
         echo "ulimit -s unlimited" >> /home/nonroot/.bashrc && \

configs/containers/docker-ubuntu-gcc-openmpi.yaml

@@ -35,12 +35,11 @@ spack:
       externals:
       - spec: [email protected]
         prefix: /usr
-    # Try this next time, already used for clang/intel
-    #gmake:
-    #  buildable: false
-    #  externals:
-    #  - spec: [email protected]
-    #    prefix: /usr
+    gmake:
+      buildable: false
+      externals:
+      - spec: [email protected]
+        prefix: /usr
     diffutils:
       buildable: false
       externals:
@@ -150,8 +149,13 @@ spack:
 
     # Extra instructions
     extra_instructions:
+      pre_build: |
+        # Copy the spack-stack extension repo into the spack code
+        COPY spack-ext-SPACK_STACK_HASH/repos/spack-stack $SPACK_ROOT/var/spack/repos/spack-stack
+        # Register spack-stack extension repo
+        RUN spack repo add $SPACK_ROOT/var/spack/repos/spack-stack --scope defaults
       pre_final: |
-        #Set environment variables for installing tzdata
+        # Set environment variables for installing tzdata
         ENV DEBIAN_FRONTEND=noninteractive
         ENV TZ=Etc/UTC
         ENV CC=gcc

configs/containers/docker-ubuntu-intel-impi.yaml

@@ -171,13 +171,18 @@ spack:
     # Extra instructions
     extra_instructions:
       pre_build: |
+        # Copy the spack-stack extension repo into the spack code
+        COPY spack-ext-SPACK_STACK_HASH/repos/spack-stack $SPACK_ROOT/var/spack/repos/spack-stack
+        # Register spack-stack extension repo
+        RUN spack repo add $SPACK_ROOT/var/spack/repos/spack-stack --scope defaults
+        # Install Intel compilers and MPI library
         RUN apt update && apt install apt-utils && \
         wget -O- https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS.PUB | gpg --dearmor | tee /usr/share/keyrings/oneapi-archive-keyring.gpg > /dev/null && \
         echo "deb [signed-by=/usr/share/keyrings/oneapi-archive-keyring.gpg] https://apt.repos.intel.com/oneapi all main" | tee /etc/apt/sources.list.d/oneAPI.list && \
         apt update && \
         apt install intel-oneapi-compiler-dpcpp-cpp-and-cpp-classic-2022.1.0 intel-oneapi-compiler-fortran-2022.1.0 intel-oneapi-mkl-devel-2022.1.0 intel-oneapi-mpi-devel-2021.6.0 -y
       pre_final: |
-        #Set environment variables for installing tzdata
+        # Set environment variables for installing tzdata
         ENV DEBIAN_FRONTEND=noninteractive
         ENV TZ=Etc/UTC
         ENV CC=icc
@@ -189,6 +194,7 @@ spack:
         spack env activate -d . && \
         spack find 2>&1 | tee /root/spack_find.out
       final: |
+        # Install Intel compilers and MPI library
         RUN apt update  && apt install apt-utils && \
         wget -O- https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS.PUB | gpg --dearmor | tee /usr/share/keyrings/oneapi-archive-keyring.gpg > /dev/null && \
         echo "deb [signed-by=/usr/share/keyrings/oneapi-archive-keyring.gpg] https://apt.repos.intel.com/oneapi all main" | tee /etc/apt/sources.list.d/oneAPI.list && \

configs/sites/aws-pcluster/compilers.yaml

@@ -16,24 +16,26 @@ compilers:
       set:
         I_MPI_PMI_LIBRARY: '/opt/slurm/lib/libpmi.so'
     extra_rpaths: []
-# Spack gets confused if there is an Intel and a OneAPI compiler ...
-#- compiler:
-#    spec: [email protected]
-#    paths:
-#      cc: /opt/intel/oneapi/compiler/2022.1.0/linux/bin/icx
-#      cxx: /opt/intel/oneapi/compiler/2022.1.0/linux/bin/icpx
-#      f77: /opt/intel/oneapi/compiler/2022.1.0/linux/bin/ifx
-#      fc: /opt/intel/oneapi/compiler/2022.1.0/linux/bin/ifx
-#    flags: {}
-#    operating_system: ubuntu20.04
-#    target: x86_64
-#    modules: []
-#    environment:
-#      #prepend_path:
-#      #  LD_LIBRARY_PATH: '/opt/intel/oneapi/compiler/2022.1.0/linux/compiler/lib/intel64_lin'
-#      set:
-#        I_MPI_PMI_LIBRARY: '/opt/slurm/lib/libpmi.so'
-#    extra_rpaths: []
+- compiler:
+    spec: [email protected]
+    paths:
+      cc: /mnt/experiments-efs/intel-oneapi-2024.0.2/compiler/2024.0/bin/icx
+      cxx: /mnt/experiments-efs/intel-oneapi-2024.0.2/compiler/2024.0/bin/icpx
+      f77: /mnt/experiments-efs/intel-oneapi-2024.0.2/compiler/2024.0/bin/ifort
+      fc: /mnt/experiments-efs/intel-oneapi-2024.0.2/compiler/2024.0/bin/ifort
+    flags: {}
+    operating_system: ubuntu20.04
+    target: x86_64
+    modules:
+    - tbb/2021.11
+    - compiler-rt/2024.0.2
+    - oclfpga/2024.0.0
+    - compiler/2024.0.2
+    - ifort/2024.0.2
+    environment:
+      set:
+        I_MPI_PMI_LIBRARY: '/opt/slurm/lib/libpmi.so'
+    extra_rpaths: []
 - compiler:
     spec: [email protected]
     paths:

configs/sites/aws-pcluster/packages.yaml

@@ -1,9 +1,10 @@
 packages:
   all:
     compiler: [[email protected], [email protected]]
-    #compiler: [oneapi@2022.1.0]
+    #compiler: [oneapi@2024.0.2]
     providers:
       mpi: [[email protected], [email protected]]
+      #mpi: [[email protected]]
 
 ### MPI, Python, MKL
   mpi:
@@ -16,11 +17,11 @@ packages:
       - libfabric-aws/1.16.0~amzn4.0
       - intelmpi
     #externals:
-    #- spec: intel-oneapi-mpi@2021.6.0%oneapi@2022.1.0
-    #  prefix: /opt/intel
+    #- spec: intel-oneapi-mpi@2021.11%oneapi@2024.0.2
+    #  prefix: /mnt/experiments-efs/intel-oneapi-2024.0.2
     #  modules:
     #  - libfabric-aws/1.16.0~amzn4.0
-    #  - intelmpi
+    #  - mpi/2021.11
   openmpi:
     externals:
     - spec: [email protected]%[email protected]~cuda~cxx~cxx_exceptions~java~memchecker+pmi~static~wrapper-rpath
@@ -32,10 +33,6 @@ packages:
 
 ### Modification of common packages
 
-  # https://github.com/spack/spack/issues/42137
-  pflogger:
-    variants: +mpi
-
 ### All other external packages listed alphabetically
   autoconf:
     externals:

setup.sh

@@ -1,15 +1,3 @@
-# Use '-i' when sourcing this file to attempt to reset $PATH to system defaults (i.e., based on /etc/profile),
-# then load any modules (e.g., python) critical for loading and working with Spack.
-# This may have unwanted/unexpected behavior on some systems, so use carefully.
-if [ "$1" == "-i" ]; then
-  echo "Resetting PATH to system defaults"
-  PATH=$(env -u PATH -u PROFILEREAD bash -i -c 'echo $PATH')
-  case "$(hostname -s)" in
-    Orion-login-[1-4]) module load python/3.7.5 ;;
-    alogin0[1-3]) module load intel/19.1.3.304 python/3.8.6 ;;
-  esac
-fi
-
 # https://stackoverflow.com/questions/59895/how-can-i-get-the-source-directory-of-a-bash-script-from-within-the-script-itsel
 # Portable way to get current directory
 SPACK_STACK_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
@@ -20,6 +8,11 @@ echo "Setting environment variable SPACK_STACK_DIR to ${SPACK_STACK_DIR}"
 source ${SPACK_STACK_DIR:?}/spack/share/spack/setup-env.sh
 echo "Sourcing spack environment ${SPACK_STACK_DIR}/spack/share/spack/setup-env.sh"
 
+# Get the current hash of the spack-stack code
+export SPACK_STACK_HASH=`git rev-parse --short HEAD`
+echo "Current hash of spack-stack is ${SPACK_STACK_HASH}"
+
+# Register the spack extension
 libpath=${SPACK_STACK_DIR}/spack-ext/lib/jcsda-emc/spack-stack
 if [ -d ${libpath}/stack/cmd ]; then
   spack config --scope defaults add "config:extensions:$libpath"
@@ -31,7 +24,7 @@ fi
 # Register the jcsda-emc repos
 msg1="Added repo with namespace"
 msg2="Repository is already registered with Spack"
-for repo in jcsda-emc jcsda-emc-bundles; do
+for repo in spack-stack; do
   repodir=${SPACK_STACK_DIR}/spack-ext/repos/$repo
   othererrors=$( ( spack repo add $repodir --scope defaults |& grep -v -e "$msg1" -e "$msg2" ) || true )
   if [ $(echo "$othererrors" | grep -c .) -ne 0 ]; then

spack-ext/lib/jcsda-emc/spack-stack/stack/container_env.py

@@ -1,3 +1,4 @@
+import shutil
 import os
 
 import spack
@@ -72,5 +73,23 @@ def write(self):
 
         os.makedirs(self.env_dir, exist_ok=True)
 
+        # Replace the placeholder SPACK_STACK_HASH with the hash stored in the env variable
+        # in the container extra_instructions pre_build string, if applicable.
+        if "pre_build" in \
+                container_yaml["spack"]["container"]["extra_instructions"].keys():
+            old = container_yaml["spack"]["container"]["extra_instructions"]["pre_build"]
+            try:
+                new = old.replace("SPACK_STACK_HASH", os.environ['SPACK_STACK_HASH'])
+            except KeyError:
+                raise Exception("Environment variable 'SPACK_STACK_HASH' not defined")
+            container_yaml["spack"]["container"]["extra_instructions"]["pre_build"] = new
+
         with open(os.path.join(self.env_dir, "spack.yaml"), "w") as f:
             syaml.dump_config(container_yaml, stream=f)
+
+        # Copy the spack-stack extension into the spack/docker env directory
+        current_wdir = os.getcwd()
+        os.chdir(self.env_dir)
+        shutil.copytree(os.path.join(os.environ['SPACK_STACK_DIR'], "spack-ext"), \
+            "spack-ext-" + os.environ['SPACK_STACK_HASH'])
+        os.chdir(current_wdir)

spack-ext/repos/spack-stack/packages/jedi-geos-env/package.py

@@ -0,0 +1,32 @@
+# Copyright 2013-2022 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+from spack.package import *
+
+
+class JediGeosEnv(BundlePackage):
+    """Development environment for JEDI-GEOS"""
+
+    homepage = "https://github.com/JCSDA/jedi-bundle"
+    git = "https://github.com/JCSDA/jedi-bundle.git"
+
+    maintainers("climbfuji", "srherbener")
+
+    version("1.0.0")
+
+    depends_on("jedi-base-env", type="run")
+    #depends_on("blas", type="run")
+    depends_on("mepo", type="run")
+    depends_on("esmf", type="run")
+    # mapl is built as part of GEOS, don't load;
+    # needs external gftl-shared/fargparse/pflogger
+    # depends_on("mapl", type="run")
+    depends_on("gftl-shared", type="run")
+    depends_on("fargparse", type="run")
+    depends_on("pflogger", type="run")
+    #
+    depends_on("py-numpy", type="run")
+
+    # There is no need for install() since there is no code.

spack-ext/repos/spack-stack/repo.yaml

@@ -0,0 +1,2 @@
+repo:
+  namespace: spack-stack