From e6f2974812d5455ef21a4d235710c8b399dea8ac Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Wed, 28 Aug 2024 11:48:36 -0600 Subject: [PATCH 01/26] Update .gitmodules: use spack branch release/1.8.0 --- .gitmodules | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitmodules b/.gitmodules index a69f3e4f1..5c7383f2a 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,7 +1,7 @@ [submodule "spack"] path = spack url = https://github.com/jcsda/spack - branch = spack-stack-dev + branch = release/1.8.0 [submodule "doc/CMakeModules"] path = doc/CMakeModules url = https://github.com/noaa-emc/cmakemodules From a5c5a2a8bb654368b9f730a26ec5cae9b127e6ee Mon Sep 17 00:00:00 2001 From: Stephen Herbener <32968781+srherbener@users.noreply.github.com> Date: Tue, 3 Sep 2024 05:43:11 -0600 Subject: [PATCH 02/26] spack-stack-1.8.0 config for Discover SCU16, SCU17 (#1282) Co-authored-by: role-r2d2-admin --- configs/sites/tier1/discover-scu16/modules.yaml | 2 -- configs/sites/tier1/discover-scu16/packages.yaml | 8 +------- configs/sites/tier1/discover-scu17/modules.yaml | 2 -- configs/sites/tier1/discover-scu17/packages.yaml | 8 +------- 4 files changed, 2 insertions(+), 18 deletions(-) diff --git a/configs/sites/tier1/discover-scu16/modules.yaml b/configs/sites/tier1/discover-scu16/modules.yaml index 4c530d42d..65801706f 100644 --- a/configs/sites/tier1/discover-scu16/modules.yaml +++ b/configs/sites/tier1/discover-scu16/modules.yaml @@ -3,7 +3,5 @@ modules: enable:: - lmod lmod: - exclude: - - ecflow include: - python diff --git a/configs/sites/tier1/discover-scu16/packages.yaml b/configs/sites/tier1/discover-scu16/packages.yaml index 4cfc9f64e..d47b8cbaa 100644 --- a/configs/sites/tier1/discover-scu16/packages.yaml +++ b/configs/sites/tier1/discover-scu16/packages.yaml @@ -38,13 +38,6 @@ packages: externals: - spec: diffutils@3.3 prefix: /usr - ecflow:: - buildable: False - externals: - - spec: ecflow@5.8.4+ui+static_boost - prefix: /discover/swdev/jcsda/spack-stack/ecflow-5.8.4 - modules: - - ecflow/5.8.4 file: externals: - spec: file@5.22 @@ -129,6 +122,7 @@ packages: - spec: pkg-config@0.28 prefix: /usr qt: + buildable: False externals: - spec: qt@5.15.2 prefix: /discover/swdev/jcsda/spack-stack/qt-5.15.2/5.15.2/gcc_64 diff --git a/configs/sites/tier1/discover-scu17/modules.yaml b/configs/sites/tier1/discover-scu17/modules.yaml index 4c530d42d..65801706f 100644 --- a/configs/sites/tier1/discover-scu17/modules.yaml +++ b/configs/sites/tier1/discover-scu17/modules.yaml @@ -3,7 +3,5 @@ modules: enable:: - lmod lmod: - exclude: - - ecflow include: - python diff --git a/configs/sites/tier1/discover-scu17/packages.yaml b/configs/sites/tier1/discover-scu17/packages.yaml index 086f342a5..906a2af31 100644 --- a/configs/sites/tier1/discover-scu17/packages.yaml +++ b/configs/sites/tier1/discover-scu17/packages.yaml @@ -46,13 +46,6 @@ packages: externals: - spec: diffutils@3.6 prefix: /usr - ecflow:: - buildable: False - externals: - - spec: ecflow@5.11.4+ui+static_boost - prefix: /discover/swdev/jcsda/spack-stack/scu17/ecflow-5.11.4 - modules: - - ecflow/5.11.4 findutils: externals: - spec: findutils@4.8.0 @@ -119,6 +112,7 @@ packages: - spec: pkg-config@0.29.2 prefix: /usr qt: + buildable: False externals: - spec: qt@5.15.10 prefix: /usr/local/other/xpdf/4.04/Deps From e689737b9c4e82f8269a4ce910d2610aee532fc5 Mon Sep 17 00:00:00 2001 From: David Huber <69919478+DavidHuber-NOAA@users.noreply.github.com> Date: Wed, 4 Sep 2024 02:44:56 +0000 Subject: [PATCH 03/26] Pin matplotlib and numpy to 3.7.4 and 1.23.5 (#1283) During testing of an early 1.8.0 candidate, it was found that compiling numpy v1.25.x and matplotlib v3.8.x together with Intel caused matplotlib to fail to generate basic plots that included tick marks. Reverting the versions to 1.23.5 and 3.7.4 resolved the issue. This version does not work on Acorn with Intel 19, but other than for newer versions of Intel, numpy-1.24.4 works there. Therefore, there will be a follow-up PR that changes the default py-numpy version on Acorn for Intel 19 only to 1.24.4. --- configs/common/packages.yaml | 6 +++++- configs/sites/tier1/acorn/packages_intel.yaml | 2 +- configs/sites/tier1/aws-pcluster/packages_intel.yaml | 2 +- configs/sites/tier1/derecho/packages_intel.yaml | 2 +- configs/sites/tier1/gaea-c5/packages.yaml | 2 +- configs/sites/tier1/gaea-c6/packages.yaml | 2 +- configs/sites/tier1/hera/packages_intel.yaml | 2 +- configs/sites/tier1/hercules/packages_intel.yaml | 2 +- configs/sites/tier1/nautilus/packages_gcc.yaml | 2 +- configs/sites/tier1/noaa-aws/packages_intel.yaml | 2 +- configs/sites/tier1/noaa-azure/packages_intel.yaml | 2 +- configs/sites/tier1/noaa-gcloud/packages_intel.yaml | 2 +- configs/sites/tier1/orion/packages_intel.yaml | 2 +- configs/sites/tier1/s4/packages.yaml | 2 +- 14 files changed, 18 insertions(+), 14 deletions(-) diff --git a/configs/common/packages.yaml b/configs/common/packages.yaml index 4e78711ea..1deb17e35 100644 --- a/configs/common/packages.yaml +++ b/configs/common/packages.yaml @@ -214,6 +214,9 @@ packages: ## To avoid duplicate packages #py-jinja2: # require: '@3.1.2' + # https://github.com/JCSDA/spack-stack/issues/1276 + py-matplotlib: + require: '@3.7.4' py-meson-python: require: '@0.15.0' # Comment out for now until build problems are solved @@ -226,9 +229,10 @@ packages: require: '@1.5.8 ~mpi' # py-numpy@1.26 causes many build problems with older Python packages # also check Nautilus site config when making changes here + # https://github.com/JCSDA/spack-stack/issues/1276 py-numpy: require: - - '@:1.25' + - '@:1.23.5' py-pandas: require: '+excel' py-pybind11: diff --git a/configs/sites/tier1/acorn/packages_intel.yaml b/configs/sites/tier1/acorn/packages_intel.yaml index 34065c870..5868e5481 100644 --- a/configs/sites/tier1/acorn/packages_intel.yaml +++ b/configs/sites/tier1/acorn/packages_intel.yaml @@ -31,7 +31,7 @@ - '@1.2.1 ~mkl' py-numpy: require:: - - '@:1.25 ^openblas' + - '@:1.23 ^openblas' # *DH eckit: require: diff --git a/configs/sites/tier1/aws-pcluster/packages_intel.yaml b/configs/sites/tier1/aws-pcluster/packages_intel.yaml index 1363ff58f..7fb3d8477 100644 --- a/configs/sites/tier1/aws-pcluster/packages_intel.yaml +++ b/configs/sites/tier1/aws-pcluster/packages_intel.yaml @@ -31,5 +31,5 @@ packages: - '@1.2.1 ~mkl' py-numpy: require:: - - '@:1.25 ^openblas' + - '@:1.23 ^openblas' # *DH diff --git a/configs/sites/tier1/derecho/packages_intel.yaml b/configs/sites/tier1/derecho/packages_intel.yaml index f37ddcacf..dc534a575 100644 --- a/configs/sites/tier1/derecho/packages_intel.yaml +++ b/configs/sites/tier1/derecho/packages_intel.yaml @@ -37,6 +37,6 @@ packages: - '@1.2.1 ~mkl' py-numpy: require:: - - '@:1.25 ^openblas' + - '@:1.23 ^openblas' # *DH diff --git a/configs/sites/tier1/gaea-c5/packages.yaml b/configs/sites/tier1/gaea-c5/packages.yaml index da5173fad..6e13f710e 100644 --- a/configs/sites/tier1/gaea-c5/packages.yaml +++ b/configs/sites/tier1/gaea-c5/packages.yaml @@ -35,7 +35,7 @@ packages: - '@1.2.1 ~mkl' py-numpy: require:: - - '@:1.25 ^openblas' + - '@:1.23 ^openblas' # *DH ### All other external packages listed alphabetically diff --git a/configs/sites/tier1/gaea-c6/packages.yaml b/configs/sites/tier1/gaea-c6/packages.yaml index 92f7d0a7c..75473846c 100644 --- a/configs/sites/tier1/gaea-c6/packages.yaml +++ b/configs/sites/tier1/gaea-c6/packages.yaml @@ -34,7 +34,7 @@ packages: - '@1.2.1 ~mkl' py-numpy: require:: - - '@:1.25 ^openblas' + - '@:1.23 ^openblas' # *DH ### All other external packages listed alphabetically diff --git a/configs/sites/tier1/hera/packages_intel.yaml b/configs/sites/tier1/hera/packages_intel.yaml index 4f8025c5f..5ac52dd8b 100644 --- a/configs/sites/tier1/hera/packages_intel.yaml +++ b/configs/sites/tier1/hera/packages_intel.yaml @@ -30,7 +30,7 @@ packages: - '@1.2.1 ~mkl' py-numpy: require:: - - '@:1.25 ^openblas' + - '@:1.23 ^openblas' # *DH zlib-ng: require: diff --git a/configs/sites/tier1/hercules/packages_intel.yaml b/configs/sites/tier1/hercules/packages_intel.yaml index 78339c1c5..eb4a22dc7 100644 --- a/configs/sites/tier1/hercules/packages_intel.yaml +++ b/configs/sites/tier1/hercules/packages_intel.yaml @@ -30,4 +30,4 @@ packages: - '@1.2.1 ~mkl' py-numpy: require:: - - '@:1.25 ^openblas' + - '@:1.23 ^openblas' diff --git a/configs/sites/tier1/nautilus/packages_gcc.yaml b/configs/sites/tier1/nautilus/packages_gcc.yaml index 841ff05b1..723bfd315 100644 --- a/configs/sites/tier1/nautilus/packages_gcc.yaml +++ b/configs/sites/tier1/nautilus/packages_gcc.yaml @@ -31,4 +31,4 @@ packages: - '@1.2.1 +mkl' py-numpy: require:: - - '@:1.25 ^intel-oneapi-mkl' + - '@:1.23 ^intel-oneapi-mkl' diff --git a/configs/sites/tier1/noaa-aws/packages_intel.yaml b/configs/sites/tier1/noaa-aws/packages_intel.yaml index 995404d2d..2c98fa98f 100644 --- a/configs/sites/tier1/noaa-aws/packages_intel.yaml +++ b/configs/sites/tier1/noaa-aws/packages_intel.yaml @@ -30,5 +30,5 @@ packages: - '@1.2.1 ~mkl' py-numpy: require:: - - '@:1.25 ^openblas' + - '@:1.23 ^openblas' # *DH diff --git a/configs/sites/tier1/noaa-azure/packages_intel.yaml b/configs/sites/tier1/noaa-azure/packages_intel.yaml index 995404d2d..2c98fa98f 100644 --- a/configs/sites/tier1/noaa-azure/packages_intel.yaml +++ b/configs/sites/tier1/noaa-azure/packages_intel.yaml @@ -30,5 +30,5 @@ packages: - '@1.2.1 ~mkl' py-numpy: require:: - - '@:1.25 ^openblas' + - '@:1.23 ^openblas' # *DH diff --git a/configs/sites/tier1/noaa-gcloud/packages_intel.yaml b/configs/sites/tier1/noaa-gcloud/packages_intel.yaml index 995404d2d..2c98fa98f 100644 --- a/configs/sites/tier1/noaa-gcloud/packages_intel.yaml +++ b/configs/sites/tier1/noaa-gcloud/packages_intel.yaml @@ -30,5 +30,5 @@ packages: - '@1.2.1 ~mkl' py-numpy: require:: - - '@:1.25 ^openblas' + - '@:1.23 ^openblas' # *DH diff --git a/configs/sites/tier1/orion/packages_intel.yaml b/configs/sites/tier1/orion/packages_intel.yaml index 88131d85c..8c1e33385 100644 --- a/configs/sites/tier1/orion/packages_intel.yaml +++ b/configs/sites/tier1/orion/packages_intel.yaml @@ -32,4 +32,4 @@ packages: - '@1.2.1 ~mkl' py-numpy: require:: - - '@:1.25 ^openblas' + - '@:1.23 ^openblas' diff --git a/configs/sites/tier1/s4/packages.yaml b/configs/sites/tier1/s4/packages.yaml index 3650e0678..e0312b799 100644 --- a/configs/sites/tier1/s4/packages.yaml +++ b/configs/sites/tier1/s4/packages.yaml @@ -33,7 +33,7 @@ packages: - '@1.2.1 ~mkl' py-numpy: require:: - - '@:1.25 ^openblas' + - '@:1.23 ^openblas' ### All other external packages listed alphabetically autoconf: From 9950bc3cc18aef4855a880eedc73e70615508655 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Thu, 5 Sep 2024 10:45:53 -0600 Subject: [PATCH 04/26] release/1.8.0: Update site configs for Nautilus, Blackpearl, Casper; move release-specific site documentation for preconfigured sites to Wiki (#1280) 1. Site config updates: - Add AOCC config for Nautilus, update oneAPI config; for AOCC, add configs/common/packages_aocc.yaml (identical to packages_gcc.yaml; both use intel-oneapi-mkl from the new oneAPI compilers 2024.2.1) - Update oneAPI config for Blackpearl - Move Casper site config from tier1 to tier2 (long agreed upon and already in the documentation) 2. In neptune-env virtual package, use sp with AOCC and ip with all other compilers (so that we can build the NEPTUNE standalone environment with AOCC) 3. Documentation updates: move release-specific site documentation to Wiki - see https://github.com/JCSDA/spack-stack/wiki , https://github.com/JCSDA/spack-stack/wiki/spack%E2%80%90stack%E2%80%901.8.0-release-documentation , and https://spack-stack--1280.org.readthedocs.build/en/1280/ --- configs/common/packages_aocc.yaml | 15 + configs/sites/tier1/atlantis/compilers.yaml | 1 - configs/sites/tier1/atlantis/modules.yaml | 2 - configs/sites/tier1/narwhal/modules.yaml | 2 - configs/sites/tier1/narwhal/packages.yaml | 8 +- configs/sites/tier1/nautilus/compilers.yaml | 29 +- configs/sites/tier1/nautilus/modules.yaml | 2 - configs/sites/tier1/nautilus/packages.yaml | 12 +- .../sites/tier1/nautilus/packages_aocc.yaml | 35 ++ .../sites/tier1/nautilus/packages_oneapi.yaml | 29 +- configs/sites/tier1/sandy/modules.yaml | 2 - configs/sites/tier2/blackpearl/compilers.yaml | 30 +- .../tier2/blackpearl/packages_oneapi.yaml | 8 +- .../{tier1 => tier2}/casper/compilers.yaml | 0 .../sites/{tier1 => tier2}/casper/config.yaml | 0 .../{tier1 => tier2}/casper/mirrors.yaml | 0 .../{tier1 => tier2}/casper/modules.yaml | 0 .../{tier1 => tier2}/casper/packages.yaml | 0 doc/source/PreConfiguredSites.rst | 483 ++++++------------ .../packages/neptune-env/package.py | 6 +- 20 files changed, 275 insertions(+), 389 deletions(-) create mode 100644 configs/common/packages_aocc.yaml create mode 100644 configs/sites/tier1/nautilus/packages_aocc.yaml rename configs/sites/{tier1 => tier2}/casper/compilers.yaml (100%) rename configs/sites/{tier1 => tier2}/casper/config.yaml (100%) rename configs/sites/{tier1 => tier2}/casper/mirrors.yaml (100%) rename configs/sites/{tier1 => tier2}/casper/modules.yaml (100%) rename configs/sites/{tier1 => tier2}/casper/packages.yaml (100%) diff --git a/configs/common/packages_aocc.yaml b/configs/common/packages_aocc.yaml new file mode 100644 index 000000000..70012d1a0 --- /dev/null +++ b/configs/common/packages_aocc.yaml @@ -0,0 +1,15 @@ +packages: + all: + providers: + blas: [openblas] + fftw-api: [fftw] + lapack: [openblas] + ectrans: + require: + - '~mkl +fftw' + gsibec: + require: + - '~mkl' + py-numpy: + require: + - '^openblas' diff --git a/configs/sites/tier1/atlantis/compilers.yaml b/configs/sites/tier1/atlantis/compilers.yaml index 15e8de396..d6ffdec29 100644 --- a/configs/sites/tier1/atlantis/compilers.yaml +++ b/configs/sites/tier1/atlantis/compilers.yaml @@ -54,4 +54,3 @@ compilers: append_path: CPATH: /cm/shared/apps/intel/oneapi-2024.2.1/compiler/2024.2/opt/compiler/include/intel64/ extra_rpaths: [] -# Todo: Define gcc compiler (default for Atlantis) for Intel (ecflow) diff --git a/configs/sites/tier1/atlantis/modules.yaml b/configs/sites/tier1/atlantis/modules.yaml index e5e02cbd2..11c5514e2 100644 --- a/configs/sites/tier1/atlantis/modules.yaml +++ b/configs/sites/tier1/atlantis/modules.yaml @@ -6,5 +6,3 @@ modules: include: # List of packages for which we need modules that are blacklisted by default - python - exclude: - - ecflow diff --git a/configs/sites/tier1/narwhal/modules.yaml b/configs/sites/tier1/narwhal/modules.yaml index f83e5dc2c..b134d3669 100644 --- a/configs/sites/tier1/narwhal/modules.yaml +++ b/configs/sites/tier1/narwhal/modules.yaml @@ -3,7 +3,5 @@ modules: enable:: - tcl tcl: - exclude: - - ecflow include: - python diff --git a/configs/sites/tier1/narwhal/packages.yaml b/configs/sites/tier1/narwhal/packages.yaml index 9351731bb..49fad5ebe 100644 --- a/configs/sites/tier1/narwhal/packages.yaml +++ b/configs/sites/tier1/narwhal/packages.yaml @@ -19,13 +19,6 @@ packages: externals: - spec: diffutils@3.6 prefix: /usr - ecflow:: - buildable: false - externals: - - spec: ecflow@5.8.4+ui+static_boost - prefix: /p/app/projects/NEPTUNE/spack-stack/ecflow-5.8.4-cray-python-3.9.7.1 - modules: - - ecflow/5.8.4-cray-python-3.9.7.1 findutils: externals: - spec: findutils@4.6.0 @@ -80,6 +73,7 @@ packages: - spec: pkg-config@0.29.2 prefix: /usr qt: + buildable: False externals: - spec: qt@5.15.2 prefix: /p/app/projects/NEPTUNE/spack-stack/qt-5.15.2/5.15.2/gcc_64 diff --git a/configs/sites/tier1/nautilus/compilers.yaml b/configs/sites/tier1/nautilus/compilers.yaml index c4460ac05..aff3f05dd 100644 --- a/configs/sites/tier1/nautilus/compilers.yaml +++ b/configs/sites/tier1/nautilus/compilers.yaml @@ -6,13 +6,11 @@ compilers: cxx: /p/app/compilers/amd/aocc/4.0.0/bin/clang++ f77: /p/app/compilers/amd/aocc/4.0.0/bin/flang fc: /p/app/compilers/amd/aocc/4.0.0/bin/flang - flags: - cflags: null - cxxflags: null - fflags: null + flags: {} operating_system: rhel8 target: x86_64 modules: + - slurm - amd/aocc/4.0.0 - amd/aocl/aocc/4.0 environment: {} @@ -28,6 +26,7 @@ compilers: operating_system: rhel8 target: x86_64 modules: + - slurm - intel/compiler/2022.0.2 environment: prepend_path: @@ -36,23 +35,29 @@ compilers: LD_LIBRARY_PATH: '/opt/scyld/slurm/lib64:/opt/scyld/slurm/lib64/slurm:/p/app/compilers/intel/oneapi/compiler/2022.0.2/linux/compiler/lib/intel64_lin:/opt/rh/gcc-toolset-11/root/usr/lib64:/opt/rh/gcc-toolset-11/root/usr/lib' extra_rpaths: [] - compiler: - spec: oneapi@2024.1.2 + spec: oneapi@2024.2.1 paths: - cc: /p/app/projects/NEPTUNE/spack-stack/oneapi-2024.1.2/compiler/2024.1/bin/icx - cxx: /p/app/projects/NEPTUNE/spack-stack/oneapi-2024.1.2/compiler/2024.1/bin/icpx - f77: /p/app/projects/NEPTUNE/spack-stack/oneapi-2024.1.2/compiler/2024.1/bin/ifort - fc: /p/app/projects/NEPTUNE/spack-stack/oneapi-2024.1.2/compiler/2024.1/bin/ifort + cc: /p/app/projects/NEPTUNE/spack-stack/oneapi-2024.2.1/compiler/2024.2/bin/icx + cxx: /p/app/projects/NEPTUNE/spack-stack/oneapi-2024.2.1/compiler/2024.2/bin/icpx + f77: /p/app/projects/NEPTUNE/spack-stack/oneapi-2024.2.1/compiler/2024.2/bin/ifort + fc: /p/app/projects/NEPTUNE/spack-stack/oneapi-2024.2.1/compiler/2024.2/bin/ifort flags: {} operating_system: rhel8 target: x86_64 - modules: [] + modules: + - slurm + - tbb/2021.13 + - compiler-rt/2024.2.1 + - compiler/2024.2.1 + - ifort/2024.2.1 environment: prepend_path: PATH: '/opt/rh/gcc-toolset-11/root/usr/bin' CPATH: '/opt/rh/gcc-toolset-11/root/usr/include' LD_LIBRARY_PATH: '/opt/scyld/slurm/lib64:/opt/scyld/slurm/lib64/slurm:/opt/rh/gcc-toolset-11/root/usr/lib64:/opt/rh/gcc-toolset-11/root/usr/lib' + MODULEPATH: '/p/app/projects/NEPTUNE/spack-stack/oneapi-2024.2.1/modulefiles' append_path: - CPATH: '/p/app/projects/NEPTUNE/spack-stack/oneapi-2024.1.2/compiler/2024.1/opt/compiler/include/intel64' + CPATH: '/p/app/projects/NEPTUNE/spack-stack/oneapi-2024.2.1/compiler/2024.2/opt/compiler/include/intel64' extra_rpaths: [] - compiler: spec: gcc@12.2.1 @@ -65,6 +70,7 @@ compilers: operating_system: rhel8 target: x86_64 modules: + - slurm - scl/gcc-toolset-12 environment: {} extra_rpaths: [] @@ -80,6 +86,7 @@ compilers: operating_system: rhel8 target: x86_64 modules: + - slurm - scl/gcc-toolset-11 environment: {} extra_rpaths: [] diff --git a/configs/sites/tier1/nautilus/modules.yaml b/configs/sites/tier1/nautilus/modules.yaml index fb279947b..aeb254c2e 100644 --- a/configs/sites/tier1/nautilus/modules.yaml +++ b/configs/sites/tier1/nautilus/modules.yaml @@ -6,5 +6,3 @@ modules: include: # List of packages for which we need modules that are blacklisted by default - python - exclude: - - ecflow diff --git a/configs/sites/tier1/nautilus/packages.yaml b/configs/sites/tier1/nautilus/packages.yaml index f8dbc3501..2a07d88c3 100644 --- a/configs/sites/tier1/nautilus/packages.yaml +++ b/configs/sites/tier1/nautilus/packages.yaml @@ -26,13 +26,6 @@ packages: externals: - spec: diffutils@3.6 prefix: /usr - ecflow:: - buildable: False - externals: - - spec: ecflow@5.8.4+ui+static_boost - prefix: /p/app/projects/NEPTUNE/spack-stack/ecflow-5.8.4 - modules: - - ecflow/5.8.4 findutils: externals: - spec: findutils@4.6.0 @@ -84,6 +77,11 @@ packages: externals: - spec: pkgconf@1.4.2 prefix: /usr + qt: + buildable: False + externals: + - spec: qt@5.15.3 + prefix: /usr sed: externals: - spec: sed@4.5 diff --git a/configs/sites/tier1/nautilus/packages_aocc.yaml b/configs/sites/tier1/nautilus/packages_aocc.yaml new file mode 100644 index 000000000..380d51b6e --- /dev/null +++ b/configs/sites/tier1/nautilus/packages_aocc.yaml @@ -0,0 +1,35 @@ +packages: +# On Nautilus, use intel-oneapi-mkl as provider +# for blas, lapack, fftw-api with AOCC because of +# problems using openblas in downstream applications +# (e.g. py-pandas): +# ELF load command address/offset not properly aligned" when loading libopenblas.so +# https://github.com/OpenMathLib/OpenBLAS/wiki/Faq#ELFoffset + all: + compiler:: [aocc@4.0.0] + providers: + mpi:: [openmpi@4.1.6] + blas:: [intel-oneapi-mkl] + fftw-api:: [intel-oneapi-mkl] + lapack:: [intel-oneapi-mkl] + mpi: + buildable: False + openmpi: + externals: + - spec: openmpi@4.1.6%aocc@4.0.0~cuda~java~memchecker~static~wrapper-rpath fabrics=ucx schedulers=slurm + prefix: /p/app/penguin/openmpi/4.1.6/aocc-4.0.0 + modules: + - penguin/openmpi/4.1.6/aocc-4.0.0 + - penguin/mpi-vars/aocc + - slurm + openblas: + buildable: False + ectrans: + require:: + - '@1.2.0 +mkl ~fftw' + gsibec: + require:: + - '@1.2.1 +mkl' + py-numpy: + require:: + - '@:1.25 ^intel-oneapi-mkl' diff --git a/configs/sites/tier1/nautilus/packages_oneapi.yaml b/configs/sites/tier1/nautilus/packages_oneapi.yaml index ce40a89e7..53254442f 100644 --- a/configs/sites/tier1/nautilus/packages_oneapi.yaml +++ b/configs/sites/tier1/nautilus/packages_oneapi.yaml @@ -1,19 +1,32 @@ packages: all: - compiler:: [oneapi@2024.1.2,gcc@11.2.1] + compiler:: [oneapi@2024.2.1,gcc@11.2.1] providers: - mpi:: [intel-oneapi-mpi@2021.12] + mpi:: [intel-oneapi-mpi@2021.13] mpi: buildable: False intel-oneapi-mpi: externals: - - spec: intel-oneapi-mpi@2021.12%oneapi@2024.1.2 - prefix: /p/app/projects/NEPTUNE/spack-stack/oneapi-2024.1.2 + - spec: intel-oneapi-mpi@2021.13%oneapi@2024.2.1 + prefix: /p/app/projects/NEPTUNE/spack-stack/oneapi-2024.2.1 + modules: + - mpi/2021.13 intel-oneapi-mkl: externals: - - spec: intel-oneapi-mkl@2024.1%oneapi@2024.1.2 - prefix: /p/app/projects/NEPTUNE/spack-stack/oneapi-2024.1.2 + - spec: intel-oneapi-mkl@2024.2%oneapi@2024.2.1 + prefix: /p/app/projects/NEPTUNE/spack-stack/oneapi-2024.2.1 + modules: + - mkl/2024.2 + intel-oneapi-tbb: + externals: + - spec: intel-oneapi-tbb@2021.13%oneapi@2024.2.1 + prefix: /p/app/projects/NEPTUNE/spack-stack/oneapi-2024.2.1 + modules: + - tbb/2021.13 intel-oneapi-runtime: externals: - - spec: intel-oneapi-runtime@2024.1.2%oneapi@2024.1.2 - prefix: /p/app/projects/NEPTUNE/spack-stack/oneapi-2024.1.2 + - spec: intel-oneapi-runtime@2024.2.1%oneapi@2024.2.1 + prefix: /p/app/projects/NEPTUNE/spack-stack/oneapi-2024.2.1 + modules: + - tbb/2021.13 + - compiler-rt/2024.2.1 diff --git a/configs/sites/tier1/sandy/modules.yaml b/configs/sites/tier1/sandy/modules.yaml index fb279947b..aeb254c2e 100644 --- a/configs/sites/tier1/sandy/modules.yaml +++ b/configs/sites/tier1/sandy/modules.yaml @@ -6,5 +6,3 @@ modules: include: # List of packages for which we need modules that are blacklisted by default - python - exclude: - - ecflow diff --git a/configs/sites/tier2/blackpearl/compilers.yaml b/configs/sites/tier2/blackpearl/compilers.yaml index f04aef5bb..420ff1307 100644 --- a/configs/sites/tier2/blackpearl/compilers.yaml +++ b/configs/sites/tier2/blackpearl/compilers.yaml @@ -1,17 +1,4 @@ compilers: -- compiler: - spec: clang@=14.0.6 - paths: - cc: /usr/bin/clang - cxx: /usr/bin/clang++ - f77: null - fc: null - flags: {} - operating_system: oracle9 - target: x86_64 - modules: [] - environment: {} - extra_rpaths: [] - compiler: spec: gcc@=11.4.1 paths: @@ -39,14 +26,15 @@ compilers: environment: {} extra_rpaths: [] - compiler: - spec: oneapi@=2024.1.2 + spec: oneapi@=2024.2.1 paths: - cc: /opt/intel/oneapi/compiler/2024.1/bin/icx - cxx: /opt/intel/oneapi/compiler/2024.1/bin/icpx - f77: /opt/intel/oneapi/compiler/2024.1/bin/ifort - fc: /opt/intel/oneapi/compiler/2024.1/bin/ifort - #f77: /opt/intel/oneapi/compiler/2024.1/bin/ifx - #fc: /opt/intel/oneapi/compiler/2024.1/bin/ifx + cc: /opt/intel/oneapi/2024.2/bin/icx + cxx: /opt/intel/oneapi/2024.2/bin/icpx + # https://github.com/JCSDA/spack-stack/issues/1279 + f77: /opt/intel/oneapi/2024.2/bin/ifort + fc: /opt/intel/oneapi/2024.2/bin/ifort + #f77: /opt/intel/oneapi/2024.2/bin/ifx + #fc: /opt/intel/oneapi/2024.2/bin/ifx flags: {} operating_system: oracle9 target: x86_64 @@ -54,7 +42,7 @@ compilers: environment: # change to append_path once relevant PR is merged append_path: - CPATH: /opt/intel/oneapi/compiler/2024.1/opt/compiler/include/intel64 + CPATH: /opt/intel/oneapi/compiler/2024.2/opt/compiler/include/intel64 extra_rpaths: [] - compiler: spec: aocc@=4.2.0 diff --git a/configs/sites/tier2/blackpearl/packages_oneapi.yaml b/configs/sites/tier2/blackpearl/packages_oneapi.yaml index f4c243c1b..34b40c943 100644 --- a/configs/sites/tier2/blackpearl/packages_oneapi.yaml +++ b/configs/sites/tier2/blackpearl/packages_oneapi.yaml @@ -1,6 +1,6 @@ packages: all: - compiler:: [oneapi@2024.1.2,gcc@11.4.1] + compiler:: [oneapi@2024.2.1,gcc@11.4.1] providers: mpi:: [intel-oneapi-mpi] @@ -8,13 +8,13 @@ packages: buildable: False intel-oneapi-mpi: externals: - - spec: intel-oneapi-mpi@2021.12%oneapi@2024.1.2 + - spec: intel-oneapi-mpi@2021.13%oneapi@2024.2.1 prefix: /opt/intel/oneapi intel-oneapi-mkl: externals: - - spec: intel-oneapi-mkl@2024.1%oneapi@2024.1.2 + - spec: intel-oneapi-mkl@2024.2%oneapi@2024.2.1 prefix: /opt/intel/oneapi intel-oneapi-runtime: externals: - - spec: intel-oneapi-runtime@2024.1.2%oneapi@2024.1.2 + - spec: intel-oneapi-runtime@2024.2.1%oneapi@2024.2.1 prefix: /opt/intel/oneapi diff --git a/configs/sites/tier1/casper/compilers.yaml b/configs/sites/tier2/casper/compilers.yaml similarity index 100% rename from configs/sites/tier1/casper/compilers.yaml rename to configs/sites/tier2/casper/compilers.yaml diff --git a/configs/sites/tier1/casper/config.yaml b/configs/sites/tier2/casper/config.yaml similarity index 100% rename from configs/sites/tier1/casper/config.yaml rename to configs/sites/tier2/casper/config.yaml diff --git a/configs/sites/tier1/casper/mirrors.yaml b/configs/sites/tier2/casper/mirrors.yaml similarity index 100% rename from configs/sites/tier1/casper/mirrors.yaml rename to configs/sites/tier2/casper/mirrors.yaml diff --git a/configs/sites/tier1/casper/modules.yaml b/configs/sites/tier2/casper/modules.yaml similarity index 100% rename from configs/sites/tier1/casper/modules.yaml rename to configs/sites/tier2/casper/modules.yaml diff --git a/configs/sites/tier1/casper/packages.yaml b/configs/sites/tier2/casper/packages.yaml similarity index 100% rename from configs/sites/tier1/casper/packages.yaml rename to configs/sites/tier2/casper/packages.yaml diff --git a/doc/source/PreConfiguredSites.rst b/doc/source/PreConfiguredSites.rst index 319b53b4a..4e6769568 100644 --- a/doc/source/PreConfiguredSites.rst +++ b/doc/source/PreConfiguredSites.rst @@ -3,103 +3,11 @@ Pre-configured sites ************************* -.. note:: - The information on this page usually refers to the latest stable release of spack-stack. There are usually no updates for the latest developmental code for preconfigured sites. Further, current and previous releases of spack-stack are often updated with new packages, and sometimes it is necessary to rebuild spack-stack environments, for example in case an HPC receives a major software update. Also in this case, the documentation available on readthedocs is not updated. Be sure to check the spack-stack Wiki for updates to current and previous releases before using the information below! - -Directory ``configs/sites`` contains site configurations for several HPC systems, as well as minimal configurations for macOS and Linux. The macOS and Linux configurations are **not** meant to be used as is, as user setups and package versions vary considerably. Instructions for adding this information can be found in :numref:`Section %s `. - Pre-configured sites are split into two categories: Tier 1 with officially supported spack-stack installations (see :numref:`Section %s `), and Tier 2 (sites with configuration files that were tested or contributed by others in the past, but that are not officially supported by the spack-stack team; see :numref:`Section %s `). -============================================================= -Officially supported spack-stack 1.7.0 installations (tier 1) -============================================================= +Directories ``configs/sites/tier1`` and ``configs/sites/tier2`` contain site configurations for several HPC systems, as well as minimal configurations for macOS and Linux. The macOS and Linux configurations are **not** meant to be used as is, as user setups and package versions vary considerably. Instructions for adding this information can be found in :numref:`Section %s `. -Ready-to-use spack-stack 1.7.0 installations are available on the following, fully supported platforms. This version supports JEDI-Skylab and various UFS and related applications (UFS Weather Model, EMC Global Workflow, GSI, UFS Short Range Weather Application). Amazon Web Services AMI are available in the US East 1 or 2 regions. - -+---------------------+----------------------------------+-----------------+------------------------------------------------------------------------------+-------------------------------+ -| Organization | System | Compilers | Location | Maintainers (principal/backup)| -+=====================+==================================+=================+==============================================================================+===============================+ -| **HPC platforms** | -+---------------------+----------------------------------+-----------------+------------------------------------------------------------------------------+-------------------------------+ -| | Hercules | GCC, Intel | ``/work/noaa/epic/role-epic/spack-stack/hercules/spack-stack-1.7.0/envs`` | EPIC / JCSDA | -| MSU +----------------------------------+-----------------+------------------------------------------------------------------------------+-------------------------------+ -| | Orion | GCC, Intel | ``/work/noaa/epic/role-epic/spack-stack/orion/spack-stack-1.7.0/envs`` | EPIC / JCSDA | -+---------------------+----------------------------------+-----------------+------------------------------------------------------------------------------+-------------------------------+ -| | Discover SCU16 | GCC, Intel | ``/gpfsm/dswdev/jcsda/spack-stack/scu16/spack-stack-1.7.0/envs`` | JCSDA | -| NASA +----------------------------------+-----------------+------------------------------------------------------------------------------+-------------------------------+ -| | Discover SCU17 | GCC, Intel | ``/gpfsm/dswdev/jcsda/spack-stack/scu17/spack-stack-1.7.0/envs`` | JCSDA | -+---------------------+----------------------------------+-----------------+------------------------------------------------------------------------------+-------------------------------+ -| NCAR-Wyoming + Derecho | GCC, Intel | ``/glade/work/epicufsrt/contrib/spack-stack/derecho/spack-stack-1.7.0/envs`` | EPIC / JCSDA | -+---------------------+----------------------------------+-----------------+------------------------------------------------------------------------------+-------------------------------+ -| NOAA (NCEP) | Acorn | Intel | ``/lfs/h1/emc/nceplibs/noscrub/spack-stack/spack-stack-1.7.0/envs`` | NOAA-EMC | -+---------------------+----------------------------------+-----------------+------------------------------------------------------------------------------+-------------------------------+ -| | Gaea | Intel | ``/ncrc/proj/epic/spack-stack/spack-stack-1.7.0/envs`` | EPIC / NOAA-EMC | -| +----------------------------------+-----------------+------------------------------------------------------------------------------+-------------------------------+ -| NOAA (RDHPCS) | Hera | GCC, Intel | ``/scratch1/NCEPDEV/nems/role.epic/spack-stack/spack-stack-1.7.0/envs`` | EPIC / NOAA-EMC | -| +----------------------------------+-----------------+------------------------------------------------------------------------------+-------------------------------+ -| | Jet | GCC, Intel | ``/mnt/lfs4/HFIP/hfv3gfs/role.epic/spack-stack/spack-stack-1.7.0/envs`` | EPIC / NOAA-EMC | -+---------------------+----------------------------------+-----------------+------------------------------------------------------------------------------+-------------------------------+ -| | Narwhal | GCC, Intel | ``/p/app/projects/NEPTUNE/spack-stack/spack-stack-1.7.0/envs`` | NRL | -| U.S. Navy (HPCMP) +----------------------------------+-----------------+------------------------------------------------------------------------------+-------------------------------+ -| | Nautilus | Intel | ``/p/app/projects/NEPTUNE/spack-stack/spack-stack-1.7.0/envs`` | NRL | -+---------------------+----------------------------------+-----------------+------------------------------------------------------------------------------+-------------------------------+ -| Univ. of Wisconsin | S4 | Intel | ``/data/prod/jedi/spack-stack/spack-stack-1.7.0/envs`` | JCSDA | -+---------------------+----------------------------------+-----------------+------------------------------------------------------------------------------+-------------------------------+ -| **Cloud platforms** | -+---------------------+----------------------------------+-----------------+------------------------------------------------------------------------------+-------------------------------+ -| | AMI Red Hat 8 | GCC | ``/home/ec2-user/spack-stack/spack-stack-1.7.0/envs`` | JCSDA | -+ Amazon Web Services +----------------------------------+-----------------+------------------------------------------------------------------------------+-------------------------------+ -| | Parallelcluster JCSDA R&D | GCC, Intel | *currently unavailable* | JCSDA | -+---------------------+----------------------------------+-----------------+------------------------------------------------------------------------------+-------------------------------+ -| NOAA (RDHPCS) | RDHPCS Cloud (Parallel Works) | Intel | ``/contrib/spack-stack/spack-stack-1.7.0/envs`` | EPIC / JCSDA | -+---------------------+----------------------------------+-----------------+------------------------------------------------------------------------------+-------------------------------+ - -.. note:: - This release of spack-stack uses different versions of ``mapl`` with different variants, depending on the version of the compiler and whether the system is used for UFS or GEOS. Please see the following table. - -+----------------------------+--------------------------------------+-----------------------------------------------------------------------+ -| Compiler | mapl configuration | Affected systems | -+============================+======================================+=======================================================================+ -| gcc (any) | ``mapl@2.40.3 +pflogger +extdata2g`` | All systems with GCC stacks | -+----------------------------+--------------------------------------+-----------------------------------------------------------------------+ -| intel@2021.6.0 and earlier | ``mapl@2.40.3 +pflogger +extdata2g`` | Discover SCU16, Acorn, Hera, Jet, Narwhal, Nautilus, S4, RDHPCS Cloud | -+----------------------------+--------------------------------------+-----------------------------------------------------------------------+ -| intel@2021.7.0 and later | ``mapl@2.40.3 ~pflogger ~extdata2g`` | Hercules, Orion, Acorn, Gaea and Derecho | -+----------------------------+--------------------------------------+-----------------------------------------------------------------------+ -| intel@2021.7.0 and later | ``mapl@2.43.0 +pflogger +extdata2g`` | Discover SCU17 | -+----------------------------+--------------------------------------+-----------------------------------------------------------------------+ - -.. note:: - We have noted problems on some - not all - platforms with ``intel@2021.5.0`` when we switched from ``zlib`` to ``zlib-ng`` in spack-stack-1.7.0. These issues went away when using a different version of the compiler (anything between 2021.3.0 and 2021.11.0). It is therefore recommended to avoid using ``intel@2021.5.0`` unless it is the only option. - -**To use one of the above installations** via the system default environment module system, and adding certain modules first (see individual sections below), add ``/install/modulefiles/Core`` to the path from the above table and prepend that path to $MODULEPATH, e.g., - -.. code-block:: console - - # On Gaea: - module use /ncrc/proj/epic/spack-stack/spack-stack-1.6.0/envs/unified-env/install/modulefiles/Core - module load stack-intel - module load bacio netcdf-c ... - -For more information about a specific platform, please see the individual sections below. - -For questions or problems, please consult the known issues in :numref:`Section %s `, the currently open GitHub `issues `_ and `discussions `_ first. - -.. _supplemental_environments: - -========================= -Supplemental environments -========================= - -The following is a list of supplemental or "add-on" environments that are maintained through spack-stack. Note that not all are included with every release; see the third column to determine release location and look under ``envs/`` subdirectory (i.e., same parent directory as ``ue-*`` directories per the above table). Check the installation directories to verify which package versions are available before using them. - -+------------------+---------------------------------------------------------+------------------------+-------------------------------------------+ -| Environment name | Description | spack-stack release(s) | Platforms | -+==================+=========================================================+========================+===========================================+ -| gsi-addon-* | Supports GSI and related applications | 1.6.0, 1.7.0 | Hera, Hercules, Gaea, Jet, S4 | -+------------------+---------------------------------------------------------+------------------------+-------------------------------------------+ -| ufswm-* | Supports UFS Weather Model with WCOSS2 package versions | 1.6.0 | Acorn, Hera, Hercules, Jet, Orion | -+------------------+---------------------------------------------------------+------------------------+-------------------------------------------+ +As of spack-stack-1.8.0, this page provides general information on the supported platforms, such as the location of the spack-stack installations on tier 1 platforms and instructions on how to set up an environment for **building** spack-stack environments. Information on **using** spack-stack environments for development of downstream applications is available on the spack-stack wiki: https://github.com/JCSDA/spack-stack/wiki .. _Preconfigured_Sites_Tier1: @@ -107,45 +15,69 @@ The following is a list of supplemental or "add-on" environments that are mainta Pre-configured sites (tier 1) ============================================================= ++---------------------+-----------------------+--------------------+--------------------------------------------------------+-----------------+ +| Organization | System | Compilers | Location of top-level spack-stack directory | Maintainers | ++=====================+=======================+====================+========================================================+=================+ +| **HPC platforms** | ++---------------------+-----------------------+--------------------+--------------------------------------------------------+-----------------+ +| | Hercules | GCC, Intel | ``/work/noaa/epic/role-epic/spack-stack/hercules/`` | EPIC / JCSDA | +| MSU +-----------------------+--------------------+--------------------------------------------------------+-----------------+ +| | Orion | GCC, Intel | ``/work/noaa/epic/role-epic/spack-stack/orion/`` | EPIC / JCSDA | ++---------------------+-----------------------+--------------------+--------------------------------------------------------+-----------------+ +| | Discover SCU16 | GCC, Intel | ``/gpfsm/dswdev/jcsda/spack-stack/scu16/`` | JCSDA | +| NASA +-----------------------+--------------------+--------------------------------------------------------+-----------------+ +| | Discover SCU17 | GCC, Intel | ``/gpfsm/dswdev/jcsda/spack-stack/scu17/`` | JCSDA | ++---------------------+-----------------------+--------------------+--------------------------------------------------------+-----------------+ +| NCAR-Wyoming + Derecho | GCC, Intel | ``/glade/work/epicufsrt/contrib/spack-stack/derecho/`` | EPIC / JCSDA | ++---------------------+-----------------------+--------------------+--------------------------------------------------------+-----------------+ +| NOAA (NCEP) | Acorn | Intel | ``/lfs/h1/emc/nceplibs/noscrub/spack-stack/`` | NOAA-EMC | ++---------------------+-----------------------+--------------------+--------------------------------------------------------+-----------------+ +| | Gaea | Intel | ``/ncrc/proj/epic/spack-stack/`` | EPIC / NOAA-EMC | +| +-----------------------+--------------------+--------------------------------------------------------+-----------------+ +| NOAA (RDHPCS) | Hera | GCC, Intel | ``/scratch1/NCEPDEV/nems/role.epic/spack-stack/`` | EPIC / NOAA-EMC | +| +-----------------------+--------------------+--------------------------------------------------------+-----------------+ +| | Jet | GCC, Intel | ``/mnt/lfs4/HFIP/hfv3gfs/role.epic/spack-stack/`` | EPIC / NOAA-EMC | ++---------------------+-----------------------+--------------------+--------------------------------------------------------+-----------------+ +| | Narwhal | GCC, Intel, oneAPI | ``/p/app/projects/NEPTUNE/spack-stack/`` | NRL | +| U.S. Navy (HPCMP) +-----------------------+--------------------+--------------------------------------------------------+-----------------+ +| | Nautilus | Intel | ``/p/app/projects/NEPTUNE/spack-stack/`` | NRL | ++---------------------+-----------------------+--------------------+--------------------------------------------------------+-----------------+ +| Univ. of Wisconsin | S4 | Intel | ``/data/prod/jedi/spack-stack/`` | JCSDA | ++---------------------+-----------------------+--------------------+--------------------------------------------------------+-----------------+ +| **Cloud platforms** | ++---------------------+-----------------------+--------------------+--------------------------------------------------------+-----------------+ +| | AMI Red Hat 8 | GCC | ``/home/ec2-user/spack-stack/`` | JCSDA | ++ Amazon Web Services +-----------------------+--------------------+--------------------------------------------------------+-----------------+ +| | Parallelcluster JCSDA | GCC, Intel | *currently unavailable* | JCSDA | ++---------------------+-----------------------+--------------------+--------------------------------------------------------+-----------------+ +| NOAA (RDHPCS) | RDHPCS Parallel Works | Intel | ``/contrib/spack-stack/`` | EPIC / JCSDA | ++---------------------+-----------------------+--------------------+--------------------------------------------------------+-----------------+ + .. _Preconfigured_Sites_Orion: ------------------------------ MSU Orion ------------------------------ -The following is required for building new spack environments and for using spack to build and run software. +The following is required for building new spack environments with any supported compiler on this platform. -.. note:: For spack-stack develop, use module path ``/work/noaa/epic/role-epic/spack-stack/orion/modulefiles-rocky9`` instead of ``/work/noaa/epic/role-epic/spack-stack/orion/modulefiles``. Note also that the ``openmpi`` version for ``gcc@12.2.0`` changes to ``4.1.4``. +**NEEDS UPDATING** .. code-block:: console module purge - module use /work/noaa/epic/role-epic/spack-stack/orion/modulefiles - module load ecflow/5.8.4 - -For ``spack-stack-1.7.0`` with Intel, load the following modules after loading miniconda and ecflow: + module use /work/noaa/epic/role-epic/spack-stack/orion/modulefiles-rocky9 -.. code-block:: console - - module use /work/noaa/epic/role-epic/spack-stack/orion/spack-stack-1.7.0/envs/ue-intel/install/modulefiles/Core - module load stack-intel/2021.9.0 - module load stack-intel-oneapi-mpi/2021.9.0 - module load stack-python/3.10.13 - -For ``spack-stack-1.7.0`` with GNU, load the following modules after loading miniconda and ecflow: -.. code-block:: console - - module use /work/noaa/epic/role-epic/spack-stack/orion/spack-stack-1.7.0/envs/ue-gcc/install/modulefiles/Core - module load stack-gcc/12.2.0 - module load stack-openmpi/4.1.6 - module load stack-python/3.10.13 +.. _Preconfigured_Sites_Hercules: ------------------------------ MSU Hercules ------------------------------ -The following is required for building new spack environments and for using spack to build and run software. +The following is required for building new spack environments with any supported compiler on this platform. + +**NEEDS UPDATING** .. code-block:: console @@ -154,23 +86,6 @@ The following is required for building new spack environments and for using spac module load ecflow/5.8.4 module load git-lfs/3.1.2 -For ``spack-stack-1.7.0`` with Intel, proceed with loading the following modules: - -.. code-block:: console - - module use /work/noaa/epic/role-epic/spack-stack/hercules/spack-stack-1.7.0/envs/ue-intel/install/modulefiles/Core - module load stack-intel/2021.9.0 - module load stack-intel-oneapi-mpi/2021.9.0 - module load stack-python/3.10.13 - -For ``spack-stack-1.7.0`` with GNU, proceed with loading the following modules: - -.. code-block:: console - - module use /work/noaa/epic/role-epic/spack-stack/hercules/spack-stack-1.7.0/envs/ue-gcc/install/modulefiles/Core - module load stack-gcc/12.2.0 - module load stack-openmpi/4.1.6 - module load stack-python/3.10.13 .. _Preconfigured_Sites_Discover_SCU16: @@ -178,7 +93,9 @@ For ``spack-stack-1.7.0`` with GNU, proceed with loading the following modules: NASA Discover SCU16 ------------------------------ -The following is required for building new spack environments and for using spack to build and run software. +The following is required for building new spack environments with any supported compiler on this platform. + +**NEEDS UPDATING** .. code-block:: console @@ -188,29 +105,16 @@ The following is required for building new spack environments and for using spac module load miniconda/3.9.7 module load ecflow/5.8.4 -For ``spack-stack-1.7.0`` with Intel, proceed with loading the following modules: - -.. code-block:: console - - module use /gpfsm/dswdev/jcsda/spack-stack/scu16/spack-stack-1.7.0/envs/ue-intel-2021.6.0/install/modulefiles/Core - module load stack-intel/2021.6.0 - module load stack-intel-oneapi-mpi/2021.6.0 - module load stack-python/3.10.13 - -For ``spack-stack-1.7.0`` with GNU, proceed with loading the following modules: -.. code-block:: console - - module use /gpfsm/dswdev/jcsda/spack-stack/scu16/spack-stack-1.7.0/envs/ue-gcc-12.1.0/install/modulefiles/Core - module load stack-gcc/12.1.0 - module load stack-openmpi/4.1.3 - module load stack-python/3.10.13 +.. _Preconfigured_Sites_Discover_SCU17: ------------------------------ NASA Discover SCU17 ------------------------------ -The following is required for building new spack environments and for using spack to build and run software. +The following is required for building new spack environments with any supported compiler on this platform. + +**NEEDS UPDATING** .. code-block:: console @@ -219,23 +123,6 @@ The following is required for building new spack environments and for using spac module use /discover/swdev/jcsda/spack-stack/scu17/modulefiles module load ecflow/5.11.4 -For ``spack-stack-1.7.0`` with Intel, load the following modules after loading ecflow: - -.. code-block:: console - - module use /gpfsm/dswdev/jcsda/spack-stack/scu17/spack-stack-1.7.0/envs/ue-intel-2021.10.0/install/modulefiles/Core - module load stack-intel/2021.10.0 - module load stack-intel-oneapi-mpi/2021.10.0 - module load stack-python/3.10.13 - -For ``spack-stack-1.7.0`` with GNU, load the following modules after loading ecflow: - -.. code-block:: console - - module use /gpfsm/dswdev/jcsda/spack-stack/scu17/spack-stack-1.7.0/envs/ue-gcc-12.3.0/install/modulefiles/Core - module load stack-gcc/12.3.0 - module load stack-openmpi/4.1.6 - module load stack-python/3.10.13 .. _Preconfigured_Sites_Narwhal: @@ -243,7 +130,7 @@ For ``spack-stack-1.7.0`` with GNU, load the following modules after loading ecf NAVY HPCMP Narwhal ------------------------------ -With Intel, the following is required for building new spack environments and for using spack to build and run software. Don't use ``module purge`` on Narwhal! +The following is required for building new spack environments with Intel on this platform.. Don't use ``module purge`` on Narwhal! .. code-block:: console @@ -266,20 +153,7 @@ With Intel, the following is required for building new spack environments and fo module unload cray-libsci module load cray-libsci/23.05.1.4 -THIS SECTION IS OUT OF DATE, REFER TO 1.7.0 RELEASE DOCUMENTATION - - For ``spack-stack-1.7.0`` with Intel, proceed with loading the following modules: - -.. code-block:: console - - # These extra steps are required for performance reason, ofi is about 30% slower than ucx - # Note we can't load craype-network-ucx for building spack-stack environments, must do here - module unload craype-network-ofi - module load craype-network-ucx - module use /p/app/projects/NEPTUNE/spack-stack/spack-stack-1.7.0/envs/ue-intel-2021.4.0/install/modulefiles/Core - module load stack-intel/2021.10.0 - module load stack-cray-mpich/8.1.21 - module load stack-python/3.10.13 - -With GNU, the following is required for building new spack environments and for using spack to build and run software. Don't use ``module purge`` on Narwhal! +The following is required for building new spack environments with GNU on this platform.. Don't use ``module purge`` on Narwhal! .. code-block:: console @@ -302,18 +176,6 @@ With GNU, the following is required for building new spack environments and for module unload cray-libsci module load cray-libsci/23.05.1.4 -THIS SECTION IS OUT OF DATE, REFER TO 1.7.0 RELEASE DOCUMENTATION - For ``spack-stack-1.7.0`` with GNU, proceed with loading the following modules: - -.. code-block:: console - - # These extra steps are required for performance reason, ofi is about 30% slower than ucx - # Note we can't load craype-network-ucx for building spack-stack environments, must do here - module unload craype-network-ofi - module load craype-network-ucx - module use /p/app/projects/NEPTUNE/spack-stack/spack-stack-1.7.0/envs/ue-gcc-10.3.0/install/modulefiles/Core - module load stack-gcc/10.3.0 - module load stack-cray-mpich/8.1.21 - module load stack-python/3.10.13 .. _Preconfigured_Sites_Nautilus: @@ -321,32 +183,13 @@ THIS SECTION IS OUT OF DATE, REFER TO 1.7.0 RELEASE DOCUMENTATION - For ``spack- NAVY HPCMP Nautilus ------------------------------ -With Intel, the following is required for building new spack environments and for using spack to build and run software. +The following is required for building new spack environments with any supported compiler on this platform. .. code-block:: console umask 0022 module purge -For ``spack-stack-1.7.0`` with Intel, proceed with loading the following modules: - -.. code-block:: console - - module use /p/app/projects/NEPTUNE/spack-stack/spack-stack-1.7.0/envs/ue-intel-2021.5.0/install/modulefiles/Core - module load stack-intel/2021.5.0 - module load stack-openmpi/4.1.6 - module load stack-python/3.10.13 - -With AMD clang/flang (aocc), the following is required for building new spack environments and for using spack to build and run software. - -.. code-block:: console - - umask 0022 - module purge - -.. note:: - - ``spack-stack-1.7.0`` is not yet supported with the Arm clang/flang compilers. Use Intel instead. .. _Preconfigured_Sites_Derecho: @@ -354,7 +197,9 @@ With AMD clang/flang (aocc), the following is required for building new spack en NCAR-Wyoming Derecho -------------------- -The following is required for building new spack environments and for using spack to build and run software. +The following is required for building new spack environments with any supported compiler on this platform. + +**NEEDS UPDATING** .. code-block:: console @@ -365,26 +210,6 @@ The following is required for building new spack environments and for using spac module use /glade/work/epicufsrt/contrib/spack-stack/derecho/modulefiles module load ecflow/5.8.4 -For ``spack-stack-1.7.0`` with Intel, proceed with loading the following modules: - -.. code-block:: console - - module use /glade/work/epicufsrt/contrib/spack-stack/derecho/spack-stack-1.7.0/envs/ue-intel/install/modulefiles/Core - module load stack-intel/2021.10.0 - module load stack-cray-mpich/8.1.25 - module load stack-python/3.10.13 - -For ``spack-stack-1.7.0`` with GNU, proceed with loading the following modules: - -.. code-block:: console - - module use /glade/work/epicufsrt/contrib/spack-stack/derecho/spack-stack-1.7.0/envs/ue-gcc/install/modulefiles/Core - module load stack-gcc/12.2.0 - module load stack-cray-mpich/8.1.25 - module load stack-python/3.10.13 - -.. note:: - CISL restricts the amount of memory available for processes on the login nodes. For example, it is impossible to compile JEDI with even one task (``make -j1``) with the Intel compiles in release mode (``-O2``). We therefore recommend compiling on compute nodes using interactive jobs, if possible. .. _Preconfigured_Sites_Acorn: @@ -392,6 +217,8 @@ For ``spack-stack-1.7.0`` with GNU, proceed with loading the following modules: NOAA Acorn (WCOSS2 test system) ------------------------------- +**NEEDS UPDATING** + For spack-stack-1.7.0, the meta modules are in ``/lfs/h1/emc/nceplibs/noscrub/spack-stack/spack-stack-1.7.0/envs/ue-intel{19,2022}/modulefiles/Core``. On WCOSS2 OpenSUSE sets ``CONFIG_SITE`` which causes libraries to be installed in ``lib64``, breaking the ``lib`` assumption made by some packages. Therefore, ``CONFIG_SITE`` should be set to empty in ``compilers.yaml``. Also, don't use ``module purge`` on Acorn! @@ -413,7 +240,9 @@ Note that certain packages, such as recent versions of `py-scipy`, cannot be com NOAA Parallel Works (AWS, Azure, Gcloud) ---------------------------------------- -The following is required for building new spack environments and for using spack to build and run software. The default module path needs to be removed, otherwise spack detects the system as Cray. +The following is required for building new spack environments with any supported compiler on this platform. The default module path needs to be removed, otherwise spack detects the system as Cray. + +**NEEDS UPDATING** .. code-block:: console @@ -425,22 +254,16 @@ The following is required for building new spack environments and for using spac module load ecflow/5.8.4 module load git-lfs/2.4.1 -For ``spack-stack-1.7.0`` with Intel, proceed with loading the following modules: - -.. code-block:: console - - module use /contrib/spack-stack/spack-stack-1.7.0/envs/ue-intel-2021.3.0/install/modulefiles/Core - module load stack-intel/2021.3.0 - module load stack-intel-oneapi-mpi/2021.3.0 - module load stack-python/3.10.13 -.. _Preconfigured_Sites_Gaea: +.. _Preconfigured_Sites_Gaea_C5: ------------------------------ -NOAA RDHPCS Gaea +NOAA RDHPCS Gaea C5 ------------------------------ -The following is required for building new spack environments and for using spack to build and run software. Log into a head node, and don't use ``module purge`` on Gaea! +The following is required for building new spack environments with Intel on this platform.. Don't use ``module purge`` on Gaea! + +**NEEDS UPDATING** .. code-block:: console @@ -452,18 +275,41 @@ The following is required for building new spack environments and for using spac module use /ncrc/proj/epic/spack-stack/modulefiles module load ecflow/5.8.4 -For ``spack-stack-1.7.0`` with Intel, proceed with loading the following modules: + +.. note:: + On Gaea, running ``module available`` without the option ``-t`` can lead to an error: ``/usr/bin/lua5.3: /opt/cray/pe/lmod/lmod/libexec/Spider.lua:568: stack overflow`` + +.. note:: + On Gaea, a current limitation is that any executable that is linked against the MPI library (``cray-mpich``) must be run through ``srun`` on a compute node, even if it is run serially (one process). This is in particular a problem when using ``ctest`` for unit testing created by the ``ecbuild add_test`` macro. A workaround is to use the `cmake` cross-compiling emulator for this: + +.. code-block:: console + + cmake -DCMAKE_CROSSCOMPILING_EMULATOR="/usr/bin/srun;-n;1" -DMPIEXEC_EXECUTABLE="/usr/bin/srun" -DMPIEXEC_NUMPROC_FLAG="-n" PATH_TO_SOURCE + + +.. _Preconfigured_Sites_Gaea_C6: + +------------------------------ +NOAA RDHPCS Gaea C6 +------------------------------ + +The following is required for building new spack environments with Intel on this platform.. Don't use ``module purge`` on Gaea! + +**NEEDS UPDATING** .. code-block:: console - module use /ncrc/proj/epic/spack-stack/spack-stack-1.7.0/envs/ue-intel/install/modulefiles/Core - module load stack-intel/2023.1.0 - module load stack-cray-mpich/8.1.25 - module load stack-python/3.10.13 - module -t available + module load PrgEnv-intel/8.3.3 + module load intel-classic/2023.1.0 + module load cray-mpich/8.1.25 + module load python/3.9.12 + + module use /ncrc/proj/epic/spack-stack/modulefiles + module load ecflow/5.8.4 + .. note:: - On Gaea, running ``module available`` without the option ``-t`` leads to an error: ``/usr/bin/lua5.3: /opt/cray/pe/lmod/lmod/libexec/Spider.lua:568: stack overflow`` + On Gaea, running ``module available`` without the option ``-t`` can lead to an error: ``/usr/bin/lua5.3: /opt/cray/pe/lmod/lmod/libexec/Spider.lua:568: stack overflow`` .. note:: On Gaea, a current limitation is that any executable that is linked against the MPI library (``cray-mpich``) must be run through ``srun`` on a compute node, even if it is run serially (one process). This is in particular a problem when using ``ctest`` for unit testing created by the ``ecbuild add_test`` macro. A workaround is to use the `cmake` cross-compiling emulator for this: @@ -472,13 +318,16 @@ For ``spack-stack-1.7.0`` with Intel, proceed with loading the following modules cmake -DCMAKE_CROSSCOMPILING_EMULATOR="/usr/bin/srun;-n;1" -DMPIEXEC_EXECUTABLE="/usr/bin/srun" -DMPIEXEC_NUMPROC_FLAG="-n" PATH_TO_SOURCE + .. _Preconfigured_Sites_Hera: ------------------------------ NOAA RDHPCS Hera ------------------------------ -The following is required for building new spack environments and for using spack to build and run software. +The following is required for building new spack environments with any supported compiler on this platform. + +**NEEDS UPDATING** .. code-block:: console @@ -487,25 +336,9 @@ The following is required for building new spack environments and for using spac module load miniconda3/4.12.0 module load ecflow/5.8.4 -For ``spack-stack-1.7.0`` with Intel, proceed with loading the following modules: - -.. code-block:: console - - module use /scratch1/NCEPDEV/nems/role.epic/spack-stack/spack-stack-1.7.0/envs/ue-intel/install/modulefiles/Core - module load stack-intel/2021.5.0 - module load stack-intel-oneapi-mpi/2021.5.1 - module load stack-python/3.10.13 - -For ``spack-stack-1.7.0`` with GNU, proceed with loading the following modules: - -.. code-block:: console - - module use /scratch1/NCEPDEV/nems/role.epic/spack-stack/spack-stack-1.7.0/envs/ue-gcc/install/modulefiles/Core - module load stack-gcc/9.2.0 - module load stack-openmpi/4.1.5 - module load stack-python/3.10.13 +.. note:: + On Hera, a dedicated node exists for ``ecflow`` server jobs (``hecflow01``). Users starting ``ecflow_server`` on the regular login nodes will see their servers being killed every few minutes, and may be barred from accessing the system. -Note that on Hera, a dedicated node exists for ``ecflow`` server jobs (``hecflow01``). Users starting ``ecflow_server`` on the regular login nodes will see their servers being killed every few minutes, and may be barred from accessing the system. .. _Preconfigured_Sites_Jet: @@ -513,7 +346,9 @@ Note that on Hera, a dedicated node exists for ``ecflow`` server jobs (``hecflow NOAA RDHPCS Jet ------------------------------ -The following is required for building new spack environments and for using spack to build and run software. +The following is required for building new spack environments with any supported compiler on this platform. + +**NEEDS UPDATING** .. code-block:: console @@ -523,29 +358,16 @@ The following is required for building new spack environments and for using spac module load ecflow/5.5.3 module use /lfs4/HFIP/hfv3gfs/role.epic/modulefiles -For ``spack-stack-1.7.0`` with Intel, proceed with loading the following modules: - -.. code-block:: console - module use /mnt/lfs4/HFIP/hfv3gfs/role.epic/spack-stack/spack-stack-1.7.0/envs/ue-intel/install/modulefiles/Core - module load stack-intel/2021.5.0 - module load stack-intel-oneapi-mpi/2021.5.1 - module load stack-python/3.10.8 - -For ``spack-stack-1.7.0`` with GNU, proceed with loading the following modules: - -.. code-block:: console - - module use /mnt/lfs4/HFIP/hfv3gfs/role.epic/spack-stack/spack-stack-1.7.0/envs/ue-gcc/install/modulefiles/Core - module load stack-gcc/9.2.0 - module load stack-openmpi/3.1.4 - module load stack-python/3.10.8 +.. _Preconfigured_Sites_S4: ------------------------------ UW (Univ. of Wisconsin) S4 ------------------------------ -The following is required for building new spack environments and for using spack to build and run software. +The following is required for building new spack environments with any supported compiler on this platform. + +**NEEDS UPDATING** .. code-block:: console @@ -554,44 +376,28 @@ The following is required for building new spack environments and for using spac module load miniconda/3.9.12 module load ecflow/5.8.4 -For ``spack-stack-1.7.0`` with Intel, proceed with loading the following modules: -.. code-block:: console - - module use /data/prod/jedi/spack-stack/spack-stack-1.7.0/envs/ue-intel-2021.5.0/install/modulefiles/Core - module load stack-intel/2021.5.0 - module load stack-intel-oneapi-mpi/2021.5.0 - module load stack-python/3.10.13 - module unuse /opt/apps/modulefiles/Compiler/intel/non-default/22 - module unuse /opt/apps/modulefiles/Compiler/intel/22 - -Note the two `module unuse` commands, that need to be run after the stack metamodules are loaded. Loading the Intel compiler meta module loads the Intel compiler module provided by the sysadmins, which adds those two directories to the module path. These contain duplicate libraries that are not compatible with our stack, such as ``hdf4``. - -.. note:: - There is currently no support for GNU on S4, because recent updates to ``hdf5`` require a newer version of ``mpich`` (or other MPI library) than available on the system. Also, for spack-stack-1.7.0, S4 is the only system that uses ``zlib`` instead of ``zlib-ng`` due to the issues described in https://github.com/JCSDA/spack-stack/issues/1055. +.. _Preconfigured_Sites_AWS_Parallelcluster: ------------------------------------------------ Amazon Web Services Parallelcluster Ubuntu 20.04 ------------------------------------------------ +**NEEDS UPDATING** + The JCSDA-managed AWS Parallel Cluster is currently unavailable. ------------------------------ -Amazon Web Services Red Hat 8 ------------------------------ -Use a c6i.4xlarge instance or larger if running out of memory with AMI "skylab-8.0.0-redhat8" (see JEDI documentation at https://jointcenterforsatellitedataassimilation-jedi-docs.readthedocs-hosted.com/en/latest for more information). +.. _Preconfigured_Sites_AWS_SingleNode_RH8: -For ``spack-stack-1.7.0``, run: +----------------------------------------- +Amazon Web Services Single Node Red Hat 8 +----------------------------------------- -.. code-block:: console +**NEEDS UPDATING** + +Use a c6i.4xlarge instance or larger if running out of memory with AMI "skylab-8.0.0-redhat8" (see JEDI documentation at https://jointcenterforsatellitedataassimilation-jedi-docs.readthedocs-hosted.com/en/latest for more information). - ulimit -s unlimited - scl_source enable gcc-toolset-11 - module use /home/ec2-user/spack-stack/spack-stack-1.7.0/envs/unified-env-gcc-11.2.1/install/modulefiles/Core - module load stack-gcc/11.2.1 - module load stack-openmpi/5.0.1 - module load stack-python/3.10.13 .. _Preconfigured_Sites_Tier2: @@ -601,9 +407,15 @@ Pre-configured sites (tier 2) Tier 2 preconfigured site are not officially supported by spack-stack. As such, instructions for these systems may be provided here, in form of a `README.md` in the site directory, or may not be available. Also, these site configs are not updated on the same regular basis as those of the tier 1 systems and therefore may be out of date and/or not working. -The following sites have site configurations in directory `configs/sites/`: -- TACC Frontera (`configs/sites/frontera/`) -- AWS Single Node with Nvidia (NVHPC) compilers (`configs/sites/aws-nvidia/`) + +.. _Preconfigured_Sites_Blackpearl: + +------------------------------ +Blackpearl +------------------------------ + +Blackpearl is an Oracle Linux 9 installation running under Windows Subsystem for Linux (WSL2) on Windows 11. This is the development system of one of the spack-stack developers and maybe useful as an example configuration for users with a similar setup. + .. _Preconfigured_Sites_Casper: @@ -611,7 +423,9 @@ The following sites have site configurations in directory `configs/sites/`: NCAR-Wyoming Casper ------------------------------ -The following is required for building new spack environments and for using spack to build and run software. +The following is required for building new spack environments with any supported compiler on this platform. + +**NEEDS UPDATING** .. code-block:: console @@ -622,6 +436,32 @@ The following is required for building new spack environments and for using spac module use /glade/work/epicufsrt/contrib/spack-stack/casper/modulefiles module load ecflow/5.8.4 + +.. _Preconfigured_Sites_EMC_RHEL: + +------------------------------ +EMC RedHat Enterprise Linux 8 +------------------------------ + +**NEEDS UPDATING** + + +.. _Preconfigured_Sites_Frontera: + +------------------------------ +??? Frontera +------------------------------ + +**NEEDS UPDATING** + + +------------------------------ +Linux/macOS default configs +------------------------------ + +The Linux and macOS configurations are **not** meant to be used as is, as user setups and package versions vary considerably. Instructions for adding this information can be found in :numref:`Section %s `. + + .. _Configurable_Sites_CreateEnv: ======================== @@ -687,6 +527,7 @@ The following instructions install a new spack environment on a pre-configured s spack concretize 2>&1 | tee log.concretize spack install [--verbose] [--fail-fast] 2>&1 | tee log.install + .. _Preconfigured_Sites_ExtendingEnvironments: ====================== diff --git a/spack-ext/repos/spack-stack/packages/neptune-env/package.py b/spack-ext/repos/spack-stack/packages/neptune-env/package.py index 917ea8865..f79968eac 100644 --- a/spack-ext/repos/spack-stack/packages/neptune-env/package.py +++ b/spack-ext/repos/spack-stack/packages/neptune-env/package.py @@ -34,7 +34,11 @@ class NeptuneEnv(BundlePackage): depends_on("p4est", type="run") depends_on("w3emc", type="run") depends_on("w3nco", type="run") - depends_on("ip@5:", type="run") + depends_on("sp", type="run", when="%aocc") + depends_on("ip@5:", type="run", when="%apple-clang") + depends_on("ip@5:", type="run", when="%gcc") + depends_on("ip@5:", type="run", when="%intel") + depends_on("ip@5:", type="run", when="%oneapi") depends_on("esmf", type="run") depends_on("nco", type="run") depends_on("mct", type="run") From bd00b5e1c1a8530c2a7763d395b6f94a4a145d39 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Thu, 5 Sep 2024 17:21:04 -0600 Subject: [PATCH 05/26] release/1.8.0: bug fix for compiling g2@3.5.1 with gfortran < 10 in spack package, remove from container configs (#1287) For release/1.8.0: Update submodule pointer for spack for the changes in JCSDA/spack#477. Remove g2 compiler flags in container configs, since these are now in the g2 package itself. Also include a bug fix to adjust the py-numpy version to match what is in configs/common/packages.yaml since yesterday (1.23.5). --- .github/workflows/ubuntu-ci-containers-x86_64.yaml | 2 +- configs/containers/docker-ubuntu-clang-mpich.yaml | 6 ------ configs/containers/docker-ubuntu-gcc-openmpi.yaml | 6 ------ configs/containers/specs/jedi-ci.yaml | 2 +- spack | 2 +- 5 files changed, 3 insertions(+), 15 deletions(-) diff --git a/.github/workflows/ubuntu-ci-containers-x86_64.yaml b/.github/workflows/ubuntu-ci-containers-x86_64.yaml index b9265c094..b69d0d324 100644 --- a/.github/workflows/ubuntu-ci-containers-x86_64.yaml +++ b/.github/workflows/ubuntu-ci-containers-x86_64.yaml @@ -8,7 +8,7 @@ on: # - '**.md' # - '.github/ISSUE_TEMPLATE/*' # - '.gitignore' - # + # schedule: - cron: '0 8 * * *' workflow_dispatch: diff --git a/configs/containers/docker-ubuntu-clang-mpich.yaml b/configs/containers/docker-ubuntu-clang-mpich.yaml index f55603577..6ac7f83e8 100644 --- a/configs/containers/docker-ubuntu-clang-mpich.yaml +++ b/configs/containers/docker-ubuntu-clang-mpich.yaml @@ -42,12 +42,6 @@ spack: externals: - spec: gcc-runtime@12.3.0 prefix: /usr - g2: - # https://github.com/NOAA-EMC/NCEPLIBS-g2/issues/745 - require: - - any_of: ['fflags="-fno-range-check"'] - when: "@3.5.1 %clang" - message: "Extra g2 compile options for version 3.5.1 with clang" g2tmpl: # https://github.com/JCSDA/spack-stack/issues/525 require: diff --git a/configs/containers/docker-ubuntu-gcc-openmpi.yaml b/configs/containers/docker-ubuntu-gcc-openmpi.yaml index d7bfe3f1b..f0e495073 100644 --- a/configs/containers/docker-ubuntu-gcc-openmpi.yaml +++ b/configs/containers/docker-ubuntu-gcc-openmpi.yaml @@ -45,12 +45,6 @@ spack: externals: - spec: gmake@4.3 prefix: /usr - g2: - # https://github.com/NOAA-EMC/NCEPLIBS-g2/issues/745 - require: - - any_of: ['fflags="-fno-range-check"'] - when: "@3.5.1 %gcc" - message: "Extra g2 compile options for version 3.5.1 with gcc/gfortran" diffutils: buildable: false externals: diff --git a/configs/containers/specs/jedi-ci.yaml b/configs/containers/specs/jedi-ci.yaml index 6b110ea7d..dea11f800 100644 --- a/configs/containers/specs/jedi-ci.yaml +++ b/configs/containers/specs/jedi-ci.yaml @@ -38,7 +38,7 @@ py-f90nml@1.4.3, py-gitpython@3.1.40, py-h5py@3.11.0, - py-numpy@1.25.2, + py-numpy@1.23.5, py-pandas@1.5.3, py-pip, py-pyyaml@6.0, diff --git a/spack b/spack index 33eea0479..78e0c1d5b 160000 --- a/spack +++ b/spack @@ -1 +1 @@ -Subproject commit 33eea04798da9317b9e48862291f3dfa9788d0e7 +Subproject commit 78e0c1d5b83ff2874ec803ce25d08b014b5c8cc3 From d152232528c7c55b758dd3e77d717b6f43523c1e Mon Sep 17 00:00:00 2001 From: RatkoVasic-NOAA <37597874+RatkoVasic-NOAA@users.noreply.github.com> Date: Thu, 5 Sep 2024 22:48:37 -0400 Subject: [PATCH 06/26] Add mirror path for Hera (#1290) --- configs/sites/tier1/hera/mirrors.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/configs/sites/tier1/hera/mirrors.yaml b/configs/sites/tier1/hera/mirrors.yaml index f6c5a8d1e..392059b66 100644 --- a/configs/sites/tier1/hera/mirrors.yaml +++ b/configs/sites/tier1/hera/mirrors.yaml @@ -1,7 +1,7 @@ mirrors: local-source: fetch: - url: file:///scratch1/NCEPDEV/nems/role.epic/spack-stack/source-cache + url: file:///contrib/spack-stack/mirror access_pair: - null - null @@ -9,7 +9,7 @@ mirrors: profile: null endpoint_url: null push: - url: file:///scratch1/NCEPDEV/nems/role.epic/spack-stack/source-cache + url: file:///contrib/spack-stack/mirror access_pair: - null - null From e3bda2a7a2a2c778b76e9c0239fbe113c6700357 Mon Sep 17 00:00:00 2001 From: Stephen Herbener <32968781+srherbener@users.noreply.github.com> Date: Mon, 9 Sep 2024 22:23:48 -0600 Subject: [PATCH 07/26] Config for MacOS Sonoma 14.6.1 with Command Line Tools 15.3 (#1281) This PR adds in configuration to enable building on the latest versions of the MacOS and Command Line Tools (14.6.1 and 15.3 respectively). The NewSiteConfig.rst documentation has been updated for building on the Mac. --------- Co-authored-by: Dom Heinzeller --- configs/common/packages.yaml | 8 +++++--- configs/sites/tier2/macos.default/packages.yaml | 7 ++++++- doc/source/NewSiteConfigs.rst | 6 ++++++ spack | 2 +- 4 files changed, 18 insertions(+), 5 deletions(-) diff --git a/configs/common/packages.yaml b/configs/common/packages.yaml index 1deb17e35..7f08bdf9b 100644 --- a/configs/common/packages.yaml +++ b/configs/common/packages.yaml @@ -75,8 +75,9 @@ packages: # Pin version to avoid duplicates require: '@2.6.4' fms: - require: '@2024.02' - variants: 'precision=32,64 +quad_precision +gfs_phys +openmp +pic constants=GFS build_type=Release +deprecated_io' + require: + - '@2024.02' + - 'precision=32,64 +quad_precision +gfs_phys +openmp +pic constants=GFS build_type=Release +deprecated_io' fontconfig: require: '+pic' freetype: @@ -274,7 +275,8 @@ packages: w3nco: require: '@2.4.1' wgrib2: - require: '@3.1.1' + require: + - '@3.1.1' wrf-io: require: '@1.2.0' zstd: diff --git a/configs/sites/tier2/macos.default/packages.yaml b/configs/sites/tier2/macos.default/packages.yaml index f08498c65..ddb2d126f 100644 --- a/configs/sites/tier2/macos.default/packages.yaml +++ b/configs/sites/tier2/macos.default/packages.yaml @@ -1,9 +1,14 @@ packages: cairo: variants: ~png ~svg + # https://github.com/JCSDA/spack-stack/issues/1211 + fms: + require: + - '+shared' git: buildable: false libiconv: buildable: false wgrib2: - variants: ~openmp + require: + - '~openmp' diff --git a/doc/source/NewSiteConfigs.rst b/doc/source/NewSiteConfigs.rst index b4116538c..afd5305b0 100644 --- a/doc/source/NewSiteConfigs.rst +++ b/doc/source/NewSiteConfigs.rst @@ -152,6 +152,9 @@ These instructions are meant to be a reference that users can follow to set up t brew install libiconv # Note - need to pin to version 5 brew install qt@5 + # Note - For Sonoma only, install m4 and pkg-config + brew install m4 + brew install pkg-config # Note - only needed for running JCSDA's # JEDI-Skylab system (using R2D2 localhost) @@ -248,6 +251,9 @@ Remember to activate the ``lua`` module environment and have MacTeX in your sear PATH="$HOMEBREW_ROOT/opt/qt@5/bin:$PATH" \ spack external find --scope system qt + PATH="$HOMEBREW_ROOT/opt/m4/bin:$PATH" \ + spack external find --scope system m4 + # Optional, only if planning to build jedi-tools environment with LaTeX support # The texlive bin directory must have been added to PATH (see above) spack external find --scope system texlive diff --git a/spack b/spack index 78e0c1d5b..1b3e3e525 160000 --- a/spack +++ b/spack @@ -1 +1 @@ -Subproject commit 78e0c1d5b83ff2874ec803ce25d08b014b5c8cc3 +Subproject commit 1b3e3e5252e5b0e4cb28f9e4255e84ea1a06ff5f From f3f0ac12b13656a3c7c0bf0b722b0d71e3461de9 Mon Sep 17 00:00:00 2001 From: Alex Richert Date: Tue, 10 Sep 2024 06:42:55 -0700 Subject: [PATCH 08/26] Acorn updates for 1.8.0 (#1285) This PR provides updates for the Acorn site config for the 1.8.0 release, which should eventually get merged into develop. --- configs/sites/tier1/acorn/compilers.yaml | 22 ++++++++++--------- configs/sites/tier1/acorn/packages.yaml | 6 +---- configs/sites/tier1/acorn/packages_intel.yaml | 12 ++++++++-- 3 files changed, 23 insertions(+), 17 deletions(-) diff --git a/configs/sites/tier1/acorn/compilers.yaml b/configs/sites/tier1/acorn/compilers.yaml index e118f32c3..179fcbdbf 100644 --- a/configs/sites/tier1/acorn/compilers.yaml +++ b/configs/sites/tier1/acorn/compilers.yaml @@ -6,18 +6,17 @@ compilers: cxx: CC f77: ftn fc: ftn - flags: {} + flags: + cflags: -static-libgcc -static-libstdc++ -Bstatic -lstdc++ -Bdynamic -lm -lpthread + cxxflags: -static-libgcc -static-libstdc++ -Bstatic -lstdc++ -Bdynamic -lm -lpthread + fflags: -static-libgcc -static-libstdc++ -Bstatic -lstdc++ -Bdynamic -lm -lpthread operating_system: sles15 modules: - - PrgEnv-intel/8.3.3 - - craype/2.7.13 + - PrgEnv-intel/8.5.0 + - craype/2.7.17 - intel/19.1.3.304 - libfabric environment: - prepend_path: - PATH: /opt/cray/pe/gcc/10.2.0/bin - LD_LIBRARY_PATH: /opt/cray/pe/gcc/10.2.0/snos/lib64 - CPATH: /opt/cray/pe/gcc/10.2.0/snos/include set: # OpenSUSE on WCOSS2 machines sets CONFIG_SITE so # Automake-based builds are installed in lib64 @@ -31,11 +30,14 @@ compilers: cxx: CC f77: ftn fc: ftn - flags: {} + flags: + cflags: -static-libgcc -static-libstdc++ -Bstatic -lstdc++ -Bdynamic -lm -lpthread + cxxflags: -static-libgcc -static-libstdc++ -Bstatic -lstdc++ -Bdynamic -lm -lpthread + fflags: -static-libgcc -static-libstdc++ -Bstatic -lstdc++ -Bdynamic -lm -lpthread operating_system: sles15 modules: - - PrgEnv-intel/8.3.3 - - craype/2.7.13 + - PrgEnv-intel/8.5.0 + - craype/2.7.17 - intel-classic/2022.2.0.262 - libfabric environment: diff --git a/configs/sites/tier1/acorn/packages.yaml b/configs/sites/tier1/acorn/packages.yaml index fcd79f254..6bd55d67c 100644 --- a/configs/sites/tier1/acorn/packages.yaml +++ b/configs/sites/tier1/acorn/packages.yaml @@ -9,11 +9,6 @@ externals: - spec: git-lfs@2.11.0 modules: [git-lfs/2.11.0] - perl: - buildable: false - externals: - - spec: perl@5.26.1~cpanm+shared+threads - prefix: /usr mysql: buildable: false externals: @@ -38,6 +33,7 @@ gdal: variants: ~curl flex: + buildable: false externals: - spec: flex@2.6.4+lex prefix: /usr diff --git a/configs/sites/tier1/acorn/packages_intel.yaml b/configs/sites/tier1/acorn/packages_intel.yaml index 5868e5481..b9cfc096c 100644 --- a/configs/sites/tier1/acorn/packages_intel.yaml +++ b/configs/sites/tier1/acorn/packages_intel.yaml @@ -31,7 +31,10 @@ - '@1.2.1 ~mkl' py-numpy: require:: - - '@:1.23 ^openblas' + - '^[virtuals=lapack,blas] openblas' + - '@:1.25' + - any_of: ['@:1.24'] + when: '%intel@19.1.3.304' # *DH eckit: require: @@ -39,13 +42,18 @@ when: "%intel@19.1.3.304" eccodes: require: - - any_of: ["@2.27.0"] + - any_of: ["@2.25.0"] when: "%intel@19.1.3.304" + message: "2.25.0 is the last version to use C++11 (as opposed to C++17)" py-scipy: require: - any_of: ["@1.10.1"] when: "%intel@19.1.3.304" cdo: require: + - any_of: ["@2.3.0"] + when: "%intel@2022.0.2.262" + message: "2.3.0 is the last version to use C++17" - any_of: ["@2.0.5"] when: "%intel@19.1.3.304" + message: "2.0.5 is the last version to use C++14" From c8164acb690615c98cebb48a6eadc7ad916bf00c Mon Sep 17 00:00:00 2001 From: AlexanderRichert-NOAA Date: Thu, 1 Aug 2024 00:04:46 +0000 Subject: [PATCH 09/26] allow setting compiler version with --compiler --- spack-ext/lib/jcsda-emc/spack-stack/stack/stack_env.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/spack-ext/lib/jcsda-emc/spack-stack/stack/stack_env.py b/spack-ext/lib/jcsda-emc/spack-stack/stack/stack_env.py index b63dd93f1..85b701839 100644 --- a/spack-ext/lib/jcsda-emc/spack-stack/stack/stack_env.py +++ b/spack-ext/lib/jcsda-emc/spack-stack/stack/stack_env.py @@ -100,7 +100,7 @@ def __init__(self, **kwargs): if not self.name: # site = self.site if self.site else 'default' - self.name = "{}.{}.{}".format(self.template, self.site, self.compiler) + self.name = "{}.{}.{}".format(self.template, self.site, self.compiler.replace("@", "-")) def env_dir(self): """env_dir is /""" @@ -170,7 +170,7 @@ def _copy_common_includes(self): self._copy_or_merge_includes("modules", modules_yaml_path, modules_yaml_modulesys_path, destination) # Merge or copy common package config(s) packages_yaml_path = os.path.join(common_path, "packages.yaml") - packages_compiler_yaml_path = os.path.join(common_path, f"packages_{self.compiler}.yaml") + packages_compiler_yaml_path = os.path.join(common_path, f"packages_{self.compiler.split('@')[0]}.yaml") destination = os.path.join(env_common_dir, "packages.yaml") self._copy_or_merge_includes("packages", packages_yaml_path, packages_compiler_yaml_path, destination) @@ -203,7 +203,7 @@ def _copy_site_includes(self): self._copy_or_merge_includes("modules", modules_yaml_path, modules_yaml_modulesys_path, destination) # Merge or copy site package config(s) packages_yaml_path = os.path.join(env_path, "packages.yaml") - packages_compiler_yaml_path = os.path.join(env_path, f"packages_{self.compiler}.yaml") + packages_compiler_yaml_path = os.path.join(env_path, f"packages_{self.compiler.split('@')[0]}.yaml") destination = os.path.join(env_site_dir, "packages.yaml") self._copy_or_merge_includes("packages", packages_yaml_path, packages_compiler_yaml_path, destination) @@ -260,7 +260,7 @@ def write(self): # DH I am too stupid to do this the "spack way" ... definitions = spack.config.get("definitions", scope=env_scope) if definitions: - target_compiler = f"%{self.compiler}" + target_compiler = f"%{self.compiler}".split("@")[0] for i in range(len(definitions)): if "compilers" in definitions[i]: j = len(definitions[i]["compilers"])-1 From c23955183817724ee55e18bc168a746d94720ee0 Mon Sep 17 00:00:00 2001 From: AlexanderRichert-NOAA Date: Thu, 1 Aug 2024 02:34:11 +0000 Subject: [PATCH 10/26] allow compilers with version in matrix defs --- spack-ext/lib/jcsda-emc/spack-stack/stack/stack_env.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/spack-ext/lib/jcsda-emc/spack-stack/stack/stack_env.py b/spack-ext/lib/jcsda-emc/spack-stack/stack/stack_env.py index 85b701839..83757629a 100644 --- a/spack-ext/lib/jcsda-emc/spack-stack/stack/stack_env.py +++ b/spack-ext/lib/jcsda-emc/spack-stack/stack/stack_env.py @@ -260,14 +260,10 @@ def write(self): # DH I am too stupid to do this the "spack way" ... definitions = spack.config.get("definitions", scope=env_scope) if definitions: - target_compiler = f"%{self.compiler}".split("@")[0] + target_compiler = f"%{self.compiler}" for i in range(len(definitions)): if "compilers" in definitions[i]: - j = len(definitions[i]["compilers"])-1 - while j>=0: - if not definitions[i]["compilers"][j] == target_compiler: - definitions[i]["compilers"].pop(j) - j -= 1 + definitions[i] = {"compilers": [target_compiler]} spack.config.set("definitions", definitions, scope=env_scope) if self.install_prefix: From 02859ac8b49d6f731e27dd8385144be60569f4f5 Mon Sep 17 00:00:00 2001 From: Alex Richert Date: Wed, 11 Sep 2024 11:26:22 -0700 Subject: [PATCH 11/26] add 'nco' template --- configs/templates/nco/spack.yaml | 98 ++++++++++++++++++++++++++++++++ 1 file changed, 98 insertions(+) create mode 100644 configs/templates/nco/spack.yaml diff --git a/configs/templates/nco/spack.yaml b/configs/templates/nco/spack.yaml new file mode 100644 index 000000000..a017ba507 --- /dev/null +++ b/configs/templates/nco/spack.yaml @@ -0,0 +1,98 @@ +spack: + concretizer: + unify: when_possible + + config: + install_tree: + root: $env/install + modules: + default: + roots: + lmod: $env/install/modulefiles + tcl: $env/install/modulefiles + + view: false + include: [] + + specs: + - awscli + - bacio + - boost + - bufr + - cairo + - cdo + - cfitsio + - cmake + - crtm + - curl + - ecflow + - eigen + - esmf + - fms + - g2 + - g2c + - g2tmpl + - gdal + - geos + - gfsio + - git + - grib-util + - gsl + - hdf5 + - imagemagick + - ip + - ip2 + - jasper + - landsfcutil + - libgeotiff + - libiconv + - libjpeg-turbo + - libpng + - libtiff + - libxml2 + - libxrender + - madis + - mapl + - mbedtls + - met + - metis + - metplus + - nco + - ncivew + - nemsio + - nemsiogfs + - netcdf-c +fismahigh + - netcdf-fortran + - perl + - pigz + - parallelio + - pixman + - proj + - python + - readline + - rsync + - scotch + - sfcio + - sigio + - sp + - subversion + - udunits + - upp + - w3emc + - w3enco + - wgrib2 + - wrf-io + + packages: + all: + prefer: 'intel@19.1.3.304' + mpi: + require: cray-mpich + fftw-api: + require: fftw + blas: + require: openblas + lapack: + require: openblas + jpeg: + require: libjpeg-turbo From 4614f873368a30ede7bba3062e83f37a1b8f4a0f Mon Sep 17 00:00:00 2001 From: Alex Richert Date: Wed, 11 Sep 2024 12:48:17 -0700 Subject: [PATCH 12/26] Allow specifying compiler version for 'spack stack create env' (#1294) This PR borrows the logic from #1213 that allows a compiler version to be specified in spack stack create env. It's a couple of tweaks to stack_env.py. This isn't strictly urgent, but I'm hoping to put this in the 1.8.0 release in anticipation of supporting installations by NCO (plus maybe others will find it useful; I know it will be useful for Acorn, and maybe also for NRL?). --- .../lib/jcsda-emc/spack-stack/stack/stack_env.py | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/spack-ext/lib/jcsda-emc/spack-stack/stack/stack_env.py b/spack-ext/lib/jcsda-emc/spack-stack/stack/stack_env.py index b63dd93f1..83757629a 100644 --- a/spack-ext/lib/jcsda-emc/spack-stack/stack/stack_env.py +++ b/spack-ext/lib/jcsda-emc/spack-stack/stack/stack_env.py @@ -100,7 +100,7 @@ def __init__(self, **kwargs): if not self.name: # site = self.site if self.site else 'default' - self.name = "{}.{}.{}".format(self.template, self.site, self.compiler) + self.name = "{}.{}.{}".format(self.template, self.site, self.compiler.replace("@", "-")) def env_dir(self): """env_dir is /""" @@ -170,7 +170,7 @@ def _copy_common_includes(self): self._copy_or_merge_includes("modules", modules_yaml_path, modules_yaml_modulesys_path, destination) # Merge or copy common package config(s) packages_yaml_path = os.path.join(common_path, "packages.yaml") - packages_compiler_yaml_path = os.path.join(common_path, f"packages_{self.compiler}.yaml") + packages_compiler_yaml_path = os.path.join(common_path, f"packages_{self.compiler.split('@')[0]}.yaml") destination = os.path.join(env_common_dir, "packages.yaml") self._copy_or_merge_includes("packages", packages_yaml_path, packages_compiler_yaml_path, destination) @@ -203,7 +203,7 @@ def _copy_site_includes(self): self._copy_or_merge_includes("modules", modules_yaml_path, modules_yaml_modulesys_path, destination) # Merge or copy site package config(s) packages_yaml_path = os.path.join(env_path, "packages.yaml") - packages_compiler_yaml_path = os.path.join(env_path, f"packages_{self.compiler}.yaml") + packages_compiler_yaml_path = os.path.join(env_path, f"packages_{self.compiler.split('@')[0]}.yaml") destination = os.path.join(env_site_dir, "packages.yaml") self._copy_or_merge_includes("packages", packages_yaml_path, packages_compiler_yaml_path, destination) @@ -263,11 +263,7 @@ def write(self): target_compiler = f"%{self.compiler}" for i in range(len(definitions)): if "compilers" in definitions[i]: - j = len(definitions[i]["compilers"])-1 - while j>=0: - if not definitions[i]["compilers"][j] == target_compiler: - definitions[i]["compilers"].pop(j) - j -= 1 + definitions[i] = {"compilers": [target_compiler]} spack.config.set("definitions", definitions, scope=env_scope) if self.install_prefix: From f96ad564154ebee39882693669a1c50cfc4ad6b6 Mon Sep 17 00:00:00 2001 From: AlexanderRichert-NOAA Date: Thu, 12 Sep 2024 15:27:46 +0000 Subject: [PATCH 13/26] add list of WCOSS2 approved packages --- configs/templates/nco/approved_list.txt | 225 ++++++++++++++++++++++++ 1 file changed, 225 insertions(+) create mode 100644 configs/templates/nco/approved_list.txt diff --git a/configs/templates/nco/approved_list.txt b/configs/templates/nco/approved_list.txt new file mode 100644 index 000000000..3fcdd8216 --- /dev/null +++ b/configs/templates/nco/approved_list.txt @@ -0,0 +1,225 @@ +awscli +bacio +boost +bufr +bufr_dump +bufr_tranjb +bufr_util +cairo +cdo +cfitsio +cfp +cmake +cray-parallel-netcdf +crtm +curl +degrib +dumpmb +ecflow +eigen +emapf-c +esmf +fms +g2 +g2c +g2tmpl +gdal +gempak +geos +gfsio +git +grib_util +gsl +hdf5 +imagemagick +ip +ip2 +jasper +lamp_shared +landsfcutil +libdap4 +libgeotiff +libiconv +libjpeg +libjpeg-turbo +libpng +libtiff +libxml2 +libxmlparse +libxrender +madis +manic +mapl +mbedtls +met +metis +metplus +mos_shared +nco +ncview +nemsio +nemsiogfs +netcdf +perl +pigz +pio +pixman +prepobs +prod_envir +prod_util +proj +python +readline +rsync +schism +scotch +sfcio +sigio +sp +subversion +udunits +upp +util_shared +w3emc +w3nco +wgrib2 +wrf_io +xyz2shp +xz +zlib +antlr +apr +apr-util +autoconf +automake +bdftopcf +berkeley-db +bison +bzip2 +ca-certificates-mozilla +diffutils +elfutils +expat +fftw +findutils +flex +font-util +fontconfig +fontsproto +freetype +fribidi +gdbm +gettext +glib +gmake +gnuconfig +gobject-introspection +gperf +harfbuzz +help2man +icu4c +inputproto +json-c +kbproto +libaec +libbsd +libedit +libffi +libfontenc +libice +libidn2 +libmd +libpthread-stubs +libsigsegv +libsm +libtool +libunistring +libwebp +libx11 +libxau +libxaw +libxcb +libxcrypt +libxdmcp +libxext +libxfont +libxmu +libxpm +libxt +libyaml +lz4 +m4 +meson +mkfontdir +mkfontscale +nasm +ncurses +netcdf-cxx4 +ninja +openblas +openssh +openssl +pango +pcre +pcre2 +pkgconf +popt +py-botocore +py-bottleneck +py-cftime +py-colorama +py-cython +py-docutils +py-flit-core +py-jmespath +py-netcdf4 +py-numexpr +py-numpy +py-packaging +py-pandas +py-pip +py-pyasn1 +py-python-dateutil +py-pytz +py-pyyaml +py-rsa +py-s3transfer +py-setuptools +py-setuptools-scm +py-setuptools-scm-git-archive +py-six +py-tomli +py-typing-extensions +py-urllib3 +py-versioneer +py-wheel +py-xarray +qt +re2c +renderproto +sqlite +tar +unixodbc +utf8proc +util-linux-uuid +util-macros +which +xcb-proto +xextproto +xproto +xtrans +xxhash +zstd +atlas +eckit +fckit +gsl-lite +qhull +gftl-shared +nccmp +cray-mpich +gcc-runtime +glibc +parallelio +pkg-config +ecbuild From 01837a27112e34adbb8cc5be49014b1a3283cf72 Mon Sep 17 00:00:00 2001 From: AlexanderRichert-NOAA Date: Thu, 12 Sep 2024 15:31:52 +0000 Subject: [PATCH 14/26] update NCO template --- configs/templates/nco/spack.yaml | 52 +++++++++++++++++++++++++++++--- 1 file changed, 47 insertions(+), 5 deletions(-) diff --git a/configs/templates/nco/spack.yaml b/configs/templates/nco/spack.yaml index a017ba507..12787de13 100644 --- a/configs/templates/nco/spack.yaml +++ b/configs/templates/nco/spack.yaml @@ -26,8 +26,11 @@ spack: - crtm - curl - ecflow + - eckit + - ecmwf-atlas - eigen - esmf + - fckit - fms - g2 - g2c @@ -35,9 +38,11 @@ spack: - gdal - geos - gfsio + - gftl-shared - git - grib-util - gsl + - gsl-lite - hdf5 - imagemagick - ip @@ -57,11 +62,13 @@ spack: - met - metis - metplus + - nccmp + - gsi-ncdiag - nco - - ncivew + - ncview - nemsio - nemsiogfs - - netcdf-c +fismahigh + - netcdf-c - netcdf-fortran - perl - pigz @@ -69,6 +76,7 @@ spack: - pixman - proj - python + - qhull - readline - rsync - scotch @@ -79,13 +87,12 @@ spack: - udunits - upp - w3emc - - w3enco + - w3nco - wgrib2 - wrf-io packages: - all: - prefer: 'intel@19.1.3.304' + # Strictly set virtual package providers: mpi: require: cray-mpich fftw-api: @@ -96,3 +103,38 @@ spack: require: openblas jpeg: require: libjpeg-turbo + # Individual package settings. + # Use `require::` to override require's from {common,site}/packages.yaml, + # and `require: - '...'` to add to existing ones. + cdo: + require: + - 'grib2=none' # avoids eccodes/grib-api dependency + curl: + require: + - '~nghttp2' # avoid nghttp2 dependency + ecmwf-atlas: + require:: '@0.33.0 +fckit ~trans ~fftw' # disabling trans and fftw to minimize dependencies; older version because newer eckit doesn't build with intel@19 + esmf: + require: + - 'snapshot=none' # make sure spack-stack doesn't accidentally give us a beta snapshot + fckit: + require:: '@0.10.1 +eckit' # older version needed for older eckit & ecmwf-atlas + glib: + require: '@:2.74.3' # newer versions don't build with intel@19 + mapl: + require: + - '~pflogger ~fargparse ~extdata2g' # minimize dependencies + netcdf-c: + require:: '@4.9.2 +mpi ~parallel-netcdf ~dap ~blosc ~szip build_system=autotools' # disabling dap prevents network access; disabling blosc and szip reduces dependencies + py-pandas: + require:: '~excel' # minimize dependencies + subversion: + require: + - '~serf +pic' # avoid serf dependency + # Explicitly require gcc when needed so Spack won't do anything weird: + boost: + require: '%gcc' + gcc-runtime: + require: '%gcc' + harfbuzz: + require: '%gcc' From 0e2afdadd204fd38de791300388e950fc1851a71 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Thu, 12 Sep 2024 16:40:29 -0600 Subject: [PATCH 15/26] Last minute updates (round 1 of ?) for NRL sites for spack-stack-1.8.0 (#1298) Next and hopefully last round of NRL site config and doc updates for release/1.8.0 --- configs/sites/tier1/atlantis/packages_oneapi.yaml | 4 ++-- doc/source/PreConfiguredSites.rst | 10 ++++++---- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/configs/sites/tier1/atlantis/packages_oneapi.yaml b/configs/sites/tier1/atlantis/packages_oneapi.yaml index 191223298..95ed8242f 100644 --- a/configs/sites/tier1/atlantis/packages_oneapi.yaml +++ b/configs/sites/tier1/atlantis/packages_oneapi.yaml @@ -2,12 +2,12 @@ packages: all: compiler:: [oneapi@2024.2.1,gcc@11.2.0] providers: - mpi:: [intel-oneapi-mpi@2021.13.0] + mpi:: [intel-oneapi-mpi@2021.13] mpi: buildable: False intel-oneapi-mpi: externals: - - spec: intel-oneapi-mpi@2021.13%2024.2.1 + - spec: intel-oneapi-mpi@2021.13%oneapi@2024.2.1 prefix: /cm/shared/apps/intel/oneapi-2024.2.1 modules: - mpi/2021.13 diff --git a/doc/source/PreConfiguredSites.rst b/doc/source/PreConfiguredSites.rst index 4e6769568..2992a16ce 100644 --- a/doc/source/PreConfiguredSites.rst +++ b/doc/source/PreConfiguredSites.rst @@ -141,13 +141,14 @@ The following is required for building new spack environments with Intel on this module load intel-classic/2023.2.0 module unload cray-mpich module unload craype-network-ofi - module load craype-network-ucx - # Warning. Do not load cray-mpich-ucx/8.1.21 + # Warning. Do not load craype-network-ucx + # or cray-mpich-ucx/8.1.21! # There is a bug in the modulefile that prevents # spack from setting the environment for its # build steps when the module is already # loaded. Instead, let spack load it when the # package requires it. + #module load craype-network-ucx #module load cray-mpich-ucx/8.1.21 module load libfabric/1.12.1.2.2.1 module unload cray-libsci @@ -164,13 +165,14 @@ The following is required for building new spack environments with GNU on this p module load gcc/10.3.0 module unload cray-mpich module unload craype-network-ofi - module load craype-network-ucx - # Warning. Do not load cray-mpich-ucx/8.1.21 + # Warning. Do not load craype-network-ucx + # or cray-mpich-ucx/8.1.21! # There is a bug in the modulefile that prevents # spack from setting the environment for its # build steps when the module is already # loaded. Instead, let spack load it when the # package requires it. + #module load craype-network-ucx #module load cray-mpich-ucx/8.1.21 module load libfabric/1.12.1.2.2.1 module unload cray-libsci From 00fbc7abd494d8358d307eb27714cb4ef294e2fe Mon Sep 17 00:00:00 2001 From: Stephen Herbener <32968781+srherbener@users.noreply.github.com> Date: Fri, 13 Sep 2024 08:33:46 -0600 Subject: [PATCH 16/26] Updated documentation for building spack-stack on Mac Sonoma OS (#1296) --- doc/source/NewSiteConfigs.rst | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/doc/source/NewSiteConfigs.rst b/doc/source/NewSiteConfigs.rst index afd5305b0..556b5d938 100644 --- a/doc/source/NewSiteConfigs.rst +++ b/doc/source/NewSiteConfigs.rst @@ -152,22 +152,31 @@ These instructions are meant to be a reference that users can follow to set up t brew install libiconv # Note - need to pin to version 5 brew install qt@5 - # Note - For Sonoma only, install m4 and pkg-config brew install m4 + # Note - For Sonoma only: install pkg-config, diffutils and clingo brew install pkg-config + brew install diffutils + brew install clingo # Note - only needed for running JCSDA's # JEDI-Skylab system (using R2D2 localhost) brew install mysql .. note:: - On an Intel based Mac, you will need to also install pkg-config using homebrew. + On an Intel based Mac, regardless of MacOS version, you will need to also install pkg-config using homebrew. This is done to work around an issue where libraries (eg, openssl) cannot be properly found during code compilation. .. code-block:: console brew install pkg-config # Intel based Mac only +.. note:: + For spack-stack-1.8.0 and newer, you must be using cmake 3.26+. + Make sure you upgrade cmake in homebrew. + +.. code-block:: console + brew upgrade cmake + 4. Configure your terminal to use the homebrew installed bash After installing bash with homebrew, you need to change your terminal application's default command to use :code:`$HOMEBREW_ROOT/bin/bash`. @@ -232,7 +241,8 @@ Remember to activate the ``lua`` module environment and have MacTeX in your sear spack external find --scope system \ --exclude bison --exclude openssl \ - --exclude python --exclude gettext + --exclude python --exclude gettext \ + --exclude m4 spack external find --scope system perl spack external find --scope system wget @@ -248,7 +258,7 @@ Remember to activate the ``lua`` module environment and have MacTeX in your sear PATH="$HOMEBREW_ROOT/opt/curl/bin:$PATH" \ spack external find --scope system curl - PATH="$HOMEBREW_ROOT/opt/qt@5/bin:$PATH" \ + PATH="$HOMEBREW_ROOT/opt/qt5/bin:$PATH" \ spack external find --scope system qt PATH="$HOMEBREW_ROOT/opt/m4/bin:$PATH" \ From 2d3a1f86d41d3329d0e0cbe6dc824c06db50b5c1 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Mon, 16 Sep 2024 20:12:05 -0600 Subject: [PATCH 17/26] release/1.8.0: document environment naming conventions (#1300) * Update doc/source/PreConfiguredSites.rst with environment naming conventions table --- doc/source/PreConfiguredSites.rst | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/doc/source/PreConfiguredSites.rst b/doc/source/PreConfiguredSites.rst index 2992a16ce..84bf1cacb 100644 --- a/doc/source/PreConfiguredSites.rst +++ b/doc/source/PreConfiguredSites.rst @@ -9,6 +9,29 @@ Directories ``configs/sites/tier1`` and ``configs/sites/tier2`` contain site con As of spack-stack-1.8.0, this page provides general information on the supported platforms, such as the location of the spack-stack installations on tier 1 platforms and instructions on how to set up an environment for **building** spack-stack environments. Information on **using** spack-stack environments for development of downstream applications is available on the spack-stack wiki: https://github.com/JCSDA/spack-stack/wiki +.. _EnvironmentNamingConventions: + +============================================================= +Environment naming conventions +============================================================= + +The following naming conventions are used on all fully-supported (tier 1) sites. Environments are named using an abbreviated prefix that depends on the template/purpose, followed by the compiler name and version: ``prefix-compiler-version``. The following table lists the prefices and gives a few examples. + ++----------------------------------+---------------------------------------------------------+-------------------+------------------------------+ +| Template (``configs/templates``) | Description | Prefix | Examples | ++==================================+=========================================================+===================+==============================+ +| ``unified-dev`` | Unified environment for all organizations/applications | ``ue`` | ``ue-intel-2021.10.0`` | ++----------------------------------+---------------------------------------------------------+-------------------+------------------------------+ +| ``skylab-dev`` | JEDI/Skylab environment for JEDI, models, EWOK | ``se`` | ``se-apple-clang-14.0.6`` | ++----------------------------------+---------------------------------------------------------+-------------------+------------------------------+ +| ``neptune-dev`` | NEPTUNE standalone environment (with xNRL Python) | ``ne`` | ``ne-oneapi-2024.2.1`` | ++----------------------------------+---------------------------------------------------------+-------------------+------------------------------+ +| ``gsi-addon-dev`` | GSI addon (chained) environment on top of unified env. | ``gsi`` | ``gsi-gcc-13.3.0`` | ++----------------------------------+---------------------------------------------------------+-------------------+------------------------------+ +| ``unified-dev`` with new ESMF | Unified environment with new ESMF (chained from ``ue``) | ``ue-esmf870b99`` | ``ue-esmf870b99-aocc-4.2.0`` | ++----------------------------------+---------------------------------------------------------+-------------------+------------------------------+ + + .. _Preconfigured_Sites_Tier1: ============================================================= From fcc634964e1e872c13e67a96657bb03c4624b13f Mon Sep 17 00:00:00 2001 From: Alex Richert Date: Tue, 17 Sep 2024 09:11:40 -0700 Subject: [PATCH 18/26] update acorn doco for 1.8.0 --- doc/source/PreConfiguredSites.rst | 16 ++++------------ 1 file changed, 4 insertions(+), 12 deletions(-) diff --git a/doc/source/PreConfiguredSites.rst b/doc/source/PreConfiguredSites.rst index 84bf1cacb..74ed5fd33 100644 --- a/doc/source/PreConfiguredSites.rst +++ b/doc/source/PreConfiguredSites.rst @@ -242,22 +242,14 @@ The following is required for building new spack environments with any supported NOAA Acorn (WCOSS2 test system) ------------------------------- -**NEEDS UPDATING** - -For spack-stack-1.7.0, the meta modules are in ``/lfs/h1/emc/nceplibs/noscrub/spack-stack/spack-stack-1.7.0/envs/ue-intel{19,2022}/modulefiles/Core``. - -On WCOSS2 OpenSUSE sets ``CONFIG_SITE`` which causes libraries to be installed in ``lib64``, breaking the ``lib`` assumption made by some packages. Therefore, ``CONFIG_SITE`` should be set to empty in ``compilers.yaml``. Also, don't use ``module purge`` on Acorn! - -When installing an official ``spack-stack`` on Acorn, be mindful of umask and group ownership, as these can be finicky. The umask value should be 002, otherwise various files can be assigned to the wrong group. In any case, running something to the effect of ``chgrp nceplibs -R`` and ``chmod o+rX -R`` after the whole installation is done is a good idea. - -Due to a combined quirk of Cray and Spack, the ``PrgEnv-gnu`` and ``gcc`` modules must be loaded when `ESMF` is being installed with ``gcc``. +On WCOSS2 OpenSUSE sets ``CONFIG_SITE`` which causes libraries to be installed in ``lib64``, breaking the ``lib`` assumption made by some packages. Therefore, ``CONFIG_SITE`` should remain set to empty in ``compilers.yaml``. -As of spring 2023, there is an inconsistency in ``libstdc++`` versions on Acorn between the login and compute nodes. It is advisable to compile on the compute nodes, which requires running ``spack fetch`` prior to installing through a batch job. +For official deployments on ``spack-stack`` on Acorn, be mindful of umask and group ownership, as these can be finicky. The umask value should be 002, otherwise various files can be assigned to the wrong group. In any case, running something to the effect of ``chgrp nceplibs -R`` and ``chmod o+rX -R`` after the whole installation is done is a good idea. -Note that certain packages, such as recent versions of `py-scipy`, cannot be compiled on compute nodes because their build systems require internet access. +Note that for the installation using Intel 19, the system GCC, 7.5.0, is used on the backend for the Intel compiler. More recent versions of GCC are not reliably compatible. Likewise, for Intel 2022, GCC 10.2.0 is used on the backend. Intel 19 is not reliably compatible with C++17 standards, and Intel 2022 is not reliably compatible with C++20. Without a handful of package version restrictions, certain package builds will break, usually in the configure stage. .. note:: - System-wide ``spack`` software installations are maintained by NCO on this platform which are not associated with spack-stack. The spack-stack official installations use those installations for one dependency (git-lfs). + System-wide ``spack`` software installations are maintained by NCO on this platform, which are not associated with spack-stack. .. _Preconfigured_Sites_Parallel_Works: From 9aa4f7e1938068504efc41f526290e0bc9075a35 Mon Sep 17 00:00:00 2001 From: Ashley Griffin Date: Wed, 25 Sep 2024 16:18:03 -0600 Subject: [PATCH 19/26] update for discover-scu17 mkl and docs (#1306) * update mkl discover-scu17 package * doc updates for discover --- configs/sites/tier1/discover-scu17/packages_intel.yaml | 2 +- doc/source/PreConfiguredSites.rst | 8 -------- 2 files changed, 1 insertion(+), 9 deletions(-) diff --git a/configs/sites/tier1/discover-scu17/packages_intel.yaml b/configs/sites/tier1/discover-scu17/packages_intel.yaml index 3512790ab..d971e2387 100644 --- a/configs/sites/tier1/discover-scu17/packages_intel.yaml +++ b/configs/sites/tier1/discover-scu17/packages_intel.yaml @@ -13,5 +13,5 @@ packages: - mpi/impi/2021.10.0 intel-oneapi-mkl: externals: - - spec: intel-oneapi-mkl@2032.2.0%intel@2021.10.0 + - spec: intel-oneapi-mkl@2023.2.0%intel@2021.10.0 prefix: /usr/local/intel/oneapi/2021 diff --git a/doc/source/PreConfiguredSites.rst b/doc/source/PreConfiguredSites.rst index 74ed5fd33..97b532daa 100644 --- a/doc/source/PreConfiguredSites.rst +++ b/doc/source/PreConfiguredSites.rst @@ -118,16 +118,12 @@ NASA Discover SCU16 The following is required for building new spack environments with any supported compiler on this platform. -**NEEDS UPDATING** - .. code-block:: console module purge module use /discover/swdev/gmao_SIteam/modulefiles-SLES12 module use /discover/swdev/jcsda/spack-stack/scu16/modulefiles module load miniconda/3.9.7 - module load ecflow/5.8.4 - .. _Preconfigured_Sites_Discover_SCU17: @@ -137,15 +133,11 @@ NASA Discover SCU17 The following is required for building new spack environments with any supported compiler on this platform. -**NEEDS UPDATING** - .. code-block:: console module purge module use /discover/swdev/gmao_SIteam/modulefiles-SLES15 module use /discover/swdev/jcsda/spack-stack/scu17/modulefiles - module load ecflow/5.11.4 - .. _Preconfigured_Sites_Narwhal: From 185593164a628e02eea2f89dcbd08ecf17282e39 Mon Sep 17 00:00:00 2001 From: Evan Parker Date: Fri, 27 Sep 2024 05:30:01 -0600 Subject: [PATCH 20/26] remove external ecflow install for aws-pcluster (#1314) --- configs/sites/tier1/aws-pcluster/README.md | 77 +++---------------- .../sites/tier1/aws-pcluster/packages.yaml | 5 -- 2 files changed, 12 insertions(+), 70 deletions(-) diff --git a/configs/sites/tier1/aws-pcluster/README.md b/configs/sites/tier1/aws-pcluster/README.md index 5c3c0d86e..b1ee5e1c0 100644 --- a/configs/sites/tier1/aws-pcluster/README.md +++ b/configs/sites/tier1/aws-pcluster/README.md @@ -116,7 +116,7 @@ apt install -y \ liblcms2-dev \ liblcms2-2 -# Install QT5 for ecflow +# Install QT5, needed for install of ecflow apt install -y \ qtcreator \ qtbase5-dev \ @@ -217,51 +217,7 @@ sudo usermod -aG docker $USER exit ``` -4. Build ecflow outside of spack to be able to link against OS boost -``` -mkdir -p /home/ubuntu/jedi/ecflow-5.8.4/src -cd /home/ubuntu/jedi/ecflow-5.8.4/src -wget https://github.com/ecmwf/ecflow/archive/refs/tags/5.8.4.tar.gz -tar -xvzf 5.8.4.tar.gz -export WK=/home/ubuntu/jedi/ecflow-5.8.4/src/ecflow-5.8.4 -export BOOST_ROOT=/usr - -# Build ecFlow -cd $WK -mkdir build -cd build -cmake .. -DPython3_EXECUTABLE=/usr/bin/python3 -DENABLE_STATIC_BOOST_LIBS=OFF -DCMAKE_INSTALL_PREFIX=/home/ubuntu/jedi/ecflow-5.8.4 2>&1 | tee log.cmake -make -j10 2>&1 | tee log.make -make install 2>&1 | tee log.install - -# Create a modulefiles directory and the "ecflow/5.8.4" module file. -mkdir -p /home/ubuntu/jedi/modulefiles/ecflow - -cat << 'EOF' > /home/ubuntu/jedi/modulefiles/ecflow/5.8.4 -#%Module1.0 - -module-whatis "Provides an ecflow-5.8.4 server+ui installation for use with spack." - -conflict ecflow - -proc ModulesHelp { } { -puts stderr "Provides an ecflow-5.8.4 server+ui installation for use with spack." -} - -# Set this value -set ECFLOW_PATH "/home/ubuntu/jedi/ecflow-5.8.4" - -prepend-path PATH "${ECFLOW_PATH}/bin" -prepend-path LD_LIBRARY_PATH "${ECFLOW_PATH}/lib" -prepend-path LIBRARY_PATH "${ECFLOW_PATH}/lib" -prepend-path CPATH "${ECFLOW_PATH}/include" -prepend-path CMAKE_PREFIX_PATH "${ECFLOW_PATH}" -prepend-path PYTHONPATH "${ECFLOW_PATH}/lib/python3.10/site-packages" -EOF - -``` - -5. Install lmod. This step must be done as `root`. +4. Install lmod. This step must be done as `root`. ``` # Install lua/lmod manually because apt only has older versions # that are not compatible with the modern lua modules spack produces @@ -343,7 +299,7 @@ module purge module list ``` -6. Install msql community server +5. Install msql community server ``` cd /home/ubuntu/jedi mkdir -p mysql-8.0.31/src @@ -365,7 +321,7 @@ exit rm *.deb ``` -7. Option 1: Testing existing site config in spack-stack (skip steps +6. Option 1: Testing existing site config in spack-stack (skip steps 8-9 afterwards) this install is done directly on the NFS drive. If you are testing an update to the configuration, do this on the faster EBS volume (use a directory in /home/ubuntu) in order to ensure a faster build. Once you have @@ -398,7 +354,7 @@ spack module lmod refresh spack stack setup-meta-modules ``` -8. Option 2: Test configuring site from scratch +7. Option 2: Test configuring site from scratch ``` mkdir -p /home/ubuntu/jedi && cd /home/ubuntu/jedi git clone -b develop --recursive https://github.com/jcsda/spack-stack spack-stack @@ -450,15 +406,6 @@ cat << 'EOF' >> ${SPACK_SYSTEM_CONFIG_PATH}/packages.yaml prefix: /usr EOF -# Add external ecflow -cat << 'EOF' >> ${SPACK_SYSTEM_CONFIG_PATH}/packages.yaml - ecflow: - buildable: False - externals: - - spec: ecflow@5.8.4 +ui - prefix: /home/ubuntu/jedi/ecflow-5.8.4 -EOF - spack compiler find --scope system export -n SPACK_SYSTEM_CONFIG_PATH @@ -479,12 +426,12 @@ spack config add "packages:all:compiler:[intel@2021.10.0, gcc@11.4.0]" # I_MPI_PMI_LIBRARY: '/opt/slurm/lib/libpmi.so' ``` -9. Option 2: To avoid duplicate library versions edit ``envs/unified-dev/site/packages.yaml`` +8. Option 2: To avoid duplicate library versions edit ``envs/unified-dev/site/packages.yaml`` and remove entries for meson, ninja, hdf5, cmake and remove the external `cmake` and `openssl` entries. -10. Concretize and install +9. Concretize and install ``` spack concretize 2>&1 | tee log.concretize.unified-env.001 ./util/show_duplicate_packages.py -d log.concretize.unified-env.001 @@ -493,7 +440,7 @@ spack module lmod refresh spack stack setup-meta-modules ``` -11. Test spack-stack installation using your favorite application. +10. Test spack-stack installation using your favorite application. ``` # Example given for building jedi-bundle @@ -515,12 +462,12 @@ make -j10 ctest ``` -12. (Optional) Remove test installs of spack-stack environments, if desired. +11. (Optional) Remove test installs of spack-stack environments, if desired. -13. Create the AMI for use in the AWS parallelcluster config. You can follow +12. Create the AMI for use in the AWS parallelcluster config. You can follow the official instructions for [Modifying an AWS ParallelCluster AMI](https://docs.aws.amazon.com/parallelcluster/latest/ug/building-custom-ami-v3.html#modify-an-aws-parallelcluster-ami-v3) -14. Use the install to build +13. Use the install to build ``` # Load the intel toolchain into your environment. @@ -551,7 +498,7 @@ make -j10 ctest ``` -15. Once the parallel cluster head node image is fully configured and tested you +14. Once the parallel cluster head node image is fully configured and tested you can create an AMI snapshot based on the configured instance using the [instructions](https://docs.aws.amazon.com/parallelcluster/latest/ug/building-custom-ami-v3.html) published by AWS. Included here is a short summary of those instructions. diff --git a/configs/sites/tier1/aws-pcluster/packages.yaml b/configs/sites/tier1/aws-pcluster/packages.yaml index 17d5b64a6..6158d0119 100644 --- a/configs/sites/tier1/aws-pcluster/packages.yaml +++ b/configs/sites/tier1/aws-pcluster/packages.yaml @@ -19,11 +19,6 @@ packages: externals: - spec: diffutils@3.8 prefix: /usr - ecflow: - buildable: False - externals: - - spec: ecflow@5.8.4+ui+static_boost - prefix: /home/ubuntu/jedi/ecflow-5.8.4 findutils: externals: - spec: findutils@4.8.0 From 0e56296c6f75801c545ec0855d13ada0ba771ba3 Mon Sep 17 00:00:00 2001 From: Evan Parker Date: Fri, 27 Sep 2024 13:51:24 -0600 Subject: [PATCH 21/26] remove ecflow module exclusion (#1317) --- configs/sites/tier1/aws-pcluster/modules.yaml | 2 -- 1 file changed, 2 deletions(-) diff --git a/configs/sites/tier1/aws-pcluster/modules.yaml b/configs/sites/tier1/aws-pcluster/modules.yaml index d7e195ab6..65801706f 100644 --- a/configs/sites/tier1/aws-pcluster/modules.yaml +++ b/configs/sites/tier1/aws-pcluster/modules.yaml @@ -5,5 +5,3 @@ modules: lmod: include: - python - exclude: - - ecflow \ No newline at end of file From 1eb0a5d188fa734ed989228dd7d2cdf2a9e1d208 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Mon, 30 Sep 2024 19:28:43 -0600 Subject: [PATCH 22/26] release/1.8.0: Update derecho site config (#1315) Update Derecho site config for release/1.8.0: 1. Use only three build jobs to avoid getting booted off the system by the arbiter 2. Use gcc@11.2.0 as backend for Intel to fix build errors for met and py-scipy 3. Pin py-scipy to 1.11.4 4. ecflow/qt changes as discussed earlier --- configs/sites/tier1/derecho/compilers.yaml | 25 ++++++++++++++++--- configs/sites/tier1/derecho/config.yaml | 2 +- configs/sites/tier1/derecho/packages.yaml | 21 +++++++++------- .../sites/tier1/derecho/packages_intel.yaml | 2 +- doc/source/PreConfiguredSites.rst | 1 - 5 files changed, 36 insertions(+), 15 deletions(-) diff --git a/configs/sites/tier1/derecho/compilers.yaml b/configs/sites/tier1/derecho/compilers.yaml index 20cf2574c..db05f7f08 100644 --- a/configs/sites/tier1/derecho/compilers.yaml +++ b/configs/sites/tier1/derecho/compilers.yaml @@ -14,9 +14,9 @@ compilers:: - intel-classic/2023.2.1 environment: prepend_path: - PATH: '/opt/cray/pe/gcc/12.2.0/bin' - CPATH: '/opt/cray/pe/gcc/12.2.0/snos/include' - LD_LIBRARY_PATH: '/glade/u/apps/common/23.08/spack/opt/spack/intel-oneapi-compilers/2023.2.1/compiler/2023.2.1/linux/compiler/lib/intel64_lin:/opt/cray/pe/gcc/12.2.0/snos/lib:/opt/cray/pe/gcc/12.2.0/lib64' + PATH: '/opt/cray/pe/gcc/11.2.0/snos/bin' + CPATH: '/opt/cray/pe/gcc/11.2.0/snos/include' + LD_LIBRARY_PATH: '/glade/u/apps/common/23.08/spack/opt/spack/intel-oneapi-compilers/2023.2.1/compiler/2023.2.1/linux/compiler/lib/intel64_lin:/opt/cray/pe/gcc/11.2.0/snos/lib:/opt/cray/pe/gcc/11.2.0/snos/lib64' set: # https://github.com/JCSDA/spack-stack/issues/957 FI_CXI_RX_MATCH_MODE: 'hybrid' @@ -41,4 +41,23 @@ compilers:: # https://github.com/JCSDA/spack-stack/issues/957 FI_CXI_RX_MATCH_MODE: 'hybrid' extra_rpaths: [] +# Needed for intel +- compiler: + spec: gcc@11.2.0 + paths: + cc: /opt/cray/pe/gcc/11.2.0/bin/gcc + cxx: /opt/cray/pe/gcc/11.2.0/bin/g++ + f77: /opt/cray/pe/gcc/11.2.0/bin/gfortran + fc: /opt/cray/pe/gcc/11.2.0/bin/gfortran + flags: {} + operating_system: sles15 + target: x86_64 + modules: + - ncarenv/23.09 + - gcc/11.2.0 + environment: + set: + # https://github.com/JCSDA/spack-stack/issues/957 + FI_CXI_RX_MATCH_MODE: 'hybrid' + extra_rpaths: [] diff --git a/configs/sites/tier1/derecho/config.yaml b/configs/sites/tier1/derecho/config.yaml index 84fe05ab9..fd1f8aa47 100644 --- a/configs/sites/tier1/derecho/config.yaml +++ b/configs/sites/tier1/derecho/config.yaml @@ -1,2 +1,2 @@ config: - build_jobs: 8 + build_jobs: 3 diff --git a/configs/sites/tier1/derecho/packages.yaml b/configs/sites/tier1/derecho/packages.yaml index 8ea313601..7105a030a 100644 --- a/configs/sites/tier1/derecho/packages.yaml +++ b/configs/sites/tier1/derecho/packages.yaml @@ -7,6 +7,11 @@ packages: # Need to force spack to build python with ssl variant when using older openssl from OS python: require: "+ssl" + # py-scipy 1.12 does not build with intel on Derecho, pin to 1.11.4 + # use same version with gcc for consistency on the machine + py-scipy: + require: + - '@1.11.4' ### All other external packages listed alphabetically autoconf: externals: @@ -26,8 +31,8 @@ packages: prefix: /usr curl: externals: - - spec: curl@7.79.1+gssapi+ldap+nghttp2 - prefix: /usr + - spec: curl@8.1.2+gssapi+ldap+nghttp2 + prefix: /glade/u/apps/derecho/23.09/spack/opt/spack/curl/8.1.2/gcc/7.5.0/uq6y cvs: externals: - spec: cvs@1.12.13 @@ -36,13 +41,6 @@ packages: externals: - spec: diffutils@3.6 prefix: /usr - ecflow:: - buildable: False - externals: - - spec: ecflow@5.8.4+ui+static_boost - prefix: /glade/work/epicufsrt/contrib/spack-stack/derecho/ecflow-5.8.4 - modules: - - ecflow/5.8.4 findutils: externals: - spec: findutils@4.8.0 @@ -96,6 +94,11 @@ packages: externals: - spec: pkg-config@0.29.2 prefix: /usr + qt: + buildable: False + externals: + - spec: qt@5.14.2 + prefix: /glade/u/apps/derecho/23.09/spack/opt/spack/qt/5.14.2/gcc/7.5.0/eqcj sed: externals: - spec: sed@4.4 diff --git a/configs/sites/tier1/derecho/packages_intel.yaml b/configs/sites/tier1/derecho/packages_intel.yaml index dc534a575..06f45d62d 100644 --- a/configs/sites/tier1/derecho/packages_intel.yaml +++ b/configs/sites/tier1/derecho/packages_intel.yaml @@ -1,6 +1,6 @@ packages: all: - compiler:: [intel@2021.10.0,gcc@12.2.0] + compiler:: [intel@2021.10.0,gcc@11.2.0] providers: mpi:: [cray-mpich@8.1.25] # Remove the next three lines to switch to intel-oneapi-mkl diff --git a/doc/source/PreConfiguredSites.rst b/doc/source/PreConfiguredSites.rst index 97b532daa..7aa53a360 100644 --- a/doc/source/PreConfiguredSites.rst +++ b/doc/source/PreConfiguredSites.rst @@ -225,7 +225,6 @@ The following is required for building new spack environments with any supported export LMOD_TMOD_FIND_FIRST=yes module load ncarenv/23.09 module use /glade/work/epicufsrt/contrib/spack-stack/derecho/modulefiles - module load ecflow/5.8.4 .. _Preconfigured_Sites_Acorn: From 801362f2d0c3a4005196de687d60f6e81be0769d Mon Sep 17 00:00:00 2001 From: RatkoVasic-NOAA <37597874+RatkoVasic-NOAA@users.noreply.github.com> Date: Wed, 2 Oct 2024 20:18:18 -0400 Subject: [PATCH 23/26] Update changes from tier1 machines for 1.8.0 release (#1310) Update changes for NOAA RDHPCS tier1 machines for 1.8.0 --- configs/sites/tier1/gaea-c5/compilers.yaml | 14 +- configs/sites/tier1/gaea-c6/compilers.yaml | 14 +- configs/sites/tier1/hera/mirrors.yaml | 4 +- configs/sites/tier1/jet/packages.yaml | 4 - configs/sites/tier1/jet/packages_intel.yaml | 3 +- configs/sites/tier1/noaa-aws/README.md | 187 ++++------------- configs/sites/tier1/noaa-aws/compilers.yaml | 22 +- configs/sites/tier1/noaa-aws/packages.yaml | 73 ++++--- .../sites/tier1/noaa-aws/packages_intel.yaml | 8 +- configs/sites/tier1/noaa-azure/README.md | 189 ++++------------- configs/sites/tier1/noaa-azure/compilers.yaml | 22 +- configs/sites/tier1/noaa-azure/packages.yaml | 100 +++------ .../tier1/noaa-azure/packages_intel.yaml | 8 +- configs/sites/tier1/noaa-gcloud/README.md | 191 ++++-------------- .../sites/tier1/noaa-gcloud/compilers.yaml | 22 +- configs/sites/tier1/noaa-gcloud/packages.yaml | 79 ++++---- .../tier1/noaa-gcloud/packages_intel.yaml | 8 +- doc/source/PreConfiguredSites.rst | 52 +---- 18 files changed, 291 insertions(+), 709 deletions(-) diff --git a/configs/sites/tier1/gaea-c5/compilers.yaml b/configs/sites/tier1/gaea-c5/compilers.yaml index e60d7fece..ad1fd416d 100644 --- a/configs/sites/tier1/gaea-c5/compilers.yaml +++ b/configs/sites/tier1/gaea-c5/compilers.yaml @@ -23,4 +23,16 @@ compilers: # which confuses some packages. CONFIG_SITE: '' extra_rpaths: [] -# Todo: Define gcc compiler 10.3.0 for Intel backend (ecflow) +- compiler: + spec: gcc@12.2.0 + paths: + cc: /opt/cray/pe/gcc/12.2.0/bin/gcc + cxx: /opt/cray/pe/gcc/12.2.0/bin/g++ + f77: /opt/cray/pe/gcc/12.2.0/bin/gfortran + fc: /opt/cray/pe/gcc/12.2.0/bin/gfortran + flags: {} + operating_system: sles15 + modules: + - gcc/12.2.0 + environment: {} + extra_rpaths: [] diff --git a/configs/sites/tier1/gaea-c6/compilers.yaml b/configs/sites/tier1/gaea-c6/compilers.yaml index 1bca116f0..aac06161c 100644 --- a/configs/sites/tier1/gaea-c6/compilers.yaml +++ b/configs/sites/tier1/gaea-c6/compilers.yaml @@ -23,4 +23,16 @@ compilers: # which confuses some packages. CONFIG_SITE: '' extra_rpaths: [] -# Todo: Define gcc compiler 12.x.y for Intel backend (ecflow) +- compiler: + spec: gcc@12.3.0 + paths: + cc: /usr/bin/gcc-12 + cxx: /usr/bin/g++-12 + f77: /usr/bin/gfortran-12 + fc: /usr/bin/gfortran-12 + flags: {} + operating_system: sles15 + modules: + - gcc-native-mixed/12.3 + environment: {} + extra_rpaths: [] diff --git a/configs/sites/tier1/hera/mirrors.yaml b/configs/sites/tier1/hera/mirrors.yaml index 392059b66..f6c5a8d1e 100644 --- a/configs/sites/tier1/hera/mirrors.yaml +++ b/configs/sites/tier1/hera/mirrors.yaml @@ -1,7 +1,7 @@ mirrors: local-source: fetch: - url: file:///contrib/spack-stack/mirror + url: file:///scratch1/NCEPDEV/nems/role.epic/spack-stack/source-cache access_pair: - null - null @@ -9,7 +9,7 @@ mirrors: profile: null endpoint_url: null push: - url: file:///contrib/spack-stack/mirror + url: file:///scratch1/NCEPDEV/nems/role.epic/spack-stack/source-cache access_pair: - null - null diff --git a/configs/sites/tier1/jet/packages.yaml b/configs/sites/tier1/jet/packages.yaml index 665e659ac..04633817d 100644 --- a/configs/sites/tier1/jet/packages.yaml +++ b/configs/sites/tier1/jet/packages.yaml @@ -57,10 +57,6 @@ packages: externals: - spec: gawk@4.0.2 prefix: /usr - gettext: - externals: - - spec: gettext@0.19.8.1 - prefix: /usr ghostscript: externals: - spec: ghostscript@9.25 diff --git a/configs/sites/tier1/jet/packages_intel.yaml b/configs/sites/tier1/jet/packages_intel.yaml index d9cdd7209..cc0cac045 100644 --- a/configs/sites/tier1/jet/packages_intel.yaml +++ b/configs/sites/tier1/jet/packages_intel.yaml @@ -8,8 +8,7 @@ packages: intel-oneapi-mpi: externals: - spec: intel-oneapi-mpi@2021.5.1%intel@2021.5.0 - modules: - - impi/2022.1.2 + prefix: /apps/oneapi intel-oneapi-mkl: externals: - spec: intel-oneapi-mkl@2022.0.2%intel@2021.5.0 diff --git a/configs/sites/tier1/noaa-aws/README.md b/configs/sites/tier1/noaa-aws/README.md index 3fd415810..1910fb757 100644 --- a/configs/sites/tier1/noaa-aws/README.md +++ b/configs/sites/tier1/noaa-aws/README.md @@ -1,164 +1,51 @@ # Provisiong ParallelWorks AWS clusters -## Steps to perform when provisioning a cluster for the very first time - -This section provides step by step instructions for installing the basic packages (OS packages, external packages) for spack-stack. Following these steps ensures that the site configuration files in `configs/sites/noaa-aws` work out of the box. - -### Basic system packages -``` -sudo su -chmod 777 /contrib - -# The following three commands were necessary to fix failures accessing these repos -yum-config-manager --disable intel-clck-2019-repo -yum-config-manager --disable intel-hpc-platform -yum-config-manager --disable intelpython - -yum install -y qt5-qtbase-devel -yum install -y qt5-qtsvg-devel -yum install -y xorg-x11-xauth -yum install -y xorg-x11-apps -yum install -y perl-IPC-Cmd -yum install -y gettext-devel -yum install -y m4 -exit -``` -Create a script that can be added to the cluster resource config so that these packages get installed automatically when provisioning new clusters later: -``` -mkdir -p /contrib/admin -cat < /contrib/admin/basic_setup.sh -#!/bin/bash +## Steps to perform before installing spack-stack version 1.8.0 +sudo su - chmod 777 /contrib yum install -y qt5-qtbase-devel yum install -y qt5-qtsvg-devel -yum install -y xorg-x11-xauth -yum install -y xorg-x11-apps -yum install -y perl-IPC-Cmd -yum install -y gettext-devel -yum install -y m4 -EOF - -chmod a+x /contrib/admin/basic_setup.sh -``` - -### Create a mysql config for local R2D2 use (if applicable) -``` -sudo su -cat < /contrib/admin/my.cnf -[mysqld] -datadir=/mysql_local/data -socket=/mysql_local/data/mysql.sock -symbolic-links=0 -default-authentication-plugin=mysql_native_password - -[mysqld_safe] -log-error=/mysql_local/log/mariadb.log -pid-file=/mysql_local/run/mariadb.pid -EOF -chmod 644 /contrib/admin/my.cnf -exit -``` - -### Build external packages for spack-stack -``` -mkdir -p /contrib/spack-stack -mkdir /contrib/spack-stack/modulefiles -cd /contrib/spack-stack/ - -mkdir -p git-lfs-2.4.1/src -cd git-lfs-2.4.1/src -wget http://mirror.centos.org/centos/7/sclo/x86_64/rh/Packages/r/rh-git218-git-lfs-2.4.1-3.el7.x86_64.rpm -rpm2cpio rh-git218-git-lfs-2.4.1-3.el7.x86_64.rpm | cpio -idmv -mv opt/rh/rh-git218/root/usr/* .. -rm -fr opt -cd /contrib/spack-stack/modulefiles -mkdir git-lfs -# Create the modulefile from the template in doc/modulefile_templates -cd /contrib/spack-stack/ -mkdir -p mysql-8.0.31/src -cd mysql-8.0.31/src -ldd --version -wget https://dev.mysql.com/get/Downloads/MySQL-8.0/mysql-8.0.31-linux-glibc2.17-x86_64-minimal.tar.xz -tar -xvf mysql-8.0.31-linux-glibc2.17-x86_64-minimal.tar.xz -mv mysql-8.0.31-linux-glibc2.17-x86_64-minimal/* .. -rmdir mysql-8.0.31-linux-glibc2.17-x86_64-minimal -cd /contrib/spack-stack/modulefiles -mkdir mysql -# Create the modulefile from the template in doc/modulefile_templates -cd /contrib/spack-stack/ -mkdir -p cmake-3.27.2/src -cd cmake-3.27.2/src -wget https://github.com/Kitware/CMake/releases/download/v3.27.2/cmake-3.27.2-linux-x86_64.tar.gz -tar -xvzf cmake-3.27.2-linux-x86_64.tar.gz -mv cmake-3.27.2-linux-x86_64/* .. -rmdir cmake-3.27.2-linux-x86_64 -cd /contrib/spack-stack/modulefiles -mkdir cmake -# Create the modulefile from the template in doc/modulefile_templates +## Steps to install spack-stack version 1.8.0 -# Set up basic modules for building the external ecflow package -module unuse /opt/cray/craype/default/modulefiles -module unuse /opt/cray/modulefiles +sudo su - +chmod 777 /contrib module purge -module load gnu/9.2.0 -module use /contrib/spack-stack/modulefiles -module load cmake/3.27.2 - -mkdir -p /contrib/spack-stack/ecflow-5.8.4/src -cd /contrib/spack-stack/ecflow-5.8.4/src -wget https://confluence.ecmwf.int/download/attachments/8650755/ecFlow-5.8.4-Source.tar.gz?api=v2 -wget https://boostorg.jfrog.io/artifactory/main/release/1.78.0/source/boost_1_78_0.tar.gz -mv ecFlow-5.8.4-Source.tar.gz\?api\=v2 ecFlow-5.8.4-Source.tar.gz -tar -xvzf boost_1_78_0.tar.gz -tar -xvzf ecFlow-5.8.4-Source.tar.gz -export WK=/contrib/spack-stack/ecflow-5.8.4/src/ecFlow-5.8.4-Source -export BOOST_ROOT=/contrib/spack-stack/ecflow-5.8.4/src/boost_1_78_0 - -# Build static boost (to not interfere with spack-stack boost) -cd $BOOST_ROOT -./bootstrap.sh 2>&1 | tee bootstrap.log -$WK/build_scripts/boost_build.sh 2>&1 | tee boost_build.log - -# Build ecFlow -cd $WK -mkdir build -cd build -cmake .. -DCMAKE_INSTALL_PREFIX=/contrib/spack-stack/ecflow-5.8.4 2>&1 | tee log.cmake -make -j4 2>&1 | tee log.make -make install 2>&1 | tee log.install - -cd /contrib/spack-stack/modulefiles -mkdir ecflow -# Create the modulefile from the template in doc/modulefile_templates -``` - -## Steps to perform when starting a new cluster - -This should be done automatically, but doesn't hurt to run again just in case: -``` -sudo /contrib/admin/basic_setup.sh -``` -Configure `git` and `aws` command line utilities -``` module unuse /opt/cray/craype/default/modulefiles module unuse /opt/cray/modulefiles -module use /contrib/spack-stack/modulefiles -module load cmake/3.27.2 -module load ecflow/5.8.4 -module load mysql/8.0.31 -module load git-lfs/2.4.1 - -git lfs install -git config --global credential.helper cache # or store -git config --global user.email "your.name@domain.com" -git config --global user.name "Your Name" +### For noaa-aws, run the line below as well: +module unuse /opt/intel/impi/2019.5.281/intel64/modulefiles +module load gnu +module load intel/2023.2.0 +module load impi/2023.2.0 +module unload gnu + +cd /contrib/spack-stack-rocky8/ +git clone --recursive https://github.com/JCSDA/spack-stack -b release/1.8.0 spack-stack-1.8.0 +cd spack-stack-1.8.0 +. setup.sh +spack stack create env --name ue-intel-2021.10.0 --template unified-dev --site noaa-aws --compiler intel +cd envs/ue-intel-2021.10.0 +spack env activate . +spack concretize 2>&1 | tee log.concretize +spack install --verbose 2>&1 | tee log.install +spack module lmod refresh -y +spack stack setup-meta-modules + +## Steps to install GSI addon + +sudo su - +chmod 777 /contrib -mkdir ~/.aws -# Create ~/.aws/config -# Create ~/.aws/credentials -chmod 400 ~/.aws/credentials -``` +cd /contrib/spack-stack-rocky8/spack-stack-1.8.0 +. setup.sh +spack stack create env --name gsi-intel-2021.10.0 --template gsi-addon-dev --site noaa-aws --upstream /contrib/spack-stack-rocky8/spack-stack-1.8.0/envs/ue-intel-2021.10.0/install --compiler intel +cd envs/gsi-intel-2021.10.0 +spack env activate . +spack concretize 2>&1 | tee log.concretize +spack install --verbose 2>&1 | tee log.install +spack module lmod refresh --upstream-modules +spack stack setup-meta-modules diff --git a/configs/sites/tier1/noaa-aws/compilers.yaml b/configs/sites/tier1/noaa-aws/compilers.yaml index aec6336cf..343520e4c 100644 --- a/configs/sites/tier1/noaa-aws/compilers.yaml +++ b/configs/sites/tier1/noaa-aws/compilers.yaml @@ -1,20 +1,19 @@ compilers: - compiler: - spec: intel@2021.3.0 + spec: intel@2021.10.0 paths: - cc: /apps/oneapi/compiler/2021.3.0/linux/bin/intel64/icc - cxx: /apps/oneapi/compiler/2021.3.0/linux/bin/intel64/icpc - f77: /apps/oneapi/compiler/2021.3.0/linux/bin/intel64/ifort - fc: /apps/oneapi/compiler/2021.3.0/linux/bin/intel64/ifort - flags: {} - operating_system: centos7 + cc: /apps/oneapi/compiler/2023.2.0/linux/bin/intel64/icc + cxx: /apps/oneapi/compiler/2023.2.0/linux/bin/intel64/icpc + f77: /apps/oneapi/compiler/2023.2.0/linux/bin/intel64/ifort + fc: /apps/oneapi/compiler/2023.2.0/linux/bin/intel64/ifort + operating_system: rocky8 target: x86_64 modules: - - intel/2021.3.0 + - intel/2023.2.0 environment: prepend_path: - PATH: '/apps/gnu/gcc-9.2.0/bin' - LD_LIBRARY_PATH: '/apps/oneapi/compiler/2021.3.0/linux/compiler/lib/intel64_lin:/apps/gnu/gcc-9.2.0/lib64' + LD_LIBRARY_PATH: '/apps/oneapi/compiler/2023.2.0/linux/compiler/lib/intel64_lin:/apps/gnu/gcc-13.2.0/lib64' + flags: {} extra_rpaths: [] - compiler: spec: gcc@9.2.0 @@ -24,8 +23,7 @@ compilers: f77: /apps/gnu/gcc-9.2.0/bin/gfortran fc: /apps/gnu/gcc-9.2.0/bin/gfortran flags: {} - operating_system: centos7 - target: x86_64 + operating_system: rocky8 modules: - gnu/9.2.0 environment: {} diff --git a/configs/sites/tier1/noaa-aws/packages.yaml b/configs/sites/tier1/noaa-aws/packages.yaml index a16a87b90..ee153e8a5 100644 --- a/configs/sites/tier1/noaa-aws/packages.yaml +++ b/configs/sites/tier1/noaa-aws/packages.yaml @@ -1,115 +1,114 @@ packages: -### Modifications of common packages - # Pin flex to avoid duplicate packages - flex: - version: ['2.6.4'] -### All other external packages listed alphabetically bash: externals: - - spec: bash@4.2.46 + - spec: bash@4.4.20 prefix: /usr berkeley-db: externals: - - spec: berkeley-db@5.3.21 + - spec: berkeley-db@5.3.28 prefix: /usr cpio: externals: - - spec: cpio@2.11 + - spec: cpio@2.12 prefix: /usr diffutils: externals: - - spec: diffutils@3.3 + - spec: diffutils@3.6 prefix: /usr ecflow:: buildable: False externals: - spec: ecflow@5.8.4+ui+static_boost - prefix: /contrib/spack-stack/ecflow-5.8.4 + prefix: /contrib/spack-stack-rocky8/ecflow-5.8.4 + modules: + - ecflow/5.8.4 file: externals: - - spec: file@5.11 + - spec: file@5.33 prefix: /usr findutils: externals: - - spec: findutils@4.5.11 + - spec: findutils@4.6.0 prefix: /usr - gawk: + flex: externals: - - spec: gawk@4.0.2 + - spec: flex@2.6.1 prefix: /usr - gettext: + gawk: externals: - - spec: gettext@0.19.8.1 + - spec: gawk@4.2.1 prefix: /usr git: externals: - - spec: git@1.8.3.1~tcltk + - spec: git@2.39.3~tcltk prefix: /usr git-lfs: externals: - - spec: git-lfs@2.4.1 - prefix: /contrib/spack-stack/git-lfs-2.4.1 + - spec: git-lfs@3.2.0 + prefix: /usr gmake: externals: - - spec: gmake@3.82 + - spec: gmake@4.2.1 prefix: /usr groff: externals: - - spec: groff@1.22.2 + - spec: groff@1.21 prefix: /usr hwloc: externals: - - spec: hwloc@1.11.8 + - spec: hwloc@2.2.0 prefix: /usr krb5: externals: - - spec: krb5@1.15.1 - prefix: /usr - lustre: - externals: - - spec: lustre@2.12.7 + - spec: krb5@1.18.2 prefix: /usr mysql: externals: - spec: mysql@8.0.31 - prefix: /contrib/spack-stack/mysql-8.0.31 + prefix: /contrib/spack-stack-rocky8/mysql-8.0.31 + ncurses: + externals: + - spec: ncurses@6.1-10.20180224+termlib abi=5 + prefix: /usr openjdk: externals: - - spec: openjdk@1.8.0_322-b06 + - spec: openjdk@1.8.0_402-b06 prefix: /usr perl: externals: - - spec: perl@5.16.3~cpanm+shared+threads + - spec: perl@5.26.3~cpanm+shared+threads prefix: /usr pkg-config: externals: - - spec: pkg-config@0.27.1 + - spec: pkg-config@1.4.2 prefix: /usr rsync: externals: - - spec: rsync@3.1.2 + - spec: rsync@3.1.3 prefix: /usr ruby: externals: - - spec: ruby@2.0.0 + - spec: ruby@2.5.9p229 prefix: /usr sed: externals: - - spec: sed@4.2.2 + - spec: sed@4.5 prefix: /usr tar: externals: - - spec: tar@1.26 + - spec: tar@1.30 prefix: /usr texinfo: externals: - - spec: texinfo@5.1 + - spec: texinfo@6.5-7 prefix: /usr wget: externals: - - spec: wget@1.14 + - spec: wget@1.19.5 prefix: /usr zip: externals: - spec: zip@3.0 prefix: /usr + zlib: + require: [~shared] diff --git a/configs/sites/tier1/noaa-aws/packages_intel.yaml b/configs/sites/tier1/noaa-aws/packages_intel.yaml index 2c98fa98f..1ffe7cf2f 100644 --- a/configs/sites/tier1/noaa-aws/packages_intel.yaml +++ b/configs/sites/tier1/noaa-aws/packages_intel.yaml @@ -1,8 +1,8 @@ packages: all: - compiler:: [intel@2021.3.0,gcc@9.2.0] + compiler:: [intel@2021.10.0] providers: - mpi:: [intel-oneapi-mpi@2021.3.0] + mpi:: [intel-oneapi-mpi@2021.10.0] # Remove the next three lines to switch to intel-oneapi-mkl blas:: [openblas] fftw-api:: [fftw] @@ -11,10 +11,10 @@ packages: buildable: False intel-oneapi-mpi: externals: - - spec: intel-oneapi-mpi@2021.3.0%intel@2021.3.0 + - spec: intel-oneapi-mpi@2021.10.0%intel@2021.10.0 prefix: /apps/oneapi modules: - - impi/2021.3.0 + - impi/2023.2.0 intel-oneapi-mkl: # Remove buildable: False and configure+uncomment externals section below to use intel-oneapi-mkl buildable: False diff --git a/configs/sites/tier1/noaa-azure/README.md b/configs/sites/tier1/noaa-azure/README.md index 2253cda9f..eff315412 100644 --- a/configs/sites/tier1/noaa-azure/README.md +++ b/configs/sites/tier1/noaa-azure/README.md @@ -1,168 +1,49 @@ # Provisiong ParallelWorks Azure clusters -## Steps to perform when provisioning a cluster for the very first time - -This section provides step by step instructions for installing the basic packages (OS packages, external packages) for spack-stack. Following these steps ensures that the site configuration files in `configs/sites/noaa-aws` work out of the box. - -### Basic system packages -``` -sudo su -chmod 777 /contrib - -# The following three commands were necessary to fix failures accessing these repos -yum-config-manager --disable intel-clck-2019-repo -yum-config-manager --disable intel-hpc-platform -yum-config-manager --disable intelpython - -yum install -y qt5-qtbase-devel -yum install -y qt5-qtsvg-devel -yum install -y xorg-x11-xauth -yum install -y xorg-x11-apps -yum install -y perl-IPC-Cmd -yum install -y gettext-devel -yum install -y ncurses-devel -yum install -y ncurses-static -yum install -y m4 -exit -``` -Create a script that can be added to the cluster resource config so that these packages get installed automatically when provisioning new clusters later: -``` -mkdir -p /contrib/admin -cat < /contrib/admin/basic_setup.sh -#!/bin/bash +## Steps to perform before installing spack-stack version 1.8.0 +sudo su - chmod 777 /contrib yum install -y qt5-qtbase-devel yum install -y qt5-qtsvg-devel -yum install -y xorg-x11-xauth -yum install -y xorg-x11-apps -yum install -y perl-IPC-Cmd -yum install -y gettext-devel -yum install -y ncurses-devel -yum install -y ncurses-static -yum install -y m4 -EOF - -chmod a+x /contrib/admin/basic_setup.sh -``` - -### Create a mysql config for local R2D2 use (if applicable) -``` -sudo su -cat < /contrib/admin/my.cnf -[mysqld] -datadir=/mysql_local/data -socket=/mysql_local/data/mysql.sock -symbolic-links=0 -default-authentication-plugin=mysql_native_password - -[mysqld_safe] -log-error=/mysql_local/log/mariadb.log -pid-file=/mysql_local/run/mariadb.pid -EOF -chmod 644 /contrib/admin/my.cnf -exit -``` - -### Build external packages for spack-stack -``` -mkdir -p /contrib/spack-stack -mkdir /contrib/spack-stack/modulefiles -cd /contrib/spack-stack/ - -mkdir -p git-lfs-2.4.1/src -cd git-lfs-2.4.1/src -wget http://mirror.centos.org/centos/7/sclo/x86_64/rh/Packages/r/rh-git218-git-lfs-2.4.1-3.el7.x86_64.rpm -rpm2cpio rh-git218-git-lfs-2.4.1-3.el7.x86_64.rpm | cpio -idmv -mv opt/rh/rh-git218/root/usr/* .. -rm -fr opt -cd /contrib/spack-stack/modulefiles -mkdir git-lfs -# Create the modulefile from the template in doc/modulefile_templates -cd /contrib/spack-stack/ -mkdir -p mysql-8.0.31/src -cd mysql-8.0.31/src -ldd --version -wget https://dev.mysql.com/get/Downloads/MySQL-8.0/mysql-8.0.31-linux-glibc2.17-x86_64-minimal.tar.xz -tar -xvf mysql-8.0.31-linux-glibc2.17-x86_64-minimal.tar.xz -mv mysql-8.0.31-linux-glibc2.17-x86_64-minimal/* .. -rmdir mysql-8.0.31-linux-glibc2.17-x86_64-minimal -cd /contrib/spack-stack/modulefiles -mkdir mysql -# Create the modulefile from the template in doc/modulefile_templates -cd /contrib/spack-stack/ -mkdir -p cmake-3.27.2/src -cd cmake-3.27.2/src -wget https://github.com/Kitware/CMake/releases/download/v3.27.2/cmake-3.27.2-linux-x86_64.tar.gz -tar -xvzf cmake-3.27.2-linux-x86_64.tar.gz -mv cmake-3.27.2-linux-x86_64/* .. -rmdir cmake-3.27.2-linux-x86_64 -cd /contrib/spack-stack/modulefiles -mkdir cmake -# Create the modulefile from the template in doc/modulefile_templates +## Steps to install spack-stack version 1.8.0 -# Set up basic modules for building the external ecflow package -module unuse /opt/cray/craype/default/modulefiles -module unuse /opt/cray/modulefiles +sudo su - +chmod 777 /contrib module purge -module load gnu/9.2.0 -module use /contrib/spack-stack/modulefiles -module load cmake/3.27.2 - -mkdir -p /contrib/spack-stack/ecflow-5.8.4/src -cd /contrib/spack-stack/ecflow-5.8.4/src -wget https://confluence.ecmwf.int/download/attachments/8650755/ecFlow-5.8.4-Source.tar.gz?api=v2 -wget https://boostorg.jfrog.io/artifactory/main/release/1.78.0/source/boost_1_78_0.tar.gz -mv ecFlow-5.8.4-Source.tar.gz\?api\=v2 ecFlow-5.8.4-Source.tar.gz -tar -xvzf boost_1_78_0.tar.gz -tar -xvzf ecFlow-5.8.4-Source.tar.gz -export WK=/contrib/spack-stack/ecflow-5.8.4/src/ecFlow-5.8.4-Source -export BOOST_ROOT=/contrib/spack-stack/ecflow-5.8.4/src/boost_1_78_0 - -# Build static boost (to not interfere with spack-stack boost) -cd $BOOST_ROOT -./bootstrap.sh 2>&1 | tee bootstrap.log -$WK/build_scripts/boost_build.sh 2>&1 | tee boost_build.log - -# Build ecFlow -cd $WK -mkdir build -cd build -cmake .. -DCMAKE_INSTALL_PREFIX=/contrib/spack-stack/ecflow-5.8.4 2>&1 | tee log.cmake -make -j4 2>&1 | tee log.make -make install 2>&1 | tee log.install - -cd /contrib/spack-stack/modulefiles -mkdir ecflow -# Create the modulefile from the template in doc/modulefile_templates -``` - -## Steps to perform when starting a new cluster - -This should be done automatically, but doesn't hurt to run again just in case: -``` -sudo /contrib/admin/basic_setup.sh -``` -Configure `git` and `aws` command line utilities -``` module unuse /opt/cray/craype/default/modulefiles module unuse /opt/cray/modulefiles -module use /contrib/spack-stack/modulefiles -module load cmake/3.27.2 -module load ecflow/5.8.4 -module load mysql/8.0.31 -module load git-lfs/2.4.1 - -git lfs install -git config --global credential.helper cache # or store -git config --global user.email "your.name@domain.com" -git config --global user.name "Your Name" +module load gnu +module load intel/2023.2.0 +module load impi/2023.2.0 +module unload gnu + +cd /contrib/spack-stack-rocky8/ +git clone --recursive https://github.com/JCSDA/spack-stack -b release/1.8.0 spack-stack-1.8.0 +cd spack-stack-1.8.0 +. setup.sh +spack stack create env --name ue-intel-2021.10.0 --template unified-dev --site noaa-azure --compiler intel +cd envs/ue-intel-2021.10.0 +spack env activate . +spack concretize 2>&1 | tee log.concretize +spack install --verbose 2>&1 | tee log.install +spack module lmod refresh -y +spack stack setup-meta-modules + +## Steps to install GSI addon + +sudo su - +chmod 777 /contrib -mkdir ~/.aws -# Create ~/.aws/config -# Create ~/.aws/credentials -chmod 400 ~/.aws/credentials -``` +cd /contrib/spack-stack-rocky8/spack-stack-1.8.0 +. setup.sh +spack stack create env --name gsi-intel-2021.10.0 --template gsi-addon-dev --site noaa-azure --upstream /contrib/spack-stack-rocky8/spack-stack-1.8.0/envs/ue-intel-2021.10.0/install --compiler intel +cd envs/gsi-intel-2021.10.0 +spack env activate . +spack concretize 2>&1 | tee log.concretize +spack install --verbose 2>&1 | tee log.install +spack module lmod refresh --upstream-modules +spack stack setup-meta-modules diff --git a/configs/sites/tier1/noaa-azure/compilers.yaml b/configs/sites/tier1/noaa-azure/compilers.yaml index aec6336cf..343520e4c 100644 --- a/configs/sites/tier1/noaa-azure/compilers.yaml +++ b/configs/sites/tier1/noaa-azure/compilers.yaml @@ -1,20 +1,19 @@ compilers: - compiler: - spec: intel@2021.3.0 + spec: intel@2021.10.0 paths: - cc: /apps/oneapi/compiler/2021.3.0/linux/bin/intel64/icc - cxx: /apps/oneapi/compiler/2021.3.0/linux/bin/intel64/icpc - f77: /apps/oneapi/compiler/2021.3.0/linux/bin/intel64/ifort - fc: /apps/oneapi/compiler/2021.3.0/linux/bin/intel64/ifort - flags: {} - operating_system: centos7 + cc: /apps/oneapi/compiler/2023.2.0/linux/bin/intel64/icc + cxx: /apps/oneapi/compiler/2023.2.0/linux/bin/intel64/icpc + f77: /apps/oneapi/compiler/2023.2.0/linux/bin/intel64/ifort + fc: /apps/oneapi/compiler/2023.2.0/linux/bin/intel64/ifort + operating_system: rocky8 target: x86_64 modules: - - intel/2021.3.0 + - intel/2023.2.0 environment: prepend_path: - PATH: '/apps/gnu/gcc-9.2.0/bin' - LD_LIBRARY_PATH: '/apps/oneapi/compiler/2021.3.0/linux/compiler/lib/intel64_lin:/apps/gnu/gcc-9.2.0/lib64' + LD_LIBRARY_PATH: '/apps/oneapi/compiler/2023.2.0/linux/compiler/lib/intel64_lin:/apps/gnu/gcc-13.2.0/lib64' + flags: {} extra_rpaths: [] - compiler: spec: gcc@9.2.0 @@ -24,8 +23,7 @@ compilers: f77: /apps/gnu/gcc-9.2.0/bin/gfortran fc: /apps/gnu/gcc-9.2.0/bin/gfortran flags: {} - operating_system: centos7 - target: x86_64 + operating_system: rocky8 modules: - gnu/9.2.0 environment: {} diff --git a/configs/sites/tier1/noaa-azure/packages.yaml b/configs/sites/tier1/noaa-azure/packages.yaml index c3522a095..004aea213 100644 --- a/configs/sites/tier1/noaa-azure/packages.yaml +++ b/configs/sites/tier1/noaa-azure/packages.yaml @@ -1,146 +1,110 @@ packages: -### Modifications of common packages - # Pin flex to avoid duplicate packages - flex: - version: ['2.6.4'] -### All other external packages listed alphabetically - autoconf: - externals: - - spec: autoconf@2.69 - prefix: /usr - automake: - externals: - - spec: automake@1.13.4 - prefix: /usr bash: externals: - - spec: bash@4.2.46 + - spec: bash@4.4.20 prefix: /usr berkeley-db: externals: - - spec: berkeley-db@5.3.21 + - spec: berkeley-db@5.3.28 prefix: /usr cpio: externals: - - spec: cpio@2.11 + - spec: cpio@2.12 prefix: /usr diffutils: externals: - - spec: diffutils@3.3 - prefix: /usr - doxygen: - externals: - - spec: doxygen@1.8.5~graphviz~mscgen + - spec: diffutils@3.6 prefix: /usr ecflow:: buildable: False externals: - spec: ecflow@5.8.4+ui+static_boost - prefix: /contrib/spack-stack/ecflow-5.8.4 + prefix: /contrib/spack-stack-rocky8/ecflow-5.8.4 + modules: + - ecflow/5.8.4 file: externals: - - spec: file@5.11 + - spec: file@5.33 prefix: /usr findutils: externals: - - spec: findutils@4.5.11 + - spec: findutils@4.6.0 prefix: /usr - gawk: + flex: externals: - - spec: gawk@4.0.2 + - spec: flex@2.6.1 prefix: /usr - gettext: + gawk: externals: - - spec: gettext@0.19.8.1 + - spec: gawk@4.2.1 prefix: /usr git: externals: - - spec: git@1.8.3.1~tcltk + - spec: git@2.39.3~tcltk prefix: /usr git-lfs: externals: - - spec: git-lfs@2.4.1 - prefix: /contrib/spack-stack/git-lfs-2.4.1 - gmake: - externals: - - spec: gmake@3.82 + - spec: git-lfs@3.2.0 prefix: /usr - groff: + gmake: externals: - - spec: groff@1.22.2 + - spec: gmake@4.2.1 prefix: /usr hwloc: externals: - - spec: hwloc@1.11.8 + - spec: hwloc@2.2.0 prefix: /usr krb5: externals: - - spec: krb5@1.15.1 - prefix: /usr - libtool: - externals: - - spec: libtool@2.4.2 - prefix: /usr - # Problem untarring libxml2 on the Azure filesystem - # so mark it as external - libxml2: - buildable: False - externals: - - spec: libxml2@2.9.1 - prefix: /usr - lustre: - externals: - - spec: lustre@2.12.7 - prefix: /usr - m4: - externals: - - spec: m4@1.4.16 + - spec: krb5@1.18.2 prefix: /usr mysql: externals: - spec: mysql@8.0.31 - prefix: /contrib/spack-stack/mysql-8.0.31 + prefix: /contrib/spack-stack-rocky8/mysql-8.0.31 ncurses: externals: - - spec: ncurses@5.9-14.20130511+termlib abi=5 + - spec: ncurses@6.1-10.20180224+termlib abi=5 prefix: /usr openjdk: externals: - - spec: openjdk@1.8.0_322-b06 + - spec: openjdk@1.8.0_402-b06 prefix: /usr perl: externals: - - spec: perl@5.16.3~cpanm+shared+threads + - spec: perl@5.26.3~cpanm+shared+threads prefix: /usr pkg-config: externals: - - spec: pkg-config@0.27.1 + - spec: pkg-config@1.4.2 prefix: /usr rsync: externals: - - spec: rsync@3.1.2 + - spec: rsync@3.1.3 prefix: /usr ruby: externals: - - spec: ruby@2.0.0 + - spec: ruby@2.5.9p229 prefix: /usr sed: externals: - - spec: sed@4.2.2 + - spec: sed@4.5 prefix: /usr tar: externals: - - spec: tar@1.26 + - spec: tar@1.30 prefix: /usr texinfo: externals: - - spec: texinfo@5.1 + - spec: texinfo@6.5-7 prefix: /usr wget: externals: - - spec: wget@1.14 + - spec: wget@1.19.5 prefix: /usr zip: externals: - spec: zip@3.0 prefix: /usr + zlib: + require: [~shared] diff --git a/configs/sites/tier1/noaa-azure/packages_intel.yaml b/configs/sites/tier1/noaa-azure/packages_intel.yaml index 2c98fa98f..1ffe7cf2f 100644 --- a/configs/sites/tier1/noaa-azure/packages_intel.yaml +++ b/configs/sites/tier1/noaa-azure/packages_intel.yaml @@ -1,8 +1,8 @@ packages: all: - compiler:: [intel@2021.3.0,gcc@9.2.0] + compiler:: [intel@2021.10.0] providers: - mpi:: [intel-oneapi-mpi@2021.3.0] + mpi:: [intel-oneapi-mpi@2021.10.0] # Remove the next three lines to switch to intel-oneapi-mkl blas:: [openblas] fftw-api:: [fftw] @@ -11,10 +11,10 @@ packages: buildable: False intel-oneapi-mpi: externals: - - spec: intel-oneapi-mpi@2021.3.0%intel@2021.3.0 + - spec: intel-oneapi-mpi@2021.10.0%intel@2021.10.0 prefix: /apps/oneapi modules: - - impi/2021.3.0 + - impi/2023.2.0 intel-oneapi-mkl: # Remove buildable: False and configure+uncomment externals section below to use intel-oneapi-mkl buildable: False diff --git a/configs/sites/tier1/noaa-gcloud/README.md b/configs/sites/tier1/noaa-gcloud/README.md index f1fe42a35..f0fab7b91 100644 --- a/configs/sites/tier1/noaa-gcloud/README.md +++ b/configs/sites/tier1/noaa-gcloud/README.md @@ -1,169 +1,48 @@ -# Provisiong ParallelWorks Gcloud clusters +# Provisiong ParallelWorks GCP clusters -## Steps to perform when provisioning a cluster for the very first time +## Steps to perform before installing spack-stack version 1.8.0 -This section provides step by step instructions for installing the basic packages (OS packages, external packages) for spack-stack. Following these steps ensures that the site configuration files in `configs/sites/noaa-aws` work out of the box. - -### Basic system packages -``` -sudo su -chmod 777 /contrib - -# The following three commands were necessary to fix failures accessing these repos -yum-config-manager --disable intel-clck-2019-repo -yum-config-manager --disable intel-hpc-platform -yum-config-manager --disable intelpython - -yum install -y qt5-qtbase-devel -yum install -y qt5-qtsvg-devel -yum install -y xorg-x11-xauth -yum install -y xorg-x11-apps -yum install -y perl-IPC-Cmd -yum install -y gettext-devel -yum install -y m4 -yum install -y finger -exit -``` -Create a script that can be added to the cluster resource config so that these packages get installed automatically when provisioning new clusters later: -``` -mkdir -p /contrib/admin -cat < /contrib/admin/basic_setup.sh -#!/bin/bash - -yum-config-manager --disable intel-clck-2019-repo -yum-config-manager --disable intel-hpc-platform -yum-config-manager --disable intelpython +sudo su - chmod 777 /contrib yum install -y qt5-qtbase-devel yum install -y qt5-qtsvg-devel -yum install -y xorg-x11-xauth -yum install -y xorg-x11-apps -yum install -y perl-IPC-Cmd -yum install -y gettext-devel -yum install -y m4 -yum install -y finger -EOF - -chmod a+x /contrib/admin/basic_setup.sh -``` - -### Create a mysql config for local R2D2 use (if applicable) -``` -sudo su -cat < /contrib/admin/my.cnf -[mysqld] -datadir=/mysql_local/data -socket=/mysql_local/data/mysql.sock -symbolic-links=0 -default-authentication-plugin=mysql_native_password - -[mysqld_safe] -log-error=/mysql_local/log/mariadb.log -pid-file=/mysql_local/run/mariadb.pid -EOF -chmod 644 /contrib/admin/my.cnf -exit -``` - -### Build external packages for spack-stack -``` -mkdir -p /contrib/spack-stack -mkdir /contrib/spack-stack/modulefiles -cd /contrib/spack-stack/ - -mkdir -p git-lfs-2.4.1/src -cd git-lfs-2.4.1/src -wget http://mirror.centos.org/centos/7/sclo/x86_64/rh/Packages/r/rh-git218-git-lfs-2.4.1-3.el7.x86_64.rpm -rpm2cpio rh-git218-git-lfs-2.4.1-3.el7.x86_64.rpm | cpio -idmv -mv opt/rh/rh-git218/root/usr/* .. -rm -fr opt -cd /contrib/spack-stack/modulefiles -mkdir git-lfs -# Create the modulefile from the template in doc/modulefile_templates -cd /contrib/spack-stack/ -mkdir -p mysql-8.0.31/src -cd mysql-8.0.31/src -ldd --version -wget https://dev.mysql.com/get/Downloads/MySQL-8.0/mysql-8.0.31-linux-glibc2.17-x86_64-minimal.tar.xz -tar -xvf mysql-8.0.31-linux-glibc2.17-x86_64-minimal.tar.xz -mv mysql-8.0.31-linux-glibc2.17-x86_64-minimal/* .. -rmdir mysql-8.0.31-linux-glibc2.17-x86_64-minimal -cd /contrib/spack-stack/modulefiles -mkdir mysql -# Create the modulefile from the template in doc/modulefile_templates -cd /contrib/spack-stack/ -mkdir -p cmake-3.27.2/src -cd cmake-3.27.2/src -wget https://github.com/Kitware/CMake/releases/download/v3.27.2/cmake-3.27.2-linux-x86_64.tar.gz -tar -xvzf cmake-3.27.2-linux-x86_64.tar.gz -mv cmake-3.27.2-linux-x86_64/* .. -rmdir cmake-3.27.2-linux-x86_64 -cd /contrib/spack-stack/modulefiles -mkdir cmake -# Create the modulefile from the template in doc/modulefile_templates +## Steps to install spack-stack version 1.8.0 -# Set up basic modules for building the external ecflow package -module unuse /opt/cray/craype/default/modulefiles -module unuse /opt/cray/modulefiles +sudo su - +chmod 777 /contrib module purge -module load gnu/9.2.0 -module use /contrib/spack-stack/modulefiles -module load cmake/3.27.2 - -mkdir -p /contrib/spack-stack/ecflow-5.8.4/src -cd /contrib/spack-stack/ecflow-5.8.4/src -wget https://confluence.ecmwf.int/download/attachments/8650755/ecFlow-5.8.4-Source.tar.gz?api=v2 -wget https://boostorg.jfrog.io/artifactory/main/release/1.78.0/source/boost_1_78_0.tar.gz -mv ecFlow-5.8.4-Source.tar.gz\?api\=v2 ecFlow-5.8.4-Source.tar.gz -tar -xvzf boost_1_78_0.tar.gz -tar -xvzf ecFlow-5.8.4-Source.tar.gz -export WK=/contrib/spack-stack/ecflow-5.8.4/src/ecFlow-5.8.4-Source -export BOOST_ROOT=/contrib/spack-stack/ecflow-5.8.4/src/boost_1_78_0 - -# Build static boost (to not interfere with spack-stack boost) -cd $BOOST_ROOT -./bootstrap.sh 2>&1 | tee bootstrap.log -$WK/build_scripts/boost_build.sh 2>&1 | tee boost_build.log - -# Build ecFlow -cd $WK -mkdir build -cd build -cmake .. -DCMAKE_INSTALL_PREFIX=/contrib/spack-stack/ecflow-5.8.4 2>&1 | tee log.cmake -make -j4 2>&1 | tee log.make -make install 2>&1 | tee log.install - -cd /contrib/spack-stack/modulefiles -mkdir ecflow -# Create the modulefile from the template in doc/modulefile_templates -``` - -## Steps to perform when starting a new cluster - -This should be done automatically, but doesn't hurt to run again just in case: -``` -sudo /contrib/admin/basic_setup.sh -``` -Configure `git` and `aws` command line utilities -``` module unuse /opt/cray/craype/default/modulefiles module unuse /opt/cray/modulefiles -module use /contrib/spack-stack/modulefiles -module load cmake/3.27.2 -module load ecflow/5.8.4 -module load mysql/8.0.31 -module load git-lfs/2.4.1 - -git lfs install -git config --global credential.helper cache # or store -git config --global user.email "your.name@domain.com" -git config --global user.name "Your Name" +module load gnu +module load intel/2023.2.0 +module load impi/2023.2.0 +module unload gnu + +cd /contrib/spack-stack-rocky8/ +git clone --recursive https://github.com/JCSDA/spack-stack -b release/1.8.0 spack-stack-1.8.0 +cd spack-stack-1.8.0 +. setup.sh +spack stack create env --name ue-intel-2021.10.0 --template unified-dev --site noaa-gcloud --compiler intel +cd envs/ue-intel-2021.10.0 +spack env activate . +spack concretize 2>&1 | tee log.concretize +spack install --verbose 2>&1 | tee log.install +spack module lmod refresh -y +spack stack setup-meta-modules + +## Steps to install GSI addon + +sudo su - +chmod 777 /contrib -mkdir ~/.aws -# Create ~/.aws/config -# Create ~/.aws/credentials -chmod 400 ~/.aws/credentials -``` +cd /contrib/spack-stack-rocky8/spack-stack-1.8.0 +. setup.sh +spack stack create env --name gsi-intel-2021.10.0 --template gsi-addon-dev --site noaa-gcloud --upstream /contrib/spack-stack-rocky8/spack-stack-1.8.0/envs/ue-intel-2021.10.0/install --compiler intel +cd envs/gsi-intel-2021.10.0 +spack concretize 2>&1 | tee log.concretize +spack install --verbose 2>&1 | tee log.install +spack module lmod refresh --upstream-modules +spack stack setup-meta-modules diff --git a/configs/sites/tier1/noaa-gcloud/compilers.yaml b/configs/sites/tier1/noaa-gcloud/compilers.yaml index aec6336cf..343520e4c 100644 --- a/configs/sites/tier1/noaa-gcloud/compilers.yaml +++ b/configs/sites/tier1/noaa-gcloud/compilers.yaml @@ -1,20 +1,19 @@ compilers: - compiler: - spec: intel@2021.3.0 + spec: intel@2021.10.0 paths: - cc: /apps/oneapi/compiler/2021.3.0/linux/bin/intel64/icc - cxx: /apps/oneapi/compiler/2021.3.0/linux/bin/intel64/icpc - f77: /apps/oneapi/compiler/2021.3.0/linux/bin/intel64/ifort - fc: /apps/oneapi/compiler/2021.3.0/linux/bin/intel64/ifort - flags: {} - operating_system: centos7 + cc: /apps/oneapi/compiler/2023.2.0/linux/bin/intel64/icc + cxx: /apps/oneapi/compiler/2023.2.0/linux/bin/intel64/icpc + f77: /apps/oneapi/compiler/2023.2.0/linux/bin/intel64/ifort + fc: /apps/oneapi/compiler/2023.2.0/linux/bin/intel64/ifort + operating_system: rocky8 target: x86_64 modules: - - intel/2021.3.0 + - intel/2023.2.0 environment: prepend_path: - PATH: '/apps/gnu/gcc-9.2.0/bin' - LD_LIBRARY_PATH: '/apps/oneapi/compiler/2021.3.0/linux/compiler/lib/intel64_lin:/apps/gnu/gcc-9.2.0/lib64' + LD_LIBRARY_PATH: '/apps/oneapi/compiler/2023.2.0/linux/compiler/lib/intel64_lin:/apps/gnu/gcc-13.2.0/lib64' + flags: {} extra_rpaths: [] - compiler: spec: gcc@9.2.0 @@ -24,8 +23,7 @@ compilers: f77: /apps/gnu/gcc-9.2.0/bin/gfortran fc: /apps/gnu/gcc-9.2.0/bin/gfortran flags: {} - operating_system: centos7 - target: x86_64 + operating_system: rocky8 modules: - gnu/9.2.0 environment: {} diff --git a/configs/sites/tier1/noaa-gcloud/packages.yaml b/configs/sites/tier1/noaa-gcloud/packages.yaml index b86026f11..004aea213 100644 --- a/configs/sites/tier1/noaa-gcloud/packages.yaml +++ b/configs/sites/tier1/noaa-gcloud/packages.yaml @@ -1,119 +1,110 @@ packages: -### Modifications of common packages - # Pin flex to avoid duplicate packages - flex: - version: ['2.6.4'] -### All other external packages listed alphabetically bash: externals: - - spec: bash@4.2.46 + - spec: bash@4.4.20 prefix: /usr berkeley-db: externals: - - spec: berkeley-db@5.3.21 + - spec: berkeley-db@5.3.28 prefix: /usr cpio: externals: - - spec: cpio@2.11 + - spec: cpio@2.12 prefix: /usr diffutils: externals: - - spec: diffutils@3.3 + - spec: diffutils@3.6 prefix: /usr ecflow:: buildable: False externals: - spec: ecflow@5.8.4+ui+static_boost - prefix: /contrib/spack-stack/ecflow-5.8.4 + prefix: /contrib/spack-stack-rocky8/ecflow-5.8.4 + modules: + - ecflow/5.8.4 file: externals: - - spec: file@5.11 + - spec: file@5.33 prefix: /usr findutils: externals: - - spec: findutils@4.5.11 + - spec: findutils@4.6.0 prefix: /usr - gawk: + flex: externals: - - spec: gawk@4.0.2 + - spec: flex@2.6.1 prefix: /usr - gettext: + gawk: externals: - - spec: gettext@0.19.8.1 + - spec: gawk@4.2.1 prefix: /usr git: externals: - - spec: git@1.8.3.1~tcltk + - spec: git@2.39.3~tcltk prefix: /usr git-lfs: externals: - - spec: git-lfs@2.4.1 - prefix: /contrib/spack-stack/git-lfs-2.4.1 - gmake: - externals: - - spec: gmake@3.82 + - spec: git-lfs@3.2.0 prefix: /usr - groff: + gmake: externals: - - spec: groff@1.22.2 + - spec: gmake@4.2.1 prefix: /usr hwloc: externals: - - spec: hwloc@1.11.8 + - spec: hwloc@2.2.0 prefix: /usr krb5: externals: - - spec: krb5@1.15.1 - prefix: /usr - lustre: - externals: - - spec: lustre@2.12.7 - prefix: /usr - m4: - externals: - - spec: m4@1.4.16 + - spec: krb5@1.18.2 prefix: /usr mysql: externals: - spec: mysql@8.0.31 - prefix: /contrib/spack-stack/mysql-8.0.31 + prefix: /contrib/spack-stack-rocky8/mysql-8.0.31 + ncurses: + externals: + - spec: ncurses@6.1-10.20180224+termlib abi=5 + prefix: /usr openjdk: externals: - - spec: openjdk@1.8.0_322-b06 + - spec: openjdk@1.8.0_402-b06 prefix: /usr perl: externals: - - spec: perl@5.16.3~cpanm+shared+threads + - spec: perl@5.26.3~cpanm+shared+threads prefix: /usr pkg-config: externals: - - spec: pkg-config@0.27.1 + - spec: pkg-config@1.4.2 prefix: /usr rsync: externals: - - spec: rsync@3.1.2 + - spec: rsync@3.1.3 prefix: /usr ruby: externals: - - spec: ruby@2.0.0 + - spec: ruby@2.5.9p229 prefix: /usr sed: externals: - - spec: sed@4.2.2 + - spec: sed@4.5 prefix: /usr tar: externals: - - spec: tar@1.26 + - spec: tar@1.30 prefix: /usr texinfo: externals: - - spec: texinfo@5.1 + - spec: texinfo@6.5-7 prefix: /usr wget: externals: - - spec: wget@1.14 + - spec: wget@1.19.5 prefix: /usr zip: externals: - spec: zip@3.0 prefix: /usr + zlib: + require: [~shared] diff --git a/configs/sites/tier1/noaa-gcloud/packages_intel.yaml b/configs/sites/tier1/noaa-gcloud/packages_intel.yaml index 2c98fa98f..1ffe7cf2f 100644 --- a/configs/sites/tier1/noaa-gcloud/packages_intel.yaml +++ b/configs/sites/tier1/noaa-gcloud/packages_intel.yaml @@ -1,8 +1,8 @@ packages: all: - compiler:: [intel@2021.3.0,gcc@9.2.0] + compiler:: [intel@2021.10.0] providers: - mpi:: [intel-oneapi-mpi@2021.3.0] + mpi:: [intel-oneapi-mpi@2021.10.0] # Remove the next three lines to switch to intel-oneapi-mkl blas:: [openblas] fftw-api:: [fftw] @@ -11,10 +11,10 @@ packages: buildable: False intel-oneapi-mpi: externals: - - spec: intel-oneapi-mpi@2021.3.0%intel@2021.3.0 + - spec: intel-oneapi-mpi@2021.10.0%intel@2021.10.0 prefix: /apps/oneapi modules: - - impi/2021.3.0 + - impi/2023.2.0 intel-oneapi-mkl: # Remove buildable: False and configure+uncomment externals section below to use intel-oneapi-mkl buildable: False diff --git a/doc/source/PreConfiguredSites.rst b/doc/source/PreConfiguredSites.rst index 7aa53a360..2ea670ac2 100644 --- a/doc/source/PreConfiguredSites.rst +++ b/doc/source/PreConfiguredSites.rst @@ -55,11 +55,13 @@ Pre-configured sites (tier 1) +---------------------+-----------------------+--------------------+--------------------------------------------------------+-----------------+ | NOAA (NCEP) | Acorn | Intel | ``/lfs/h1/emc/nceplibs/noscrub/spack-stack/`` | NOAA-EMC | +---------------------+-----------------------+--------------------+--------------------------------------------------------+-----------------+ -| | Gaea | Intel | ``/ncrc/proj/epic/spack-stack/`` | EPIC / NOAA-EMC | +| | Gaea C5 | Intel | ``/ncrc/proj/epic/spack-stack/`` | EPIC / NOAA-EMC | | +-----------------------+--------------------+--------------------------------------------------------+-----------------+ -| NOAA (RDHPCS) | Hera | GCC, Intel | ``/scratch1/NCEPDEV/nems/role.epic/spack-stack/`` | EPIC / NOAA-EMC | +| | Gaea C6 | Intel | ``/ncrc/proj/epic/spack-stack/c6/`` | EPIC / NOAA-EMC | +| NOAA (RDHPCS) +-----------------------+--------------------+--------------------------------------------------------+-----------------+ +| | Hera | GCC, Intel | ``/contrib/spack-stack/`` | EPIC / NOAA-EMC | | +-----------------------+--------------------+--------------------------------------------------------+-----------------+ -| | Jet | GCC, Intel | ``/mnt/lfs4/HFIP/hfv3gfs/role.epic/spack-stack/`` | EPIC / NOAA-EMC | +| | Jet | GCC, Intel | ``/contrib/spack-stack`` | EPIC / NOAA-EMC | +---------------------+-----------------------+--------------------+--------------------------------------------------------+-----------------+ | | Narwhal | GCC, Intel, oneAPI | ``/p/app/projects/NEPTUNE/spack-stack/`` | NRL | | U.S. Navy (HPCMP) +-----------------------+--------------------+--------------------------------------------------------+-----------------+ @@ -73,7 +75,7 @@ Pre-configured sites (tier 1) + Amazon Web Services +-----------------------+--------------------+--------------------------------------------------------+-----------------+ | | Parallelcluster JCSDA | GCC, Intel | *currently unavailable* | JCSDA | +---------------------+-----------------------+--------------------+--------------------------------------------------------+-----------------+ -| NOAA (RDHPCS) | RDHPCS Parallel Works | Intel | ``/contrib/spack-stack/`` | EPIC / JCSDA | +| NOAA (RDHPCS) | RDHPCS Parallel Works | Intel | ``/contrib/spack-stack-rocky8/`` | EPIC / JCSDA | +---------------------+-----------------------+--------------------+--------------------------------------------------------+-----------------+ .. _Preconfigured_Sites_Orion: @@ -84,12 +86,9 @@ MSU Orion The following is required for building new spack environments with any supported compiler on this platform. -**NEEDS UPDATING** - .. code-block:: console module purge - module use /work/noaa/epic/role-epic/spack-stack/orion/modulefiles-rocky9 .. _Preconfigured_Sites_Hercules: @@ -100,14 +99,10 @@ MSU Hercules The following is required for building new spack environments with any supported compiler on this platform. -**NEEDS UPDATING** - .. code-block:: console module purge - module use /work/noaa/epic/role-epic/spack-stack/hercules/modulefiles - module load ecflow/5.8.4 - module load git-lfs/3.1.2 + # No need to load modules, spack-stack-1.8.0 have no dependences .. _Preconfigured_Sites_Discover_SCU16: @@ -250,17 +245,9 @@ NOAA Parallel Works (AWS, Azure, Gcloud) The following is required for building new spack environments with any supported compiler on this platform. The default module path needs to be removed, otherwise spack detects the system as Cray. -**NEEDS UPDATING** - .. code-block:: console module purge - module unuse /opt/cray/craype/default/modulefiles - module unuse /opt/cray/modulefiles - module use /contrib/spack-stack/modulefiles - module load cmake/3.27.2 - module load ecflow/5.8.4 - module load git-lfs/2.4.1 .. _Preconfigured_Sites_Gaea_C5: @@ -271,18 +258,14 @@ NOAA RDHPCS Gaea C5 The following is required for building new spack environments with Intel on this platform.. Don't use ``module purge`` on Gaea! -**NEEDS UPDATING** - .. code-block:: console + # These modules should be loaded by default, if not load (swap) with: module load PrgEnv-intel/8.3.3 module load intel-classic/2023.1.0 module load cray-mpich/8.1.25 module load python/3.9.12 - module use /ncrc/proj/epic/spack-stack/modulefiles - module load ecflow/5.8.4 - .. note:: On Gaea, running ``module available`` without the option ``-t`` can lead to an error: ``/usr/bin/lua5.3: /opt/cray/pe/lmod/lmod/libexec/Spider.lua:568: stack overflow`` @@ -303,18 +286,14 @@ NOAA RDHPCS Gaea C6 The following is required for building new spack environments with Intel on this platform.. Don't use ``module purge`` on Gaea! -**NEEDS UPDATING** - .. code-block:: console + # These modules should be loaded by default, if not load (swap) with: module load PrgEnv-intel/8.3.3 - module load intel-classic/2023.1.0 + module load intel-classic/2023.2.0 module load cray-mpich/8.1.25 module load python/3.9.12 - module use /ncrc/proj/epic/spack-stack/modulefiles - module load ecflow/5.8.4 - .. note:: On Gaea, running ``module available`` without the option ``-t`` can lead to an error: ``/usr/bin/lua5.3: /opt/cray/pe/lmod/lmod/libexec/Spider.lua:568: stack overflow`` @@ -335,14 +314,9 @@ NOAA RDHPCS Hera The following is required for building new spack environments with any supported compiler on this platform. -**NEEDS UPDATING** - .. code-block:: console module purge - module use /scratch1/NCEPDEV/nems/role.epic/modulefiles - module load miniconda3/4.12.0 - module load ecflow/5.8.4 .. note:: On Hera, a dedicated node exists for ``ecflow`` server jobs (``hecflow01``). Users starting ``ecflow_server`` on the regular login nodes will see their servers being killed every few minutes, and may be barred from accessing the system. @@ -356,15 +330,9 @@ NOAA RDHPCS Jet The following is required for building new spack environments with any supported compiler on this platform. -**NEEDS UPDATING** - .. code-block:: console module purge - module use /lfs4/HFIP/hfv3gfs/spack-stack/modulefiles - module load miniconda/3.9.12 - module load ecflow/5.5.3 - module use /lfs4/HFIP/hfv3gfs/role.epic/modulefiles .. _Preconfigured_Sites_S4: From b016e18a6c1e0d450b57c82104553f232c2334d1 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Wed, 2 Oct 2024 18:33:26 -0600 Subject: [PATCH 24/26] Update spack tag in .gitmodules and finalize doc/source/conf.py --- .gitmodules | 2 +- doc/source/conf.py | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.gitmodules b/.gitmodules index 5c7383f2a..d48593e8a 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,7 +1,7 @@ [submodule "spack"] path = spack url = https://github.com/jcsda/spack - branch = release/1.8.0 + branch = spack-stack-1.8.0 [submodule "doc/CMakeModules"] path = doc/CMakeModules url = https://github.com/noaa-emc/cmakemodules diff --git a/doc/source/conf.py b/doc/source/conf.py index 82920fd02..310fe60fb 100644 --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -21,13 +21,13 @@ project = 'spack-stack' copyright = '2024 ' -author = 'Dominikus Heinzeller, Alexander Richert, Cameron Book' +author = 'Dominikus Heinzeller, Alexander Richert, Steve Herbener, Ratko Vasic' # The short X.Y version -version = 'dev' +version = '1.8' # The full version, including alpha/beta/rc tags -release = 'develop' +release = '1.8.0' numfig = True @@ -143,7 +143,7 @@ # Latex figure (float) alignment # # 'figure_align': 'htbp', - 'maketitle': r'\newcommand\sphinxbackoftitlepage{For referencing this document please use: \newline \break Heinzeller, D., A. Richert, C. Book, E. Hartnett, H. Lei, N. Perlin, R. Vasic, S. Herbener, 2024. spack-stack documentation develop. Available at https://spack-stack.readthedocs.io/\textunderscore/downloads/en/latest/pdf/.}\sphinxmaketitle' + 'maketitle': r'\newcommand\sphinxbackoftitlepage{For referencing this document please use: \newline \break Heinzeller, D., A. Richert, C. Book, E. Hartnett, H. Lei, N. Perlin, R. Vasic, S. Herbener, 2024. spack-stack documentation 1.8.0. Available at https://spack-stack.readthedocs.io/\textunderscore/downloads/en/latest/pdf/.}\sphinxmaketitle' } # Grouping the document tree into LaTeX files. List of tuples From a19c782a70dd283f875a46659e1c24e8c0d45256 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Wed, 2 Oct 2024 20:57:24 -0600 Subject: [PATCH 25/26] Revert release-1.8.0 specific changes to .gitmodules and doc/source/conf.py --- .gitmodules | 2 +- doc/source/conf.py | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.gitmodules b/.gitmodules index d48593e8a..a69f3e4f1 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,7 +1,7 @@ [submodule "spack"] path = spack url = https://github.com/jcsda/spack - branch = spack-stack-1.8.0 + branch = spack-stack-dev [submodule "doc/CMakeModules"] path = doc/CMakeModules url = https://github.com/noaa-emc/cmakemodules diff --git a/doc/source/conf.py b/doc/source/conf.py index 310fe60fb..8f33a6da8 100644 --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -24,10 +24,10 @@ author = 'Dominikus Heinzeller, Alexander Richert, Steve Herbener, Ratko Vasic' # The short X.Y version -version = '1.8' +version = 'dev' # The full version, including alpha/beta/rc tags -release = '1.8.0' +release = 'develop' numfig = True @@ -143,7 +143,7 @@ # Latex figure (float) alignment # # 'figure_align': 'htbp', - 'maketitle': r'\newcommand\sphinxbackoftitlepage{For referencing this document please use: \newline \break Heinzeller, D., A. Richert, C. Book, E. Hartnett, H. Lei, N. Perlin, R. Vasic, S. Herbener, 2024. spack-stack documentation 1.8.0. Available at https://spack-stack.readthedocs.io/\textunderscore/downloads/en/latest/pdf/.}\sphinxmaketitle' + 'maketitle': r'\newcommand\sphinxbackoftitlepage{For referencing this document please use: \newline \break Heinzeller, D., A. Richert, C. Book, E. Hartnett, H. Lei, N. Perlin, R. Vasic, S. Herbener, 2024. spack-stack documentation develop. Available at https://spack-stack.readthedocs.io/\textunderscore/downloads/en/latest/pdf/.}\sphinxmaketitle' } # Grouping the document tree into LaTeX files. List of tuples From d4af74ea3e3df59405b471a78f8ffe52af4352ee Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Thu, 3 Oct 2024 10:33:37 -0600 Subject: [PATCH 26/26] Update submodule pointer for spack --- spack | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spack b/spack index 5c1d706c9..6f9a6ba4e 160000 --- a/spack +++ b/spack @@ -1 +1 @@ -Subproject commit 5c1d706c9088a5d0d9d9d5bcd1f38eb069a83612 +Subproject commit 6f9a6ba4e5850494ffb7e7d592fe982036c32fb9