diff --git a/configs/sites/nautilus/compilers.yaml b/configs/sites/nautilus/compilers.yaml index bd9b05f16..3e16961a9 100644 --- a/configs/sites/nautilus/compilers.yaml +++ b/configs/sites/nautilus/compilers.yaml @@ -33,7 +33,8 @@ compilers: prepend_path: PATH: '/opt/rh/gcc-toolset-11/root/usr/bin' CPATH: '/opt/rh/gcc-toolset-11/root/usr/include' - LD_LIBRARY_PATH: '/p/app/compilers/intel/oneapi/compiler/2022.0.2/linux/compiler/lib/intel64_lin:/opt/rh/gcc-toolset-11/root/usr/lib64:/opt/rh/gcc-toolset-11/root/usr/lib' - set: - I_MPI_ROOT: '/p/app/compilers/intel/oneapi/mpi/2021.5.1' + LD_LIBRARY_PATH: '/opt/scyld/slurm/lib64:/opt/scyld/slurm/lib64/slurm:/p/app/compilers/intel/oneapi/compiler/2022.0.2/linux/compiler/lib/intel64_lin:/opt/rh/gcc-toolset-11/root/usr/lib64:/opt/rh/gcc-toolset-11/root/usr/lib' + # For intel-oneapi-mpi only + #set: + # I_MPI_ROOT: '/p/app/compilers/intel/oneapi/mpi/2021.5.1' extra_rpaths: [] diff --git a/configs/sites/nautilus/packages.yaml b/configs/sites/nautilus/packages.yaml index d6683c0d3..f664ac383 100644 --- a/configs/sites/nautilus/packages.yaml +++ b/configs/sites/nautilus/packages.yaml @@ -2,21 +2,45 @@ packages: all: compiler:: [intel@2021.5.0, aocc@4.0.0] providers: - mpi:: [intel-oneapi-mpi@2021.5.1, openmpi@4.1.4] + # For now need to enable one or the other; + # see https://github.com/JCSDA/spack-stack/issues/659 + mpi:: [openmpi@4.1.5rc2] + #mpi:: [openmpi@4.1.4] + blas:: [intel-oneapi-mkl] + fftw-api:: [intel-oneapi-mkl] + lapack:: [intel-oneapi-mkl] + ectrans: + variants:: +mkl + gsibec: + variants:: +mkl ### MPI, Python, MKL mpi: buildable: False - intel-oneapi-mpi: - externals: - - spec: intel-oneapi-mpi@2021.5.1%intel@2021.5.0 - modules: - - intel/mpi/2021.5.1 - prefix: /p/app/compilers/intel/oneapi + #intel-oneapi-mpi: + # externals: + # - spec: intel-oneapi-mpi@2021.5.1%intel@2021.5.0 + # modules: + # - intel/mpi/2021.5.1 + # prefix: /p/app/compilers/intel/oneapi openmpi: externals: + - spec: openmpi@4.1.5rc2%intel@2021.5.0~cuda~cxx~cxx_exceptions~java~memchecker+pmi~static~wrapper-rpath fabrics=ucx schedulers=slurm + prefix: /p/app/penguin/openmpi/4.1.5rc2/intel + modules: + - penguin/openmpi/4.1.5rc2/intel + - slurm - spec: openmpi@4.1.4%aocc@4.0.0~cuda~cxx~cxx_exceptions~java~memchecker+pmi~static~wrapper-rpath fabrics=ucx schedulers=slurm prefix: /p/app/penguin/openmpi/4.1.4/aoc + modules: + - penguin/openmpi/4.1.4/aocc + - slurm + intel-oneapi-mkl: + externals: + - spec: intel-oneapi-mkl@2022.0.2 + prefix: /p/app/compilers/intel/oneapi + modules: + - intel-oneapi-mkl@2022.0.2 ### Modifications of common packages # Version 2.0.8 doesn't compile on Nautilus diff --git a/doc/source/PreConfiguredSites.rst b/doc/source/PreConfiguredSites.rst index 92a961c1f..37adbb85c 100644 --- a/doc/source/PreConfiguredSites.rst +++ b/doc/source/PreConfiguredSites.rst @@ -30,7 +30,7 @@ Ready-to-use spack-stack 1.4.0 installations are available on the following, ful +------------------------------------------------------------+-------------------------------+--------------------------------------------------------------------------------------------------------------+ | NAVY HPCMP Narwhal GNU^** | Dom Heinzeller / ??? | ``/p/app/projects/NEPTUNE/spack-stack/spack-stack-1.4.0/envs/unified-env-gcc-10.3.0`` | +------------------------------------------------------------+-------------------------------+--------------------------------------------------------------------------------------------------------------+ -| NAVY HPCMP Nautilus Intel^* | Dom Heinzeller / ??? | ``/p/app/projects/NEPTUNE/spack-stack/spack-stack-1.4.0/envs/unified-env-intel-2021.5.0-hdf5-1.14.0`` | +| NAVY HPCMP Nautilus Intel^* | Dom Heinzeller / ??? | ``/p/app/projects/NEPTUNE/spack-stack/spack-stack-1.4.0/envs/unified-env-intel-2021.5.0-openmpi-4.1.5`` | +------------------------------------------------------------+-------------------------------+--------------------------------------------------------------------------------------------------------------+ | NAVY HPCMP Nautilus AMD clang/flang | Dom Heinzeller / ??? | **currently not supported** | +------------------------------------------------------------+-------------------------------+--------------------------------------------------------------------------------------------------------------+ @@ -188,6 +188,7 @@ With Intel, the following is required for building new spack environments and fo .. code-block:: console + umask 0022 module unload PrgEnv-cray module load PrgEnv-intel/8.3.2 module unload intel @@ -216,6 +217,7 @@ With GNU, the following is required for building new spack environments and for .. code-block:: console + umask 0022 module unload PrgEnv-cray module load PrgEnv-gnu/8.3.2 module unload gcc @@ -254,11 +256,12 @@ With Intel, the following is required for building new spack environments and fo .. code-block:: console + umask 0022 module purge module load slurm module load intel/compiler/2022.0.2 - module load intel/mpi/2021.5.1 + module load penguin/openmpi/4.1.5rc2/intel module use /p/app/projects/NEPTUNE/spack-stack/modulefiles module load ecflow/5.8.4 @@ -268,15 +271,16 @@ For ``spack-stack-1.4.0`` with Intel, load the following modules after loading t .. code-block:: console - module use /p/app/projects/NEPTUNE/spack-stack/spack-stack-1.4.0/envs/unified-env-intel-2021.5.0-hdf5-1.14.0/install/modulefiles/Core + module use /p/app/projects/NEPTUNE/spack-stack/spack-stack-dev-20230628/envs/unified-env/install/modulefiles/Core module load stack-intel/2021.5.0 - module load stack-intel-oneapi-mpi/2021.5.1 + module load stack-openmpi/4.1.5rc2 module load stack-python/3.10.8 With AMD clang/flang (aocc), the following is required for building new spack environments and for using spack to build and run software. .. code-block:: console + umask 0022 module purge module load slurm diff --git a/spack b/spack index 5f8f7c59e..204f82600 160000 --- a/spack +++ b/spack @@ -1 +1 @@ -Subproject commit 5f8f7c59ec3ac4882756edeb24cc8a41b0c27eb6 +Subproject commit 204f82600bda589358e1d2957ca3a23a65453c23