Merge pull request JCSDA#671 from climbfuji/feature/merge_141_into_jc…

…sda_emc_spack_stack

Merge release/1.4.1 into develop

configs/common/packages.yaml

@@ -20,7 +20,7 @@
       version: [1.78.0]
       variants: ~atomic +chrono +date_time +exception +filesystem ~graph ~iostreams ~locale ~log ~math ~mpi ~numpy +pic +program_options +python ~random +regex +serialization ~signals +system +test +thread +timer ~wave cxxstd=14 visibility=hidden
     bufr:
-      version: [11.7.1]
+      version: [12.0.0]
       variants: +python
     # Newer versions of CDO require the C++-17 standard, which doesn't
     # work with all compilers that are currently in use in spack-stack
@@ -42,7 +42,7 @@
       version: [5.8.4]
       variants: +ui
     eckit:
-      version: [1.23.0]
+      version: [1.23.1]
       variants: linalg=eigen,lapack compression=lz4,bzip2
     ecmwf-atlas:
       version: [0.33.0]
@@ -70,7 +70,7 @@
       version: [1.1.0]
     fms:
       version: [2023.01]
-      variants: precision=32,64 +quad_precision +gfs_phys +openmp +pic constants=GFS
+      variants: precision=32,64 +quad_precision +gfs_phys +openmp +pic constants=GFS build_type=Release
     g2:
       version: [3.4.5]
     g2c:
@@ -248,6 +248,7 @@
       version: [5.15.3]
     scotch:
       version: [7.0.3]
+      variants: +mpi+metis~shared~threads~mpi_thread+noarch
     sfcio:
       version: [1.4.1]
     shumlib:

configs/containers/README.md

@@ -2,11 +2,11 @@
 
 To avoid hardcoding specs in the generic container recipes, we keep the specs list empty (`specs: []`) and manually add the specs for the particular spack-stack release and application as listed below, *after* running `spack stack create ctr`.
 
-### spack-stack-1.4.0 / skylab-5.0.0 containers for fv3-jedi and mpas-jedi (but not for ufs-jedi)
+### spack-stack-1.4.1 / skylab-5.0.0 containers for fv3-jedi and mpas-jedi (but not for ufs-jedi)
 ```
   specs: [[email protected], [email protected] ~fftw, [email protected], [email protected],
-    [email protected], [email protected], [email protected], bufr@11.7.1, [email protected], [email protected], ecflow@5,
-    [email protected].0, [email protected] +trans ~fftw, [email protected] ~fftw, [email protected],
+    [email protected], [email protected], [email protected], bufr@12.0.0, [email protected], [email protected], ecflow@5,
+    [email protected].1, [email protected] +trans ~fftw, [email protected] ~fftw, [email protected],
     [email protected], fms@release-jcsda, [email protected], [email protected], [email protected],
     [email protected], [email protected], [email protected], [email protected], [email protected], [email protected],
     [email protected], [email protected], [email protected], [email protected],
@@ -27,14 +27,14 @@ To avoid hardcoding specs in the generic container recipes, we keep the specs li
     # [email protected]
 ```
 
-### spack-stack-1.4.0 / ufs-weather-model-x.y.z containers for ufs-weather-model as of May 18, 2023
+### spack-stack-1.4.1 / ufs-weather-model-x.y.z containers for ufs-weather-model as of July 5, 2023
 
 **Note. This is not yet working correctly, some libraries are missing. Please do not use yet! Also, if using the clang-mpich container, need to disable openmp for fms, not clear how to do this cleanly.**
 
 ```
   specs: [[email protected],
-    [email protected], [email protected], bufr@11.7.1, [email protected], [email protected], ecflow@5,
-    [email protected].0, [email protected] +trans ~fftw, [email protected] ~fftw, [email protected],
+    [email protected], [email protected], bufr@12.0.0, [email protected], [email protected], ecflow@5,
+    [email protected].1, [email protected] +trans ~fftw, [email protected] ~fftw, [email protected],
     [email protected], [email protected], [email protected], [email protected], [email protected],
     [email protected], [email protected], [email protected], [email protected], [email protected], [email protected],
     [email protected], [email protected], [email protected], [email protected],

configs/sites/gaea-c5/packages.yaml

@@ -80,7 +80,7 @@ packages:
     buildable: False
     externals:
     - spec: [email protected]+ui+static_boost
-      prefix: /lustre/f2/dev/wpo/role.epic/contrib/spack-stack/ecflow-5.8.4-c5
+      prefix: /lustre/f2/dev/wpo/role.epic/contrib/spack-stack/c5/ecflow-5.8.4
       modules: [ecflow/5.8.4]
   file:
     externals:
@@ -174,7 +174,7 @@ packages:
     buildable: False
     externals:
     - spec: [email protected]
-      prefix: /lustre/f2/dev/wpo/role.epic/contrib/spack-stack/mysql-8.0.31-c5
+      prefix: /lustre/f2/dev/wpo/role.epic/contrib/spack-stack/c5/mysql-8.0.31
       modules: [mysql/8.0.31]
   ncurses:
     externals:
@@ -204,7 +204,7 @@ packages:
   qt:
     externals:
     - spec: [email protected]
-      prefix: /lustre/f2/dev/wpo/role.epic/contrib/spack-stack/qt-5.15.2-c5/5.15.2/gcc_64
+      prefix: /lustre/f2/dev/wpo/role.epic/contrib/spack-stack/c5/qt-5.15.2/5.15.2/gcc_64
   rdma-core:
     externals:
     - spec: [email protected]

configs/sites/hercules/mirrors.yaml

@@ -1,15 +1,15 @@
 mirrors:
   local-source:
     fetch:
-      url: file:///work/noaa/da/role-da/spack-stack/source-cache
+      url: file:///work/noaa/epic/role-epic/spack-stack/source-cache
       access_pair:
       - null
       - null
       access_token: null
       profile: null
       endpoint_url: null
     push:
-      url: file:///work/noaa/da/role-da/spack-stack/source-cache
+      url: file:///work/noaa/epic/role-epic/spack-stack/source-cache
       access_pair:
       - null
       - null

configs/sites/hercules/packages.yaml

@@ -3,7 +3,7 @@ packages:
     compiler:: [[email protected], [email protected]]
     #compiler:: [[email protected]]
     providers:
-      mpi:: [[email protected], [email protected].4]
+      mpi:: [[email protected], [email protected].5]
 
 ### MPI, Python, MKL
   mpi:
@@ -20,11 +20,11 @@ packages:
     #  - intel-oneapi-mpi/2021.7.1
   openmpi:
     externals:
-    - spec: [email protected].4%[email protected] ~cuda~cxx~cxx_exceptions~java~memchecker+pmi+static~wrapper-rpath
+    - spec: [email protected].5%[email protected]~cuda~cxx~cxx_exceptions~java~memchecker+pmi~static~wrapper-rpath
         fabrics=ucx schedulers=slurm
-      prefix: /apps/spack-managed/gcc-11.3.1/openmpi-4.1.4-ruvlmb6yyvzbzbiqaov4zk75ogthczsp
+      prefix: /work/noaa/epic/role-epic/spack-stack/hercules/openmpi-4.1.5/gcc-11.3.1
       modules:
-      - openmpi/4.1.4
+      - openmpi/4.1.5
   python:
     buildable: False
     externals:
@@ -74,9 +74,9 @@ packages:
     buildable: False
     externals:
     - spec: [email protected]+ui+static_boost
-      prefix: /work/noaa/epic-ps/role-epic-ps/spack-stack/ecflow-5.8.4-hercules
+      prefix: /work/noaa/epic/role-epic/spack-stack/hercules/ecflow-5.8.4
       modules:
-      - ecflow/5.8.4-hercules
+      - ecflow/5.8.4
   findutils:
     externals:
     - spec: [email protected]
@@ -119,9 +119,9 @@ packages:
     buildable: False
     externals:
     - spec: [email protected]
-      prefix: /work/noaa/epic-ps/role-epic-ps/spack-stack/mysql-8.0.31-hercules
+      prefix: /work/noaa/epic/role-epic/spack-stack/hercules/mysql-8.0.31
       modules:
-      - mysql/8.0.31-hercules
+      - mysql/8.0.31
   openssh:
     externals:
     - spec: [email protected]
@@ -164,3 +164,10 @@ packages:
     externals:
     - spec: [email protected]
       prefix: /usr
+  # Need to use external zlib, because of qt dependence on it (otherwise issues with tar command)
+  zlib:
+    externals:
+    - spec: [email protected]
+      prefix: /apps/spack-managed/gcc-11.3.1/zlib-1.2.13-ltp4c3zzde3zi3gf7x4b7c7nj5ww4i4g
+      modules:
+      - zlib/1.2.13

configs/sites/orion/mirrors.yaml

@@ -1,15 +1,15 @@
 mirrors:
   local-source:
     fetch:
-      url: file:///work/noaa/da/role-da/spack-stack/source-cache
+      url: file:///work/noaa/epic/role-epic/spack-stack/source-cache
       access_pair:
       - null
       - null
       access_token: null
       profile: null
       endpoint_url: null
     push:
-      url: file:///work/noaa/da/role-da/spack-stack/source-cache
+      url: file:///work/noaa/epic/role-epic/spack-stack/source-cache
       access_pair:
       - null
       - null

doc/modulefile_templates/openmpi

@@ -31,7 +31,7 @@ unsetenv SLURM_EXPORT_ENV
 setenv PSM2_PATH_SELECTION "static_base"
 setenv SLURM_CPU_BIND "none"
 
-# Settings specific for Cheyenne
+# Settings specific for Cheyenne and Hercules
 setenv MPI_ROOT ${OPENMPI_PATH}
 setenv UCX_MAX_RNDV_RAILS "1"
 setenv OMPI_MCA_btl "^openib"

doc/source/MaintainersSection.rst

@@ -63,6 +63,9 @@ Sign into qt, select customized installation, choose [email protected] only (uncheck all
 .. note::
    On air-gapped systems, the above method may not work (we have not encountered such a system so far).
 
+.. note::
+   If ``./qt-unified-linux-x64-online.run`` fails to start with the error ``qt.qpa.xcb: could not connect to display`` and a role account is being used, follow the procedure described in https://www.thegeekdiary.com/how-to-set-x11-forwarding-export-remote-display-for-users-who-switch-accounts-using-sudo to export the display. A possible warning ``xauth:  file /ncrc/home1/role.epic/.Xauthority does not exist`` can be ignored, since this file gets created by the ``xauth`` command.
+
 ..  _MaintainersSection_ecFlow:
 
 ------------------------------
@@ -243,7 +246,7 @@ MSU Hercules
 ------------------------------
 
 ecflow
-  ``ecFlow`` must be built manually using the GNU compilers and linked against a static ``boost`` library, using an available ``Qt5`` installation. After loading the following modules, follow the instructions in :numref:`Section %s <MaintainersSection_ecFlow>` to install ``ecflow`` in ``/work/noaa/epic-ps/role-epic-ps/spack-stack/ecflow-5.8.4-hercules``.
+  ``ecFlow`` must be built manually using the GNU compilers and linked against a static ``boost`` library, using an available ``Qt5`` installation. After loading the following modules, follow the instructions in :numref:`Section %s <MaintainersSection_ecFlow>` to install ``ecflow`` in ``/work/noaa/epic/role-epic/spack-stack/hercules/ecflow-5.8.4``.
 
 .. code-block:: console
 
@@ -253,6 +256,22 @@ ecflow
 mysql
   ``mysql`` must be installed separately from ``spack`` using a binary tarball provided by the MySQL community. Follow the instructions in :numref:`Section %s <MaintainersSection_MySQL>` to install ``mysql`` in ``/work/noaa/epic-ps/role-epic-ps/spack-stack/mysql-8.0.31-hercules``.
 
+openmpi
+  need to load qt so to get consistent zlib (or just load zlib directly, check qt module)
+
+.. code-block:: console
+
+   module purge
+   module load zlib/1.2.13
+   module load ucx/1.13.1
+   ./configure \
+       --prefix=/work/noaa/epic/role-epic/spack-stack/hercules/openmpi-4.1.5/gcc-11.3.1  \
+       --with-ucx=$UCX_ROOT \
+       --with-zlib=$ZLIB_ROOT
+   make VERBOSE=1 -j4
+   make check
+   make install
+
 .. _MaintainersSection_Discover:
 
 ------------------------------
@@ -538,17 +557,8 @@ NOAA RDHPCS Gaea C5
 
 On Gaea C5, ``miniconda``, ``qt``, ``ecflow``, and ``mysql`` need to be installed as a one-off before spack can be used.
 
-miniconda
-   Follow the instructions in :numref:`Section %s <MaintainersSection_Miniconda>` to create a basic ``miniconda`` installation and associated modulefile for working with spack. Don't forget to log off and back on to forget about the conda environment. Use the following workaround to avoid the terminal being spammed by error messages about missing version information (``/usr/bin/lua5.3: /lustre/f2/dev/wpo/role.epic/contrib/spack-stack/miniconda-3.9.12-c5/lib/libtinfo.so.6: no version information available (required by /lib64/libreadline.so.7)``):
-
-.. code-block:: console
-
-   cd /lustre/f2/dev/wpo/role.epic/contrib/spack-stack/miniconda-3.9.12-c5/lib
-   mv libtinfow.so.6.3 libtinfow.so.6.3.conda.original
-   ln -sf /lib64/libtinfo.so.6 libtinfow.so.6.3
-
 qt (qt@5)
-   The default ``qt@5`` in ``/usr`` is incomplete and thus insufficient for building ``ecflow``. After loading/unloading the modules as shown below, refer to :numref:`Section %s <MaintainersSection_Qt5>` to install ``[email protected]`` in ``/lustre/f2/dev/wpo/role.epic/contrib/spack-stack/qt-5.15.2-c5``. Note that the installation must be done as a regular user due to problems with graphical applications for role accounts.
+   The default ``qt@5`` in ``/usr`` is incomplete and thus insufficient for building ``ecflow``. After loading/unloading the modules as shown below, refer to :numref:`Section %s <MaintainersSection_Qt5>` to install ``[email protected]`` in ``/lustre/f2/dev/wpo/role.epic/contrib/spack-stack/c5/qt-5.15.2``. :numref:`Section %s <MaintainersSection_Qt5>` describes how to export the X windows environment in order to install ``qt@5`` using the role account.
 
 .. code-block:: console
 
@@ -557,7 +567,7 @@ qt (qt@5)
    module load PrgEnv-gnu/8.3.3
 
 ecflow
-  ``ecFlow`` must be built manually using the GNU compilers and linked against a static ``boost`` library. After installing `qt5` and loading the following modules, follow the instructions in :numref:`Section %s <MaintainersSection_ecFlow>`. Because of the dependency on ``miniconda``, that module must be loaded automatically in the ``ecflow`` module (similar to ``[email protected]``).  Ensure to follow the extra instructions in that section for Gaea C5.
+  ``ecFlow`` must be built manually using the GNU compilers and linked against a static ``boost`` library. After installing `qt5` and loading the following modules, follow the instructions in :numref:`Section %s <MaintainersSection_ecFlow>`. Because of the dependency on ``miniconda``, that module must be loaded automatically in the ``ecflow`` module (similar to ``[email protected]``).  Ensure to follow the extra instructions in that section for Gaea C5 in ``/lustre/f2/dev/wpo/role.epic/contrib/spack-stack/c5/ecflow-5.8.4``.
 
    Ensure to follow the extra instructions in that section for Gaea.
 
@@ -568,11 +578,11 @@ ecflow
    module load PrgEnv-gnu/8.3.3
    module load python/3.9.12
 
-   module use /lustre/f2/dev/wpo/role.epic/contrib/spack-stack/modulefiles-c5
+   module use /lustre/f2/dev/wpo/role.epic/contrib/spack-stack/c5/modulefiles
    module load qt/5.15.2
 
 mysql
-  ``mysql`` must be installed separately from ``spack`` using a binary tarball provided by the MySQL community. Follow the instructions in :numref:`Section %s <MaintainersSection_MySQL>` to install ``mysql`` in ``/lustre/f2/dev/wpo/role.epic/contrib/spack-stack/mysql-8.0.31-c5``.
+  ``mysql`` must be installed separately from ``spack`` using a binary tarball provided by the MySQL community. Follow the instructions in :numref:`Section %s <MaintainersSection_MySQL>` to install ``mysql`` in ``/lustre/f2/dev/wpo/role.epic/contrib/spack-stack/c5/mysql-8.0.31``.
 
 .. _MaintainersSection_Hera: