Merge branch 'develop' into feature/cleanup_crtm

.github/workflows/macos-ci-aarch64.yaml

@@ -67,7 +67,7 @@ jobs:
           export SPACK_SYSTEM_CONFIG_PATH="${ENVDIR}/site"
 
           # Find external packages
-          spack external find --scope system
+          spack external find --scope system --exclude bison
           spack external find --scope system perl
           spack external find --scope system wget
           PATH="/opt/homebrew/opt/curl/bin:$PATH" \
@@ -93,6 +93,7 @@ jobs:
           sed -i '' "s/\['\%aocc', '\%apple-clang', '\%gcc', '\%intel'\]/\['\%apple-clang'\]/g" $ENVDIR/spack.yaml
 
           spack concretize 2>&1 | tee log.concretize.apple-clang-14.0.0
+          ${SPACK_STACK_DIR}/util/show_duplicate_packages.py -d log.concretize.apple-clang-14.0.0 -i fms -i crtm
 
           # Add and update source cache
           spack mirror add local-source file:///Users/ec2-user/spack-stack/source-cache/

.github/workflows/macos-ci-x86_64.yaml

@@ -60,7 +60,7 @@ jobs:
           export SPACK_SYSTEM_CONFIG_PATH="${ENVDIR}/site"
 
           # Find external packages
-          spack external find --scope system
+          spack external find --scope system --exclude bison
           spack external find --scope system perl
           spack external find --scope system wget
           PATH="/usr/local/opt/curl/bin:$PATH" \
@@ -86,6 +86,7 @@ jobs:
           sed -i '' "s/\['\%aocc', '\%apple-clang', '\%gcc', '\%intel'\]/\['\%apple-clang'\]/g" $ENVDIR/spack.yaml
 
           spack concretize 2>&1 | tee log.concretize.apple-clang-14.0.0
+          ${SPACK_STACK_DIR}/util/show_duplicate_packages.py -d log.concretize.apple-clang-14.0.0 -i fms -i crtm
 
           # Add and update source cache
           spack mirror add local-source file:///Users/ec2-user/spack-stack/source-cache/

.github/workflows/ubuntu-ci-x86_64.yaml

@@ -124,6 +124,7 @@ jobs:
           sed -i "s/\['\%aocc', '\%apple-clang', '\%gcc', '\%intel'\]/\['\%intel'\]/g" $ENVDIR/spack.yaml
 
           spack concretize 2>&1 | tee log.concretize.intel-2021.4.0
+          ${SPACK_STACK_DIR}/util/show_duplicate_packages.py -d log.concretize.intel-2021.4.0 -i fms -i crtm
 
           # Add and update source cache
           spack mirror add local-source file:///home/ubuntu/spack-stack/source-cache/

doc/source/NewSiteConfigs.rst

@@ -30,7 +30,7 @@ It is also instructive to peruse the GitHub actions scripts in ``.github/workflo
 macOS
 ------------------------------
 
-On macOS, it is important to use certain Homebrew packages as external packages, because the native macOS packages are incomplete (e.g. missing the development header files): ``curl``, ``qt``, etc. The instructions provided in the following have been tested extensively on many macOS installations.
+On macOS, it is important to use certain Homebrew packages as external packages, because the native macOS packages are incomplete (e.g. missing the development header files): ``curl``, ``qt``, etc. The instructions provided in the following have been tested extensively on many macOS installations. Occasionally, the use of external packages may lead to concretization issues in the form of duplicate packages (i.e., more than one spec per package). This is the case with ``bison``, therefore the package should be installed by ``spack``.
 
 Unlike in previous versions, the instructions below assume that ``Python`` is built by ``spack``. That means that when using the ``spack`` environments (i.e., loading the modules for building or running code), the ``spack`` installation of ``Python`` with its available ``Python`` modules should be used to ensure consistency. However, a Homebrew ``Python`` installation may still be needed to build new ``spack`` environments. It can also be beneficial for the user to have a version of ``Python`` installed with Homebrew that can be used for virtual environments that are completely independent of any ``spack``-built environment.
 
@@ -203,7 +203,7 @@ Remember to activate the ``lua`` module environment and have MacTeX in your sear
 
 .. code-block:: console
 
-   spack external find --scope system
+   spack external find --scope system # use '--exclude' for troublesome packages like bison@:3.3
    spack external find --scope system perl
    # Don't use any external Python, let spack build it
    #spack external find --scope system python

doc/source/PreConfiguredSites.rst

@@ -742,8 +742,8 @@ The following instructions install a new spack environment on a pre-configured s
    emacs envs/unified-dev.hera/common/*.yaml
    emacs envs/unified-dev.hera/site/*.yaml
 
-   # Process/concretize the specs
-   spack concretize
+   # Process/concretize the specs; optionally check for duplicate packages
+   spack concretize | ${SPACK_STACK_DIR}/util/show_duplicate_packages.py -d log.concretize
 
    # Optional step for systems with a pre-configured spack mirror, see below.
 

doc/source/Utilities.rst

@@ -0,0 +1,28 @@
+.. _Utilities:
+
+Miscellaneous utilities
+*************************
+
+.. _Duplicate_Checker:
+
+------------------------------
+show_duplicate_packages.py
+------------------------------
+
+The utility located at util/show_duplicate_packages.py parses the output of ``spack concretize`` and detects duplicates. Usage is as follows:
+
+.. code-block:: console
+
+   spack concretize | ${SPACK_STACK_DIR}/util/show_duplicate_packages.py
+   # - OR -
+   spack concretize |& tee log.concretize
+   ${SPACK_STACK_DIR}/util/show_duplicate_packages.py log.concretize
+
+The ``-d`` option shows only a list of the duplicates, as opposed to the default behavior, which is to show a print-out of all packages with colorized duplicates. In any case, the identification of any duplicates will yield a return code of 1. The ``-i`` option can be invoked multiple times to skip specific package names.
+
+.. _Acorn_Utilities:
+
+------------------------------
+Acorn utilities
+------------------------------
+The util/acorn/ directory provides scripting for spack-stack builds through PBS Pro on Acorn. To use them, copy them into the directory of the Spack environment you wish to build, set the number of nodes to use (make sure ``#PBS -l select=X`` and ``mpiexec -n X`` use the same value for ``X``), and run ``qsub build.pbs``. Note that the temporary directory specification uses a soft link where the referent location depends on the node (this is to avoid compiling directly on LFS, which frequently fails when working with small files as when cloning git repositories). For parallel installations on Acorn, 2-6 is a reasonable range for the number of nodes (MPI proc analogs), and 6-8 is a reasonable number for the number of threads (note that for ``#PBS -l ncpus=Y`` in build.pbs, ``Y`` should match the ``-j`` argument for ``spack install`` in spackinstall.sh).

doc/source/index.rst

@@ -15,6 +15,7 @@ Table of Contents
    Environments
    Prerequisites
    SpackStackExtension
+   Utilities
    KnownIssues
    MaintainersSection
    Documentation

util/show_duplicate_packages.py

@@ -7,6 +7,7 @@
 #   spack concretize | tee log.concretize | show_duplicate_packages.py > list_of_duplicates.txt
 #
 # '-d' argument prints only duplicates (and disables highlighting).
+# '-i' argument ignores a specific package, and can be invoked multiple times.
 #
 # Alex Richert, June 2023
 
@@ -20,12 +21,13 @@ def colorize_spec(line, package_name, colorize=False):
     c1 = r'\033[93m' ; c2 = r'\033[0m'
     return re.sub("(\w{7}\s+)(%s)@"%package_name, f"\\1{c1}\\2{c2}@", line)
 
-def show_duplicate_packages(txt_to_check, only_show_dups=False):
+def show_duplicate_packages(txt_to_check, ignore_list=[], only_show_dups=False):
     dd = defaultdict(set)
     for line in txt_to_check.split("\n"):
         line = line.replace("^", "")
         package_name = re.findall("\s\w{7}\s+(\^?[^\s@]+)@", line)
         if not package_name: continue
+        if [package_name[0]] in ignore_list: continue
         line = "  ".join(line.split()[1:])
         dd[package_name[0]].add(line)
     duplicates_found = False
@@ -39,16 +41,19 @@ def show_duplicate_packages(txt_to_check, only_show_dups=False):
             print(colorize_spec(line, key, colorize=colorize))
     sys.stderr.write("===\n%suplicates found%s\n" % (("D","!") if duplicates_found else ("No d",".")))
     sys.stderr.flush()
+    return int(duplicates_found)
 
 if __name__ == "__main__":
     parser = argparse.ArgumentParser(description="Check output of `spack concretize` for duplicate packages")
-    if sys.stdin.isatty():
-        parser.add_argument("filename")
+    parser.add_argument("filename", nargs="?")
     parser.add_argument("-d", action="store_true")
+    parser.add_argument("-i", nargs="*", action="append")
     args = parser.parse_args()
-    if sys.stdin.isatty():
+    if args.filename:
         with open(args.filename, "r") as f:
             txt_to_check = f.read()
     else:
         txt_to_check = sys.stdin.read()
-    show_duplicate_packages(txt_to_check, only_show_dups=args.d)
+    ret = show_duplicate_packages(txt_to_check, only_show_dups=args.d, ignore_list=args.i)
+    sys.exit(ret)
+