diff --git a/.github/workflows/macos-ci-aarch64.yaml b/.github/workflows/macos-ci-aarch64.yaml index 639f7df6c..c9e818137 100644 --- a/.github/workflows/macos-ci-aarch64.yaml +++ b/.github/workflows/macos-ci-aarch64.yaml @@ -67,7 +67,7 @@ jobs: export SPACK_SYSTEM_CONFIG_PATH="${ENVDIR}/site" # Find external packages - spack external find --scope system + spack external find --scope system --exclude bison spack external find --scope system perl spack external find --scope system wget PATH="/opt/homebrew/opt/curl/bin:$PATH" \ @@ -93,6 +93,7 @@ jobs: sed -i '' "s/\['\%aocc', '\%apple-clang', '\%gcc', '\%intel'\]/\['\%apple-clang'\]/g" $ENVDIR/spack.yaml spack concretize 2>&1 | tee log.concretize.apple-clang-14.0.0 + ${SPACK_STACK_DIR}/util/show_duplicate_packages.py -d log.concretize.apple-clang-14.0.0 -i fms -i crtm # Add and update source cache spack mirror add local-source file:///Users/ec2-user/spack-stack/source-cache/ diff --git a/.github/workflows/macos-ci-x86_64.yaml b/.github/workflows/macos-ci-x86_64.yaml index cc3360174..6de991947 100644 --- a/.github/workflows/macos-ci-x86_64.yaml +++ b/.github/workflows/macos-ci-x86_64.yaml @@ -60,7 +60,7 @@ jobs: export SPACK_SYSTEM_CONFIG_PATH="${ENVDIR}/site" # Find external packages - spack external find --scope system + spack external find --scope system --exclude bison spack external find --scope system perl spack external find --scope system wget PATH="/usr/local/opt/curl/bin:$PATH" \ @@ -86,6 +86,7 @@ jobs: sed -i '' "s/\['\%aocc', '\%apple-clang', '\%gcc', '\%intel'\]/\['\%apple-clang'\]/g" $ENVDIR/spack.yaml spack concretize 2>&1 | tee log.concretize.apple-clang-14.0.0 + ${SPACK_STACK_DIR}/util/show_duplicate_packages.py -d log.concretize.apple-clang-14.0.0 -i fms -i crtm # Add and update source cache spack mirror add local-source file:///Users/ec2-user/spack-stack/source-cache/ diff --git a/.github/workflows/ubuntu-ci-x86_64.yaml b/.github/workflows/ubuntu-ci-x86_64.yaml index b34e36028..61dba08fb 100644 --- a/.github/workflows/ubuntu-ci-x86_64.yaml +++ b/.github/workflows/ubuntu-ci-x86_64.yaml @@ -124,6 +124,7 @@ jobs: sed -i "s/\['\%aocc', '\%apple-clang', '\%gcc', '\%intel'\]/\['\%intel'\]/g" $ENVDIR/spack.yaml spack concretize 2>&1 | tee log.concretize.intel-2021.4.0 + ${SPACK_STACK_DIR}/util/show_duplicate_packages.py -d log.concretize.intel-2021.4.0 -i fms -i crtm # Add and update source cache spack mirror add local-source file:///home/ubuntu/spack-stack/source-cache/ diff --git a/doc/source/NewSiteConfigs.rst b/doc/source/NewSiteConfigs.rst index c68d2e3cd..eb43888d0 100644 --- a/doc/source/NewSiteConfigs.rst +++ b/doc/source/NewSiteConfigs.rst @@ -30,7 +30,7 @@ It is also instructive to peruse the GitHub actions scripts in ``.github/workflo macOS ------------------------------ -On macOS, it is important to use certain Homebrew packages as external packages, because the native macOS packages are incomplete (e.g. missing the development header files): ``curl``, ``qt``, etc. The instructions provided in the following have been tested extensively on many macOS installations. +On macOS, it is important to use certain Homebrew packages as external packages, because the native macOS packages are incomplete (e.g. missing the development header files): ``curl``, ``qt``, etc. The instructions provided in the following have been tested extensively on many macOS installations. Occasionally, the use of external packages may lead to concretization issues in the form of duplicate packages (i.e., more than one spec per package). This is the case with ``bison``, therefore the package should be installed by ``spack``. Unlike in previous versions, the instructions below assume that ``Python`` is built by ``spack``. That means that when using the ``spack`` environments (i.e., loading the modules for building or running code), the ``spack`` installation of ``Python`` with its available ``Python`` modules should be used to ensure consistency. However, a Homebrew ``Python`` installation may still be needed to build new ``spack`` environments. It can also be beneficial for the user to have a version of ``Python`` installed with Homebrew that can be used for virtual environments that are completely independent of any ``spack``-built environment. @@ -203,7 +203,7 @@ Remember to activate the ``lua`` module environment and have MacTeX in your sear .. code-block:: console - spack external find --scope system + spack external find --scope system # use '--exclude' for troublesome packages like bison@:3.3 spack external find --scope system perl # Don't use any external Python, let spack build it #spack external find --scope system python diff --git a/doc/source/PreConfiguredSites.rst b/doc/source/PreConfiguredSites.rst index 349097e1e..380b3d998 100644 --- a/doc/source/PreConfiguredSites.rst +++ b/doc/source/PreConfiguredSites.rst @@ -742,8 +742,8 @@ The following instructions install a new spack environment on a pre-configured s emacs envs/unified-dev.hera/common/*.yaml emacs envs/unified-dev.hera/site/*.yaml - # Process/concretize the specs - spack concretize + # Process/concretize the specs; optionally check for duplicate packages + spack concretize | ${SPACK_STACK_DIR}/util/show_duplicate_packages.py -d log.concretize # Optional step for systems with a pre-configured spack mirror, see below. diff --git a/doc/source/Utilities.rst b/doc/source/Utilities.rst new file mode 100644 index 000000000..a9a2b6129 --- /dev/null +++ b/doc/source/Utilities.rst @@ -0,0 +1,28 @@ +.. _Utilities: + +Miscellaneous utilities +************************* + +.. _Duplicate_Checker: + +------------------------------ +show_duplicate_packages.py +------------------------------ + +The utility located at util/show_duplicate_packages.py parses the output of ``spack concretize`` and detects duplicates. Usage is as follows: + +.. code-block:: console + + spack concretize | ${SPACK_STACK_DIR}/util/show_duplicate_packages.py + # - OR - + spack concretize |& tee log.concretize + ${SPACK_STACK_DIR}/util/show_duplicate_packages.py log.concretize + +The ``-d`` option shows only a list of the duplicates, as opposed to the default behavior, which is to show a print-out of all packages with colorized duplicates. In any case, the identification of any duplicates will yield a return code of 1. The ``-i`` option can be invoked multiple times to skip specific package names. + +.. _Acorn_Utilities: + +------------------------------ +Acorn utilities +------------------------------ +The util/acorn/ directory provides scripting for spack-stack builds through PBS Pro on Acorn. To use them, copy them into the directory of the Spack environment you wish to build, set the number of nodes to use (make sure ``#PBS -l select=X`` and ``mpiexec -n X`` use the same value for ``X``), and run ``qsub build.pbs``. Note that the temporary directory specification uses a soft link where the referent location depends on the node (this is to avoid compiling directly on LFS, which frequently fails when working with small files as when cloning git repositories). For parallel installations on Acorn, 2-6 is a reasonable range for the number of nodes (MPI proc analogs), and 6-8 is a reasonable number for the number of threads (note that for ``#PBS -l ncpus=Y`` in build.pbs, ``Y`` should match the ``-j`` argument for ``spack install`` in spackinstall.sh). diff --git a/doc/source/index.rst b/doc/source/index.rst index 3a4391a6c..9b03fc01f 100644 --- a/doc/source/index.rst +++ b/doc/source/index.rst @@ -15,6 +15,7 @@ Table of Contents Environments Prerequisites SpackStackExtension + Utilities KnownIssues MaintainersSection Documentation diff --git a/util/show_duplicate_packages.py b/util/show_duplicate_packages.py index 4dcd8892f..2ae1412d8 100755 --- a/util/show_duplicate_packages.py +++ b/util/show_duplicate_packages.py @@ -7,6 +7,7 @@ # spack concretize | tee log.concretize | show_duplicate_packages.py > list_of_duplicates.txt # # '-d' argument prints only duplicates (and disables highlighting). +# '-i' argument ignores a specific package, and can be invoked multiple times. # # Alex Richert, June 2023 @@ -20,12 +21,13 @@ def colorize_spec(line, package_name, colorize=False): c1 = r'\033[93m' ; c2 = r'\033[0m' return re.sub("(\w{7}\s+)(%s)@"%package_name, f"\\1{c1}\\2{c2}@", line) -def show_duplicate_packages(txt_to_check, only_show_dups=False): +def show_duplicate_packages(txt_to_check, ignore_list=[], only_show_dups=False): dd = defaultdict(set) for line in txt_to_check.split("\n"): line = line.replace("^", "") package_name = re.findall("\s\w{7}\s+(\^?[^\s@]+)@", line) if not package_name: continue + if [package_name[0]] in ignore_list: continue line = " ".join(line.split()[1:]) dd[package_name[0]].add(line) duplicates_found = False @@ -39,16 +41,19 @@ def show_duplicate_packages(txt_to_check, only_show_dups=False): print(colorize_spec(line, key, colorize=colorize)) sys.stderr.write("===\n%suplicates found%s\n" % (("D","!") if duplicates_found else ("No d","."))) sys.stderr.flush() + return int(duplicates_found) if __name__ == "__main__": parser = argparse.ArgumentParser(description="Check output of `spack concretize` for duplicate packages") - if sys.stdin.isatty(): - parser.add_argument("filename") + parser.add_argument("filename", nargs="?") parser.add_argument("-d", action="store_true") + parser.add_argument("-i", nargs="*", action="append") args = parser.parse_args() - if sys.stdin.isatty(): + if args.filename: with open(args.filename, "r") as f: txt_to_check = f.read() else: txt_to_check = sys.stdin.read() - show_duplicate_packages(txt_to_check, only_show_dups=args.d) + ret = show_duplicate_packages(txt_to_check, only_show_dups=args.d, ignore_list=args.i) + sys.exit(ret) +