Add utility for verifying that version/variant/externals are respected

doc/source/Utilities.rst

@@ -20,6 +20,21 @@ The utility located at util/show_duplicate_packages.py parses the output of ``sp
 
 The ``-d`` option shows only a list of the duplicates, as opposed to the default behavior, which is to show a print-out of all packages with colorized duplicates. In any case, the identification of any duplicates will yield a return code of 1. The ``-i`` option can be invoked multiple times to skip specific package names. The ``-c`` option can be used to ignore duplicates associated with different compilers; in an environment with, say, GCC and Intel copies of any given package, those two copies of a package will not be reported as duplicates.
 
+.. _Package_Config_Checker:
+
+------------------------------
+check_package_config.py
+------------------------------
+
+The utility at util/check_package_config.py is run after concretization in an active spack-stack environment (i.e., `$SPACK_ENV` is set) to confirm that the packages versions and variants in common/packages.yaml are respected in the concretization, as well as that any externals specified in site/packages.yaml are not being omitted. It does this by reading common/packages.yaml (for the version and variant settings), site/packages.yaml (for the external settings), and spack.lock. Usage is as follows:
+
+.. code-block:: console
+   spack env active envs/unified-env/
+   # To verify versions, variants, and externals:
+   ${SPACK_STACK_DIR}/util/check_package_config.py
+   # To ignore a known mismatch in version, variant, or external status for package 'esmf':
+   ${SPACK_STACK_DIR}/util/check_package_config.py esmf
+
 .. _Permissions_Checker:
 
 ------------------------------

util/check_package_config.py

@@ -0,0 +1,82 @@
+#!/usr/bin/env python3
+# This utility checks whether the package versions and variants set in
+# common/packages.yaml are being respected in the concretization, and
+# whether the externals in site/packages.yaml are being used.
+#
+# To use this script, run it in a loaded spack-stack environment.
+# Package names to be ignored can be provided as optional arguments.
+#
+# Alex Richert, Jan 2024
+
+import json
+import os
+import sys
+import yaml
+
+SPACK_ENV = os.getenv("SPACK_ENV")
+assert SPACK_ENV, "$SPACK_ENV must be set but is not!"
+
+## Load common/packages.yaml and site/packages.yaml for versions and externals, respectively
+packages_versions_path = os.path.join(SPACK_ENV, "common", "packages.yaml")
+with open(packages_versions_path, "r") as f:
+    packages_versions = yaml.safe_load(f)
+
+packages_externals_path = os.path.join(SPACK_ENV, "site", "packages.yaml")
+with open(packages_externals_path, "r") as f:
+    packages_externals = yaml.safe_load(f)
+
+## Load spack.lock
+spack_lock_path = os.path.join(SPACK_ENV, "spack.lock")
+with open(spack_lock_path, "r") as f:
+    spack_lock = json.load(f)
+
+iret = 0
+
+## Iterate over concretized packages
+for concrete_spec in spack_lock["concrete_specs"].values():
+    concrete_name = concrete_spec["name"]
+    if concrete_name in sys.argv[1:]:
+      continue
+    concrete_version = concrete_spec["version"]
+    if concrete_name in packages_versions["packages"].keys():
+        # Check whether concretized package has specified version from common/packages.yaml
+        config_version = packages_versions["packages"][concrete_name]["version"][0]
+        if concrete_version != config_version:
+            iret = 1
+            print(
+                f"WARNING: '{concrete_name}' concretized version {concrete_version} does not match {config_version} specified in $SPACK_ENV/common/packages.yaml"
+            )
+        # Check whether concretized variants match settings from common/packages.yaml
+        config_variants = packages_versions["packages"][concrete_name]["variants"].split()
+        for config_variant in config_variants:
+            variant_mismatch = False
+            # Boolean variant
+            if config_variant[0] in ("+", "~"):
+                config_value = config_variant[0]=="+"
+                if concrete_spec["parameters"][config_variant[1:]] != config_value:
+                    variant_mismatch = True
+            # Named variant
+            elif "=" in config_variant:
+                config_variant, config_value = config_variant.split("=")
+                concrete_values = concrete_spec["parameters"][config_variant]
+                if type(concrete_values) is str:
+                     concrete_values = [concrete_values]
+                if set(config_value.split(",")) != set(concrete_values):
+                     variant_mismatch = True
+            if variant_mismatch:
+                iret = 1
+                print(f"WARNING: '{concrete_name}' concretized variant '{config_variant}' does not match configured value in $SPACK_ENV/common/packages.yaml")
+    # Check whether concretized package is an external based on site/packages.yaml
+    if concrete_name in packages_externals["packages"].keys():
+        is_external_config = "externals" in packages_externals["packages"][concrete_name].keys()
+    else:
+        is_external_config = False
+    is_external_concrete = "external" in concrete_spec.keys()
+    if is_external_config != is_external_concrete:
+        iret = 1
+        print(
+            f"WARNING: '{concrete_name}' is %sconfigured as external in $SPACK_ENV/site/packages.yaml but was %sconcretized as external"
+            % ((not is_external_config) * "not ", (not is_external_concrete) * "not ")
+        )
+
+sys.exit(iret)

util/test_env/README

@@ -0,0 +1 @@
+This directory provides test files to verify the check_package_config.py utility.

util/test_env/common/packages.yaml

@@ -0,0 +1,286 @@
+# Pin versions and specs when building packages
+  packages:
+    #
+    all:
+      providers:
+        blas: [openblas]
+        fftw-api: [fftw]
+        gl: [opengl]
+        glu: [openglu]
+        jpeg: [libjpeg-turbo]
+        lapack: [openblas]
+        yacc: [bison]
+        zlib-api: [zlib]
+    #
+    # This version of awscli goes with [email protected]
+    awscli:
+      version: ['1.27.84']
+    bacio:
+      version: ['2.4.1']
+    bison:
+      version: ['3.8.2']
+    boost:
+      version: ['1.83.0']
+      variants: ~atomic +chrono +date_time +exception +filesystem ~graph ~iostreams ~locale ~log ~math ~mpi ~numpy +pic +program_options +python ~random +regex +serialization ~signals +system +test +thread +timer ~wave cxxstd=17 visibility=hidden
+    bufr:
+      version: ['12.0.1']
+      variants: +python
+    cairo:
+      variants: +pic
+    cdo:
+      version: ['2.2.0']
+      variants: ~openmp
+    cmake:
+      version: ['3.23.1']
+      variants: +ownlibs
+    # Attention - when updating also check the various jcsda-emc-bundles env packages
+    crtm:
+      version: ['2.4.0.1']
+      variants: +fix
+    ecbuild:
+      version: ['3.7.2']
+    eccodes:
+      version: ['2.32.0']
+      variants: +png
+    ecflow:
+      version: ['5.11.4']
+      variants: +ui
+    eckit:
+      version: ['1.24.5']
+      variants: linalg=eigen,lapack compression=lz4,bzip2
+    ecmwf-atlas:
+      version: ['0.35.1']
+      variants: +fckit +trans +tesselation +fftw
+    ectrans:
+      version: ['1.2.0']
+      variants: ~mkl +fftw
+    eigen:
+      version: ['3.4.0']
+    # Attention - when updating the version also check the common modules.yaml
+    # config and update the projections for lmod/tcl.
+    # Also, check the acorn and derecho site configs which have esmf modifications.
+    esmf:
+      version: ['8.6.0']
+      variants: ~xerces ~pnetcdf snapshot=none +shared +external-parallelio
+      require:
+        - any_of: ['fflags="-fp-model precise" cxxflags="-fp-model precise"']
+          when: "%intel"
+          message: "Extra ESMF compile options for Intel"
+        - any_of: ['']
+          when: "%gcc"
+          message: "Extra ESMF compile options for GCC"
+    fckit:
+      version: ['0.11.0']
+      variants: +eckit
+    fftw:
+      version: ['3.3.10']
+    fiat:
+      version: ['1.2.0']
+    fms:
+      version: ['2023.04']
+      variants: precision=32,64 +quad_precision +gfs_phys +openmp +pic constants=GFS build_type=Release +deprecated_io
+    fontconfig:
+      variants: +pic
+    freetype:
+      variants: +pic
+    g2:
+      version: ['3.4.5']
+    g2c:
+      version: ['1.6.4']
+    g2tmpl:
+      version: ['1.10.2']
+    gettext:
+      version: ['0.21.1']
+    gfsio:
+      version: ['1.4.1']
+    gftl-shared:
+      version: ['1.6.1']
+    #git-lfs:
+      # Assume git-lfs is provided, hard to install
+      # because of dependencies on go/go-bootstrap.
+      # Note: Uncommenting this entry will break
+      # the container builds.
+      #version: ['2.11.0']
+    grib-util:
+      version: ['1.3.0']
+    gsibec:
+      version: ['1.1.3']
+    gsi-ncdiag:
+      version: ['1.1.2']
+    gsl-lite:
+      version: ['0.37.0']
+    hdf:
+      version: ['4.2.15']
+      variants: +external-xdr ~fortran ~netcdf
+    hdf5:
+      version: ['1.14.0']
+      variants: +hl +fortran +mpi ~threadsafe +szip
+    ip:
+      version: ['4.3.0']
+      variants: precision=4,d,8
+    ip2:
+      version: ['1.1.2']
+    jasper:
+      version: ['2.0.32']
+    jedi-cmake:
+      version: ['1.4.0']
+    jpeg:
+      version: ['9.1.0']
+    landsfcutil:
+      version: ['2.4.1']
+    libjpeg-turbo:
+      version: ['2.1.0']
+    libpng:
+      version: ['1.6.37']
+      variants: +pic
+    libyaml:
+      version: ['0.2.5']
+    mapl:
+      version: ['2.40.3']
+      variants: +shared +pflogger ~f2py
+    # If making changes here, also check the Discover site config and the CI workflows
+    met:
+      version: ['11.1.0']
+      variants: +python +grib2
+    metplus:
+      version: ['5.1.0']
+    metis:
+      require: "+int64 +real64"
+    mpich:
+      variants: ~hwloc +two_level_namespace
+    mysql:
+      variants: +download_boost
+    nco:
+      version: ['5.0.6']
+      variants: ~doc
+    # ncview - when adding information here, also check Orion
+    # and Discover site configs
+    nemsio:
+      version: ['2.5.4']
+    nemsiogfs:
+      version: ['2.5.3']
+    nccmp:
+      version: ['1.9.0.1']
+    ncio:
+      version: ['1.1.2']
+    netcdf-c:
+      version: ['4.9.2']
+      # If using 4.9.1, turn off byterange variant to fix compile error: ~byterange
+      variants: +dap +mpi ~parallel-netcdf
+    netcdf-cxx4:
+      version: ['4.3.1']
+    netcdf-fortran:
+      version: ['4.6.1']
+    # ninja - when adding information here, also check Discover site config
+    nlohmann-json:
+      version: ['3.10.5']
+    nlohmann-json-schema-validator:
+      version: ['2.1.0']
+    odc:
+      version: ['1.4.6']
+      variants: ~fortran
+    openblas:
+      version: ['0.3.24']
+      variants: +noavx512
+    openmpi:
+      variants: +internal-hwloc +two_level_namespace
+    # Pin openssl to avoid duplicate packages being built
+    openssl:
+      variants: +shared
+    p4est:
+      version: ['2.8']
+    parallelio:
+      version: ['2.5.10']
+      variants: +pnetcdf
+    parallel-netcdf:
+      version: ['1.12.2']
+    pflogger:
+      version: ['1.12.0']
+      variants: +mpi
+    pixman:
+      variants: +pic
+    # Do not build pkgconf - https://github.com/jcsda/spack-stack/issues/123
+    pkgconf:
+      buildable: False
+    prod-util:
+      version: ['2.1.1']
+    proj:
+      version: ['8.1.0']
+      variants: ~tiff
+    python:
+      require: "@3.10.13"
+    py-attrs:
+      # https://github.com/JCSDA/spack-stack/issues/740
+      version: ['21.4.0']
+    py-cartopy:
+      variants: +plotting
+      require: "@0.21.1"
+    py-cryptography:
+      variants: +rust_bootstrap
+    # Introduced in https://github.com/JCSDA/spack-stack/pull/894, pin py-cython
+    # to avoid duplicate packages being built (cylc dependencies soft-want @3:)
+    py-cython:
+      require: "@0.29.36"
+    py-h5py:
+      version: ['3.7.0']
+      variants: ~mpi
+    # Comment out for now until build problems are solved
+    # https://github.com/jcsda/spack-stack/issues/522
+    # see also ewok-env virtual package and container
+    # README.md
+    #py-mysql-connector-python:
+    #  version: ['8.0.32']
+    py-netcdf4:
+      version: ['1.5.8']
+      variants: ~mpi
+    py-numpy:
+      require: ['@1.22.3']
+    py-pandas:
+      variants: +excel
+    # To avoid pip._vendor.pep517.wrappers.BackendInvalid errors with newer
+    # versions of py-poetry-core when using external/homebrew Python as
+    # we do at the moment in spack-stack.
+    # Pin the py-setuptools version to avoid duplicate Python packages
+    py-setuptools:
+      require: ['@63.4.3']
+    py-setuptools-rust:
+      variants: +rust_bootstrap
+    py-shapely:
+      require: ['@1.8.0']
+    qt:
+      version: ['5.15.3']
+    scotch:
+      version: ['7.0.4']
+      variants: +mpi+metis~shared~threads~mpi_thread+noarch
+    sfcio:
+      version: ['1.4.1']
+    shumlib:
+      version: ['macos_clang_linux_intel_port']
+    sigio:
+      version: ['2.3.2']
+    sp:
+      version: ['2.5.0']
+      variants: precision=4,d,8 derp=auto
+    udunits:
+      version: ['2.2.28']
+    upp:
+      version: ['10.0.10']
+    w3emc:
+      version: ['2.10.0']
+      variants: precision=4,d,8
+    w3nco:
+      version: ['2.4.1']
+    wget:
+      version: ['1.21.2']
+    # When changing wgrib2, also check Hercules and Nautilus site configs
+    wgrib2:
+      version: ['2.0.8']
+    wrf-io:
+      version: ['1.2.0']
+    yafyaml:
+      version: ['0.5.1']
+    zlib:
+      version: ['1.2.13']
+    zstd:
+      version: ['1.5.2']
+      variants: +programs