diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index c02b239373..b748728bc2 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,6 +1,23 @@ -# --------------------------------------------------------------- +############################################################################### +# Copyright (c) 2022-23, Lawrence Livermore National Security, LLC and RADIUSS +# project contributors. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# SPDX-License-Identifier: (MIT) +############################################################################### +# Modifications to this file have been made under the folllowing copyright. +# ----------------------------------------------------------------------------- # SUNDIALS Copyright Start -# Copyright (c) 2002-2021, Lawrence Livermore National Security +# Copyright (c) 2002-2024, Lawrence Livermore National Security # and Southern Methodist University. # All rights reserved. # @@ -8,129 +25,97 @@ # # SPDX-License-Identifier: BSD-3-Clause # SUNDIALS Copyright End -# --------------------------------------------------------------- +# ----------------------------------------------------------------------------- +# DESCRIPTION: ############################################################################### # General GitLab pipelines configurations for supercomputers and Linux clusters # at Lawrence Livermore National Laboratory (LLNL). -############################################################################### - -# We define the following GitLab pipeline variables (all of which can be -# set from the GitLab CI "run pipeline" UI): -# -# GIT_SUBMODULE_STRATEGY: -# Tells Gitlab to recursively update the submodules when cloning. -# -# ALLOC_NAME: -# Allocation unique name. -# -# BUILD_ROOT: -# The path to the shared resources between all jobs. The BUILD_ROOT is unique to -# the pipeline, preventing any form of concurrency with other pipelines. This -# also means that the BUILD_ROOT directory will never be cleaned. -# -# DEFAULT_TIME: -# Default time to let the jobs run will be 30 minutes. -# -# BUILD_JOBS: -# Number of threads to use for builds. -# -# NCPU: -# Number of CPUs to alloc. +# This entire pipeline is LLNL-specific # -# VERBOSE_BUILD: -# Sets CMAKE_VERBOSE_MAKEFILE to TRUE when "ON". +# Important note: This file is a template provided by llnl/radiuss-shared-ci. +# Remains to set variable values, change the reference to the radiuss-shared-ci +# repo, opt-in and out optional features. The project can then extend it with +# additional stages. # -# VERBOSE_TEST: -# Passes --verbose to CTest when "ON". +# In addition, each project should copy over and complete: +# - .gitlab/custom-jobs-and-variables.yml +# - .gitlab/subscribed-pipelines.yml # -# ON_LASSEN: -# Should the Lassen pipeline run? Set to "ON" or "OFF" to enable/disable. -# -# ON_QUARTZ: -# Should the Quartz pipeline run? Set to "ON" or "OFF" to enable/disable. -# -# SPACK_PREFIX: prefix used for shared spack installation. -# Usually this a spack version number that matches the version set in the uberenv_config.json file. -# Spack installs go in /usr/workspace/sundials/spack_installs/${SPACK_PREFIX}/$(hostname). -# -# SHARED_SPACK: -# If "ON", then a shared spack install that has been pre-configured is utilized. -# If "OFF", then a new spack instance is created for every build (meaning all TPLs have to be installed). -# If "UPSTREAM" (the default), then the shared spack is used as an upstream for a build specific spack. -# -# BENCHMARK: -# If "ON", then the SUNDIALS benchmark problems are run and generate profiles. -# -# BENCHMARK_NNODES: -# Number of nodes to use for benchmarks. Default is 4. -# -# BENCHMARK_QUEUE: -# What queue to submit the benchmarks too. Default is pbatch (for Livermore). +# The jobs should be specified in a file local to the project, +# - .gitlab/jobs/${CI_MACHINE}.yml +# or generated (see LLNL/Umpire for an example). +############################################################################### +# We define the following GitLab pipeline variables: variables: +##### LC GITLAB CONFIGURATION +# Use an LLNL service user to run CI. This prevents from running pipelines as +# an actual user. + LLNL_SERVICE_USER: "" +# Use the service user workspace. Solves permission issues, stores everything +# at the same location whoever triggers a pipeline. + CUSTOM_CI_BUILDS_DIR: "/usr/workspace/sundials/ci/.builds/" +# Tells Gitlab to recursively update the submodules when cloning the project. GIT_SUBMODULE_STRATEGY: recursive - ALLOC_NAME: ${CI_PROJECT_NAME}_ci_${CI_PIPELINE_ID} - BUILD_ROOT: ${CI_PROJECT_DIR} - DEFAULT_TIME: 30 - BUILD_JOBS: 32 - NCPUS: 12 - VERBOSE_BUILD: "OFF" - VERBOSE_TEST: "OFF" - ON_LASSEN: "ON" - ON_QUARTZ: "ON" - ON_CORONA: "ON" - SPACK_PREFIX: "v0.19.1" - SHARED_SPACK: "UPSTREAM" - BENCHMARK: "OFF" - BENCHMARK_NNODES: 4 - BENCHMARK_QUEUE: "pbatch" - -# Normally, stages are blocking in Gitlab. However, using the keyword "needs" we -# can express dependencies between job that break the ordering of stages, in -# favor of a DAG. -# In practice q_*, l_* and c_* stages are independently run and start immediately. -stages: - - q_build_and_test - - l_build_and_test - - l_build_and_bench - - c_build_and_test +##### PROJECT VARIABLES +# We build the projects in the CI clone directory. +# Used in script/gitlab/build_and_test.sh script. +# TODO: add a clean-up mechanism. + BUILD_ROOT: ${CI_PROJECT_DIR} + SPACK_REF: "594a376c521cc746978571b1181a47bbcff30a21" # v0.22.2 -# These are also templates (.name) that define project specific build commands. -# If an allocation exist with the name defined in this pipeline, the job will -# use it (slurm specific). -.build_toss_3_x86_64_ib_script: - script: - - echo ${ALLOC_NAME} - - srun -p pdebug -N 1 -n ${NCPUS} --interactive -t ${DEFAULT_TIME} - --job-name=${ALLOC_NAME} .gitlab/build_and_test.sh +##### SHARED_CI CONFIGURATION +# Required information about GitHub repository + GITHUB_PROJECT_NAME: "sundials" + GITHUB_PROJECT_ORG: "LLNL" +# Set the build-and-test command. +# Nested variables are allowed and useful to customize the job command. We +# prevent variable expansion so that you can define them at job level. + JOB_CMD: + value: ".gitlab/build_and_test.sh" + expand: false +# Override the pattern describing branches that will skip the "draft PR filter +# test". Add protected branches here. See default value in +# preliminary-ignore-draft-pr.yml. +# ALWAYS_RUN_PATTERN: "" -# Corona -.build_toss_4_x86_64_ib_corona_script: - script: - - echo ${ALLOC_NAME} - - flux alloc -N 1 -t ${DEFAULT_TIME} .gitlab/build_and_test.sh +# We organize the build-and-test stage with sub-pipelines. Each sub-pipeline +# corresponds to a test batch on a given machine. -# CORAL systems use spectrum LSF instead of SLURM -.build_blueos_3_ppc64le_ib_script: - script: - - echo ${ALLOC_NAME} - - bsub -q pdebug -J ${ALLOC_NAME} -nnodes 1 -W ${DEFAULT_TIME} -Is .gitlab/build_and_test.sh +# High level stages +stages: + - prerequisites + - build-and-test -# Benchmark job for CORAL systems -.build_blueos_3_ppc64le_ib_bench: - script: - - echo ${ALLOC_NAME} - - bsub -q ${BENCHMARK_QUEUE} -J ${ALLOC_NAME} -nnodes ${BENCHMARK_NNODES} -W ${DEFAULT_TIME} -Is .gitlab/build_and_bench.sh +# Template for jobs triggering a build-and-test sub-pipeline: +.build-and-test: + stage: build-and-test + trigger: + include: + - local: '.gitlab/custom-jobs-and-variables.yml' + - project: 'sundials/radiuss-shared-ci' # https://lc.llnl.gov/gitlab/sundials/radiuss-shared-ci + ref: 'ae1f3786591beed83abc6a0de2229f6e9532e2d4' + file: 'pipelines/${CI_MACHINE}.yml' + # Add your jobs + # you can use a local file + - local: '.gitlab/jobs/${CI_MACHINE}.yml' + # or a file generated in the previous steps + # - artifact: '${CI_MACHINE}-jobs.yml' + # job: 'generate-job-file' + # (See Umpire CI setup for an example). + strategy: depend + forward: + pipeline_variables: true -# This is where jobs are included. include: + # Sets ID tokens for every job using `default:` - project: 'lc-templates/id_tokens' file: 'id_tokens.yml' - - local: .gitlab/quartz-templates.yml - - local: .gitlab/quartz-jobs.yml - - local: .gitlab/lassen-templates.yml - - local: .gitlab/lassen-jobs.yml - - local: .gitlab/corona-templates.yml - - local: .gitlab/corona-jobs.yml + # # [Optional] checks preliminary to running the actual CI test + # - project: 'radiuss/radiuss-shared-ci' + # ref: 'v2024.04.0' + # file: 'utilities/preliminary-ignore-draft-pr.yml' + # pipelines subscribed by the project + - local: '.gitlab/subscribed-pipelines.yml' diff --git a/.gitlab/build_and_test.sh b/.gitlab/build_and_test.sh index cf7ab193fa..04abb537ea 100755 --- a/.gitlab/build_and_test.sh +++ b/.gitlab/build_and_test.sh @@ -20,9 +20,6 @@ job_unique_id=${CI_JOB_ID:-""} sys_type=${SYS_TYPE:-""} py_env_path=${PYTHON_ENVIRONMENT_PATH:-""} -spack_prefix=${SHARED_SPACK_PREFIX:-"v0.19.1"} -shared_spack=${SHARED_SPACK:-"UPSTREAM"} - # Dependencies date @@ -38,7 +35,6 @@ echo "spec = ${spec}" echo "job_unique_id = ${job_unique_id}" echo "sys_type = ${sys_type}" echo "py_env_path = ${py_env_path}" -echo "shared_spack = ${shared_spack}" # remove tailing number from hostname hostname=${hostname%%[0-9]*} @@ -46,19 +42,6 @@ hostname=${hostname%%[0-9]*} # number of parallel build jobs BUILD_JOBS=${BUILD_JOBS:-"1"} -# load newer python to try the clingo concretizer -# machine specific loads -if [[ "${hostname}" == "lassen" ]]; then - echo "module load python/3.8.2" - module load python/3.8.2 -elif [[ "${hostname}" == "corona" ]]; then - echo "module load python/3.9.12" - module load python/3.9.12 -else - echo "module load python" - module load python -fi - if [[ "${option}" != "--build-only" && "${option}" != "--test-only" ]] then echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~" @@ -97,35 +80,29 @@ then mkdir -p ${spack_user_cache} fi - if [[ -d /usr/workspace/sundials ]] - then - upstream="/usr/workspace/sundials/spack_installs/${spack_prefix}/${hostname}" - mkdir -p "${upstream}" - upstream_opt="--upstream=${upstream}" - fi + mirror_opt="" + buildcache="/usr/workspace/sundials/ci/spack_stuff/build_caches/${SPACK_REF}" - if [[ "${shared_spack}" == "UPSTREAM" ]] + if [[ ! -d "${buildcache}" ]] then - python3 .gitlab/uberenv/uberenv.py --spec="${spec}" "${prefix_opt}" "${upstream_opt}" - elif [[ "${shared_spack}" == "ON" ]] - then - python3 .gitlab/uberenv/uberenv.py --spec="${spec}" --prefix="${upstream}" - else - python3 .gitlab/uberenv/uberenv.py --spec="${spec}" "${prefix_opt}" + mkdir "${buildcache}" fi - # Ensure correct CUDA module is loaded, only works for module naming - # convention on Lassen. Only needed for CUDA 11 (unclear why). - if [[ -n "${CUDA_SPEC}" ]]; then - cuda_version="${CUDA_SPEC##*@}" - echo "module load cuda/${cuda_version}" - module load cuda/"${cuda_version}" - fi + mirror_opt=("--mirror=${buildcache}" "--mirror-autopush") - module load cmake/3.23 + key_path=/usr/workspace/sundials/ci/spack_stuff/gpg_backup + python3 .gitlab/uberenv/uberenv.py \ + --trust-key ${key_path}/pubring.gpg --trust-key ${key_path}/secring.gpg \ + --spec="${spec}" "${mirror_opt[@]}" "${prefix_opt}" \ + --spack-commit="${SPACK_REF}" fi + date +# Reload the spack environment created by uberenv +. ${prefix}/spack/share/spack/setup-env.sh +spack load + # Host config file if [[ -z ${hostconfig} ]] then @@ -173,6 +150,9 @@ then echo "~ Host-config: ${hostconfig_path}" echo "~ Build Dir: ${build_dir}" echo "~ Project Dir: ${project_dir}" + echo "~ MPIEXEC_EXECUTABLE: ${MPIEXEC_EXECUTABLE}" + echo "~ MPIEXEC_PREFLAGS: ${MPIEXEC_PREFLAGS}" + echo "~ MPIEXEC_POSTFLAGS: ${MPIEXEC_POSTFLAGS}" echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~" echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~" @@ -195,6 +175,9 @@ then $cmake_exe \ -C "${hostconfig_path}" \ -DCMAKE_INSTALL_PREFIX=${install_dir} \ + -DMPIEXEC_EXECUTABLE=$(which $MPIEXEC_EXECUTABLE) \ + -DMPIEXEC_PREFLAGS=${MPIEXEC_PREFLAGS} \ + -DMPIEXEC_POSTFLAGS=${MPIEXEC_POSTFLAGS} \ -DSUNDIALS_CALIPER_OUTPUT_DIR="${CALIPER_DIR}/Release/${hostname}/${sundials_version}" \ "${project_dir}" @@ -202,6 +185,9 @@ then $cmake_exe \ -C "${hostconfig_path}" \ -DCMAKE_INSTALL_PREFIX=${install_dir} \ + -DMPIEXEC_EXECUTABLE=$(which $MPIEXEC_EXECUTABLE) \ + -DMPIEXEC_PREFLAGS=${MPIEXEC_PREFLAGS} \ + -DMPIEXEC_POSTFLAGS=${MPIEXEC_POSTFLAGS} \ "${project_dir}" fi diff --git a/.gitlab/corona-jobs.yml b/.gitlab/corona-jobs.yml deleted file mode 100644 index f694611c39..0000000000 --- a/.gitlab/corona-jobs.yml +++ /dev/null @@ -1,48 +0,0 @@ -# ------------------------------------------------------------------------------ -# SUNDIALS Copyright Start -# Copyright (c) 2002-2021, Lawrence Livermore National Security -# and Southern Methodist University. -# All rights reserved. -# -# See the top-level LICENSE and NOTICE files for details. -# -# SPDX-License-Identifier: BSD-3-Clause -# SUNDIALS Copyright End -# ------------------------------------------------------------------------------ - -# ------------------------------------------------------------------------------ -# HIP -# ------------------------------------------------------------------------------ - -# Builds with HIP -corona_rocmcc_550: - parallel: - matrix: - - COMPILER_SPEC: rocmcc@5.5.0 - AMDGPU_TARGET: [gfx906] - variables: - SPEC: "%${COMPILER_SPEC} cstd=99 cxxstd=14 precision=double amdgpu_target=${AMDGPU_TARGET} scheduler=flux +rocm+mpi" - extends: .corona_build_and_test - -# ------------------------------------------------------------------------------ -# HIP + TPLs -# ------------------------------------------------------------------------------ -corona_rocmcc_550_tpls: - parallel: - matrix: - - COMPILER_SPEC: rocmcc@5.5.0 - AMDGPU_TARGET: [gfx906] - ROCM_VERSION: 5.5.0 - variables: - SPEC: "%${COMPILER_SPEC} cstd=99 cxxstd=14 precision=double ~int64 amdgpu_target=${AMDGPU_TARGET} scheduler=flux +rocm+mpi+magma+raja+kokkos+kokkos-kernels~ginkgo ^magma+rocm amdgpu_target=${AMDGPU_TARGET} ^raja+rocm~openmp~examples~exercises amdgpu_target=${AMDGPU_TARGET} ^kokkos+rocm~profiling amdgpu_target=${AMDGPU_TARGET} ^hipblas@${ROCM_VERSION} ^hipsparse@${ROCM_VERSION} ^hip@${ROCM_VERSION} ^hsa-rocr-dev@${ROCM_VERSION} ^llvm-amdgpu@${ROCM_VERSION}" - extends: .corona_build_and_test - -corona_rocmcc_523_tpls: - parallel: - matrix: - - COMPILER_SPEC: rocmcc@5.2.3 - AMDGPU_TARGET: [gfx906] - ROCM_VERSION: 5.2.3 - variables: - SPEC: "%${COMPILER_SPEC} cstd=99 cxxstd=14 precision=double ~int64 amdgpu_target=${AMDGPU_TARGET} scheduler=flux +rocm+mpi+magma+raja+kokkos+kokkos-kernels+ginkgo ^magma+rocm amdgpu_target=${AMDGPU_TARGET} ^raja+rocm~openmp~examples~exercises amdgpu_target=${AMDGPU_TARGET} ^kokkos+rocm~profiling amdgpu_target=${AMDGPU_TARGET} ^ginkgo+rocm amdgpu_target=${AMDGPU_TARGET} ^hipblas@${ROCM_VERSION} ^hipsparse@${ROCM_VERSION} ^rocrand@${ROCM_VERSION} ^rocthrust@${ROCM_VERSION} ^hip@${ROCM_VERSION} ^hsa-rocr-dev@${ROCM_VERSION} ^llvm-amdgpu@${ROCM_VERSION} ^rocprim@${ROCM_VERSION}" - extends: .corona_build_and_test diff --git a/.gitlab/corona-templates.yml b/.gitlab/corona-templates.yml deleted file mode 100644 index a901f4fa48..0000000000 --- a/.gitlab/corona-templates.yml +++ /dev/null @@ -1,35 +0,0 @@ -# ------------------------------------------------------------------------------ -# SUNDIALS Copyright Start -# Copyright (c) 2002-2021, Lawrence Livermore National Security -# and Southern Methodist University. -# All rights reserved. -# -# See the top-level LICENSE and NOTICE files for details. -# -# SPDX-License-Identifier: BSD-3-Clause -# SUNDIALS Copyright End -# ------------------------------------------------------------------------------ - -# ------------------------------------------------------------------------------ -# Tags and rules to run tests on Corona -# ------------------------------------------------------------------------------ - -# Generic Corona build job, extending build script for Toss 4 x86_64 Systems -.corona_build_and_test: - tags: - - shell - - corona - extends: [.build_toss_4_x86_64_ib_corona_script] - stage: c_build_and_test - needs: [] - artifacts: - paths: - - spack-*.txt - - build_*/* - when: always - rules: - # Don't run if... - - if: '$CI_COMMIT_BRANCH =~ /_cnone/ || $ON_CORONA == "OFF" || $BENCHMARK == "ON"' - when: never - # Default is to run if previous stage succeeded - - when: on_success diff --git a/.gitlab/custom-jobs-and-variables.yml b/.gitlab/custom-jobs-and-variables.yml new file mode 100644 index 0000000000..1b830e2e3c --- /dev/null +++ b/.gitlab/custom-jobs-and-variables.yml @@ -0,0 +1,84 @@ +############################################################################### +# Copyright (c) 2022-23, Lawrence Livermore National Security, LLC and RADIUSS +# project contributors. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# SPDX-License-Identifier: (MIT) +############################################################################### +# Modifications to this file have been made under the folllowing copyright. +# ----------------------------------------------------------------------------- +# SUNDIALS Copyright Start +# Copyright (c) 2002-2024, Lawrence Livermore National Security +# and Southern Methodist University. +# All rights reserved. +# +# See the top-level LICENSE and NOTICE files for details. +# +# SPDX-License-Identifier: BSD-3-Clause +# SUNDIALS Copyright End +# ----------------------------------------------------------------------------- + +# We define the following GitLab pipeline variables: +variables: +# In some pipelines we create only one allocation shared among jobs in +# order to save time and resources. This allocation has to be uniquely +# named so that we are sure to retrieve it and avoid collisions. + ALLOC_NAME: ${CI_PROJECT_NAME}_ci_${CI_PIPELINE_ID} + +# Dane +# Arguments for top level allocation + DANE_SHARED_ALLOC: "--exclusive --reservation=ci --time=60 --nodes=1" +# Arguments for job level allocation + DANE_JOB_ALLOC: "--time=55 --nodes=1 --overlap" +# Add variables that should apply to all the jobs on a machine: +# DANE_MY_VAR: "..." + +# Tioga +# Arguments for top level allocation + TIOGA_SHARED_ALLOC: "--exclusive --queue pci --time-limit=60m --nodes=1" +# Arguments for job level allocation + TIOGA_JOB_ALLOC: "--time-limit=55m --nodes=1" +# Add variables that should apply to all the jobs on a machine: +# TIOGA_MY_VAR: "..." + +# Number of threads to use for builds. + BUILD_JOBS: 32 + +# Sets CMAKE_VERBOSE_MAKEFILE to TRUE when "ON". + VERBOSE_BUILD: "ON" + +# Passes --verbose to CTest when "ON". + VERBOSE_TEST: "ON" + +# Should the Dane pipeline run? Set to "ON" or "OFF" to enable/disable. + ON_DANE: "ON" + +# Should the Tioga pipeline run? Set to "ON" or "OFF" to enable/disable. + ON_TIOGA: "ON" + +# If "ON", then the SUNDIALS benchmark problems are run and generate profiles. + BENCHMARK: "OFF" + +# Number of nodes to use for benchmarks. Default is 4. + BENCHMARK_NNODES: 4 + +# What queue to submit the benchmarks too. Default is pbatch (for Livermore). + BENCHMARK_QUEUE: "pbatch" + +# Configuration shared by build and test jobs specific to this project. +# Not all configuration can be shared. Here projects can fine tune the +# CI behavior. +# See Umpire for an example (export junit test reports). +.custom_job: + variables: + SPACK_DISABLE_LOCAL_CONFIG: "true" + SPACK_USER_CACHE_PATH: /tmp/spack diff --git a/.gitlab/jobs/dane.yml b/.gitlab/jobs/dane.yml new file mode 100644 index 0000000000..bcb31c8bc8 --- /dev/null +++ b/.gitlab/jobs/dane.yml @@ -0,0 +1,66 @@ +############################################################################## +# Copyright (c) 2022-24, Lawrence Livermore National Security, LLC and RADIUSS +# project contributors. See the COPYRIGHT file for details. +# +# SPDX-License-Identifier: (MIT) +############################################################################## +# ------------------------------------------------------------------------------ +# SUNDIALS Copyright Start +# Copyright (c) 2002-2024, Lawrence Livermore National Security +# and Southern Methodist University. +# All rights reserved. +# +# See the top-level LICENSE and NOTICE files for details. +# +# SPDX-License-Identifier: BSD-3-Clause +# SUNDIALS Copyright End +# ------------------------------------------------------------------------------ + +########################################## +# Overridden shared radiuss-shared-ci jobs +########################################## + +# We duplicate the shared jobs description and add necessary changes for this +# project. We keep ${PROJECT__VARIANTS} and ${PROJECT__DEPS} +# So that the comparison with the original job is easier. + +# No overridden jobs so far. + +######################## +# Extra jobs +######################## + +.sundials_job_on_dane: + variables: + MPIEXEC_EXECUTABLE: "srun" + extends: [.custom_job, .job_on_dane] + +dane_clang_tpls: + parallel: + matrix: + - COMPILER_SPEC: clang@14.0.6 + INDEX_SPEC: [~int64] + PRECISION_SPEC: [double] + variables: + SPEC: "%${COMPILER_SPEC} cstd=99 cxxstd=14 ${INDEX_SPEC} precision=${PRECISION_SPEC} +f2003+mpi+openmp+hypre+superlu-dist+lapack+klu+petsc+ginkgo+kokkos+kokkos-kernels~trilinos ^openblas" + extends: .sundials_job_on_dane + +dane_gcc_tpls: + parallel: + matrix: + - COMPILER_SPEC: gcc@10.3.1 + INDEX_SPEC: [~int64] + PRECISION_SPEC: [double] + variables: + SPEC: "%${COMPILER_SPEC} cstd=99 cxxstd=14 ${INDEX_SPEC} precision=${PRECISION_SPEC} +f2003+mpi+openmp+hypre+superlu-dist+lapack+klu+petsc+ginkgo+kokkos+kokkos-kernels~trilinos ^netlib-lapack" + extends: .sundials_job_on_dane + +dane_intel_tpls: + parallel: + matrix: + - COMPILER_SPEC: intel@2023.2.1 + INDEX_SPEC: [~int64] + PRECISION_SPEC: [double] + variables: + SPEC: "%${COMPILER_SPEC} cstd=99 cxxstd=14 ${INDEX_SPEC} precision=${PRECISION_SPEC} ~f2003+mpi+openmp+hypre+superlu-dist+lapack~klu+petsc+ginkgo+kokkos+kokkos-kernels~trilinos ^intel-oneapi-mkl" + extends: .sundials_job_on_dane diff --git a/.gitlab/jobs/tioga.yml b/.gitlab/jobs/tioga.yml new file mode 100644 index 0000000000..3a38476cca --- /dev/null +++ b/.gitlab/jobs/tioga.yml @@ -0,0 +1,92 @@ +############################################################################## +# Copyright (c) 2022-24, Lawrence Livermore National Security, LLC and RADIUSS +# project contributors. See the COPYRIGHT file for details. +# +# SPDX-License-Identifier: (MIT) +############################################################################## +# ------------------------------------------------------------------------------ +# SUNDIALS Copyright Start +# Copyright (c) 2002-2024, Lawrence Livermore National Security +# and Southern Methodist University. +# All rights reserved. +# +# See the top-level LICENSE and NOTICE files for details. +# +# SPDX-License-Identifier: BSD-3-Clause +# SUNDIALS Copyright End +# ------------------------------------------------------------------------------ + +########################################## +# Overridden shared radiuss-shared-ci jobs +########################################## + +# We duplicate the shared jobs description and add necessary changes for this +# project. We keep ${PROJECT__VARIANTS} and ${PROJECT__DEPS} +# So that the comparison with the original job is easier. + +# No overridden jobs so far. + +######################## +# Extra jobs +######################## + +.sundials_job_on_tioga: + variables: + MPIEXEC_EXECUTABLE: "flux" + MPIEXEC_PREFLAGS: "run" + extends: [.custom_job, .job_on_tioga] + +# --- Builds with HIP + +# Builds fine but non-mpi tests fail to execute due to a missing so: +# error while loading shared libraries: libpgmath.so: cannot open shared object file: No such file or directory +# This seems like it might be an LC environment issue, so disabling the job for now. +tioga_rocmcc_571_tpls: + parallel: + matrix: + - COMPILER_SPEC: rocmcc@5.7.1 + AMDGPU_TARGET: [gfx90a] + variables: + ON_TIOGA: "OFF" # disable until we can figure out libpgmath.so error + # have to use ginkgo@master because our spack version does not have ginkgo@1.8.0: yet (which seems to be needed) + SPEC: "%${COMPILER_SPEC} cstd=99 cxxstd=14 precision=double amdgpu_target=${AMDGPU_TARGET} +rocm+mpi+magma+ginkgo+kokkos ^magma+rocm amdgpu_target=${AMDGPU_TARGET} ^ginkgo@master+rocm amdgpu_target=${AMDGPU_TARGET} ^kokkos+rocm amdgpu_target=${AMDGPU_TARGET}" + before_script: + - module load rocmcc/5.7.1-magic + extends: [.sundials_job_on_tioga] + +tioga_rocmcc_620_tpls: + parallel: + matrix: + - COMPILER_SPEC: rocmcc@6.2.0 + AMDGPU_TARGET: [gfx90a] + variables: + # have to use ginkgo@master because our spack version does not have ginkgo@1.8.0: yet (which seems to be needed) + # similarly, we need a newer magma than available to compile with 'rocm@6:' so we turn it off + SPEC: "%${COMPILER_SPEC} cstd=99 cxxstd=14 precision=double amdgpu_target=${AMDGPU_TARGET} +rocm+mpi~magma+ginkgo+kokkos ^ginkgo@master+rocm amdgpu_target=${AMDGPU_TARGET} ^kokkos+rocm amdgpu_target=${AMDGPU_TARGET}" + before_script: + - module load rocmcc/6.2.0-magic + extends: [.sundials_job_on_tioga] + +# --- Builds without HIP + +tioga_cce_1800_mpi_only: + parallel: + matrix: + - COMPILER_SPEC: cce@18.0.0 + variables: + SPEC: "%${COMPILER_SPEC} cstd=99 cxxstd=14 precision=double +mpi" + extends: [.sundials_job_on_tioga] + + +# --- Benchmark jobs + +tioga_rocmcc_620_benchmark: + parallel: + matrix: + - COMPILER_SPEC: rocmcc@6.2.0 + AMDGPU_TARGET: [gfx90a] + CALIPER_DIR: /usr/workspace/sundials/ci/performance/califiles + variables: + ON_TIOGA: "OFF" # disable until we re-baseline on Tioga + SPEC: "%${COMPILER_SPEC} cstd=99 cxxstd=14 build_type=Release precision=double scheduler=flux caliper-dir=${CALIPER_DIR} ~int64 +benchmarks+profiling+caliper+adiak+mpi+openmp+rocm+raja amdgpu_target=${AMDGPU_TARGET} ^raja+rocm~openmp~examples~exercises amdgpu_target=${AMDGPU_TARGET} ^caliper+adiak+rocm amdgpu_target=${AMDGPU_TARGET}" + extends: [.sundials_job_on_tioga] diff --git a/.gitlab/lassen-jobs.yml b/.gitlab/lassen-jobs.yml deleted file mode 100644 index 1ffca71d46..0000000000 --- a/.gitlab/lassen-jobs.yml +++ /dev/null @@ -1,45 +0,0 @@ -# ------------------------------------------------------------------------------ -# SUNDIALS Copyright Start -# Copyright (c) 2002-2021, Lawrence Livermore National Security -# and Southern Methodist University. -# All rights reserved. -# -# See the top-level LICENSE and NOTICE files for details. -# -# SPDX-License-Identifier: BSD-3-Clause -# SUNDIALS Copyright End -# ------------------------------------------------------------------------------ - -# Builds with CUDA, RAJA and other TPLs -lassen_cuda_no_tpls: - parallel: - matrix: - - COMPILER_SPEC: [xl@16.1.1.14] - CUDA_SPEC: [cuda@10.1.243] - variables: - SPEC: "%${COMPILER_SPEC} cstd=99 cxxstd=14 precision=double ~int64 +mpi+openmp+cuda~raja~magma~superlu-dist~petsc~hypre~ginkgo cuda_arch=70 ^${CUDA_SPEC}+allow-unsupported-compilers" - extends: .lassen_build_and_test - -lassen_gcc_cuda_tpls: - parallel: - matrix: - - COMPILER_SPEC: gcc@8.3.1 - CUDA_SPEC: [cuda@11.8.0] - variables: - SPEC: "%${COMPILER_SPEC} cstd=99 cxxstd=14 precision=double ~int64 +mpi+openmp+cuda+raja+magma+superlu-dist+petsc+hypre+ginkgo cuda_arch=70 ^ginkgo+cuda cuda_arch=70 ^hypre~cuda ^petsc~cuda ^superlu-dist+cuda cuda_arch=70 ^magma+cuda cuda_arch=70 ^raja+cuda~openmp~examples~exercises cuda_arch=70 ^${CUDA_SPEC}+allow-unsupported-compilers" - extends: .lassen_build_and_test - -# ------------------------------------------------------------------------------ -# Benchmark jobs -# ------------------------------------------------------------------------------ - -lassen_gcc_cuda_bench: - parallel: - matrix: - - COMPILER_SPEC: gcc@8.3.1 - CUDA_SPEC: [cuda@11.8.0] - CALIPER_DIR: /usr/workspace/sundials/califiles - variables: - SPEC: "%${COMPILER_SPEC} cstd=99 cxxstd=14 build_type=Release precision=double scheduler=lsf caliper-dir=${CALIPER_DIR} ~int64 +benchmarks+profiling+caliper+adiak+mpi+openmp+cuda+raja cuda_arch=70 ^raja+cuda~openmp~examples~exercises cuda_arch=70 ^caliper+adiak+cuda cuda_arch=70 ^${CUDA_SPEC}+allow-unsupported-compilers" - extends: .lassen_build_and_bench - diff --git a/.gitlab/lassen-templates.yml b/.gitlab/lassen-templates.yml deleted file mode 100644 index 450a3134c2..0000000000 --- a/.gitlab/lassen-templates.yml +++ /dev/null @@ -1,56 +0,0 @@ -# ------------------------------------------------------------------------------ -# SUNDIALS Copyright Start -# Copyright (c) 2002-2021, Lawrence Livermore National Security -# and Southern Methodist University. -# All rights reserved. -# -# See the top-level LICENSE and NOTICE files for details. -# -# SPDX-License-Identifier: BSD-3-Clause -# SUNDIALS Copyright End -# ------------------------------------------------------------------------------ - -# ------------------------------------------------------------------------------ -# Tags and rules to run tests on Lassen -# ------------------------------------------------------------------------------ - -# Generic lassen build job, extending build script for IBM P9 systems -.lassen_build_and_test: - tags: - - shell - - lassen - extends: [.build_blueos_3_ppc64le_ib_script] - stage: l_build_and_test - needs: [] - artifacts: - paths: - - spack-*.txt - - build_*/* - when: always - rules: - # Don't run if... - - if: '$CI_COMMIT_BRANCH =~ /_lnone/ || $ON_LASSEN == "OFF" || $BENCHMARK == "ON"' - when: never - # Default is to run if previous stage succeeded - - when: on_success - -.lassen_build_and_bench: - tags: - - shell - - lassen - extends: [.build_blueos_3_ppc64le_ib_bench] - stage: l_build_and_bench - needs: [] - artifacts: - paths: - - spack-*.txt - - build_*/* - - '*.cali' - when: always - expire_in: never - rules: - # Don't run if... - - if: '$CI_COMMIT_BRANCH =~ /_lnone/ || $ON_LASSEN == "OFF" || $BENCHMARK == "OFF"' - when: never - # Default is to run if previous stage succeeded - - when: on_success diff --git a/.gitlab/quartz-jobs.yml b/.gitlab/quartz-jobs.yml deleted file mode 100644 index a0c96755a6..0000000000 --- a/.gitlab/quartz-jobs.yml +++ /dev/null @@ -1,85 +0,0 @@ -# --------------------------------------------------------------- -# SUNDIALS Copyright Start -# Copyright (c) 2002-2021, Lawrence Livermore National Security -# and Southern Methodist University. -# All rights reserved. -# -# See the top-level LICENSE and NOTICE files for details. -# -# SPDX-License-Identifier: BSD-3-Clause -# SUNDIALS Copyright End -# --------------------------------------------------------------- - -# ------------------------------------------------------------------------------ -# Basic builds w/o TPLs -# ------------------------------------------------------------------------------ -### Builds without TPLs -# How to easily add +f2003 when int64 and double? - -# quartz_clang: -# parallel: -# matrix: -# - COMPILER_SPEC: clang@12.0.0 -# INDEX_SPEC: [~int64, +int64] -# PRECISION_SPEC: [double] -# variables: -# SPEC: "%${COMPILER_SPEC} ${INDEX_SPEC} precision=${PRECISION_SPEC}" -# extends: .quartz_build_and_test - -# quartz_gcc: -# parallel: -# matrix: -# - COMPILER_SPEC: gcc@10.3.1 -# INDEX_SPEC: [~int64, +int64] -# PRECISION_SPEC: [double] -# variables: -# SPEC: "%${COMPILER_SPEC} ${INDEX_SPEC} precision=${PRECISION_SPEC}" -# extends: .quartz_build_and_test - -# quartz_intel: -# parallel: -# matrix: -# - COMPILER_SPEC: intel@19.1.2 -# INDEX_SPEC: [~int64, +int64] -# PRECISION_SPEC: [double] -# variables: -# SPEC: "%${COMPILER_SPEC} ${INDEX_SPEC} precision=${PRECISION_SPEC}" -# extends: .quartz_build_and_test - -# ------------------------------------------------------------------------------ -# Builds with TPLs -# ------------------------------------------------------------------------------ -# +petsc ~hypre ~superlu-dist ~int64 ~hdf5 -# +superlu-dist == segfaults - -quartz_clang_tpls: - parallel: - matrix: - - COMPILER_SPEC: clang@14.0.6 - INDEX_SPEC: [~int64] - PRECISION_SPEC: [double] - variables: - SPEC: "%${COMPILER_SPEC} cstd=99 cxxstd=14 ${INDEX_SPEC} precision=${PRECISION_SPEC} +mpi +openmp +hypre +superlu-dist +lapack +klu +petsc ^suite-sparse@5.13.0 ^openblas" - extends: .quartz_build_and_test - -quartz_gcc_tpls: - parallel: - matrix: - - COMPILER_SPEC: gcc@10.3.1 - INDEX_SPEC: [~int64] - PRECISION_SPEC: [double] - variables: - # For some reason nvhpc gets picked up for lapack w/o ^openblas - SPEC: "%${COMPILER_SPEC} cstd=99 cxxstd=14 ${INDEX_SPEC} precision=${PRECISION_SPEC} +mpi +openmp +hypre +superlu-dist +lapack +klu +petsc ^suite-sparse@5.13.0 ^openblas" - extends: .quartz_build_and_test - -quartz_intel_tpls: - parallel: - matrix: - - COMPILER_SPEC: intel@19.1.2 - INDEX_SPEC: [~int64] - PRECISION_SPEC: [double] - variables: - # perl@5.32.0 needed until spack v0.20.0: https://github.com/spack/spack/pull/35666 - SPEC: "%${COMPILER_SPEC} cstd=99 cxxstd=14 ${INDEX_SPEC} precision=${PRECISION_SPEC} +mpi +openmp +hypre ~superlu-dist +lapack +klu ^suite-sparse@5.13.0 ^perl@5.32.0" - extends: .quartz_build_and_test diff --git a/.gitlab/quartz-templates.yml b/.gitlab/quartz-templates.yml deleted file mode 100644 index bfc33a3fd5..0000000000 --- a/.gitlab/quartz-templates.yml +++ /dev/null @@ -1,35 +0,0 @@ -# ------------------------------------------------------------------------------ -# SUNDIALS Copyright Start -# Copyright (c) 2002-2021, Lawrence Livermore National Security -# and Southern Methodist University. -# All rights reserved. -# -# See the top-level LICENSE and NOTICE files for details. -# -# SPDX-License-Identifier: BSD-3-Clause -# SUNDIALS Copyright End -# ------------------------------------------------------------------------------ - -# ------------------------------------------------------------------------------ -# Tags and rules to run tests on Quartz -# ------------------------------------------------------------------------------ - -# Generic quartz build job, extending build script -.quartz_build_and_test: - tags: - - shell - - quartz - extends: [.build_toss_3_x86_64_ib_script] - stage: q_build_and_test - needs: [] - artifacts: - paths: - - spack-*.txt - - build_*/* - when: always - rules: - # Don't run if... - - if: '$CI_COMMIT_BRANCH =~ /_lnone/ || $ON_QUARTZ == "OFF" || $BENCHMARK == "ON"' - when: never - # Default is to run if previous stage succeeded - - when: on_success diff --git a/.gitlab/radiuss-spack-configs b/.gitlab/radiuss-spack-configs index 1264a6db12..89c2074e46 160000 --- a/.gitlab/radiuss-spack-configs +++ b/.gitlab/radiuss-spack-configs @@ -1 +1 @@ -Subproject commit 1264a6db12f451153a5cb22a32a1b7ba2faa5fc2 +Subproject commit 89c2074e466d454c77ec17249cbdf8555d183388 diff --git a/.gitlab/spack_packages/netlib-lapack/ibm-xl-3.9.1.patch b/.gitlab/spack_packages/netlib-lapack/ibm-xl-3.9.1.patch deleted file mode 100644 index c9e1707857..0000000000 --- a/.gitlab/spack_packages/netlib-lapack/ibm-xl-3.9.1.patch +++ /dev/null @@ -1,108 +0,0 @@ -Fixes for IBM XL and Cray CCE builds: - -* Correct path to the fallback configuration used to handle mangling for - C++/Fortran compatibility (CCE, XL) - -* Change logic for detecting recursive fortran flags to (a) Include XL -(qrecur), and (b) Be explicit, since not every compiler will correctly reject -an incorrect option (ALL) - -NOTE: This patch has been accepted upstream -(see https://github.com/Reference-LAPACK/lapack/pull/621) - -############################################################################## - -diff -Naur a/CBLAS/CMakeLists.txt b/CBLAS/CMakeLists.txt ---- a/CBLAS/CMakeLists.txt 2021-03-25 12:25:15.000000000 -0600 -+++ b/CBLAS/CMakeLists.txt 2021-09-01 16:27:23.561355382 -0600 -@@ -11,9 +11,7 @@ - MACRO_NAMESPACE "F77_" - SYMBOL_NAMESPACE "F77_") - if(NOT FortranCInterface_GLOBAL_FOUND OR NOT FortranCInterface_MODULE_FOUND) -- message(WARNING "Reverting to pre-defined include/lapacke_mangling.h") -- configure_file(include/lapacke_mangling_with_flags.h.in -- ${LAPACK_BINARY_DIR}/include/lapacke_mangling.h) -+ message(WARNING "Reverting to pre-defined include/cblas_mangling.h") - configure_file(include/cblas_mangling_with_flags.h.in - ${LAPACK_BINARY_DIR}/include/cblas_mangling.h) - endif() -diff -Naur a/CMakeLists.txt b/CMakeLists.txt ---- a/CMakeLists.txt 2021-03-25 12:25:15.000000000 -0600 -+++ b/CMakeLists.txt 2021-09-02 09:49:18.070436958 -0600 -@@ -94,16 +94,22 @@ - - # Check if recursive flag exists - include(CheckFortranCompilerFlag) --check_fortran_compiler_flag("-recursive" _recursiveFlag) --check_fortran_compiler_flag("-frecursive" _frecursiveFlag) --check_fortran_compiler_flag("-Mrecursive" _MrecursiveFlag) -+if(CMAKE_Fortran_COMPILER_ID STREQUAL Flang) -+ check_fortran_compiler_flag("-Mrecursive" _MrecursiveFlag) -+elseif(CMAKE_Fortran_COMPILER_ID STREQUAL GNU) -+ check_fortran_compiler_flag("-frecursive" _frecursiveFlag) -+elseif(CMAKE_Fortran_COMPILER_ID STREQUAL Intel) -+ check_fortran_compiler_flag("-recursive" _recursiveFlag) -+elseif(CMAKE_Fortran_COMPILER_ID STREQUAL XL) -+ check_fortran_compiler_flag("-qrecur" _qrecurFlag) -+endif() - - # Add recursive flag --if(_recursiveFlag) -- string(REGEX MATCH "-recursive" output_test "${CMAKE_Fortran_FLAGS}") -+if(_MrecursiveFlag) -+ string(REGEX MATCH "-Mrecursive" output_test "${CMAKE_Fortran_FLAGS}") - if(NOT output_test) -- set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -recursive" -- CACHE STRING "Recursive flag must be set" FORCE) -+ set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -Mrecursive" -+ CACHE STRING "Recursive flag must be set" FORCE) - endif() - elseif(_frecursiveFlag) - string(REGEX MATCH "-frecursive" output_test "${CMAKE_Fortran_FLAGS}") -@@ -111,11 +117,17 @@ - set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -frecursive" - CACHE STRING "Recursive flag must be set" FORCE) - endif() --elseif(_MrecursiveFlag) -- string(REGEX MATCH "-Mrecursive" output_test "${CMAKE_Fortran_FLAGS}") -+elseif(_recursiveFlag) -+ string(REGEX MATCH "-recursive" output_test "${CMAKE_Fortran_FLAGS}") - if(NOT output_test) -- set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -Mrecursive" -- CACHE STRING "Recursive flag must be set" FORCE) -+ set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -recursive" -+ CACHE STRING "Recursive flag must be set" FORCE) -+ endif() -+elseif(_qrecurFlag) -+ string(REGEX MATCH "-qrecur" output_test "${CMAKE_Fortran_FLAGS}") -+ if(NOT output_test) -+ set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -qrecur" -+ CACHE STRING "Recursive flag must be set" FORCE) - endif() - endif() - -@@ -124,7 +136,7 @@ - set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -fp-model strict") - endif() - if(CMAKE_Fortran_COMPILER_ID STREQUAL XL) -- set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -qnosave -qstrict=none") -+ set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -qnosave -qstrict") - endif() - # Delete libmtsk in linking sequence for Sun/Oracle Fortran Compiler. - # This library is not present in the Sun package SolarisStudio12.3-linux-x86-bin -diff -Naur a/INSTALL/make.inc.XLF b/INSTALL/make.inc.XLF ---- a/INSTALL/make.inc.XLF 2021-03-25 12:25:15.000000000 -0600 -+++ b/INSTALL/make.inc.XLF 2021-09-02 09:50:02.664646455 -0600 -@@ -14,10 +14,10 @@ - # the compiler options desired when NO OPTIMIZATION is selected. - # - FC = xlf --FFLAGS = -O3 -qfixed -qnosave -+FFLAGS = -O3 -qfixed -qnosave -qrecur - # For -O2, add -qstrict=none - FFLAGS_DRV = $(FFLAGS) --FFLAGS_NOOPT = -O0 -qfixed -qnosave -+FFLAGS_NOOPT = -O0 -qfixed -qnosave -qrecur - - # Define LDFLAGS to the desired linker options for your machine. - # diff --git a/.gitlab/spack_packages/netlib-lapack/ibm-xl.patch b/.gitlab/spack_packages/netlib-lapack/ibm-xl.patch deleted file mode 100644 index 52b5f19719..0000000000 --- a/.gitlab/spack_packages/netlib-lapack/ibm-xl.patch +++ /dev/null @@ -1,53 +0,0 @@ -Fixes for IBM XL and Cray CCE builds: - -* Avoid optimizations that would alter program semantics by changing the - qstrict activation threshold from O3 to O2 (XL) - -* Don't assume Fortran code is all in fixed source form; disable qfixed (XL) - -* Correct path to the fallback configuration used to handle mangling for - C++/Fortran compatibility (CCE, XL) -############################################################################## - ---- a/CMakeLists.txt -+++ b/CMakeLists.txt -@@ -62,7 +62,7 @@ - set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -fp-model strict") - endif() - if("${CMAKE_Fortran_COMPILER}" MATCHES "xlf") -- set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -qnosave -qstrict=none") -+ set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -qnosave -qstrict") - endif() - # Delete libmtsk in linking sequence for Sun/Oracle Fortran Compiler. - # This library is not present in the Sun package SolarisStudio12.3-linux-x86-bin - ---- a/CMAKE/CheckLAPACKCompilerFlags.cmake -+++ b/CMAKE/CheckLAPACKCompilerFlags.cmake -@@ -43,12 +43,6 @@ - if( "${CMAKE_Fortran_FLAGS}" MATCHES "-qflttrap=[a-zA-Z:]:enable" ) - set( FPE_EXIT TRUE ) - endif() -- -- if( NOT ("${CMAKE_Fortran_FLAGS}" MATCHES "-qfixed") ) -- message( STATUS "Enabling fixed format F90/F95 with -qfixed" ) -- set( CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -qfixed" -- CACHE STRING "Flags for Fortran compiler." FORCE ) -- endif() - - # HP Fortran - elseif( CMAKE_Fortran_COMPILER_ID STREQUAL "HP" ) - ---- a/CBLAS/CMakeLists.txt -+++ b/CBLAS/CMakeLists.txt -@@ -12,8 +12,8 @@ - SYMBOL_NAMESPACE "F77_") - if(NOT FortranCInterface_GLOBAL_FOUND OR NOT FortranCInterface_MODULE_FOUND) - message(WARNING "Reverting to pre-defined include/lapacke_mangling.h") -- configure_file(include/lapacke_mangling_with_flags.h.in -- ${LAPACK_BINARY_DIR}/include/lapacke_mangling.h) -+ configure_file(include/cblas_mangling_with_flags.h.in -+ ${LAPACK_BINARY_DIR}/include/cblas_mangling.h) - endif() - - include_directories(include ${LAPACK_BINARY_DIR}/include) - diff --git a/.gitlab/spack_packages/netlib-lapack/package.py b/.gitlab/spack_packages/netlib-lapack/package.py deleted file mode 100644 index cb1832d061..0000000000 --- a/.gitlab/spack_packages/netlib-lapack/package.py +++ /dev/null @@ -1,223 +0,0 @@ -# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other -# Spack Project Developers. See the top-level COPYRIGHT file for details. -# -# SPDX-License-Identifier: (Apache-2.0 OR MIT) -import spack.build_systems.cmake -from spack.package import * - - -class NetlibLapack(CMakePackage): - """LAPACK version 3.X is a comprehensive FORTRAN library that does - linear algebra operations including matrix inversions, least squared - solutions to linear sets of equations, eigenvector analysis, singular - value decomposition, etc. It is a very comprehensive and reputable - package that has found extensive use in the scientific community. - - """ - - homepage = "https://www.netlib.org/lapack/" - url = "https://www.netlib.org/lapack/lapack-3.5.0.tgz" - tags = ["windows"] - - version( - "3.10.1", - sha256="cd005cd021f144d7d5f7f33c943942db9f03a28d110d6a3b80d718a295f7f714", - url="https://github.com/Reference-LAPACK/lapack/archive/refs/tags/v3.10.1.tar.gz", - ) - version( - "3.10.0", - sha256="328c1bea493a32cac5257d84157dc686cc3ab0b004e2bea22044e0a59f6f8a19", - url="https://github.com/Reference-LAPACK/lapack/archive/refs/tags/v3.10.0.tar.gz", - ) - version( - "3.9.1", - sha256="d0085d2caf997ff39299c05d4bacb6f3d27001d25a4cc613d48c1f352b73e7e0", - url="https://github.com/Reference-LAPACK/lapack/archive/refs/tags/v3.9.1.tar.gz", - ) - version( - "3.9.0", - sha256="106087f1bb5f46afdfba7f569d0cbe23dacb9a07cd24733765a0e89dbe1ad573", - url="https://github.com/Reference-LAPACK/lapack/archive/v3.9.0.tar.gz", - ) - version( - "3.8.0", - sha256="deb22cc4a6120bff72621155a9917f485f96ef8319ac074a7afbc68aab88bcf6", - url="https://www.netlib.org/lapack/lapack-3.8.0.tar.gz", - ) - version("3.7.1", sha256="f6c53fd9f56932f3ddb3d5e24c1c07e4cd9b3b08e7f89de9c867125eecc9a1c8") - version("3.7.0", sha256="ed967e4307e986474ab02eb810eed1d1adc73f5e1e3bc78fb009f6fe766db3be") - version("3.6.1", sha256="888a50d787a9d828074db581c80b2d22bdb91435a673b1bf6cd6eb51aa50d1de") - version("3.6.0", sha256="a9a0082c918fe14e377bbd570057616768dca76cbdc713457d8199aaa233ffc3") - version("3.5.0", sha256="9ad8f0d3f3fb5521db49f2dd716463b8fb2b6bc9dc386a9956b8c6144f726352") - version("3.4.2", sha256="60a65daaf16ec315034675942618a2230521ea7adf85eea788ee54841072faf0") - version("3.4.1", sha256="93b910f94f6091a2e71b59809c4db4a14655db527cfc5821ade2e8c8ab75380f") - version("3.4.0", sha256="a7139ef97004d0e3c4c30f1c52d508fd7ae84b5fbaf0dd8e792c167dc306c3e9") - version("3.3.1", sha256="56821ab51c29369a34e5085728f92c549a9aa926f26acf7eeac87b61eed329e4") - - # netlib-lapack is the reference implementation of LAPACK - for ver in [ - "3.10.1", - "3.10.0", - "3.9.1", - "3.9.0", - "3.8.0", - "3.7.1", - "3.7.0", - "3.6.1", - "3.6.0", - "3.5.0", - "3.4.2", - "3.4.1", - "3.4.0", - "3.3.1", - ]: - provides("lapack@" + ver, when="@" + ver) - - variant("shared", default=True, description="Build shared library version") - variant("external-blas", default=False, description="Build lapack with an external blas") - - variant("lapacke", default=True, description="Activates the build of the LAPACKE C interface") - variant("xblas", default=False, description="Builds extended precision routines using XBLAS") - - # Fixes for IBM XL and Cray CCE builds: - # Avoid optimizations that alter program semantics - # Don't assume fixed source form for Fortran - # Correct path to mangling config - patch("ibm-xl.patch", when="@3.7:3.8 %xl") - patch("ibm-xl.patch", when="@3.7:3.8 %xl_r") - patch("ibm-xl.patch", when="@3.7:3.8 %cce@9:") - - # https://github.com/Reference-LAPACK/lapack/pull/621 - # Fixes for IBM XL and Cray CCE builds: - # Correct path to mangling config - # Fix logic for detecting recursive Fortran flags - patch("ibm-xl-3.9.1.patch", when="@3.9.1 %xl") - patch("ibm-xl-3.9.1.patch", when="@3.9.1 %xl_r") - patch("ibm-xl-3.9.1.patch", when="@3.9.1 %cce@13:") - - # https://github.com/Reference-LAPACK/lapack/issues/228 - patch("undefined_declarations.patch", when="@3.8.0:3.8") - - # https://github.com/Reference-LAPACK/lapack/pull/268 - patch("testing.patch", when="@3.7.0:3.8") - - # virtual dependency - provides("blas", when="~external-blas") - provides("lapack") - - depends_on("blas", when="+external-blas") - depends_on("netlib-xblas+fortran+plain_blas", when="+xblas") - depends_on("python@2.7:", type="test") - - # We need to run every phase twice in order to get static and shared - # versions of the libraries. When ~shared, we run the default - # implementations of the CMakePackage's phases and get only one building - # directory 'spack-build-static' with -DBUILD_SHARED_LIBS:BOOL=OFF (see - # implementations of self.build_directory and self.cmake_args() below). - # When +shared, we run the overridden methods for the phases, each - # running the default implementation twice with different values for - # self._building_shared. As a result, we get two building directories: - # 'spack-build-static' with -DBUILD_SHARED_LIBS:BOOL=OFF and - # 'spack-build-shared' with -DBUILD_SHARED_LIBS:BOOL=ON. - _building_shared = False - - def patch(self): - # Fix cblas CMakeLists.txt -- has wrong case for subdirectory name. - if self.spec.satisfies("@3.6.0:"): - filter_file( - "${CMAKE_CURRENT_SOURCE_DIR}/CMAKE/", - "${CMAKE_CURRENT_SOURCE_DIR}/cmake/", - "CBLAS/CMakeLists.txt", - string=True, - ) - - # Remove duplicate header file that gets generated during CMake shared - # builds: https://github.com/Reference-LAPACK/lapack/issues/583 - if self.spec.satisfies("platform=windows @0:3.9.1"): - force_remove("LAPACKE/include/lapacke_mangling.h") - - @property - def blas_libs(self): - shared = True if "+shared" in self.spec else False - query_parameters = self.spec.last_query.extra_parameters - query2libraries = { - tuple(): ["libblas"], - ("c", "fortran"): ["libcblas", "libblas"], - ("c",): ["libcblas"], - ("fortran",): ["libblas"], - } - key = tuple(sorted(query_parameters)) - libraries = query2libraries[key] - return find_libraries(libraries, root=self.prefix, shared=shared, recursive=True) - - @property - def lapack_libs(self): - shared = True if "+shared" in self.spec else False - query_parameters = self.spec.last_query.extra_parameters - query2libraries = { - tuple(): ["liblapack"], - ("c", "fortran"): ["liblapacke", "liblapack"], - ("c",): ["liblapacke"], - ("fortran",): ["liblapack"], - } - key = tuple(sorted(query_parameters)) - libraries = query2libraries[key] - return find_libraries(libraries, root=self.prefix, shared=shared, recursive=True) - - @property - def headers(self): - include_dir = self.spec.prefix.include - cblas_h = join_path(include_dir, "cblas.h") - lapacke_h = join_path(include_dir, "lapacke.h") - return HeaderList([cblas_h, lapacke_h]) - - -class CMakeBuilder(spack.build_systems.cmake.CMakeBuilder): - def cmake_args(self): - args = [ - self.define_from_variant("BUILD_SHARED_LIBS", "shared"), - self.define_from_variant("LAPACKE", "lapacke"), - self.define_from_variant("LAPACKE_WITH_TMG", "lapacke"), - self.define("CBLAS", self.spec.satisfies("@3.6.0:")), - ] - - if self.spec.satisfies("%intel"): - # Intel compiler finds serious syntax issues when trying to - # build CBLAS and LapackE - args.extend([self.define("CBLAS", False), self.define("LAPACKE", False)]) - - if self.spec.satisfies("%xl") or self.spec.satisfies("%xl_r"): - # use F77 compiler if IBM XL - args.extend( - [ - self.define("CMAKE_Fortran_COMPILER", self.pkg.compiler.f77), - self.define( - "CMAKE_Fortran_FLAGS", - " ".join(self.spec.compiler_flags["fflags"]) + " -O3 -qnohot", - ), - ] - ) - - # deprecated routines are commonly needed by, for example, suitesparse - # Note that OpenBLAS spack is built with deprecated routines - args.append(self.define("BUILD_DEPRECATED", True)) - - if self.spec.satisfies("+external-blas"): - args.extend( - [ - self.define("USE_OPTIMIZED_BLAS", True), - self.define("BLAS_LIBRARIES:PATH", self.spec["blas"].libs.joined(";")), - ] - ) - - if self.spec.satisfies("+xblas"): - args.extend( - [ - self.define("XBLAS_INCLUDE_DIR", self.spec["netlib-xblas"].prefix.include), - self.define("XBLAS_LIBRARY", self.spec["netlib-xblas"].libs.joined(";")), - ] - ) - - args.append(self.define("BUILD_TESTING", self.pkg.run_tests)) - - return args diff --git a/.gitlab/spack_packages/netlib-lapack/testing.patch b/.gitlab/spack_packages/netlib-lapack/testing.patch deleted file mode 100644 index fce18548c4..0000000000 --- a/.gitlab/spack_packages/netlib-lapack/testing.patch +++ /dev/null @@ -1,13 +0,0 @@ -diff --git a/TESTING/LIN/alahd.f b/TESTING/LIN/alahd.f -index 8f4cd58d..6a4946e0 100644 ---- a/TESTING/LIN/alahd.f -+++ b/TESTING/LIN/alahd.f -@@ -1036,7 +1036,7 @@ - 9929 FORMAT( ' Test ratios (1-3: ', A1, 'TZRZF):' ) - 9920 FORMAT( 3X, ' 7-10: same as 3-6', 3X, ' 11-14: same as 3-6' ) - 9921 FORMAT( ' Test ratios:', / ' (1-2: ', A1, 'GELS, 3-6: ', A1, -- $ 'GELSY, 7-10: ', A1, 'GELSS, 11-14: ', A1, 'GELSD, 15-16: ' -+ $ 'GELSY, 7-10: ', A1, 'GELSS, 11-14: ', A1, 'GELSD, 15-16: ', - $ A1, 'GETSLS)') - 9928 FORMAT( 7X, 'where ALPHA = ( 1 + SQRT( 17 ) ) / 8' ) - 9927 FORMAT( 3X, I2, ': ABS( Largest element in L )', / 12X, diff --git a/.gitlab/spack_packages/netlib-lapack/undefined_declarations.patch b/.gitlab/spack_packages/netlib-lapack/undefined_declarations.patch deleted file mode 100644 index 9dac2562f7..0000000000 --- a/.gitlab/spack_packages/netlib-lapack/undefined_declarations.patch +++ /dev/null @@ -1,26 +0,0 @@ -diff --git a/SRC/dsytrf_aa_2stage.f b/SRC/dsytrf_aa_2stage.f -index 2991305..f5f06cc 100644 ---- a/SRC/dsytrf_aa_2stage.f -+++ b/SRC/dsytrf_aa_2stage.f -@@ -191,7 +191,7 @@ - EXTERNAL LSAME, ILAENV - * .. - * .. External Subroutines .. -- EXTERNAL XERBLA, DCOPY, DLACGV, DLACPY, -+ EXTERNAL XERBLA, DCOPY, DLACPY, - $ DLASET, DGBTRF, DGEMM, DGETRF, - $ DSYGST, DSWAP, DTRSM - * .. -diff --git a/SRC/ssytrf_aa_2stage.f b/SRC/ssytrf_aa_2stage.f -index be6809d..a929749 100644 ---- a/SRC/ssytrf_aa_2stage.f -+++ b/SRC/ssytrf_aa_2stage.f -@@ -191,7 +191,7 @@ - EXTERNAL LSAME, ILAENV - * .. - * .. External Subroutines .. -- EXTERNAL XERBLA, SCOPY, SLACGV, SLACPY, -+ EXTERNAL XERBLA, SCOPY, SLACPY, - $ SLASET, SGBTRF, SGEMM, SGETRF, - $ SSYGST, SSWAP, STRSM - * .. diff --git a/.gitlab/subscribed-pipelines.yml b/.gitlab/subscribed-pipelines.yml new file mode 100644 index 0000000000..879dcc7be6 --- /dev/null +++ b/.gitlab/subscribed-pipelines.yml @@ -0,0 +1,77 @@ +############################################################################### +# Copyright (c) 2022-23, Lawrence Livermore National Security, LLC and RADIUSS +# project contributors. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# SPDX-License-Identifier: (MIT) +############################################################################### +# Modifications to this file have been made under the folllowing copyright. +# ----------------------------------------------------------------------------- +# SUNDIALS Copyright Start +# Copyright (c) 2002-2024, Lawrence Livermore National Security +# and Southern Methodist University. +# All rights reserved. +# +# See the top-level LICENSE and NOTICE files for details. +# +# SPDX-License-Identifier: BSD-3-Clause +# SUNDIALS Copyright End +# ----------------------------------------------------------------------------- + +# The template job to test whether a machine is up. +# Expects CI_MACHINE defined to machine name. +.machine-check: + stage: prerequisites + tags: [shell, oslic] + variables: + GIT_STRATEGY: none + script: + - | + if [[ $(jq '.[env.CI_MACHINE].total_nodes_up' /usr/global/tools/lorenz/data/loginnodeStatus) == 0 ]] + then + echo -e "\e[31mNo node available on ${CI_MACHINE}\e[0m" + curl --url "https://api.github.com/repos/${GITHUB_PROJECT_ORG}/${GITHUB_PROJECT_NAME}/statuses/${CI_COMMIT_SHA}" \ + --header 'Content-Type: application/json' \ + --header "authorization: Bearer ${GITHUB_TOKEN}" \ + --data "{ \"state\": \"failure\", \"target_url\": \"${CI_PIPELINE_URL}\", \"description\": \"GitLab ${CI_MACHINE} down\", \"context\": \"ci/gitlab/${CI_MACHINE}\" }" + exit 1 + fi + +### +# Trigger a build-and-test pipeline for a machine. +# Comment the jobs for machines you don’t need. +### + +# DANE +dane-up-check: + variables: + CI_MACHINE: "dane" + extends: [.machine-check] + +dane-build-and-test: + variables: + CI_MACHINE: "dane" + needs: [dane-up-check] + extends: [.build-and-test] + +# TIOGA +tioga-up-check: + variables: + CI_MACHINE: "tioga" + extends: [.machine-check] + +tioga-build-and-test: + variables: + CI_MACHINE: "tioga" + needs: [tioga-up-check] + extends: [.build-and-test] + diff --git a/.gitlab/uberenv b/.gitlab/uberenv index 0d00dc8e19..ec6fd144c8 160000 --- a/.gitlab/uberenv +++ b/.gitlab/uberenv @@ -1 +1 @@ -Subproject commit 0d00dc8e19a889ba07ae433590b87533c4b5b3da +Subproject commit ec6fd144c814ce30a6487db9fb8294def606afda diff --git a/.gitmodules b/.gitmodules index 659855b75f..e915f32aa6 100644 --- a/.gitmodules +++ b/.gitmodules @@ -3,7 +3,7 @@ url = https://github.com/sundials-codes/radiuss-spack-configs.git [submodule "uberenv"] path = .gitlab/uberenv - url = https://github.com/LLNL/uberenv.git + url = https://github.com/sundials-codes/uberenv.git [submodule "test/answers"] path = test/answers url = https://github.com/sundials-codes/answers.git diff --git a/.uberenv_config.json b/.uberenv_config.json index 67618cca2f..3264fe5b05 100644 --- a/.uberenv_config.json +++ b/.uberenv_config.json @@ -4,8 +4,7 @@ "package_final_phase": "initconfig", "package_source_dir": "../..", "spack_url": "https://github.com/spack/spack", - "spack_commit": "5e0d2107348eed6cbe6deca43a30f5b06c5e40af", "spack_activate": {}, "spack_configs_path": ".gitlab/radiuss-spack-configs", - "spack_packages_path": ".gitlab/spack_packages" -} + "spack_packages_path": "scripts/spack/packages" +} \ No newline at end of file diff --git a/cmake/macros/SundialsAddTest.cmake b/cmake/macros/SundialsAddTest.cmake index a1b491dc49..4b4733820e 100644 --- a/cmake/macros/SundialsAddTest.cmake +++ b/cmake/macros/SundialsAddTest.cmake @@ -185,13 +185,25 @@ macro(SUNDIALS_ADD_TEST NAME EXECUTABLE) if((SUNDIALS_ADD_TEST_MPI_NPROCS) AND ((MPIEXEC_EXECUTABLE) OR (SUNDIALS_TEST_MPIRUN_COMMAND))) if(SUNDIALS_TEST_MPIRUN_COMMAND) - set(RUN_COMMAND - "${SUNDIALS_TEST_MPIRUN_COMMAND} ${MPIEXEC_NUMPROC_FLAG} ${SUNDIALS_ADD_TEST_MPI_NPROCS} ${MPIEXEC_PREFLAGS}" - ) + if(MPIEXEC_PREFLAGS) + set(RUN_COMMAND + "${SUNDIALS_TEST_MPIRUN_COMMAND} ${MPIEXEC_PREFLAGS} ${MPIEXEC_NUMPROC_FLAG} ${SUNDIALS_ADD_TEST_MPI_NPROCS}" + ) + else() + set(RUN_COMMAND + "${SUNDIALS_TEST_MPIRUN_COMMAND} ${MPIEXEC_NUMPROC_FLAG} ${SUNDIALS_ADD_TEST_MPI_NPROCS}" + ) + endif() elseif(MPIEXEC_EXECUTABLE) - set(RUN_COMMAND - "${MPIEXEC_EXECUTABLE} ${MPIEXEC_NUMPROC_FLAG} ${SUNDIALS_ADD_TEST_MPI_NPROCS} ${MPIEXEC_PREFLAGS}" - ) + if(MPIEXEC_PREFLAGS) + set(RUN_COMMAND + "${MPIEXEC_EXECUTABLE} ${MPIEXEC_PREFLAGS} ${MPIEXEC_NUMPROC_FLAG} ${SUNDIALS_ADD_TEST_MPI_NPROCS}" + ) + else() + set(RUN_COMMAND + "${MPIEXEC_EXECUTABLE} ${MPIEXEC_NUMPROC_FLAG} ${SUNDIALS_ADD_TEST_MPI_NPROCS}" + ) + endif() endif() # remove trailing white space (empty MPIEXEC_PREFLAGS) as it can cause @@ -211,7 +223,12 @@ macro(SUNDIALS_ADD_TEST NAME EXECUTABLE) set(_run_args "${_run_args} ${_extra_args}") unset(_extra_args) endif() - if(_have_test_args OR _have_extra_test_args) + if(MPIEXEC_POSTFLAGS) + set(_run_args "${MPIEXEC_POSTFLAGS} ${_run_args}") + endif() + if(_have_test_args + OR _have_extra_test_args + OR MPIEXEC_POSTFLAGS) string(STRIP "${_run_args}" _run_args) list(APPEND TEST_ARGS "--runargs=\"${_run_args}\"") unset(_run_args) @@ -239,22 +256,25 @@ macro(SUNDIALS_ADD_TEST NAME EXECUTABLE) if(MPIEXEC_PREFLAGS) string(REPLACE " " ";" PREFLAGS "${MPIEXEC_PREFLAGS}") endif() + if(MPIEXEC_POSTFLAGS) + string(REPLACE " " ";" POSTLAGS "${MPIEXEC_POSTFLAGS}") + endif() if(SUNDIALS_TEST_MPIRUN_COMMAND) string(REPLACE " " ";" MPI_EXEC_ARGS "${SUNDIALS_TEST_MPIRUN_COMMAND}") add_test( NAME ${NAME} COMMAND - ${MPI_EXEC_ARGS} ${MPIEXEC_NUMPROC_FLAG} - ${SUNDIALS_ADD_TEST_MPI_NPROCS} ${PREFLAGS} - $ ${TEST_ARGS}) + ${MPI_EXEC_ARGS} ${PREFLAGS} ${MPIEXEC_NUMPROC_FLAG} + ${SUNDIALS_ADD_TEST_MPI_NPROCS} $ + ${POSTFLAGS} ${TEST_ARGS}) else() add_test( NAME ${NAME} COMMAND - ${MPIEXEC_EXECUTABLE} ${MPIEXEC_NUMPROC_FLAG} - ${SUNDIALS_ADD_TEST_MPI_NPROCS} ${PREFLAGS} - $ ${TEST_ARGS}) + ${MPIEXEC_EXECUTABLE} ${PREFLAGS} ${MPIEXEC_NUMPROC_FLAG} + ${SUNDIALS_ADD_TEST_MPI_NPROCS} ${POSTFLAGS} + $ ${POSTFLAGS} ${TEST_ARGS}) endif() else() add_test(NAME ${NAME} COMMAND $ ${TEST_ARGS}) diff --git a/doc/cvode/guide/source/Usage/index.rst b/doc/cvode/guide/source/Usage/index.rst index 3c56d48587..b2e793d5e3 100644 --- a/doc/cvode/guide/source/Usage/index.rst +++ b/doc/cvode/guide/source/Usage/index.rst @@ -208,15 +208,15 @@ function to be called or macro to be referenced. If a nonlinear solver requiring a linear solver is chosen (e.g., the default Newton iteration), then initialize the CVLS linear solver interface by attaching the linear solver object (and matrix object, - if applicable) with a call ``ier = CVodeSetLinearSolver(cvode_mem, NLS)`` (for details see - :numref:`CVODE.Usage.CC.callable_fct_sim.lin_solv_init`): + if applicable) with a call ``ier = CVodeSetLinearSolver(cvode_mem, LS, A)`` + (for details see :numref:`CVODE.Usage.CC.callable_fct_sim.lin_solv_init`). Alternately, if the CVODE-specific diagonal linear solver module, CVDIAG, is desired, initialize the linear solver module and attach it to CVODE with the call to :c:func:`CVodeSetLinearSolver`. #. **Set optional inputs** - Call ```CVodeSet***`` functions to change any optional inputs that control the + Call ``CVodeSet***`` functions to change any optional inputs that control the behavior of CVODE from their default values. See :numref:`CVODE.Usage.CC.optional_input` for details. diff --git a/doc/shared/sundials/Install.rst b/doc/shared/sundials/Install.rst index 2a5068e5c9..0b778bb07c 100644 --- a/doc/shared/sundials/Install.rst +++ b/doc/shared/sundials/Install.rst @@ -868,6 +868,23 @@ illustration only. .. note:: This option is triggered only if MPI is enabled (``ENABLE_MPI`` is ``ON``). +.. cmakeoption:: MPIEXEC_PREFLAGS + + Specifies flags that come directly after ``MPIEXEC_EXECUTABLE`` and before + ``MPIEXEC_NUMPROC_FLAG`` and ``MPIEXEC_MAX_NUMPROCS``. + + Default: none + + .. note:: This option is triggered only if MPI is enabled (``ENABLE_MPI`` is ``ON``). + +.. cmakeoption:: MPIEXEC_POSTFLAGS + + Specifies flags that come after the executable to run but before any other program arguments. + + Default: none + + .. note:: This option is triggered only if MPI is enabled (``ENABLE_MPI`` is ``ON``). + .. cmakeoption:: ENABLE_ONEMKL Enable oneMKL support. diff --git a/doc/superbuild/source/developers/style_guide/Documentation.rst b/doc/superbuild/source/developers/style_guide/Documentation.rst index 92b6d02e86..c21f600714 100644 --- a/doc/superbuild/source/developers/style_guide/Documentation.rst +++ b/doc/superbuild/source/developers/style_guide/Documentation.rst @@ -126,6 +126,18 @@ References All citations go into `doc/shared/sundials.bib`. TODO: add citation and reference key style. +Links +===== + +Links to websites should typically use the anonymous link syntax. + +.. code-block:: rst + + `SUNDIALS documentation `__ + +Pay special attention to the two trailing underscores - two underscores +indicates an anonymous link. + User-Callable Functions ======================= diff --git a/doc/superbuild/source/developers/testing/CI.rst b/doc/superbuild/source/developers/testing/CI.rst index a71b4e5930..e891ed3e74 100644 --- a/doc/superbuild/source/developers/testing/CI.rst +++ b/doc/superbuild/source/developers/testing/CI.rst @@ -13,7 +13,7 @@ .. _CI: -GitHub CI Testing +GitHub CI Testing ================= There are two categories of CI testing that we run on GitHub via `GitHub actions `_: @@ -111,6 +111,11 @@ For information specific to the LLNL GitLab CI see: * `LLNL GitLab Runner Tags `_ +SUNDIALS utilizes the GitLab CI pipeline code repository shared by LLNL RADIUSS +projects. The `docs `__ for +the shared project should be reviewed before reading the docs below. + + CI Pipelines and Jobs --------------------- @@ -123,124 +128,120 @@ the jobs in a stage succeed, the pipeline moves on to the next stage. If any job in a stage fails, the next stage is usually (see below) not executed and the pipeline ends early. +Some pipelines are run automatically on new commits (after they are mirrored +from GitHub to LC GitLab). Other pipelines, such as the benchmarking pipeline, +are run on a schedule that is configured through the +`GitLab UI `__. + +Structure +^^^^^^^^^ + +As previously stated, most of our code for the LC GitLab CI pipelines is sourced from +the templates provided in the +`radiuss-shared-ci `__ repo. +Here we briefly outline the relevant files: + +The ``.gitlab-ci.yml`` file in the root of the repository is the starting point for +defining the SUNDIALS GitLab CI pipelines. The only thing that is typically changed +in this file is the ``SPACK_REF`` variable in the ``variables`` section (this +is done when we update the version of Spack we use for installing dependencies). +Currently, we also override the `.build-and-test` job defined in this file so +that we can pull in some files from our fork of `radiuss-shared-ci` +(maintained `here `__) +instead of the upstream repository. + +The ``.gitlab/subscribed-pipelines.yml`` defines which machines we will test on. +This file may be modified if you need to add a new machine to test on. + +The ``.gitlab/custom-jobs-and-variables.yml`` defines variables available in all +pipelines and jobs. This file may be modified if you need to add a new variable +that needs to be accessible to all pipelines and jobs. + +The ``.gitlab/jobs/.yaml`` files define the jobs for a specific machine. +A "hidden" job of the form `.sundials_job_on_` is defined first in these +files and typically defines variables specific to that machine, such as what command +to use for executing MPI programs. The rest of the jobs in the file extend the +`.sundials_job_on_` and define the Spack spec that we will build and test. +Take for example, this Tioga job: -CI Pipeline -^^^^^^^^^^^ - -The YAML file ``.gitlab-ci.yml`` at the top-level of the repository defines the -GitLab CI pipeline. The SUNDIALS CI configuration file is organized as follows: - -* The ``variables:`` keyword defines environment variables shared by all stages - and jobs. Variables can be used to alter the CI pipeline behavior when - launching a job from the GitLab "Run pipeline" UI or from the GitLab pipeline - scheduling UI. See the ``.gitlab-ci.yml`` file for details about all of the - available variables and what they do. - - .. code-block:: YAML +.. code-block:: YAML + tioga_rocmcc_620_tpls: + parallel: + matrix: + - COMPILER_SPEC: rocmcc@6.2.0 + AMDGPU_TARGET: [gfx90a] variables: - GIT_SUBMODULE_STRATEGY: recursive - ALLOC_NAME: ${CI_PROJECT_NAME}_ci_${CI_PIPELINE_ID} - BUILD_ROOT: ${CI_PROJECT_DIR} - # ... - -* The ``stages:`` keyword defines independent CI stages targeting a specific - test machine following the prefix naming convention: + # have to use ginkgo@master because our spack version does not have ginkgo@1.8.0: yet (which seems to be needed) + # similarly, we need a newer magma than available to compile with 'rocm@6:' so we turn it off + SPEC: "%${COMPILER_SPEC} cstd=99 cxxstd=14 precision=double amdgpu_target=${AMDGPU_TARGET} +rocm+mpi~magma+ginkgo+kokkos ^ginkgo@master+rocm amdgpu_target=${AMDGPU_TARGET} ^kokkos+rocm amdgpu_target=${AMDGPU_TARGET}" + before_script: + - module load rocmcc/6.2.0-magic + extends: [.sundials_job_on_tioga] - * ``q_`` jobs run on Quartz - * ``l_`` jobs run on Lassen +The ``parallel:`` and ``matrix:`` keywords could be used to enable creating multiple jobs +with different variable values for each instance of the job, e.g., one job using +``rocmcc@6.2.0`` and another using ``rocmcc@6.2.1``. However, right now they only create +a single job (hence why ``COMPILER_SPEC`` and ``AMDGPU_TARGET`` only have one value). These +variables values are then used to create an environment variable ``SPEC`` which is the Spack spec +used by ``build_and_test.sh`` (discussed below) to configure and build SUNDIALS and the +necessary dependencies. - .. code-block:: YAML +Disabling a Job +^^^^^^^^^^^^^^^ - stages: - - q_build_and_test - - l_build_and_test - # ... +A job can be disabled by adding the variable ``.ON_: "OFF"`` to the ``variables:`` +section of the job, e.g., -* Several hidden job templates (job names start with ``.``) are defined for - specific architectures and batch queue systems. These jobs provide the batch - system command to run the ``build_and_test.sh`` script that configures, - builds, and tests SUNDIALS. - .. code-block:: YAML - - .build_toss_3_x86_64_ib_script: - script: - - echo ${ALLOC_NAME} - - srun -p pdebug -N 1 -n ${NCPUS} --interactive -t ${DEFAULT_TIME} - --job-name=${ALLOC_NAME} .gitlab/build_and_test.sh - - # ... - -* The ``include:`` keyword loads YAML files defining the jobs for specific - machines. - - .. code-block:: YAML - - include: - - local: .gitlab/quartz-jobs.yml - - local: .gitlab/lassen-jobs.yml - # ... - - -CI Jobs -^^^^^^^ +.. code-block:: YAML -As noted above, each stage in the CI pipeline corresponds to testing on a -specific machine. For example, jobs run on Lassen are associated with the -``l_build_and_test`` stage. The actual jobs to run are defined in the YAML -file ``.gitlab/lassen-jobs.yml``. + tioga_rocmcc_620_tpls: + parallel: + matrix: + - COMPILER_SPEC: rocmcc@6.2.0 + AMDGPU_TARGET: [gfx90a] + variables: + ON_TIOGA: "OFF" # disable this job + SPEC: "%${COMPILER_SPEC} cstd=99 cxxstd=14 precision=double amdgpu_target=${AMDGPU_TARGET} +rocm+mpi~magma+ginkgo+kokkos ^ginkgo@master+rocm amdgpu_target=${AMDGPU_TARGET} ^kokkos+rocm amdgpu_target=${AMDGPU_TARGET}" + before_script: + - module load rocmcc/6.2.0-magic + extends: [.sundials_job_on_tioga] -The Lassen build and test jobs inherit from three job templates: +These variables can also be set when manually or scheduling a pipeline in the GitLab UI. -* ``.build_blueos_3_ppc64le_ib_script`` executes the LSF command to run the - testing script. +Updating Spack +^^^^^^^^^^^^^^ -* ``.on_lassen`` defines the tags (``tags:`` keyword) to select a shell runner - on Lassen and the rules (``rules:`` keyword) for when a job should run. +To update the spack commit used for the CI simply replace the commit hash in the +``SPACK_REF`` variable inside the ``.gitlab-ci.yml`` file with the new commit hash. +The first time a pipeline runs with a new ``SPACK_REF`` the pipeline will take longer than +normal as a new Spack build cache must be created and populated (so all packages will be +built from source). -* ``.lassen_build_and_test`` inherits from the prior two job templates using the - ``extends:`` keyword and acts as the base jobs that all other Lassen jobs - inherit from. The base template includes: - * The ``stage:`` keyword defines which stage the jobs run in. +Benchmark Jobs +^^^^^^^^^^^^^^ - * The ``needs:`` keyword lists the job dependencies. Normally, GitLab stages - are blocking however, by providing the dependencies we can break the - ordering of stages, in favor of using a DAG. This allows jobs to be run - out-of-order rather than waiting on the jobs in other stages to complete. +See :ref:`SUNDIALS Continuous Performance Testing (CPT)` for more details. - * The ``artifacts:`` keyword defines ``files:`` and directories (``paths:``) - created by the job that should be retained and ``when:`` they should be - attached to the job. -The Lassen tests are defined by jobs that extend the ``.lassen_build_and_test`` -template and use the naming convention ``lassen__``. -For example, tests using GCC, CUDA, and third-party libraries enabled are -defined by the job: +Directories and Permissions +^^^^^^^^^^^^^^^^^^^^^^^^^^^ -.. code-block:: YAML +* ``/usr/workspace/sundials`` is the workspace directory associated with the ``sundials`` LC group. + Users must be added to this group through the LC IDM application. - lassen_gcc_cuda_tpls: - parallel: - matrix: - - COMPILER_SPEC: gcc@7.3.1 - CUDA_SPEC: [cuda@10.1.243, cuda@11.2.0] - variables: - SPEC: "%${COMPILER_SPEC} precision=double ~int64 +openmp +cuda +raja cuda_arch=70 \ - ^raja+cuda~examples~exercises cuda_arch=70 ^${CUDA_SPEC}" - extends: .lassen_build_and_test +* ``/usr/workspace/sundials/ci`` is where all GitLab CI related files are stored. + The correct permissions for this directory are ``drwxrws---``. -The ``parallel:`` and ``matrix:`` keywords enable creating multiple jobs with -different variable values for each instance of the job i.e., one job using -``cuda@10.1.243`` and another using ``cuda@11.2.0``. These variables values are -then used to create an environment variable ``SPEC`` with a Spack spec used by -``build_and_test.sh`` when configuring SUNDIALS. +* ``/usr/workspace/sundials/ci/.builds`` is where GitLab CI pipelines are run. The permissions + for this directory are ``drwxrwx---``, but directories within it must be ``drwx------``. + Files within it should be ``-rw-rw----`` (can add ``x`` for group and owner as appropriate). -Benchmark Jobs -^^^^^^^^^^^^^^ +* ``/usr/workspace/sundials/ci/spack_stuff`` contains the Spack build caches amongst other Spack + files. The permissions for this directory and directories below should be ``drwxrws---``. -See :ref:`SUNDIALS Continuous Performance Testing (CPT)` for more details. GitLab CI Test Script --------------------- @@ -257,26 +258,27 @@ build, and test SUNDIALS. This script leverages two Git submodules: * `radiuss-spack-configs `_ -- is the SUNDIALS fork of the `LLNL radiuss-spack-configs `_ - repository that provides spack configuration files for various LLNL platfornms - i.e., ``compilers.yaml`` and ``packages.yaml`` files for Quartz, Lassen, etc. + repository that provides spack environment files for various LLNL platfornms + i.e., ``spack.yaml`` for Dane, Tioga, etc. -These submodules work in conjunction with ``spack_packages/sundials/package.py`` +These submodules work in conjunction with ``scripts/sundials/package.py`` to configure and build any third-party libraries needed by the SUNDIALS configuration and generates an initial CMake cache file for building SUNDIALS. -Other packages can be added to ``spack_packages//package.py`` -if the default Spack package needs to be overridden. We do this currently for -Caliper, as we need a newer version than in the Spack commit currently used. +Other packages can be added to ``spack/packages`` if the default Spack package +needs to be overridden. -Updating Spack --------------- +Spack Build Cache +^^^^^^^^^^^^^^^^^ + +The ``build_and_test.sh`` script leverage Spack build caches in ``/usr/workspace/sundials/ci/spack_stuff/`` +to speedup builds. These caches store binaries of packages that have been built previously. Separate caches are +made for each ``SPACK_REF`` to avoid conflicts across Spack versions. -To update the spack commit used for the CI: -1. The first thing to do is update the spack commit in the -``.uberenv_config.json`` file. -2. Then, a pipeline should be manually launched from the GitLab UI with the -``SHARED_SPACK`` CI variable set to ``ON`` and the ``SPACK_PREFIX`` variable to -the version of spack being set in the uberenv_config.json. +Running Locally +^^^^^^^^^^^^^^^ -This will create a new spack installation and rebuild all of the specs. +It is possible to run these scripts locally on an LC machine. First set a ``SPACK_REF`` +environment variable to a spack commit that you want to use, and then set a ``SPEC`` +environment variable with a SUNDIALS spack spec that you want to test. diff --git a/examples/nvector/petsc/CMakeLists.txt b/examples/nvector/petsc/CMakeLists.txt index 9b18176bee..83809bab49 100644 --- a/examples/nvector/petsc/CMakeLists.txt +++ b/examples/nvector/petsc/CMakeLists.txt @@ -18,8 +18,9 @@ # develop for examples excluded from 'make test' in releases # Examples using SUNDIALS PETSc nvector -set(nvector_petsc_examples "test_nvector_petsc\;1000 0\;\;\;" # run sequentially - "test_nvector_petsc\;1000 0\;1\;4\;" # run 4 procs +set(nvector_petsc_examples + "test_nvector_petsc\;1000 0\;1\;1\;" # run sequentially + "test_nvector_petsc\;1000 0\;1\;4\;" # run 4 procs ) # Dependencies for nvector examples diff --git a/examples/sunnonlinsol/petsc/CMakeLists.txt b/examples/sunnonlinsol/petsc/CMakeLists.txt index 7146134ea8..2c366597a6 100644 --- a/examples/sunnonlinsol/petsc/CMakeLists.txt +++ b/examples/sunnonlinsol/petsc/CMakeLists.txt @@ -18,7 +18,7 @@ # examples excluded from 'make test' in releases # Example programs -set(examples "test_sunnonlinsol_petscsnes\;\;") +set(examples "test_sunnonlinsol_petscsnes\;\;1\;1\;") if(MPI_C_COMPILER) # use MPI wrapper as the compiler @@ -41,7 +41,9 @@ foreach(example_tuple ${examples}) # parse the example tuple list(GET example_tuple 0 example) list(GET example_tuple 1 example_args) - list(GET example_tuple 2 example_type) + list(GET example_tuple 2 number_of_nodes) + list(GET example_tuple 3 number_of_tasks) + list(GET example_tuple 4 example_type) # check if this example has already been added, only need to add example # source files once for testing with different inputs @@ -67,6 +69,7 @@ foreach(example_tuple ${examples}) sundials_add_test( ${test_name} ${example} TEST_ARGS ${example_args} + MPI_NPROCS ${number_of_tasks} EXAMPLE_TYPE ${example_type} NODIFF) diff --git a/examples/sunnonlinsol/petsc/test_sunnonlinsol_petscsnes.c b/examples/sunnonlinsol/petsc/test_sunnonlinsol_petscsnes.c index b9486208b8..d66fb5526a 100644 --- a/examples/sunnonlinsol/petsc/test_sunnonlinsol_petscsnes.c +++ b/examples/sunnonlinsol/petsc/test_sunnonlinsol_petscsnes.c @@ -157,6 +157,7 @@ int main(int argc, char* argv[]) N_VDestroy(y0); N_VDestroy(w); SUNNonlinSolFree(NLS); + PetscFinalize(); SUNContext_Free(&sunctx); /* Print result */ diff --git a/.gitlab/spack_packages/sundials/0001-add-missing-README-to-examples-cvode-hip.patch b/scripts/spack/packages/sundials/0001-add-missing-README-to-examples-cvode-hip.patch similarity index 100% rename from .gitlab/spack_packages/sundials/0001-add-missing-README-to-examples-cvode-hip.patch rename to scripts/spack/packages/sundials/0001-add-missing-README-to-examples-cvode-hip.patch diff --git a/.gitlab/spack_packages/sundials/5.5.0-xsdk-patches.patch b/scripts/spack/packages/sundials/5.5.0-xsdk-patches.patch similarity index 100% rename from .gitlab/spack_packages/sundials/5.5.0-xsdk-patches.patch rename to scripts/spack/packages/sundials/5.5.0-xsdk-patches.patch diff --git a/.gitlab/spack_packages/sundials/FindPackageMultipass.cmake.patch b/scripts/spack/packages/sundials/FindPackageMultipass.cmake.patch similarity index 100% rename from .gitlab/spack_packages/sundials/FindPackageMultipass.cmake.patch rename to scripts/spack/packages/sundials/FindPackageMultipass.cmake.patch diff --git a/.gitlab/spack_packages/sundials/nvector-pic.patch b/scripts/spack/packages/sundials/nvector-pic.patch similarity index 100% rename from .gitlab/spack_packages/sundials/nvector-pic.patch rename to scripts/spack/packages/sundials/nvector-pic.patch diff --git a/.gitlab/spack_packages/sundials/package.py b/scripts/spack/packages/sundials/package.py similarity index 98% rename from .gitlab/spack_packages/sundials/package.py rename to scripts/spack/packages/sundials/package.py index ce7f4ece91..2257576317 100644 --- a/.gitlab/spack_packages/sundials/package.py +++ b/scripts/spack/packages/sundials/package.py @@ -183,9 +183,9 @@ class Sundials(CachedCMakePackage, CudaPackage, ROCmPackage): # Scheduler variant( - "scheduler", - default="slurm", - description="Specify which scheduler the system runs on", + "scheduler", + default="slurm", + description="Specify which scheduler the system runs on", values=("flux", "lsf", "slurm") ) @@ -194,9 +194,9 @@ class Sundials(CachedCMakePackage, CudaPackage, ROCmPackage): # Profiling examples variant( - "profile-examples", - default=False, - when="+adiak +caliper", + "profile-examples", + default=False, + when="+adiak +caliper", description="Build examples with profiling capabilities") # Caliper Directory @@ -251,7 +251,8 @@ class Sundials(CachedCMakePackage, CudaPackage, ROCmPackage): depends_on("superlu-dist@6.3.0:", when="@5.5.0:6.3.0 +superlu-dist") depends_on("superlu-dist@6.1.1:", when="@:5.4.0 +superlu-dist") depends_on("superlu-mt+blas", when="+superlu-mt") - depends_on("trilinos+tpetra", when="+trilinos") + depends_on("trilinos+tpetra@:13", when="@:7.1.1 +trilinos") + depends_on("trilinos+tpetra@14:", when="@7.2.0: +trilinos") # Require that external libraries built with the same precision depends_on("petsc~double~complex", when="+petsc precision=single") @@ -697,7 +698,7 @@ def initconfig_mpi_entries(self): entries.append(cmake_cache_string("SUNDIALS_SCHEDULER_COMMAND", "srun")) if "scheduler=lsf" in spec: entries.append(cmake_cache_string("SUNDIALS_SCHEDULER_COMMAND", "jsrun")) - + return entries @@ -719,11 +720,10 @@ def initconfig_hardware_entries(self): entries.extend( [ self.cache_option_from_variant("ENABLE_HIP", "rocm"), - cmake_cache_path("CMAKE_C_COMPILER", spec["llvm-amdgpu"].prefix.bin.clang), - cmake_cache_path("CMAKE_CXX_COMPILER", spec["hip"].hipcc), cmake_cache_path("HIP_PATH", spec["hip"].prefix), + cmake_cache_path("HIP_DIR", spec["hip"].prefix.cmake), cmake_cache_path("HIP_CLANG_INCLUDE_PATH", spec["llvm-amdgpu"].prefix.include), - cmake_cache_path("ROCM_PATH", spec["llvm-amdgpu"].prefix), + cmake_cache_path("ROCM_PATH", spec["hsa-rocr-dev"].prefix), cmake_cache_string("AMDGPU_TARGETS", ";".join(spec.variants["amdgpu_target"].value)) ] ) @@ -763,7 +763,7 @@ def initconfig_package_entries(self): self.cache_option_from_variant("SUNDIALS_TEST_PROFILE", "profile-examples"), self.cache_option_from_variant("SUNDIALS_TEST_DEVTESTS", "profile-examples"), cmake_cache_string("SPACK_VERSION", ".".join(map(str, spack.spack_version_info))) - + ] ) @@ -799,7 +799,7 @@ def initconfig_package_entries(self): ) # Building with Adiak - if "+adiak" in spec: + if "+adiak" in spec: entries.append(cmake_cache_path("adiak_DIR", spec["adiak"].prefix.lib.cmake + "/adiak")) # Building with Caliper diff --git a/.gitlab/spack_packages/sundials/remove-links-to-OpenMP-vector.patch b/scripts/spack/packages/sundials/remove-links-to-OpenMP-vector.patch similarity index 100% rename from .gitlab/spack_packages/sundials/remove-links-to-OpenMP-vector.patch rename to scripts/spack/packages/sundials/remove-links-to-OpenMP-vector.patch diff --git a/.gitlab/spack_packages/sundials/sundials-v5.8.0.patch b/scripts/spack/packages/sundials/sundials-v5.8.0.patch similarity index 100% rename from .gitlab/spack_packages/sundials/sundials-v5.8.0.patch rename to scripts/spack/packages/sundials/sundials-v5.8.0.patch diff --git a/.gitlab/spack_packages/sundials/test_nvector_parhyp.patch b/scripts/spack/packages/sundials/test_nvector_parhyp.patch similarity index 100% rename from .gitlab/spack_packages/sundials/test_nvector_parhyp.patch rename to scripts/spack/packages/sundials/test_nvector_parhyp.patch diff --git a/scripts/spack/repo.yaml b/scripts/spack/repo.yaml new file mode 100644 index 0000000000..97f36d22aa --- /dev/null +++ b/scripts/spack/repo.yaml @@ -0,0 +1,2 @@ +repo: + namespace: 'sundials'