Skip to content

Commit

Permalink
Merge branch 'develop' into feature/mri-adaptivity
Browse files Browse the repository at this point in the history
  • Loading branch information
drreynolds authored Nov 1, 2024
2 parents 8b71729 + 8c72a8b commit 0c65bfd
Show file tree
Hide file tree
Showing 38 changed files with 645 additions and 1,025 deletions.
203 changes: 94 additions & 109 deletions .gitlab-ci.yml
Original file line number Diff line number Diff line change
@@ -1,136 +1,121 @@
# ---------------------------------------------------------------
###############################################################################
# Copyright (c) 2022-23, Lawrence Livermore National Security, LLC and RADIUSS
# project contributors.
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in all
# copies or substantial portions of the Software.
#
# SPDX-License-Identifier: (MIT)
###############################################################################
# Modifications to this file have been made under the folllowing copyright.
# -----------------------------------------------------------------------------
# SUNDIALS Copyright Start
# Copyright (c) 2002-2021, Lawrence Livermore National Security
# Copyright (c) 2002-2024, Lawrence Livermore National Security
# and Southern Methodist University.
# All rights reserved.
#
# See the top-level LICENSE and NOTICE files for details.
#
# SPDX-License-Identifier: BSD-3-Clause
# SUNDIALS Copyright End
# ---------------------------------------------------------------
# -----------------------------------------------------------------------------

# DESCRIPTION:
###############################################################################
# General GitLab pipelines configurations for supercomputers and Linux clusters
# at Lawrence Livermore National Laboratory (LLNL).
###############################################################################

# We define the following GitLab pipeline variables (all of which can be
# set from the GitLab CI "run pipeline" UI):
#
# GIT_SUBMODULE_STRATEGY:
# Tells Gitlab to recursively update the submodules when cloning.
#
# ALLOC_NAME:
# Allocation unique name.
#
# BUILD_ROOT:
# The path to the shared resources between all jobs. The BUILD_ROOT is unique to
# the pipeline, preventing any form of concurrency with other pipelines. This
# also means that the BUILD_ROOT directory will never be cleaned.
#
# DEFAULT_TIME:
# Default time to let the jobs run will be 30 minutes.
#
# BUILD_JOBS:
# Number of threads to use for builds.
#
# NCPU:
# Number of CPUs to alloc.
# This entire pipeline is LLNL-specific
#
# VERBOSE_BUILD:
# Sets CMAKE_VERBOSE_MAKEFILE to TRUE when "ON".
# Important note: This file is a template provided by llnl/radiuss-shared-ci.
# Remains to set variable values, change the reference to the radiuss-shared-ci
# repo, opt-in and out optional features. The project can then extend it with
# additional stages.
#
# VERBOSE_TEST:
# Passes --verbose to CTest when "ON".
# In addition, each project should copy over and complete:
# - .gitlab/custom-jobs-and-variables.yml
# - .gitlab/subscribed-pipelines.yml
#
# ON_LASSEN:
# Should the Lassen pipeline run? Set to "ON" or "OFF" to enable/disable.
#
# ON_QUARTZ:
# Should the Quartz pipeline run? Set to "ON" or "OFF" to enable/disable.
#
# SPACK_PREFIX: prefix used for shared spack installation.
# Usually this a spack version number that matches the version set in the uberenv_config.json file.
# Spack installs go in /usr/workspace/sundials/spack_installs/${SPACK_PREFIX}/$(hostname).
#
# SHARED_SPACK:
# If "ON", then a shared spack install that has been pre-configured is utilized.
# If "OFF", then a new spack instance is created for every build (meaning all TPLs have to be installed).
# If "UPSTREAM" (the default), then the shared spack is used as an upstream for a build specific spack.
#
# BENCHMARK:
# If "ON", then the SUNDIALS benchmark problems are run and generate profiles.
#
# BENCHMARK_NNODES:
# Number of nodes to use for benchmarks. Default is 4.
#
# BENCHMARK_QUEUE:
# What queue to submit the benchmarks too. Default is pbatch (for Livermore).
# The jobs should be specified in a file local to the project,
# - .gitlab/jobs/${CI_MACHINE}.yml
# or generated (see LLNL/Umpire for an example).
###############################################################################

# We define the following GitLab pipeline variables:
variables:
##### LC GITLAB CONFIGURATION
# Use an LLNL service user to run CI. This prevents from running pipelines as
# an actual user.
LLNL_SERVICE_USER: ""
# Use the service user workspace. Solves permission issues, stores everything
# at the same location whoever triggers a pipeline.
CUSTOM_CI_BUILDS_DIR: "/usr/workspace/sundials/ci/.builds/"
# Tells Gitlab to recursively update the submodules when cloning the project.
GIT_SUBMODULE_STRATEGY: recursive
ALLOC_NAME: ${CI_PROJECT_NAME}_ci_${CI_PIPELINE_ID}
BUILD_ROOT: ${CI_PROJECT_DIR}
DEFAULT_TIME: 30
BUILD_JOBS: 32
NCPUS: 12
VERBOSE_BUILD: "OFF"
VERBOSE_TEST: "OFF"
ON_LASSEN: "ON"
ON_QUARTZ: "ON"
ON_CORONA: "ON"
SPACK_PREFIX: "v0.19.1"
SHARED_SPACK: "UPSTREAM"
BENCHMARK: "OFF"
BENCHMARK_NNODES: 4
BENCHMARK_QUEUE: "pbatch"

# Normally, stages are blocking in Gitlab. However, using the keyword "needs" we
# can express dependencies between job that break the ordering of stages, in
# favor of a DAG.
# In practice q_*, l_* and c_* stages are independently run and start immediately.

stages:
- q_build_and_test
- l_build_and_test
- l_build_and_bench
- c_build_and_test
##### PROJECT VARIABLES
# We build the projects in the CI clone directory.
# Used in script/gitlab/build_and_test.sh script.
# TODO: add a clean-up mechanism.
BUILD_ROOT: ${CI_PROJECT_DIR}
SPACK_REF: "594a376c521cc746978571b1181a47bbcff30a21" # v0.22.2

# These are also templates (.name) that define project specific build commands.
# If an allocation exist with the name defined in this pipeline, the job will
# use it (slurm specific).
.build_toss_3_x86_64_ib_script:
script:
- echo ${ALLOC_NAME}
- srun -p pdebug -N 1 -n ${NCPUS} --interactive -t ${DEFAULT_TIME}
--job-name=${ALLOC_NAME} .gitlab/build_and_test.sh
##### SHARED_CI CONFIGURATION
# Required information about GitHub repository
GITHUB_PROJECT_NAME: "sundials"
GITHUB_PROJECT_ORG: "LLNL"
# Set the build-and-test command.
# Nested variables are allowed and useful to customize the job command. We
# prevent variable expansion so that you can define them at job level.
JOB_CMD:
value: ".gitlab/build_and_test.sh"
expand: false
# Override the pattern describing branches that will skip the "draft PR filter
# test". Add protected branches here. See default value in
# preliminary-ignore-draft-pr.yml.
# ALWAYS_RUN_PATTERN: ""

# Corona
.build_toss_4_x86_64_ib_corona_script:
script:
- echo ${ALLOC_NAME}
- flux alloc -N 1 -t ${DEFAULT_TIME} .gitlab/build_and_test.sh
# We organize the build-and-test stage with sub-pipelines. Each sub-pipeline
# corresponds to a test batch on a given machine.

# CORAL systems use spectrum LSF instead of SLURM
.build_blueos_3_ppc64le_ib_script:
script:
- echo ${ALLOC_NAME}
- bsub -q pdebug -J ${ALLOC_NAME} -nnodes 1 -W ${DEFAULT_TIME} -Is .gitlab/build_and_test.sh
# High level stages
stages:
- prerequisites
- build-and-test

# Benchmark job for CORAL systems
.build_blueos_3_ppc64le_ib_bench:
script:
- echo ${ALLOC_NAME}
- bsub -q ${BENCHMARK_QUEUE} -J ${ALLOC_NAME} -nnodes ${BENCHMARK_NNODES} -W ${DEFAULT_TIME} -Is .gitlab/build_and_bench.sh
# Template for jobs triggering a build-and-test sub-pipeline:
.build-and-test:
stage: build-and-test
trigger:
include:
- local: '.gitlab/custom-jobs-and-variables.yml'
- project: 'sundials/radiuss-shared-ci' # https://lc.llnl.gov/gitlab/sundials/radiuss-shared-ci
ref: 'ae1f3786591beed83abc6a0de2229f6e9532e2d4'
file: 'pipelines/${CI_MACHINE}.yml'
# Add your jobs
# you can use a local file
- local: '.gitlab/jobs/${CI_MACHINE}.yml'
# or a file generated in the previous steps
# - artifact: '${CI_MACHINE}-jobs.yml'
# job: 'generate-job-file'
# (See Umpire CI setup for an example).
strategy: depend
forward:
pipeline_variables: true

# This is where jobs are included.
include:
# Sets ID tokens for every job using `default:`
- project: 'lc-templates/id_tokens'
file: 'id_tokens.yml'
- local: .gitlab/quartz-templates.yml
- local: .gitlab/quartz-jobs.yml
- local: .gitlab/lassen-templates.yml
- local: .gitlab/lassen-jobs.yml
- local: .gitlab/corona-templates.yml
- local: .gitlab/corona-jobs.yml
# # [Optional] checks preliminary to running the actual CI test
# - project: 'radiuss/radiuss-shared-ci'
# ref: 'v2024.04.0'
# file: 'utilities/preliminary-ignore-draft-pr.yml'
# pipelines subscribed by the project
- local: '.gitlab/subscribed-pipelines.yml'
62 changes: 24 additions & 38 deletions .gitlab/build_and_test.sh
Original file line number Diff line number Diff line change
Expand Up @@ -20,9 +20,6 @@ job_unique_id=${CI_JOB_ID:-""}
sys_type=${SYS_TYPE:-""}
py_env_path=${PYTHON_ENVIRONMENT_PATH:-""}

spack_prefix=${SHARED_SPACK_PREFIX:-"v0.19.1"}
shared_spack=${SHARED_SPACK:-"UPSTREAM"}

# Dependencies
date

Expand All @@ -38,27 +35,13 @@ echo "spec = ${spec}"
echo "job_unique_id = ${job_unique_id}"
echo "sys_type = ${sys_type}"
echo "py_env_path = ${py_env_path}"
echo "shared_spack = ${shared_spack}"

# remove tailing number from hostname
hostname=${hostname%%[0-9]*}

# number of parallel build jobs
BUILD_JOBS=${BUILD_JOBS:-"1"}

# load newer python to try the clingo concretizer
# machine specific loads
if [[ "${hostname}" == "lassen" ]]; then
echo "module load python/3.8.2"
module load python/3.8.2
elif [[ "${hostname}" == "corona" ]]; then
echo "module load python/3.9.12"
module load python/3.9.12
else
echo "module load python"
module load python
fi

if [[ "${option}" != "--build-only" && "${option}" != "--test-only" ]]
then
echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~"
Expand Down Expand Up @@ -97,35 +80,29 @@ then
mkdir -p ${spack_user_cache}
fi

if [[ -d /usr/workspace/sundials ]]
then
upstream="/usr/workspace/sundials/spack_installs/${spack_prefix}/${hostname}"
mkdir -p "${upstream}"
upstream_opt="--upstream=${upstream}"
fi
mirror_opt=""
buildcache="/usr/workspace/sundials/ci/spack_stuff/build_caches/${SPACK_REF}"

if [[ "${shared_spack}" == "UPSTREAM" ]]
if [[ ! -d "${buildcache}" ]]
then
python3 .gitlab/uberenv/uberenv.py --spec="${spec}" "${prefix_opt}" "${upstream_opt}"
elif [[ "${shared_spack}" == "ON" ]]
then
python3 .gitlab/uberenv/uberenv.py --spec="${spec}" --prefix="${upstream}"
else
python3 .gitlab/uberenv/uberenv.py --spec="${spec}" "${prefix_opt}"
mkdir "${buildcache}"
fi

# Ensure correct CUDA module is loaded, only works for module naming
# convention on Lassen. Only needed for CUDA 11 (unclear why).
if [[ -n "${CUDA_SPEC}" ]]; then
cuda_version="${CUDA_SPEC##*@}"
echo "module load cuda/${cuda_version}"
module load cuda/"${cuda_version}"
fi
mirror_opt=("--mirror=${buildcache}" "--mirror-autopush")

module load cmake/3.23
key_path=/usr/workspace/sundials/ci/spack_stuff/gpg_backup
python3 .gitlab/uberenv/uberenv.py \
--trust-key ${key_path}/pubring.gpg --trust-key ${key_path}/secring.gpg \
--spec="${spec}" "${mirror_opt[@]}" "${prefix_opt}" \
--spack-commit="${SPACK_REF}"
fi

date

# Reload the spack environment created by uberenv
. ${prefix}/spack/share/spack/setup-env.sh
spack load

# Host config file
if [[ -z ${hostconfig} ]]
then
Expand Down Expand Up @@ -173,6 +150,9 @@ then
echo "~ Host-config: ${hostconfig_path}"
echo "~ Build Dir: ${build_dir}"
echo "~ Project Dir: ${project_dir}"
echo "~ MPIEXEC_EXECUTABLE: ${MPIEXEC_EXECUTABLE}"
echo "~ MPIEXEC_PREFLAGS: ${MPIEXEC_PREFLAGS}"
echo "~ MPIEXEC_POSTFLAGS: ${MPIEXEC_POSTFLAGS}"
echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~"

echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~"
Expand All @@ -195,13 +175,19 @@ then
$cmake_exe \
-C "${hostconfig_path}" \
-DCMAKE_INSTALL_PREFIX=${install_dir} \
-DMPIEXEC_EXECUTABLE=$(which $MPIEXEC_EXECUTABLE) \
-DMPIEXEC_PREFLAGS=${MPIEXEC_PREFLAGS} \
-DMPIEXEC_POSTFLAGS=${MPIEXEC_POSTFLAGS} \
-DSUNDIALS_CALIPER_OUTPUT_DIR="${CALIPER_DIR}/Release/${hostname}/${sundials_version}" \
"${project_dir}"

else
$cmake_exe \
-C "${hostconfig_path}" \
-DCMAKE_INSTALL_PREFIX=${install_dir} \
-DMPIEXEC_EXECUTABLE=$(which $MPIEXEC_EXECUTABLE) \
-DMPIEXEC_PREFLAGS=${MPIEXEC_PREFLAGS} \
-DMPIEXEC_POSTFLAGS=${MPIEXEC_POSTFLAGS} \
"${project_dir}"
fi

Expand Down
48 changes: 0 additions & 48 deletions .gitlab/corona-jobs.yml

This file was deleted.

Loading

0 comments on commit 0c65bfd

Please sign in to comment.