Skip to content

Commit

Permalink
Update Ubuntu CI scripts (move to new AWS pcluster 3.3.0) (JCSDA#729)
Browse files Browse the repository at this point in the history 
* Update .github/workflows/ubuntu-ci-x86_64.yaml for moving to new AWS pcluster
* Switch Ubuntu CI from tcl to lmod
  • Loading branch information
@climbfuji
climbfuji authored Aug 23, 2023
1 parent 86c06e3 commit f00d250
Showing 1 changed file with 24 additions and 18 deletions.
42 changes: 24 additions & 18 deletions .github/workflows/ubuntu-ci-x86_64.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -80,10 +80,10 @@ jobs:
echo " intel-oneapi-mpi:" >> ${SPACK_SYSTEM_CONFIG_PATH}/packages.yaml
echo " buildable: false" >> ${SPACK_SYSTEM_CONFIG_PATH}/packages.yaml
echo " externals:" >> ${SPACK_SYSTEM_CONFIG_PATH}/packages.yaml
echo " - spec: intel-oneapi-mpi@2021.4.0%intel@2021.4.0" >> ${SPACK_SYSTEM_CONFIG_PATH}/packages.yaml
echo " - spec: intel-oneapi-mpi@2021.6.0%intel@2022.1.0" >> ${SPACK_SYSTEM_CONFIG_PATH}/packages.yaml
echo " prefix: /opt/intel" >> ${SPACK_SYSTEM_CONFIG_PATH}/packages.yaml
echo " modules:" >> ${SPACK_SYSTEM_CONFIG_PATH}/packages.yaml
echo " - libfabric-aws/1.16.0~amzn3.0" >> ${SPACK_SYSTEM_CONFIG_PATH}/packages.yaml
echo " - libfabric-aws/1.16.0~amzn4.0" >> ${SPACK_SYSTEM_CONFIG_PATH}/packages.yaml
echo " - intelmpi" >> ${SPACK_SYSTEM_CONFIG_PATH}/packages.yaml
# For GNU
Expand All @@ -94,19 +94,19 @@ jobs:
echo "" >> ${SPACK_SYSTEM_CONFIG_PATH}/compilers.yaml
echo "compilers:" >> ${SPACK_SYSTEM_CONFIG_PATH}/compilers.yaml
echo "- compiler:" >> ${SPACK_SYSTEM_CONFIG_PATH}/compilers.yaml
echo " spec: intel@2021.4.0" >> ${SPACK_SYSTEM_CONFIG_PATH}/compilers.yaml
echo " spec: intel@2022.1.0" >> ${SPACK_SYSTEM_CONFIG_PATH}/compilers.yaml
echo " paths:" >> ${SPACK_SYSTEM_CONFIG_PATH}/compilers.yaml
echo " cc: /opt/intel/oneapi/compiler/2021.4.0/linux/bin/intel64/icc" >> ${SPACK_SYSTEM_CONFIG_PATH}/compilers.yaml
echo " cxx: /opt/intel/oneapi/compiler/2021.4.0/linux/bin/intel64/icpc" >> ${SPACK_SYSTEM_CONFIG_PATH}/compilers.yaml
echo " f77: /opt/intel/oneapi/compiler/2021.4.0/linux/bin/intel64/ifort" >> ${SPACK_SYSTEM_CONFIG_PATH}/compilers.yaml
echo " fc: /opt/intel/oneapi/compiler/2021.4.0/linux/bin/intel64/ifort" >> ${SPACK_SYSTEM_CONFIG_PATH}/compilers.yaml
echo " cc: /opt/intel/oneapi/compiler/2022.1.0/linux/bin/intel64/icc" >> ${SPACK_SYSTEM_CONFIG_PATH}/compilers.yaml
echo " cxx: /opt/intel/oneapi/compiler/2022.1.0/linux/bin/intel64/icpc" >> ${SPACK_SYSTEM_CONFIG_PATH}/compilers.yaml
echo " f77: /opt/intel/oneapi/compiler/2022.1.0/linux/bin/intel64/ifort" >> ${SPACK_SYSTEM_CONFIG_PATH}/compilers.yaml
echo " fc: /opt/intel/oneapi/compiler/2022.1.0/linux/bin/intel64/ifort" >> ${SPACK_SYSTEM_CONFIG_PATH}/compilers.yaml
echo " flags: {}" >> ${SPACK_SYSTEM_CONFIG_PATH}/compilers.yaml
echo " operating_system: ubuntu20.04" >> ${SPACK_SYSTEM_CONFIG_PATH}/compilers.yaml
echo " target: x86_64" >> ${SPACK_SYSTEM_CONFIG_PATH}/compilers.yaml
echo " modules: []" >> ${SPACK_SYSTEM_CONFIG_PATH}/compilers.yaml
echo " environment:" >> ${SPACK_SYSTEM_CONFIG_PATH}/compilers.yaml
echo " prepend_path:" >> ${SPACK_SYSTEM_CONFIG_PATH}/compilers.yaml
echo " LD_LIBRARY_PATH: '/opt/intel/oneapi/compiler/2021.4.0/linux/compiler/lib/intel64_lin'" >> ${SPACK_SYSTEM_CONFIG_PATH}/compilers.yaml
echo " LD_LIBRARY_PATH: '/opt/intel/oneapi/compiler/2022.1.0/linux/compiler/lib/intel64_lin'" >> ${SPACK_SYSTEM_CONFIG_PATH}/compilers.yaml
echo " set:" >> ${SPACK_SYSTEM_CONFIG_PATH}/compilers.yaml
echo " I_MPI_PMI_LIBRARY: '/opt/slurm/lib/libpmi.so'" >> ${SPACK_SYSTEM_CONFIG_PATH}/compilers.yaml
echo " extra_rpaths: []" >> ${SPACK_SYSTEM_CONFIG_PATH}/compilers.yaml
Expand All @@ -119,12 +119,15 @@ jobs:
# *DH
# Set compiler and MPI
spack config add "packages:all:providers:mpi:[intel-oneapi-mpi@2021.4.0]"
spack config add "packages:all:compiler:[intel@2021.4.0]"
spack config add "packages:all:providers:mpi:[intel-oneapi-mpi@2021.6.0]"
spack config add "packages:all:compiler:[intel@2022.1.0]"
sed -i "s/\['\%aocc', '\%apple-clang', '\%gcc', '\%intel'\]/\['\%intel'\]/g" $ENVDIR/spack.yaml
spack concretize 2>&1 | tee log.concretize.intel-2021.4.0
${SPACK_STACK_DIR}/util/show_duplicate_packages.py -d log.concretize.intel-2021.4.0 -i fms -i crtm
# Switch from default tcl to lmod modules
sed -i "s/tcl/lmod/g" $ENVDIR/site/modules.yaml
spack concretize 2>&1 | tee log.concretize.intel-2022.1.0
${SPACK_STACK_DIR}/util/show_duplicate_packages.py -d log.concretize.intel-2022.1.0 -i fms -i crtm
# Add and update source cache
spack mirror add local-source file:///home/ubuntu/spack-stack/source-cache/
Expand All @@ -145,21 +148,21 @@ jobs:
# base-env
echo "base-env ..."
# DH* 20230721 - todo remove --no-checksum
spack install --fail-fast --source --no-check-signature --no-checksum base-env 2>&1 | tee log.install.intel-2021.4.0.base-env
spack install --fail-fast --source --no-check-signature --no-checksum base-env 2>&1 | tee log.install.intel-2022.1.0.base-env
# DH* 20230721 - todo remove || true (this was here all the time, but should not be needed if spack creates buildcaches correctly)
spack buildcache create -a -u /home/ubuntu/spack-stack/build-cache/ || true
# the rest
echo "${{ inputs.template || 'unified-dev' }} ..."
# DH* 20230721 - todo remove --no-checksum
spack install --fail-fast --source --no-check-signature --no-checksum 2>&1 | tee log.install.intel@2021.4.0.${{ inputs.template || 'unified-dev' }}
spack install --fail-fast --source --no-check-signature --no-checksum 2>&1 | tee log.install.intel@2022.1.0.${{ inputs.template || 'unified-dev' }}
# DH* 20230721 - todo remove || true (this was here all the time, but should not be needed if spack creates buildcaches correctly)
spack buildcache create -a -u /home/ubuntu/spack-stack/build-cache/ || true
# Next steps: synchronize source and build cache to a central/combined mirror?
echo "Next steps ..."
spack clean -a
spack module tcl refresh -y
spack module lmod refresh -y
spack stack setup-meta-modules
spack env deactivate
Expand All @@ -177,7 +180,10 @@ jobs:
- name: test-env
run: |
source /etc/profile.d/modules.sh
source /etc/profile.d/z00_lmod.sh
module use /usr/share/modules/modulefiles
module use /opt/intel/mpi/2021.6.0/modulefiles
module use /home/ubuntu/jedi/modulefiles
# DH* 20230302 - to avoid using padded_length for build caches,
# always build in the same environment so that the length of the
Expand All @@ -189,8 +195,8 @@ jobs:
ls -l ${ENVDIR}/install/modulefiles/Core
module use ${ENVDIR}/install/modulefiles/Core
module load stack-intel/2021.4.0
module load stack-intel-oneapi-mpi/2021.4.0
module load stack-intel/2022.1.0
module load stack-intel-oneapi-mpi/2021.6.0
module load stack-python/3.10.8
module available
Expand Down

0 comments on commit f00d250

Please sign in to comment.