-
Notifications
You must be signed in to change notification settings - Fork 89
Building Albany on CCI DRP
Glen Hansen edited this page Feb 11, 2019
·
12 revisions
module purge
module load cmake/3.3.2
module load gcc/6.3.0_1
module load openmpi/2.1.1_1
tarballs/config scripts of all TPLs in /gpfs/u/home/ADDM/ADDMgnzw/barn-shared/tpls
# zlib v. 1.2.8
export tpls=/gpfs/u/home/ADDM/ADDMgnzw/scratch-shared/tpls
CC=mpicc CFLAGS="-O3 -fPIC" ./configure --64 --prefix=${tpls}
# hdf5 v 1.10.1
export tpls=/gpfs/u/home/ADDM/ADDMgnzw/scratch-shared/tpls
./configure CC=mpicc CXX=mpicxx FC=mpif90 \
CFLAGS="-O3 -fPIC -std=c99" CXXFLAGS="-O3 -fPIC" FCFLAGS="-O3 -fPIC" \
--enable-parallel \
--with-zlib=${tpls} \
--prefix=${tpls}
# netcdf v 4.5.0
export tpls=/gpfs/u/home/ADDM/ADDMgnzw/scratch-shared/tpls
./configure CC=mpicc CXX=mpicxx FC=mpif90 \
CFLAGS="-fPIC -I${tpls}/include -O3 -march=native" \
CXXFLAGS="-fPIC -I${tpls}/include -O3 -march=native" \
FCFLAGS="-fPIC -I${tpls}/include -O3 -march=native" \
LDFLAGS="-fPIC -L${tpls}/lib -O3 -march=native" \
--prefix=${tpls} \
--disable-doxygen \
--enable-netcdf4
# boost v 1.65.1
export tpls=/gpfs/u/home/ADDM/ADDMgnzw/scratch-shared/tpls
echo "using gcc : 6.3.0_1 : /gpfs/u/software/amd64-rhel6/compilers/gcc/6.3.0_1/bin/gcc ;" \
>> ~/user-config.jam
echo "using mpi : /gpfs/u/software/amd64-rhel6/mpi/openmpi/2.1.1_1/gcc-6.3.0_1/bin/mpicxx ;" \
>> ~/user-config.jam
./bootstrap.sh \
--with-libraries=signals,regex,filesystem,system,mpi,serialization,thread,program_options,exception \
--prefix=${tpls}
# lapack v 3.5.0
export tpls=/gpfs/u/home/ADDM/ADDMgnzw/scratch/tpls
mkdir build
cd build
ccmake -DCMAKE_INSTALL_PREFIX=${tpls} ..
# parmetis v 4.0.3
export tpls=/gpfs/u/home/ADDM/ADDMgnzw/scratch/tpls
export metis=/gpfs/u/home/ADDM/ADDMgnzw/scratch/parmetis-4.0.3/metis
cd build
cmake \
-D CMAKE_INSTALL_PREFIX=${tpls} \
-D CMAKE_C_COMPILER=mpicc \
-D CMAKE_CXX_COMPILER=mpicxx \
-D METIS_PATH=${metis} \
-D GKLIB_PATH=${metis}/GKlib \
..
#!/bin/bash
# Modify these paths for your system.
TOOLS=/gpfs/u/home/ADDM/ADDMgnzw/scratch-shared/tpls
MPIDIR=/gpfs/u/software/amd64-rhel6/mpi/openmpi/2.1.1_1/gcc-6.3.0_1/bin/mpicxx
TRIL_INSTALL_DIR=/gpfs/u/home/ADDM/ADDMgnzw/scratch-shared/trilinos
BOOSTDIR=$TOOLS
NETCDFDIR=$TOOLS
HDF5DIR=$TOOLS
ZLIBDIR=$TOOLS
PARMETISDIR=$TOOLS
cmake \
\
-D Trilinos_DISABLE_ENABLED_FORWARD_DEP_PACKAGES=ON \
-D CMAKE_INSTALL_PREFIX=$TRIL_INSTALL_DIR \
-D CMAKE_BUILD_TYPE=NONE \
-D TPL_ENABLE_MPI=ON \
-D MPI_BASE_DIR=$MPIDIR \
-D CMAKE_C_FLAGS="-O2 -g" \
-D CMAKE_CXX_FLAGS="-O2 -g -Wno-deprecated-declarations -Wno-sign-compare" \
-D CMAKE_VERBOSE_MAKEFILE=OFF \
-D BUILD_SHARED_LIBS=ON \
-D Trilinos_EXTRA_LINK_FLAGS="-ldl" \
-D Trilinos_ENABLE_SECONDARY_TESTED_CODE=ON \
-D CMAKE_INSTALL_RPATH_USE_LINK_PATH=True \
\
-D Trilinos_ENABLE_ALL_PACKAGES=OFF \
-D Trilinos_WARNINGS_AS_ERRORS_FLAGS="" \
-D Trilinos_ENABLE_Teuchos=ON \
-D Trilinos_ENABLE_Shards=ON \
-D Trilinos_ENABLE_Sacado=ON \
-D Trilinos_ENABLE_Epetra=ON \
-D Trilinos_ENABLE_EpetraExt=ON \
-D Trilinos_ENABLE_Ifpack=ON \
-D Trilinos_ENABLE_AztecOO=ON \
-D Trilinos_ENABLE_Amesos=ON \
-D Trilinos_ENABLE_Anasazi=ON \
-D Trilinos_ENABLE_Belos=ON \
-D Trilinos_ENABLE_ML=ON \
-D Trilinos_ENABLE_Phalanx=ON \
-D Trilinos_ENABLE_MiniTensor=ON \
-D Trilinos_ENABLE_NOX=ON \
-D Trilinos_ENABLE_Stratimikos=ON \
-D Trilinos_ENABLE_Thyra=ON \
-D Trilinos_ENABLE_Rythmos=ON \
-D Trilinos_ENABLE_Stokhos=OFF \
-D Trilinos_ENABLE_Piro=ON \
-D Trilinos_ENABLE_Teko=ON \
-D Trilinos_ENABLE_STKIO=ON \
-D Trilinos_ENABLE_STKMesh=ON \
-D Trilinos_ENABLE_Tpetra=ON \
-D Trilinos_ENABLE_Kokkos=ON \
-D Kokkos_ENABLE_Pthread=OFF \
-D HAVE_INTREPID_KOKKOSCORE=ON \
-D Trilinos_ENABLE_Ifpack2=ON \
-D Trilinos_ENABLE_Zoltan2=ON \
-D Trilinos_ENABLE_MueLu=ON \
\
-D Trilinos_ENABLE_Intrepid2=ON \
-D Trilinos_ENABLE_Intrepid=ON \
-D Intrepid2_ENABLE_KokkosDynRankView=ON \
\
-D Trilinos_ENABLE_Amesos2=ON \
-D Amesos2_ENABLE_KLU2=ON \
-D Amesos2_ENABLE_SuperLUDist=OFF \
-D Amesos2_ENABLE_Epetra=OFF \
\
-D Trilinos_ENABLE_SEACAS=ON \
-D Trilinos_ENABLE_SEACASIoss=ON \
-D Trilinos_ENABLE_SEACASExodus=ON \
-D SEACAS_ENABLE_SEACASSVDI=OFF \
-D Trilinos_ENABLE_SEACASFastq=OFF \
-D Trilinos_ENABLE_SEACASBlot=OFF \
-D Trilinos_ENABLE_SEACASPLT=OFF \
\
-D TPL_ENABLE_Matio=OFF \
-D TPL_ENABLE_X11=OFF \
\
-D TPL_ENABLE_Boost=ON \
-D Boost_INCLUDE_DIRS="$BOOSTDIR/include" \
-D Boost_LIBRARY_DIRS="$BOOSTDIR/lib" \
-D TPL_ENABLE_BoostLib=ON \
-D BoostLib_INCLUDE_DIRS="$BOOSTDIR/include" \
-D BoostLib_LIBRARY_DIRS="$BOOSTDIR/lib" \
\
-D TPL_ENABLE_Netcdf=ON \
-D Netcdf_INCLUDE_DIRS="$NETCDFDIR/include" \
-D Netcdf_LIBRARY_DIRS="$NETCDFDIR/lib" \
\
-D TPL_ENABLE_HDF5=ON \
-D HDF5_INCLUDE_DIRS="$HDF5DIR/include" \
-D HDF5_LIBRARY_DIRS="$HDF5DIR/lib" \
\
-D TPL_ENABLE_Zlib=ON \
-D Zlib_INCLUDE_DIRS="$ZLIBDIR/include" \
-D Zlib_LIBRARY_DIRS="$ZLIBDIR/lib" \
\
-D TPL_ENABLE_BLAS=ON \
-D TPL_BLAS_LIBRARIES="$TOOLS/lib/libblas.so" \
\
-D TPL_ENABLE_LAPACK=ON \
-D TPL_LAPACK_LIBRARIES="$TOOLS/lib/liblapack.so" \
\
-D TPL_ENABLE_ParMETIS=ON \
-D ParMETIS_INCLUDE_DIRS="$PARMETISDIR/include" \
-D ParMETIS_LIBRARY_DIRS="$PARMETISDIR/lib" \
\
-D TPL_ENABLE_METIS=ON \
-D METIS_INCLUDE_DIRS="$PARMETISDIR/include" \
-D METIS_LIBRARY_DIRS="$PARMETISDIR/lib" \
-D Zoltan_ENABLE_ULLONG_IDS=ON \
-D Teuchos_ENABLE_LONG_LONG_INT=ON \
\
-D Trilinos_ENABLE_EXPLICIT_INSTANTIATION=ON \
-D Tpetra_INST_FLOAT=OFF \
-D Tpetra_INST_INT_INT=ON \
-D Tpetra_INST_DOUBLE=ON \
-D Tpetra_INST_COMPLEX_FLOAT=OFF \
-D Tpetra_INST_COMPLEX_DOUBLE=OFF \
-D Tpetra_INST_INT_UNSIGNED=OFF \
-D Tpetra_INST_INT_LONG_LONG=ON \
\
..
TPL_INSTALL_DIR=/gpfs/u/home/ADDM/ADDMgnzw/scratch-shared/tpls
TRIL_INSTALL_DIR=/gpfs/u/home/ADDM/ADDMgnzw/scratch-shared/trilinos
CORE_INSTALL_DIR=/gpfs/u/home/ADDM/ADDMgnzw/scratch-shared/scorec
cmake \
-D CMAKE_C_COMPILER="mpicc" \
-D CMAKE_CXX_COMPILER="mpicxx" \
-D CMAKE_INSTALL_PREFIX=${CORE_INSTALL_DIR} \
-D BUILD_SHARED_LIBS=ON \
-D ENABLE_ZOLTAN=ON \
-D ZOLTAN_LIBRARY=${TRIL_INSTALL_DIR}/lib/libzoltan.so \
-D ZOLTAN_INCLUDE_DIR=${TRIL_INSTALL_DIR}/include \
-D PARMETIS_LIBRARY=${TPL_INSTALL_DIR}/lib/libparmetis.a \
-D METIS_LIBRARY=${TPL_INSTALL_DIR}/lib/libmetis.a \
-D PARMETIS_INCLUDE_DIR=${TPL_INSTALL_DIR}/include \
-D ENABLE_STK=ON \
-D ENABLE_STK_MESH=ON \
-D Trilinos_PREFIX=${TRIL_INSTALL_DIR} \
..
2>&1 | tee config_log
git clone https://github.com/SNLComputation/Albany
cd Albany
mkdir build
cd build
./do-config-albany.sh
where the ./do-config-albany.sh looks like:
(you will need to modify the ALB_DIR variable)
export TRI_DIR=/gpfs/u/home/ADDM/ADDMgnzw/scratch/trilinos/install
export ALB_DIR=/gpfs/u/home/ADDM/ADDMgnzw/barn-shared/albany
cmake \
-D ENABLE_INSTALL=ON \
-D CMAKE_INSTALL_PREFIX=${ALB_DIR} \
-D ALBANY_TRILINOS_DIR=${TRI_DIR} \
-D ENABLE_64BIT_INT=OFF \
-D ENABLE_SCOREC=ON \
-D ENABLE_AMP=ON \
-D ENABLE_LCM=ON \
-D ENABLE_CTM=ON \
-D ENABLE_CHECK_FPE=OFF \
-D ENABLE_ALBANY_EPETRA_EXE=OFF \
-D ENABLE_LCM_SPECULATIVE=OFF \
-D ENABLE_HYDRIDE=OFF \
-D ENABLE_SG_MP=OFF \
-D ENABLE_QCAD=OFF \
-D ENABLE_MOR=OFF \
-D ENABLE_STOKHOS=OFF \
-D ENABLE_FELIX=OFF \
../
Before submitting a job ensure you have ssh'd to either drpfen01 or drpfen02 and loaded the modules here
sbatch -t 30 -p debug -n 1 srun.sh
This is what srun.sh contains:
#!/bin/bash -x
# dirty hack
ln -s /usr/lib64/libslurm.so.30 libslurm.so.26
export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$PWD
export LD_LIBRARY_PATH=/gpfs/u/home/ADDM/ADDMgnzw/barn-shared/albany/lib:$LD_LIBRARY_PATH
export LD_LIBRARY_PATH=/gpfs/u/home/ADDM/ADDMgnzw/scratch/tpls/lib:$LD_LIBRARY_PATH
export LD_LIBRARY_PATH=/gpfs/u/home/ADDM/ADDMgnzw/scratch/trilinos/install/lib:$LD_LIBRARY_PATH
export ALB=/gpfs/u/home/ADDM/ADDMgnzw/barn-shared/albany/bin/AlbanyT
srun hostname -s > /tmp//hosts.$SLURM_JOB_ID
if [ "x$SLURM_NPROCS" = "x" ]
then
if [ "x$SLURM_NTASKS_PER_NODE" = "x" ]
then
SLURM_NTASKS_PER_NODE=1
fi
SLURM_NPROCS=`expr $SLURM_JOB_NUM_NODES \* $SLURM_NTASKS_PER_NODE`
fi
mpirun -hostfile /tmp/hosts.$SLURM_JOB_ID --bind-to core -np $SLURM_NPROCS ${ALB} input.xml
rm /tmp/hosts.$SLURM_JOB_ID