-
Notifications
You must be signed in to change notification settings - Fork 89
Building Albany on CCI DRP
Brian Granzow edited this page Mar 27, 2017
·
12 revisions
module purge
module load cmake/3.3.2
module load gcc/4.9.1_1
module load openmpi/1.8.1_1
module load simmetrix/simModSuite/11.0-170217dev
tarballs/config scripts of all TPLs in /gpfs/u/home/ADDM/ADDMgnzw/barn-shared/tpls
export tpls=/gpfs/u/home/ADDM/ADDMgnzw/scratch/tpls
CC=mpicc CFLAGS="-O3 -fPIC" ./configure --64 --prefix=${tpls}
export tpls=/gpfs/u/home/ADDM/ADDMgnzw/scratch/tpls
./configure CC=mpicc CXX=mpicxx FC=mpif90 \
CFLAGS="-O3 -fPIC -std=c99" CXXFLAGS="-O3 -fPIC" FCFLAGS="-O3 -fPIC" \
--enable-parallel \
--with-zlib=${tpls} \
--prefix=${tpls}
export tpls=/gpfs/u/home/ADDM/ADDMgnzw/scratch/tpls
./configure CC=mpicc CXX=mpicxx FC=mpif90 \
CFLAGS="-fPIC -I${tpls}/include -O3 -march=native" \
CXXFLAGS="-fPIC -I${tpls}/include -O3 -march=native" \
FCFLAGS="-fPIC -I${tpls}/include -O3 -march=native" \
LDFLAGS="-fPIC -L${tpls}/lib -O3 -march=native" \
--prefix=${tpls} \
--disable-doxygen \
--enable-netcdf4
export tpls=/gpfs/u/home/ADDM/ADDMgnzw/scratch/tpls
echo "using gcc : 4.9.1_1 : /gpfs/u/software/amd64-rhel6/compilers/gcc/4.9.1_1/bin/gcc ;" \
>> ./tools/build/v2/user-config.jam
echo "using mpi : /gpfs/u/software/amd64-rhel6/mpi/openmpi/1.8.1_1/gcc-4.9.1_1/bin/mpicxx ;" \
>> ./tools/build/v2/user-config.jam
./bootstrap.sh \
--with-libraries=signals,regex,filesystem,system,mpi,serialization,thread,program_options,exception \
--prefix=${tpls}
export tpls=/gpfs/u/home/ADDM/ADDMgnzw/scratch/tpls
ccmake -DCMAKE_INSTALL_PREFIX=${tpls} ..
export tpls=/gpfs/u/home/ADDM/ADDMgnzw/scratch/tpls
export metis=/gpfs/u/home/ADDM/ADDMgnzw/scratch/parmetis-4.0.3/metis
cmake \
-D CMAKE_INSTALL_PREFIX=${tpls} \
-D CMAKE_C_COMPILER=mpicc \
-D CMAKE_CXX_COMPILER=mpicxx \
-D METIS_PATH=${metis} \
-D GKLIB_PATH=${metis}/GKlib \
cmake .. \
-DCMAKE_CXX_COMPILER=mpicxx \
-DCMAKE_INSTALL_PREFIX=/gpfs/u/home/ADDM/ADDMgnzw/scratch/tpls \
-DCMAKE_BUILD_TYPE=RelWithDebInfo \
-DBUILD_SHARED_LIBS=ON \
-DYAML_CPP_BUILD_TOOLS=OFF \
2>&1 | tee config_log
#!/bin/bash
# Modify these paths for your system.
TOOLS=/gpfs/u/home/ADDM/ADDMgnzw/scratch/tpls
MPIDIR=/gpfs/u/software/amd64-rhel6/mpi/openmpi/1.8.1_1/gcc-4.9.1_1/bin/mpicxx
TRIL_INSTALL_DIR=../install
BOOSTDIR=$TOOLS
NETCDFDIR=$TOOLS
HDF5DIR=$TOOLS
ZLIBDIR=$TOOLS
PARMETISDIR=$TOOLS
SIM_INCLUDE=$SIM_INSTALL_DIR/include
SIM_LIB=$SIM_INSTALL_DIR/lib/x64_rhel5_gcc41
HAVE_LL=OFF
cmake \
\
-D Trilinos_DISABLE_ENABLED_FORWARD_DEP_PACKAGES=ON \
-D CMAKE_INSTALL_PREFIX=$TRIL_INSTALL_DIR \
-D CMAKE_BUILD_TYPE=NONE \
-D TPL_ENABLE_MPI=ON \
-D MPI_BASE_DIR=$MPIDIR \
-D CMAKE_C_FLAGS="-O2 -g" \
-D CMAKE_CXX_FLAGS="-O2 -g -Wno-deprecated-declarations -Wno-sign-compare" \
-D CMAKE_VERBOSE_MAKEFILE=OFF \
-D BUILD_SHARED_LIBS=ON \
-D Trilinos_EXTRA_LINK_FLAGS="-ldl" \
-D Trilinos_ENABLE_SECONDARY_TESTED_CODE=ON \
-D CMAKE_INSTALL_RPATH_USE_LINK_PATH=True \
\
-D Trilinos_ENABLE_ALL_PACKAGES=OFF \
-D Trilinos_WARNINGS_AS_ERRORS_FLAGS="" \
-D Trilinos_ENABLE_Teuchos=ON \
-D Trilinos_ENABLE_Shards=ON \
-D Trilinos_ENABLE_Sacado=ON \
-D Trilinos_ENABLE_Epetra=ON \
-D Trilinos_ENABLE_EpetraExt=ON \
-D Trilinos_ENABLE_Ifpack=ON \
-D Trilinos_ENABLE_AztecOO=ON \
-D Trilinos_ENABLE_Amesos=ON \
-D Trilinos_ENABLE_Anasazi=ON \
-D Trilinos_ENABLE_Belos=ON \
-D Trilinos_ENABLE_ML=ON \
-D Trilinos_ENABLE_Phalanx=ON \
-D Trilinos_ENABLE_MiniTensor=ON \
-D Trilinos_ENABLE_NOX=ON \
-D Trilinos_ENABLE_Stratimikos=ON \
-D Trilinos_ENABLE_Thyra=ON \
-D Trilinos_ENABLE_Rythmos=ON \
-D Trilinos_ENABLE_Stokhos=OFF \
-D Trilinos_ENABLE_Piro=ON \
-D Trilinos_ENABLE_Teko=ON \
-D Trilinos_ENABLE_STKIO=ON \
-D Trilinos_ENABLE_STKMesh=ON \
-D Trilinos_ENABLE_Tpetra=ON \
-D Trilinos_ENABLE_Kokkos=ON \
-D Kokkos_ENABLE_Pthread=OFF \
-D HAVE_INTREPID_KOKKOSCORE=ON \
-D Trilinos_ENABLE_Ifpack2=ON \
-D Trilinos_ENABLE_Zoltan2=ON \
-D Trilinos_ENABLE_MueLu=ON \
\
-D Trilinos_ENABLE_Intrepid2=ON \
-D Trilinos_ENABLE_Intrepid=ON \
-D Intrepid2_ENABLE_KokkosDynRankView=ON \
\
-D Trilinos_ENABLE_Amesos2=ON \
-D Amesos2_ENABLE_KLU2=ON \
-D Amesos2_ENABLE_SuperLUDist=OFF \
-D Amesos2_ENABLE_Epetra=OFF \
\
-D Trilinos_ENABLE_SEACAS=ON \
-D Trilinos_ENABLE_SEACASIoss=ON \
-D Trilinos_ENABLE_SEACASExodus=ON \
-D SEACAS_ENABLE_SEACASSVDI=OFF \
-D Trilinos_ENABLE_SEACASFastq=OFF \
-D Trilinos_ENABLE_SEACASBlot=OFF \
-D Trilinos_ENABLE_SEACASPLT=OFF \
\
-D TPL_ENABLE_Matio=OFF \
-D TPL_ENABLE_X11=OFF \
\
-D TPL_ENABLE_Boost=ON \
-D Boost_INCLUDE_DIRS="$BOOSTDIR/include" \
-D Boost_LIBRARY_DIRS="$BOOSTDIR/lib" \
-D TPL_ENABLE_BoostLib=ON \
-D BoostLib_INCLUDE_DIRS="$BOOSTDIR/include" \
-D BoostLib_LIBRARY_DIRS="$BOOSTDIR/lib" \
\
-D TPL_ENABLE_Netcdf=ON \
-D Netcdf_INCLUDE_DIRS="$NETCDFDIR/include" \
-D Netcdf_LIBRARY_DIRS="$NETCDFDIR/lib" \
\
-D TPL_ENABLE_HDF5=ON \
-D HDF5_INCLUDE_DIRS="$HDF5DIR/include" \
-D HDF5_LIBRARY_DIRS="$HDF5DIR/lib" \
\
-D TPL_ENABLE_Zlib=ON \
-D Zlib_INCLUDE_DIRS="$ZLIBDIR/include" \
-D Zlib_LIBRARY_DIRS="$ZLIBDIR/lib" \
\
-D TPL_ENABLE_BLAS=ON \
-D TPL_BLAS_LIBRARIES="$TOOLS/lib/libblas.so" \
\
-D TPL_ENABLE_LAPACK=ON \
-D TPL_LAPACK_LIBRARIES="$TOOLS/lib/liblapack.so" \
\
-D Trilinos_ENABLE_SCOREC=ON \
-D Trilinos_ENABLE_EXPORT_MAKEFILES=OFF \
-D Trilinos_ASSERT_MISSING_PACKAGES=OFF \
\
-D TPL_ENABLE_ParMETIS=ON \
-D ParMETIS_INCLUDE_DIRS="$PARMETISDIR/include" \
-D ParMETIS_LIBRARY_DIRS="$PARMETISDIR/lib" \
\
-D TPL_ENABLE_METIS=ON \
-D METIS_INCLUDE_DIRS="$PARMETISDIR/include" \
-D METIS_LIBRARY_DIRS="$PARMETISDIR/lib" \
\
-D Zoltan_ENABLE_ULLONG_IDS=ON \
-D Teuchos_ENABLE_LONG_LONG_INT=$HAVE_LL \
\
-D Trilinos_ENABLE_EXPLICIT_INSTANTIATION=ON \
-D Tpetra_INST_FLOAT=OFF \
-D Tpetra_INST_INT_INT=ON \
-D Tpetra_INST_DOUBLE=ON \
-D Tpetra_INST_COMPLEX_FLOAT=OFF \
-D Tpetra_INST_COMPLEX_DOUBLE=OFF \
-D Tpetra_INST_INT_LONG=OFF \
-D Tpetra_INST_INT_UNSIGNED=OFF \
-D Tpetra_INST_INT_LONG_LONG=$HAVE_LL \
\
-D SimMesh_INCLUDE_DIRS=$SIM_INCLUDE \
-D SimMesh_LIBRARY_DIRS=$SIM_LIB \
-D SimModel_INCLUDE_DIRS=$SIM_INCLUDE \
-D SimModel_LIBRARY_DIRS=$SIM_LIB \
-D TPL_ENABLE_SimParasolid=True \
-D SimParasolid_INCLUDE_DIRS=$SIM_INCLUDE \
-D SimParasolid_LIBRARY_DIRS="$SIM_LIB;$SIM_LIB/psKrnl" \
-D TPL_ENABLE_SimAcis=True \
-D SimAcis_INCLUDE_DIRS=$SIM_INCLUDE \
-D SimAcis_LIBRARY_DIRS="$SIM_LIB;$SIM_LIB/acisKrnl" \
-D TPL_ENABLE_SimField=True \
-D SimField_INCLUDE_DIRS=$SIM_INCLUDE \
-D SimField_LIBRARY_DIRS=$SIM_LIB \
-D SIM_MPI='openmpi110' \
-D SIM_BASEMPI='mpi' \
\
..
git clone https://github.com/gahansen/Albany
cd Albany
mkdir build
cd build
./do-config-albany.sh
where the ./do-config-albany.sh looks like:
(you will need to modify the ALB_DIR variable)
export TRI_DIR=/gpfs/u/home/ADDM/ADDMgnzw/scratch/trilinos/install
export ALB_DIR=/gpfs/u/home/ADDM/ADDMgnzw/barn-shared/albany
cmake \
-D ENABLE_INSTALL=ON \
-D CMAKE_INSTALL_PREFIX=${ALB_DIR} \
-D ALBANY_TRILINOS_DIR=${TRI_DIR} \
-D ENABLE_64BIT_INT=OFF \
-D ENABLE_SCOREC=ON \
-D ENABLE_AMP=ON \
-D ENABLE_LCM=ON \
-D ENABLE_CTM=ON \
-D ENABLE_CHECK_FPE=OFF \
-D ENABLE_ALBANY_EPETRA_EXE=OFF \
-D ENABLE_LCM_SPECULATIVE=OFF \
-D ENABLE_HYDRIDE=OFF \
-D ENABLE_SG_MP=OFF \
-D ENABLE_QCAD=OFF \
-D ENABLE_MOR=OFF \
-D ENABLE_STOKHOS=OFF \
-D ENABLE_FELIX=OFF \
../
Before submitting a job ensure you have ssh'd to either drpfen01 or drpfen02 and loaded the modules here
sbatch -t 30 -p debug -n 1 srun.sh
This is what srun.sh contains:
#!/bin/bash -x
# dirty hack
ln -s /usr/lib64/libslurm.so.30 libslurm.so.26
export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$PWD
export LD_LIBRARY_PATH=/gpfs/u/home/ADDM/ADDMgnzw/barn-shared/albany/lib:$LD_LIBRARY_PATH
export LD_LIBRARY_PATH=/gpfs/u/home/ADDM/ADDMgnzw/scratch/tpls/lib:$LD_LIBRARY_PATH
export LD_LIBRARY_PATH=/gpfs/u/home/ADDM/ADDMgnzw/scratch/trilinos/install/lib:$LD_LIBRARY_PATH
export ALB=/gpfs/u/home/ADDM/ADDMgnzw/barn-shared/albany/bin/AlbanyT
srun hostname -s > /tmp//hosts.$SLURM_JOB_ID
if [ "x$SLURM_NPROCS" = "x" ]
then
if [ "x$SLURM_NTASKS_PER_NODE" = "x" ]
then
SLURM_NTASKS_PER_NODE=1
fi
SLURM_NPROCS=`expr $SLURM_JOB_NUM_NODES \* $SLURM_NTASKS_PER_NODE`
fi
mpirun -hostfile /tmp/hosts.$SLURM_JOB_ID --bind-to core -np $SLURM_NPROCS ${ALB} input.xml
rm /tmp/hosts.$SLURM_JOB_ID