From 2eb212e6e250c9cc6d4580064fcd32ff154db69a Mon Sep 17 00:00:00 2001 From: Ray Culbertson Date: Sat, 17 Jul 2021 14:52:48 -0500 Subject: [PATCH] updates for Muse --- bin/valJobBuild.sh | 48 ++++++++++-------- bin/valJobCheck.sh | 67 +++++++++++--------------- bin/valJobMaster.sh | 115 ++++++++++++++++++++++++++++++-------------- bin/valJobNode.sh | 82 ++++++++++++++++++++++--------- 4 files changed, 193 insertions(+), 119 deletions(-) diff --git a/bin/valJobBuild.sh b/bin/valJobBuild.sh index 1c6bcba..0ffa224 100755 --- a/bin/valJobBuild.sh +++ b/bin/valJobBuild.sh @@ -9,8 +9,9 @@ echo_date() { echo "[$(date)] $*" } -echo_date "cd" -WORKDIR="$1" +echo_date "start build WORKDIR=$1 TBALL=$2" +echo_date "cd $1" +WORKDIR=$1 shift if [ -z "$WORKDIR" ]; then echo "ERROR - no work dir provided - exit" @@ -33,39 +34,40 @@ fi echo_date "general setups" source /cvmfs/fermilab.opensciencegrid.org/products/common/etc/setups source /cvmfs/mu2e.opensciencegrid.org/setupmu2e-art.sh +setup muse T0=$(date +%s) echo_date "clone offline" -#git clone -q http://cdcvs.fnal.gov/projects/mu2eofflinesoftwaremu2eoffline/Offline.git git clone -q https://github.com/mu2e/Offline RC=$? T1=$(date +%s) DT_CLONE=$(($T1-$T0)) echo_date "clone return code $RC time $DT_CLONE s" echo "REPORT TIME clone $DT_CLONE" +git clone -q https://github.com/mu2e/Production +RCP=$? +RC=$(($RC+$RCP)) if [ $RC -ne 0 ]; then echo "REPORT STATUS FAIL build" exit $RC fi -echo_date "cd Offline" -cd Offline - echo_date "print commit" -git show -q -git rev-parse HEAD +git -C Offline show -q +git -C Offline rev-parse HEAD -echo_date "source setup" -source setup.sh +echo_date "muse setup" +muse setup -1 +muse status -echo_date "start scons" +echo_date "start build" T0=$(date +%s) -scons -j 20 +muse build -j 20 --mu2eCompactPrint RC=$? T1=$(date +%s) DT_BUILD=$(($T1-$T0)) -echo_date "scons return code $RC time $DT_BUILD s" +echo_date "build return code $RC time $DT_BUILD s" echo "REPORT TIME build $DT_BUILD" if [ $RC -ne 0 ]; then @@ -77,15 +79,18 @@ echo_date "starting tarball" T0=$(date +%s) cp /mu2e/app/home/mu2epro/cron/val/seeds.txt . cp /mu2e/app/home/mu2epro/cron/val/recoInputFiles.txt . -cd .. -tar --exclude="*.cc" --exclude="*.os" --exclude="Offline/tmp/*" \ - -czf code.tgz Offline +#tar --exclude="*.cc" --exclude="*.os" --exclude="$MUSE_BUILD_BASE/Offline/tmp/*" \ +# -czf code.tgz Offline build *.txt +TEMPBALL=$( muse tarball recoInputFiles.txt seeds.txt | grep "Tarball:" | awk '{print $2}' ) RC=$? T1=$(date +%s) DT_TAR=$(($T1-$T0)) -ls -l code.tgz + +echo_date "ls -l $TEMPBALL" +echo_date ls -l $TEMPBALL +ls -l $TEMPBALL echo_date "tar return code $RC time $DT_TAR s" echo "REPORT TIME tar $DT_TAR" @@ -95,7 +100,9 @@ if [ $RC -ne 0 ]; then fi [ -f $TBALL ] && mv $TBALL ${TBALL}_$(date +%s) -cp code.tgz $TBALL +echo_date "cp $TEMPBALL $TBALL" +echo_date cp $TEMPBALL $TBALL +cp $TEMPBALL $TBALL RC=$? ls -l $TBALL echo_date "copy tarball $RC" @@ -105,12 +112,15 @@ if [ $RC -ne 0 ]; then exit $RC fi +# keep from building up tarballs in muse temp area +rm -rf $(dirname $TEMPBALL) + # # remove older tarballs # DD=$(dirname $TBALL) N=$(ls -1 $DD/* | wc -l) -if [ $N -gt 5 ]; then +if [ $N -gt 10 ]; then NRM=$(($N-5)) FILES=$( ls -1 $DD/* | head -$NRM ) for FF in $FILES diff --git a/bin/valJobCheck.sh b/bin/valJobCheck.sh index 41b6388..36250f7 100755 --- a/bin/valJobCheck.sh +++ b/bin/valJobCheck.sh @@ -26,21 +26,16 @@ RC=0 echo_date "general setups" source /cvmfs/fermilab.opensciencegrid.org/products/common/etc/setups source /cvmfs/mu2e.opensciencegrid.org/setupmu2e-art.sh +setup muse setup codetools - -echo_date "cd Offline" -cd Offline - -echo_date "source setup" -source setup.sh - +muse setup -1 echo_date "start root overlaps" rootOverlaps.sh RCT=${PIPESTATUS[0]} echo_date "root overlaps return code $RCT" -LEGAL=$( grep CloseGeometry ../check | awk '{print $4 " " $5 " " $6 " " $7 " " $8}' ) -ILLEGAL=$( grep illegal ../check | awk '{print $NF}' ) +LEGAL=$( grep CloseGeometry check | awk '{print $4 " " $5 " " $6 " " $7 " " $8}' ) +ILLEGAL=$( grep illegal check | awk '{print $NF}' ) if [[ $RCT -eq 0 && $ILLEGAL -eq 0 ]]; then STATUS=OK else @@ -52,7 +47,7 @@ RC=$(($RC+$RCT)) echo_date "start transportOnly" -mu2e -n 5 -c Mu2eG4/fcl/transportOnly.fcl +mu2e -n 5 -c Offline/Mu2eG4/fcl/transportOnly.fcl RCT=$? echo_date "transportOnly return code $RCT" if [ $RCT -eq 0 ]; then @@ -65,7 +60,7 @@ RC=$(($RC+$RCT)) echo_date "start g4study2" -mu2e -n 5 -c Mu2eG4/g4study/g4study.fcl +mu2e -n 5 -c Offline/Mu2eG4/g4study/g4study.fcl RCT=$? echo_date "g4study2 return code $RCT" if [ $RCT -eq 0 ]; then @@ -78,7 +73,7 @@ RC=$(($RC+$RCT)) echo_date "start g4test_03MT" -mu2e -n 20 -c Mu2eG4/fcl/g4test_03MT.fcl +mu2e -n 20 -c Offline/Mu2eG4/fcl/g4test_03MT.fcl RCT=$? echo_date "g4test_03MT return code $RCT" if [ $RCT -eq 0 ]; then @@ -93,7 +88,7 @@ RC=$(($RC+$RCT)) # 4/2021 MDC2020 production sequence echo_date "start ceSteps" -mu2e -n 50 -c Validation/test/ceSteps.fcl +mu2e -n 50 -c Production/Validation/ceSteps.fcl RCT=$? echo_date "ceSteps return code $RCT" if [ $RCT -eq 0 ]; then @@ -107,7 +102,7 @@ RC=$(($RC+$RCT)) echo_date "start ceDigi" # takes ceSteps as input -mu2e -c Validation/test/ceDigi.fcl +mu2e -c Production/Validation/ceDigi.fcl RCT=$? echo_date "ceDigi return code $RCT" if [ $RCT -eq 0 ]; then @@ -119,7 +114,7 @@ echo "REPORT STATUS $STATUS ceDigi" RC=$(($RC+$RCT)) echo_date "start muDauSteps" -mu2e -n 6000 -c Validation/test/muDauSteps.fcl +mu2e -n 6000 -c Production/Validation/muDauSteps.fcl RCT=$? echo_date "muDauSteps return code $RCT" if [ $RCT -eq 0 ]; then @@ -130,10 +125,9 @@ fi echo "REPORT STATUS $STATUS muDauSteps" RC=$(($RC+$RCT)) -# disabled until it runs.. echo_date "start ceMix" # takes ceSteps and muDauSteps as input -mu2e -c Validation/test/ceMix.fcl +mu2e -c Production/Validation/ceMix.fcl RCT=$? echo_date "ceMix return code $RCT" if [ $RCT -eq 0 ]; then @@ -144,35 +138,28 @@ fi echo "REPORT STATUS $STATUS ceMix" RC=$(($RC+$RCT)) - - -# 11/2/20 paused cemixdigi due to no mising file with new code -# echo_date "start ceMixDigi" -# mu2e -n 10 -c Validation/fcl/ceMixDigi.fcl >& ceMixDigi.log -# RCT=$? -# cat ceMixDigi.log -# echo_date "ceMixDigi return code $RCT" -# if [ $RCT -eq 0 ]; then -# STATUS=OK -# else -# STATUS=FAILED -# fi -# echo "REPORT STATUS $STATUS ceMixDigi" -# MIXCPU=$( grep "TimeReport CPU" ceMixDigi.log | awk '{print int($4)}') -# MIXMEM=$( grep "VmPeak" ceMixDigi.log | awk '{print int($4)}') -# MIXSIZ=$( ls -l dig.owner.val-ceMixDigi.dsconf.seq.art | awk '{print $5}') -# echo "REPORT EXE ceMixDigi $MIXCPU $MIXMEM $MIXSIZ" -# RC=$(($RC+$RCT)) +echo_date "start cosmicOffSpill" +# takes an explicit input file, in fcl +mu2e -n 20 -c Production/Validation/cosmicOffSpill.fcl +RCT=$? +echo_date "cosmicOffSpill return code $RCT" +if [ $RCT -eq 0 ]; then + STATUS=OK +else + STATUS=FAILED +fi +echo "REPORT STATUS $STATUS cosmicOffSpill" +RC=$(($RC+$RCT)) echo_date "start geant surfaceCheck" -mu2e -c Mu2eG4/fcl/surfaceCheck.fcl +mu2e -c Offline/Mu2eG4/fcl/surfaceCheck.fcl RCT=$? echo_date "geant surfaceCheck return code $RCT" -LEGAL=$( grep 'Checking overlaps for volume' ../check | grep -c OK ) -ILLEGAL=$( grep 'Checking overlaps for volume' ../check | grep -v OK | wc -l ) +LEGAL=$( grep 'Checking overlaps for volume' check | grep -c OK ) +ILLEGAL=$( grep 'Checking overlaps for volume' check | grep -v OK | wc -l ) # print overlaps into the log -grep 'Checking overlaps for volume' ../check | grep -v OK +grep 'Checking overlaps for volume' check | grep -v OK if [[ $RCT -eq 0 && $LEGAL -gt 0 && $ILLEGAL -eq 0 ]]; then STATUS=OK else diff --git a/bin/valJobMaster.sh b/bin/valJobMaster.sh index 76083d3..f7e55a8 100755 --- a/bin/valJobMaster.sh +++ b/bin/valJobMaster.sh @@ -11,7 +11,7 @@ echo "[$(date)] $*" define_surfaceCheck() { LABEL[$NPROJ]=surfaceCheck TARBALL[$NPROJ]=$TBALL - FCL[$NPROJ]=Mu2eG4/fcl/surfaceCheck.fcl + FCL[$NPROJ]=Offline/Mu2eG4/fcl/surfaceCheck.fcl INPUT[$NPROJ]=NULL NINPUT[$NPROJ]=0 NEV[$NPROJ]=1 @@ -21,6 +21,7 @@ define_surfaceCheck() { JID[$NPROJ]=0 SEEDS[$NPROJ]=no WATCHDOG[$NPROJ]=no + DATABASE[$NPROJ]=no CHECK[$NPROJ]=overLaps STATUS[$NPROJ]="defined" CSTATUS[$NPROJ]="undefined" @@ -33,7 +34,7 @@ define_surfaceCheck() { define_potSim() { LABEL[$NPROJ]=potSim TARBALL[$NPROJ]=$TBALL - FCL[$NPROJ]=Validation/fcl/potSim.fcl + FCL[$NPROJ]=Production/Validation/potSim.fcl INPUT[$NPROJ]=NULL NINPUT[$NPROJ]=0 NEV[$NPROJ]=500 @@ -43,6 +44,7 @@ define_potSim() { JID[$NPROJ]=0 SEEDS[$NPROJ]=yes WATCHDOG[$NPROJ]=no + DATABASE[$NPROJ]=no CHECK[$NPROJ]=statPlots STATUS[$NPROJ]="defined" CSTATUS[$NPROJ]="undefined" @@ -54,7 +56,7 @@ define_potSim() { define_ceSimReco() { LABEL[$NPROJ]=ceSimReco TARBALL[$NPROJ]=$TBALL - FCL[$NPROJ]=Validation/fcl/ceSimReco.fcl + FCL[$NPROJ]=Production/Validation/ceSimReco.fcl INPUT[$NPROJ]=NULL NINPUT[$NPROJ]=0 NEV[$NPROJ]=5000 @@ -64,6 +66,7 @@ define_ceSimReco() { JID[$NPROJ]=0 SEEDS[$NPROJ]=yes WATCHDOG[$NPROJ]=no + DATABASE[$NPROJ]=no CHECK[$NPROJ]=statPlots CSTATUS[$NPROJ]="undefined" STATUS[$NPROJ]="defined" @@ -75,7 +78,7 @@ define_ceSimReco() { define_cosmicSimReco() { LABEL[$NPROJ]=cosmicSimReco TARBALL[$NPROJ]=$TBALL - FCL[$NPROJ]=Validation/fcl/cosmicSimReco.fcl + FCL[$NPROJ]=Production/Validation/cosmicSimReco.fcl INPUT[$NPROJ]=NULL NINPUT[$NPROJ]=0 NEV[$NPROJ]=50000 @@ -85,6 +88,7 @@ define_cosmicSimReco() { JID[$NPROJ]=0 SEEDS[$NPROJ]=yes WATCHDOG[$NPROJ]=no + DATABASE[$NPROJ]=no CHECK[$NPROJ]=statPlots STATUS[$NPROJ]="defined" CSTATUS[$NPROJ]="undefined" @@ -96,9 +100,9 @@ define_cosmicSimReco() { define_reco() { LABEL[$NPROJ]=reco TARBALL[$NPROJ]=$TBALL - FCL[$NPROJ]=Validation/fcl/reco.fcl + FCL[$NPROJ]=Production/Validation/reco.fcl INPUT[$NPROJ]=recoInputFiles.txt - NINPUT[$NPROJ]=4 + NINPUT[$NPROJ]=1 NEV[$NPROJ]=999999 NJOB[$NPROJ]=40 MEM[$NPROJ]=1980MB @@ -106,6 +110,7 @@ define_reco() { JID[$NPROJ]=0 SEEDS[$NPROJ]=no WATCHDOG[$NPROJ]=yes + DATABASE[$NPROJ]=no CHECK[$NPROJ]=statPlots STATUS[$NPROJ]="defined" CSTATUS[$NPROJ]="undefined" @@ -125,25 +130,27 @@ build_code() { return 1 fi DD=$(dirname $BUILD_DIR) - rm -rf $DD/05 - mv $DD/04 $DD/05 - mv $DD/03 $DD/04 - mv $DD/02 $DD/03 - mv $DD/01 $DD/02 - mv $DD/current $DD/01 + if [ -z "$DEBUG" ]; then + rm -rf $DD/05 + mv $DD/04 $DD/05 + mv $DD/03 $DD/04 + mv $DD/02 $DD/03 + mv $DD/01 $DD/02 + mv $DD/current $DD/01 + fi mkdir -p $BUILD_DIR - ssh -n mu2ebuild01 "$HOME/cron/val/valJobBuild.sh $BUILD_DIR $TBALL >& $BUILD_DIR/build" + ssh -n mu2ebuild01 "$HOME/cron/val/valJobBuild.sh $BUILD_DIR $TBALL >& $BUILD_DIR/buildlog" RC=$? # this includes the tar command if [ $RC -ne 0 ]; then echo_date "ERROR in build $RC" - tail -20 $BUILD_DIR/build + tail -20 $BUILD_DIR/buildlog echo "FAIL build" >> $REPORT else echo_date "Done build" echo "OK build" >> $REPORT fi - grep REPORT $BUILD_DIR/build | sed 's/REPORT //' >> $WEBREPORT + grep REPORT $BUILD_DIR/buildlog | sed 's/REPORT //' >> $WEBREPORT return $RC } @@ -211,6 +218,7 @@ submit_jobs() { CMD="$CMD -e VALJOB_LABEL=${LABEL[$I]} " CMD="$CMD -e VALJOB_SEEDS=${SEEDS[$I]} " CMD="$CMD -e VALJOB_WATCHDOG=${WATCHDOG[$I]} " + CMD="$CMD -e VALJOB_DATABASE=${DATABASE[$I]} " CMD="$CMD -e VALJOB_OUTDIR=$VALJOB_OUTDIR " CMD="$CMD -e VALJOB_NEV=${NEV[$I]} " CMD="$CMD file://$PWD/$SCRIPT" @@ -296,7 +304,8 @@ recover_jobs() { fi NREC=$(( $NREC + 1 )) - local RDIR=$BUILD_DIR/rec_${LABEL[$I]}_${TT} + local RSDIR=rec_${LABEL[$I]}_${TT} + local RDIR=$BUILD_DIR/$RSDIR mkdir $RDIR local CMDF=$RDIR/rec.sh touch $CMDF @@ -309,11 +318,12 @@ recover_jobs() { echo "export VALJOB_NINPUT=${NINPUT[$I]} " >> $CMDF echo "export VALJOB_SEEDS=${SEEDS[$I]} " >> $CMDF echo "export VALJOB_WATCHDOG=no " >> $CMDF + echo "export VALJOB_DATABASE=${DATABASE[$I]} " >> $CMDF echo "export VALJOB_OUTDIR=$OUTDIR/${LABEL[$I]} " >> $CMDF echo "export VALJOB_NEV=${NEV[$I]} " >> $CMDF - echo "cd $RDIR " >> $CMDF - echo "ln -s ../Offline " >> $CMDF - echo "/mu2e/app/home/mu2epro/cron/val/valJobNode.sh >& log" >> $CMDF + echo "export VALJOB_RSDIR=$RSDIR " >> $CMDF + echo "cd $BUILD_DIR " >> $CMDF + echo "/mu2e/app/home/mu2epro/cron/val/valJobNode.sh >& $RDIR/log" >> $CMDF ssh -n -f mu2ebuild01 "$CMDF" fi @@ -396,7 +406,7 @@ wait_jobs() { # # analyze logs and return -# average median lowest highest +# average median highest lowest ana_numbers() { local TMP=$1 local TMP2=$( mktemp ) @@ -417,7 +427,7 @@ ana_numbers() { # # -# +#order CPU Wall memory ana_logs() { local I=$1 local DD=$OUTDIR/${LABEL[$I]}/log @@ -587,7 +597,7 @@ nightlyweb() { df -h fi cp $REPORT $WEB_DIR_DAY - cp $BUILD_DIR/build $WEB_DIR_DAY + cp $BUILD_DIR/buildlog $WEB_DIR_DAY cp $BUILD_DIR/check $WEB_DIR_DAY local CPWD=$PWD @@ -621,7 +631,9 @@ nightlyweb() { STAT=$( grep build $DDR/valJobWeb.txt | grep STATUS | awk '{print $2}') [ -z "$STAT" ] && STAT="-" colorByStat "$STAT" - if [ -r $DDR/build ]; then + if [ -r $DDR/buildlog ]; then + echo "log" >> $TF + elif [ -r $DDR/build ]; then echo "log" >> $TF else echo "log" >> $TF @@ -702,7 +714,7 @@ exit_proc() { local RESULT="SUCCESS" [ -n "`grep FAIL $REPORT`" ] && RESULT=FAIL cat $REPORT | mail -r valJob -s "valJob $COMPLETE and $RESULT" \ -rlc@fnal.gov,genser@fnal.gov,kutschke@fnal.gov,dave_brown@lbl.gov,david.brown@louisville.edu,gandr@fnal.gov,murat@fnal.gov,gianipez@fnal.gov,echenard@fnal.gov,ehrlich@virginia.edu +rlc@fnal.gov,genser@fnal.gov,kutschke@fnal.gov,dave_brown@lbl.gov,david.brown@louisville.edu,gandr@fnal.gov,murat@fnal.gov,gianipez@fnal.gov,echenard@fnal.gov,ehrlich@virginia.edu,macndev@fnal.gov,hcasler@fnal.gov # rlc@fnal.gov exit $RC } @@ -714,10 +726,19 @@ rlc@fnal.gov,genser@fnal.gov,kutschke@fnal.gov,dave_brown@lbl.gov,david.brown@lo # # + +# +# if there is a argument, then being run as a test, +# write t oa test area and do not run the web part +# +DEBUG="$1" +shift + echo_date "start setups" cd $HOME/cron/val source /cvmfs/fermilab.opensciencegrid.org/products/common/etc/setups source /cvmfs/mu2e.opensciencegrid.org/setupmu2e-art.sh +setup muse source $HOME/bin/authentication.sh setup jobsub_client @@ -729,16 +750,29 @@ WEBREPORT=valJobWeb.txt rm -f $WEBREPORT # all output will be written here BASE_DIR=/pnfs/mu2e/persistent/users/mu2epro/valjob -OUTDIR=$BASE_DIR/`date +%Y`/`date +%m`/`date +%d` -# the collected plots -mkdir -p $OUTDIR/summary -# the build location -BUILD_DIR=/mu2e/app/users/mu2epro/nightly/current -# tarball for submission to the grid -TBALL=/pnfs/mu2e/resilient/users/mu2epro/nightly/$(date +%Y-%m-%d).tgz # the val comparison WEB_DIR=/web/sites/mu2e.fnal.gov/htdocs/atwork/computing/ops/val/valJob/nightly -WEB_DIR_DAY=/web/sites/mu2e.fnal.gov/htdocs/atwork/computing/ops/val/valJob/nightly/$(date +%Y_%m/%d) + +if [ -n "$DEBUG" ]; then + OUTDIR=$BASE_DIR/$DEBUG + # the build location + BUILD_DIR=/mu2e/app/users/mu2epro/nightly/$DEBUG + # tarball for submission to the grid + TBALL=/pnfs/mu2e/resilient/users/mu2epro/nightly/${DEBUG}.tgz + # web subdir + WEB_DIR_DAY=$WEB_DIR/$DEBUG +else + OUTDIR=$BASE_DIR/`date +%Y`/`date +%m`/`date +%d` + # the collected plots + BUILD_DIR=/mu2e/app/users/mu2epro/nightly/current + # tarball for submission to the grid + TBALL=/pnfs/mu2e/resilient/users/mu2epro/nightly/$(date +%Y-%m-%d).tgz + # the val comparison + WEB_DIR_DAY=$WEB_DIR/$(date +%Y_%m/%d) +fi + +# the collected plots +mkdir -p $OUTDIR/summary mkdir -p $WEB_DIR_DAY RC=$? if [ $RC -ne 0 ]; then @@ -752,8 +786,12 @@ build_code RC=$? # can't continue if the build fails if [ $RC -ne 0 ]; then - nightlyweb - exit_proc 1 "build failed" + if [ -z "$DEBUG" ]; then + nightlyweb + exit_proc 1 "build failed" + else + exit 1 + fi fi # run quick tests @@ -784,12 +822,15 @@ RC=$? # collect plots and check overlaps # will need root to do concatenation -# temproraily setup explicity on 01 until it is also sl7 -#source $BUILD_DIR/Offline/setup.sh -source /cvmfs/mu2e.opensciencegrid.org/Offline/v7_4_1/SLF6/prof/Offline/setup.sh +# need to access hadd and valCompare +muse setup Offline collect_summaries RC=$? + +# quit with no comparison if running in test mode +[ -n "$DEBUG" ] && exit 0 + valcompare RC=$? nightlyweb diff --git a/bin/valJobNode.sh b/bin/valJobNode.sh index 9f61a3b..3b19127 100755 --- a/bin/valJobNode.sh +++ b/bin/valJobNode.sh @@ -3,7 +3,7 @@ # # expects: # VALJOB_RELEASE=/pnfs/mu2e/resilient/users/mu2epro/nightly/date.tgz -# VALJOB_FCL=Validation/fcl/potSim.fcl +# VALJOB_FCL=Production/Validation/potSim.fcl # VALJOB_NEV=500 # VALJOB_LABEL=potSim # VALJOB_OUTDIR=/pnfs/mu2e/persistent/users/mu2epro/valjob/2017/10/06/potSim @@ -61,6 +61,9 @@ watchdog() { initialize() { ## move input files to cwd #find $CONDOR_DIR_INPUT -type f -exec mv {} . \; + + NUMBER=$(printf "%02d" $PROCESS ) + LABEL=$(printf "%s_%02d" $VALJOB_LABEL $PROCESS ) tee_date "setup" source /cvmfs/fermilab.opensciencegrid.org/products/common/etc/setups @@ -71,7 +74,7 @@ initialize() { if [ -n "$VALJOB_TARBALL" ]; then tee_date "copy in tarball $VALJOB_TARBALL" - ifdh cp $VALJOB_TARBALL ./tball.tgz + ifdh cp $VALJOB_TARBALL ./tball RC=$? tee_date "tarball copy return code $RC" if [ $RC -ne 0 ]; then @@ -80,24 +83,51 @@ initialize() { fi tee_date "unwind tarball" - tar -xzf tball.tgz + tar -xjf tball RC=$? tee_date "tarball unwind return code $RC" if [ $RC -ne 0 ]; then tee_date "ERROR exit unwind tarball $RC" return $RC fi + + tee_date "setup" + mv Code/* . + sed -i 's/-q/-1 -q/' setup.sh + source setup.sh + else - tee_date "no tarball, assuming Offline exists" + tee_date "no tarball, checking that Offline exists" if [ ! -d Offline ]; then tee_date "ERROR no tarball or Offline directory" return 1 fi + + tee_date "setup" + setup muse + muse setup -1 + if [ -z "$VALJOB_RSDIR" ]; then + tee_date "ERROR VALJOB_RSDIR not defined, not sure what to do, quitting" + return 1 + fi + + if [ $VALJOB_INPUT != "NULL" ]; then + tee_date "cp $VALJOB_INPUT $VALJOB_RSDIR" + cp $VALJOB_INPUT $VALJOB_RSDIR + fi + + if [ "$VALJOB_SEEDS" != "no" ]; then + tee_date cp seeds.txt $VALJOB_RSDIR + cp seeds.txt $VALJOB_RSDIR + fi + + tee_date cd $VALJOB_RSDIR + cd $VALJOB_RSDIR + fi - tee_date "setup" - source Offline/setup.sh - + muse status + tee_date "printenv" printenv @@ -107,12 +137,10 @@ initialize() { tee_date "model name" cat /proc/cpuinfo | grep "model name" | head -1 + tee_date PWD=$PWD tee_date "ls start" ls -l - NUMBER=$(printf "%02d" $PROCESS ) - LABEL=$(printf "%s_%02d" $VALJOB_LABEL $PROCESS ) - return 0 } @@ -122,10 +150,11 @@ initialize() { exe() { tee_date "start exe" TIME=`date +%s` - cp Offline/$VALJOB_FCL ./local.fcl + tee_date cp $MUSE_WORK_DIR/$VALJOB_FCL ./local.fcl + cp $MUSE_WORK_DIR/$VALJOB_FCL ./local.fcl if [ "$VALJOB_SEEDS" != "no" ]; then - export VALJOB_SEED=`cat Offline/seeds.txt | awk -v n=$PROCESS '{if(NR==(n+1)) print $1}'` + export VALJOB_SEED=`cat seeds.txt | awk -v n=$PROCESS '{if(NR==(n+1)) print $1}'` echo "services.SeedService.baseSeed: $VALJOB_SEED" >> local.fcl fi @@ -136,26 +165,33 @@ exe() { echo "services.ProditionsService.strawElectronics.useDb: true" >> local.fcl fi + + # if there is a list of input files, insert the ones we need if [ "$VALJOB_INPUT" != "NULL" ]; then echo "source.fileNames : [ " >> local.fcl - if [ "$LOCALJOB" ]; then - cat Offline/$VALJOB_INPUT | \ + cat $VALJOB_INPUT | \ awk -v n=$VALJOB_NINPUT -v i=$PROCESS 'BEGIN{is=i*n+1;ie=(i+1)*n+1}{xend=",";if(NR==ie-1) xend="";if(NR>=is && NR> local.fcl echo " ]" >> local.fcl - else - cat Offline/$VALJOB_INPUT | \ - awk -v n=$VALJOB_NINPUT -v i=$PROCESS 'BEGIN{is=i*n+1;ie=(i+1)*n+1}{xend=",";if(NR==ie-1) xend="";if(NR>=is && NR> local.fcl - echo " ]" >> local.fcl - fi fi + # if not local job, switch input files to xrootd + if [ -z "$LOCALJOB" ]; then + sed -i 's|/pnfs|xroot://fndca1.fnal.gov/pnfs/fnal.gov/usr|' local.fcl + fi + + NEV="" [ $VALJOB_NEV -gt 0 ] && NEV=" -n $VALJOB_NEV " - /cvmfs/mu2e.opensciencegrid.org/bin/SLF6/mu2e_time \ + + # potsim has multiple output streams, select the main one + OUTTEXT="" + #[ "$VALJOB_LABEL" == "potSim" ] && OUTTEXT="BeamOutput:" + + + /cvmfs/mu2e.opensciencegrid.org/bin/SLF7/mu2e_time \ mu2e $NEV -c ./local.fcl \ - -o ${LABEL}.art -T ${LABEL}.root + -o ${OUTTEXT}${LABEL}.art -T ${LABEL}.root RC=$? tee_date "exe $RC" @@ -193,7 +229,7 @@ validation() { fi /cvmfs/mu2e.opensciencegrid.org/bin/SLF6/mu2e_time \ - mu2e -s $IFILE -c Validation/fcl/val.fcl -T val_${LABEL}.root + mu2e -s $IFILE -c Offline/Validation/fcl/val.fcl -T val_${LABEL}.root RC=$? tee_date "val $RC"