From fb134c1f35edfe9fb711cc4da429b24f2c3d5380 Mon Sep 17 00:00:00 2001 From: Efe Date: Fri, 12 Jan 2024 16:01:38 +0100 Subject: [PATCH] studpid changes just for the github workflow to run without accessing mcm --- ...S-RunIISummer20UL16wmLHEGEN-00768_get_test | 131 ++++++++++++++++++ bin/utils/request_fragment_check.py | 33 +++-- 2 files changed, 150 insertions(+), 14 deletions(-) create mode 100644 bin/utils/SUS-RunIISummer20UL16wmLHEGEN-00768_get_test diff --git a/bin/utils/SUS-RunIISummer20UL16wmLHEGEN-00768_get_test b/bin/utils/SUS-RunIISummer20UL16wmLHEGEN-00768_get_test new file mode 100644 index 000000000000..de29b34a9454 --- /dev/null +++ b/bin/utils/SUS-RunIISummer20UL16wmLHEGEN-00768_get_test @@ -0,0 +1,131 @@ +#!/bin/bash + +# GEN Script begin +rm -f request_fragment_check.py +wget -q https://raw.githubusercontent.com/cms-sw/genproductions/master/bin/utils/request_fragment_check.py +chmod +x request_fragment_check.py +./request_fragment_check.py --bypass_status --prepid SUS-RunIISummer20UL16wmLHEGEN-00768 +GEN_ERR=$? +if [ $GEN_ERR -ne 0 ]; then + echo "GEN Checking Script returned exit code $GEN_ERR which means there are $GEN_ERR errors" + echo "Validation WILL NOT RUN" + echo "Please correct errors in the request and run validation again" + exit $GEN_ERR +fi +echo "Running VALIDATION. GEN Request Checking Script returned no errors" +# GEN Script end + +# Download fragment from McM +curl -s -k https://cms-pdmv-prod.web.cern.ch/mcm/public/restapi/requests/get_fragment/SUS-RunIISummer20UL16wmLHEGEN-00768 --retry 3 --create-dirs -o Configuration/GenProduction/python/SUS-RunIISummer20UL16wmLHEGEN-00768-fragment.py +[ -s Configuration/GenProduction/python/SUS-RunIISummer20UL16wmLHEGEN-00768-fragment.py ] || exit $?; + +# Check if fragment contais gridpack path ant that it is in cvmfs +if grep -q "gridpacks" Configuration/GenProduction/python/SUS-RunIISummer20UL16wmLHEGEN-00768-fragment.py; then + if ! grep -q "/cvmfs/cms.cern.ch/phys_generator/gridpacks" Configuration/GenProduction/python/SUS-RunIISummer20UL16wmLHEGEN-00768-fragment.py; then + echo "Gridpack inside fragment is not in cvmfs." + exit -1 + fi +fi + +# Dump actual test code to a SUS-RunIISummer20UL16wmLHEGEN-00768_test.sh file that can be run in Singularity +cat <<'EndOfTestFile' > SUS-RunIISummer20UL16wmLHEGEN-00768_test.sh +#!/bin/bash + +export SCRAM_ARCH=slc7_amd64_gcc700 + +source /cvmfs/cms.cern.ch/cmsset_default.sh +if [ -r CMSSW_10_6_32_patch1/src ] ; then + echo release CMSSW_10_6_32_patch1 already exists +else + scram p CMSSW CMSSW_10_6_32_patch1 +fi +cd CMSSW_10_6_32_patch1/src +eval `scram runtime -sh` + +mv ../../Configuration . +scram b +cd ../.. + +# Maximum validation duration: 28800s +# Margin for validation duration: 30% +# Validation duration with margin: 28800 * (1 - 0.30) = 20160s +# Time per event for each sequence: 30.0000s +# Threads for each sequence: 1 +# Time per event for single thread for each sequence: 1 * 30.0000s = 30.0000s +# Which adds up to 30.0000s per event +# Single core events that fit in validation duration: 20160s / 30.0000s = 672 +# Produced events limit in McM is 10000 +# According to 1.0000 efficiency, validation should run 10000 / 1.0000 = 10000 events to reach the limit of 10000 +# Take the minimum of 672 and 10000, but more than 0 -> 672 +# It is estimated that this validation will produce: 672 * 1.0000 = 672 events +EVENTS=672 + + +# cmsDriver command +cmsDriver.py Configuration/GenProduction/python/SUS-RunIISummer20UL16wmLHEGEN-00768-fragment.py --python_filename SUS-RunIISummer20UL16wmLHEGEN-00768_1_cfg.py --eventcontent RAWSIM,LHE --customise Configuration/DataProcessing/Utils.addMonitoring --datatier GEN,LHE --fileout file:SUS-RunIISummer20UL16wmLHEGEN-00768.root --conditions 106X_mcRun2_asymptotic_v13 --beamspot Realistic25ns13TeV2016Collision --customise_commands process.source.numberEventsInLuminosityBlock="cms.untracked.uint32(100)" --step LHE,GEN --geometry DB:Extended --era Run2_2016 --no_exec --mc -n $EVENTS || exit $? ; + +# Run generated config +REPORT_NAME=SUS-RunIISummer20UL16wmLHEGEN-00768_report.xml +# Run the cmsRun +cmsRun -e -j $REPORT_NAME SUS-RunIISummer20UL16wmLHEGEN-00768_1_cfg.py || exit $? ; + +# Parse values from SUS-RunIISummer20UL16wmLHEGEN-00768_report.xml report +processedEvents=$(grep -Po "(?<=)" $REPORT_NAME | tail -n 1) +producedEvents=$(grep -Po "(?<=)(\d*)(?=)" $REPORT_NAME | tail -n 1) +threads=$(grep -Po "(?<=)" $REPORT_NAME | tail -n 1) +peakValueRss=$(grep -Po "(?<=)" $REPORT_NAME | tail -n 1) +peakValueVsize=$(grep -Po "(?<=)" $REPORT_NAME | tail -n 1) +totalSize=$(grep -Po "(?<=)" $REPORT_NAME | tail -n 1) +totalSizeAlt=$(grep -Po "(?<=)" $REPORT_NAME | tail -n 1) +totalJobTime=$(grep -Po "(?<=)" $REPORT_NAME | tail -n 1) +totalJobCPU=$(grep -Po "(?<=)" $REPORT_NAME | tail -n 1) +eventThroughput=$(grep -Po "(?<=)" $REPORT_NAME | tail -n 1) +avgEventTime=$(grep -Po "(?<=)" $REPORT_NAME | tail -n 1) +if [ -z "$threads" ]; then + echo "Could not find NumberOfThreads in report, defaulting to 1" + threads=1 +fi +if [ -z "$eventThroughput" ]; then + eventThroughput=$(bc -l <<< "scale=4; 1 / ($avgEventTime / $threads)") +fi +if [ -z "$totalSize" ]; then + totalSize=$totalSizeAlt +fi +if [ -z "$processedEvents" ]; then + processedEvents=$EVENTS +fi +echo "Validation report of SUS-RunIISummer20UL16wmLHEGEN-00768 sequence 1/1" +echo "Processed events: $processedEvents" +echo "Produced events: $producedEvents" +echo "Threads: $threads" +echo "Peak value RSS: $peakValueRss MB" +echo "Peak value Vsize: $peakValueVsize MB" +echo "Total size: $totalSize MB" +echo "Total job time: $totalJobTime s" +echo "Total CPU time: $totalJobCPU s" +echo "Event throughput: $eventThroughput" +echo "CPU efficiency: "$(bc -l <<< "scale=2; ($totalJobCPU * 100) / ($threads * $totalJobTime)")" %" +echo "Size per event: "$(bc -l <<< "scale=4; ($totalSize * 1024 / $producedEvents)")" kB" +echo "Time per event: "$(bc -l <<< "scale=4; (1 / $eventThroughput)")" s" +echo "Filter efficiency percent: "$(bc -l <<< "scale=8; ($producedEvents * 100) / $processedEvents")" %" +echo "Filter efficiency fraction: "$(bc -l <<< "scale=10; ($producedEvents) / $processedEvents") + +# End of SUS-RunIISummer20UL16wmLHEGEN-00768_test.sh file +EndOfTestFile + +# Make file executable +chmod +x SUS-RunIISummer20UL16wmLHEGEN-00768_test.sh + +if [ -e "/cvmfs/unpacked.cern.ch/registry.hub.docker.com/cmssw/el7:amd64" ]; then + CONTAINER_NAME="el7:amd64" +elif [ -e "/cvmfs/unpacked.cern.ch/registry.hub.docker.com/cmssw/el7:x86_64" ]; then + CONTAINER_NAME="el7:x86_64" +else + echo "Could not find amd64 or x86_64 for el7" + exit 1 +fi +# Run in singularity container +# Mount afs, eos, cvmfs +# Mount /etc/grid-security for xrootd +export SINGULARITY_CACHEDIR="/tmp/$(whoami)/singularity" +singularity run -B /afs -B /eos -B /cvmfs -B /etc/grid-security -B /etc/pki/ca-trust --home $PWD:$PWD /cvmfs/unpacked.cern.ch/registry.hub.docker.com/cmssw/$CONTAINER_NAME $(echo $(pwd)/SUS-RunIISummer20UL16wmLHEGEN-00768_test.sh) diff --git a/bin/utils/request_fragment_check.py b/bin/utils/request_fragment_check.py index 4c746398e7e5..2ef1bab17774 100755 --- a/bin/utils/request_fragment_check.py +++ b/bin/utils/request_fragment_check.py @@ -11,8 +11,8 @@ import json import ast from datetime import datetime -sys.path.append('/afs/cern.ch/cms/PPD/PdmV/tools/McM-QA/') -from rest import McM +#sys.path.append('/afs/cern.ch/cms/PPD/PdmV/tools/McM-QA/') +#from rest import McM from json import dumps parser = argparse.ArgumentParser( @@ -54,8 +54,11 @@ sys.exit() # Use no-id as identification mode in order not to use a SSO cookie -mcm = McM(id=None, dev=args.dev, debug=args.debug) -mcm_link = mcm.server +if args.local is False: + sys.path.append('/afs/cern.ch/cms/PPD/PdmV/tools/McM-QA/') + from rest import McM + mcm = McM(id=None, dev=args.dev, debug=args.debug) + mcm_link = mcm.server def get_request(prepid): if args.local is False: @@ -473,15 +476,16 @@ def root_requests_from_ticket(ticket_prepid, include_docs=False): return requests -if args.ticket is not None: - ticket = args.ticket - ticket = ticket[0] - print("------------------------------------") - print("--> Ticket = "+ticket) - print("------------------------------------") - prepid = [] - for rr in root_requests_from_ticket(ticket): - if 'GS' in rr or 'wmLHE' in rr or 'pLHE' in rr or 'FS' in rr: prepid.append(rr) +if args.local is False: + if args.ticket is not None: + ticket = args.ticket + ticket = ticket[0] + print("------------------------------------") + print("--> Ticket = "+ticket) + print("------------------------------------") + prepid = [] + for rr in root_requests_from_ticket(ticket): + if 'GS' in rr or 'wmLHE' in rr or 'pLHE' in rr or 'FS' in rr: prepid.append(rr) @@ -693,7 +697,8 @@ def root_requests_from_ticket(ticket_prepid, include_docs=False): os.system('mkdir -p '+my_path+'/eos/'+pi) os.system('mv '+pi+'_tmp '+pi) os.system('cp '+pi+' '+my_path+'/'+pi+'/.') - os.system('wget -q '+mcm_link+'public/restapi/requests/get_test/'+pi+' -O '+pi+'_get_test') + if args.local is False: + os.system('wget -q '+mcm_link+'public/restapi/requests/get_test/'+pi+' -O '+pi+'_get_test') gettest = os.popen('grep cff '+pi+'_get_test'+' | grep curl').read() if os.path.getsize(pi+'_get_test') == 0: print("public/restapi/requests/get_test/ is not acessible for this request. Exiting! Please send an email to 'cms-ppd-pdmv-dev@cern.ch'")