fix: get batch system info from the pilot #5
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
name: integration | |
on: [push] | |
jobs: | |
integration-local_install: | |
runs-on: ubuntu-latest | |
strategy: | |
matrix: | |
dirac_version: | |
- rel-v8r0 | |
- integration | |
pilot_schema: | |
- pilot_oldSchema.json | |
- pilot_newSchema.json | |
VO: | |
- dteam | |
- gridpp | |
steps: | |
- uses: actions/checkout@v4 | |
- uses: cvmfs-contrib/github-action-cvmfs@v3 | |
- name: Test CernVM-FS | |
run: ls /cvmfs/dirac.egi.eu | |
- name: Retrieve the secret and decode it to a file | |
env: | |
HOSTCERT_BASE64: ${{ secrets.HOSTCERT_BASE64 }} | |
HOSTKEY_BASE64: ${{ secrets.HOSTKEY_BASE64 }} | |
run: | | |
cd Pilot | |
mkdir -p etc/grid-security | |
echo "$HOSTCERT_BASE64" | base64 --decode > etc/grid-security/hostcert.pem | |
echo "$HOSTKEY_BASE64" | base64 --decode > etc/grid-security/hostkey.pem | |
chmod 440 etc/grid-security/hostcert.pem | |
chmod 400 etc/grid-security/hostkey.pem | |
- name: tests | |
run: | | |
cd Pilot | |
cp ../tests/CI/${{ matrix.pilot_schema }} pilot.json | |
sed -i "s/VAR_JENKINS_SITE/DIRAC.Jenkins.ch/g" pilot.json | |
sed -i "s/VAR_JENKINS_CE/jenkins.cern.ch/g" pilot.json | |
sed -i "s/VAR_JENKINS_QUEUE/jenkins-queue_not_important/g" pilot.json | |
sed -i "s/VAR_DIRAC_VERSION/${{ matrix.dirac_version }}/g" pilot.json | |
sed -i "s#VAR_CS#https://lbcertifdirac70.cern.ch:9135/Configuration/Server#g" pilot.json | |
sed -i "s#VAR_USERDN#/DC=ch/DC=cern/OU=Organic Units/OU=Users/CN=alboyer/CN=819281/CN=Alexandre Franck Boyer#g" pilot.json | |
sed -i "s#VAR_USERDN_GRIDPP#${DIRACUSERDN_GRIDPP}#g" pilot.json | |
g_job="testintegrationworkflow${GITHUB_JOB//-/}" | |
pilotUUID="${g_job//_/}""${GITHUB_RUN_ID}" | |
pilotUUID=$(echo $pilotUUID | rev | cut -c 1-32 | rev) | |
python dirac-pilot.py --modules https://github.com/DIRACGrid/DIRAC.git:::DIRAC:::${{ matrix.dirac_version }} -M 1 -S DIRAC-Certification -N jenkins.cern.ch -Q jenkins-queue_not_important -n DIRAC.Jenkins.ch --cert --certLocation=${GITHUB_WORKSPACE}/Pilot/etc/grid-security --wnVO=${{ matrix.VO }} --pilotUUID="${pilotUUID}" --debug | |
integration-cvmfs: | |
runs-on: ubuntu-latest | |
strategy: | |
matrix: | |
pilot_schema: | |
- pilot_oldSchema.json | |
- pilot_newSchema.json | |
dirac_version: | |
- v8.0.27 | |
- v8.1.0a17 | |
steps: | |
- uses: actions/checkout@v4 | |
- uses: cvmfs-contrib/github-action-cvmfs@v3 | |
- name: Test CernVM-FS | |
run: ls /cvmfs/dirac.egi.eu | |
- name: Retrieve the secret and decode it to a file | |
env: | |
HOSTCERT_BASE64: ${{ secrets.HOSTCERT_BASE64 }} | |
HOSTKEY_BASE64: ${{ secrets.HOSTKEY_BASE64 }} | |
run: | | |
cd Pilot | |
mkdir -p etc/grid-security | |
echo "$HOSTCERT_BASE64" | base64 --decode > etc/grid-security/hostcert.pem | |
echo "$HOSTKEY_BASE64" | base64 --decode > etc/grid-security/hostkey.pem | |
chmod 440 etc/grid-security/hostcert.pem | |
chmod 400 etc/grid-security/hostkey.pem | |
- name: tests | |
env: | |
X509_CERT_DIR: /cvmfs/grid.cern.ch/etc/grid-security/certificates | |
X509_VOMS_DIR: /cvmfs/grid.cern.ch/etc/grid-security/vomsdir | |
DIRAC_VOMSES: /cvmfs/grid.cern.ch/etc/grid-security/vomses | |
run: | | |
cd Pilot | |
cp ../tests/CI/${{ matrix.pilot_schema }} pilot.json | |
sed -i "s/VAR_JENKINS_SITE/DIRAC.Jenkins.ch/g" pilot.json | |
sed -i "s/VAR_JENKINS_CE/jenkins.cern.ch/g" pilot.json | |
sed -i "s/VAR_JENKINS_QUEUE/jenkins-queue_not_important/g" pilot.json | |
sed -i "s/VAR_DIRAC_VERSION/${{ matrix.dirac_version }}/g" pilot.json | |
sed -i "s#VAR_CS#https://lbcertifdirac70.cern.ch:9135/Configuration/Server#g" pilot.json | |
sed -i "s#VAR_USERDN#/DC=ch/DC=cern/OU=Organic Units/OU=Users/CN=alboyer/CN=819281/CN=Alexandre Franck Boyer#g" pilot.json | |
sed -i "s#VAR_USERDN_GRIDPP#${DIRACUSERDN_GRIDPP}#g" pilot.json | |
g_job="testintegrationworkflow${GITHUB_JOB//-/}" | |
pilotUUID="${g_job//_/}""${GITHUB_RUN_ID}" | |
pilotUUID=$(echo $pilotUUID | rev | cut -c 1-32 | rev) | |
python dirac-pilot.py -M 1 -S DIRAC-Certification -N jenkins.cern.ch -Q jenkins-queue_not_important -n DIRAC.Jenkins.ch --preinstalledEnvPrefix=/cvmfs/dirac.egi.eu/dirac --cert --certLocation=${GITHUB_WORKSPACE}/Pilot/etc/grid-security --wnVO=dteam --pilotUUID="${pilotUUID}" --debug | |
integration-cvmfs_no_env: | |
runs-on: ubuntu-latest | |
steps: | |
- uses: actions/checkout@v4 | |
- uses: cvmfs-contrib/github-action-cvmfs@v3 | |
- name: Test CernVM-FS | |
run: ls /cvmfs/dirac.egi.eu | |
- name: Retrieve the secret and decode it to a file | |
env: | |
HOSTCERT_BASE64: ${{ secrets.HOSTCERT_BASE64 }} | |
HOSTKEY_BASE64: ${{ secrets.HOSTKEY_BASE64 }} | |
run: | | |
cd Pilot | |
mkdir -p etc/grid-security | |
echo "$HOSTCERT_BASE64" | base64 --decode > etc/grid-security/hostcert.pem | |
echo "$HOSTKEY_BASE64" | base64 --decode > etc/grid-security/hostkey.pem | |
chmod 440 etc/grid-security/hostcert.pem | |
chmod 400 etc/grid-security/hostkey.pem | |
- name: tests | |
run: | | |
cd Pilot | |
cp ../tests/CI/pilot_newSchema.json pilot.json | |
sed -i "s/VAR_JENKINS_SITE/DIRAC.Jenkins.ch/g" pilot.json | |
sed -i "s/VAR_JENKINS_CE/jenkins.cern.ch/g" pilot.json | |
sed -i "s/VAR_JENKINS_QUEUE/jenkins-queue_not_important/g" pilot.json | |
sed -i "s/VAR_DIRAC_VERSION/v8.1.0a17/g" pilot.json | |
sed -i "s#VAR_CS#https://lbcertifdirac70.cern.ch:9135/Configuration/Server#g" pilot.json | |
sed -i "s#VAR_USERDN#/DC=ch/DC=cern/OU=Organic Units/OU=Users/CN=alboyer/CN=819281/CN=Alexandre Franck Boyer#g" pilot.json | |
sed -i "s#VAR_USERDN_GRIDPP#${DIRACUSERDN_GRIDPP}#g" pilot.json | |
g_job="testintegrationworkflow${GITHUB_JOB//-/}" | |
pilotUUID="${g_job//_/}""${GITHUB_RUN_ID}" | |
pilotUUID=$(echo $pilotUUID | rev | cut -c 1-32 | rev) | |
python dirac-pilot.py -M 1 -S DIRAC-Certification -N jenkins.cern.ch -Q jenkins-queue_not_important -n DIRAC.Jenkins.ch --preinstalledEnvPrefix=/cvmfs/dirac.egi.eu/dirac --cert --certLocation=${GITHUB_WORKSPACE}/Pilot/etc/grid-security --wnVO=dteam -o cvmfsOnly --pilotUUID="${pilotUUID}" --debug | |
matching: | |
runs-on: ubuntu-latest | |
strategy: | |
matrix: | |
pilot_schema: | |
- pilot_oldSchema.json | |
- pilot_newSchema.json | |
steps: | |
- uses: actions/checkout@v4 | |
- uses: cvmfs-contrib/github-action-cvmfs@v3 | |
- name: Test CernVM-FS | |
run: ls /cvmfs/dirac.egi.eu | |
- name: Retrieve the secret and decode it to a file | |
env: | |
HOSTCERT_BASE64: ${{ secrets.HOSTCERT_BASE64 }} | |
HOSTKEY_BASE64: ${{ secrets.HOSTKEY_BASE64 }} | |
run: | | |
cd Pilot | |
mkdir -p etc/grid-security | |
echo "$HOSTCERT_BASE64" | base64 --decode > etc/grid-security/hostcert.pem | |
echo "$HOSTKEY_BASE64" | base64 --decode > etc/grid-security/hostkey.pem | |
chmod 440 etc/grid-security/hostcert.pem | |
chmod 400 etc/grid-security/hostkey.pem | |
- name: tests | |
run: | | |
cd Pilot | |
cp ../tests/CI/${{ matrix.pilot_schema }} pilot.json | |
sed -i "s/VAR_JENKINS_SITE/DIRAC.Jenkins.ch/g" pilot.json | |
sed -i "s/VAR_JENKINS_CE/jenkins-full.cern.ch/g" pilot.json | |
sed -i "s/VAR_JENKINS_QUEUE/jenkins-queue_not_important/g" pilot.json | |
sed -i "s/VAR_DIRAC_VERSION/integration/g" pilot.json | |
sed -i "s#VAR_CS#https://lbcertifdirac70.cern.ch:9135/Configuration/Server#g" pilot.json | |
sed -i "s#VAR_USERDN#/DC=ch/DC=cern/OU=Organic Units/OU=Users/CN=alboyer/CN=819281/CN=Alexandre Franck Boyer#g" pilot.json | |
sed -i "s#VAR_USERDN_GRIDPP#${DIRACUSERDN_GRIDPP}#g" pilot.json | |
g_job="testintegrationworkflow${GITHUB_JOB//-/}" | |
pilotUUID="${g_job//_/}""${GITHUB_RUN_ID}" | |
pilotUUID=$(echo $pilotUUID | rev | cut -c 1-32 | rev) | |
python dirac-pilot.py --modules https://github.com/DIRACGrid/DIRAC.git:::DIRAC:::integration -M 1 -S DIRAC-Certification -N jenkins-full.cern.ch -Q jenkins-queue_not_important -n DIRAC.Jenkins.ch --wnVO=dteam --cert --certLocation=${GITHUB_WORKSPACE}/Pilot/etc/grid-security --pilotUUID="${pilotUUID}" --debug | |
integration-cvmfs_matching: | |
runs-on: ubuntu-latest | |
strategy: | |
matrix: | |
VO: | |
- dteam | |
- gridpp | |
dirac_version: | |
- v8.0.27 | |
- v8.1.0a17 | |
steps: | |
- uses: actions/checkout@v4 | |
- uses: cvmfs-contrib/github-action-cvmfs@v3 | |
- name: Test CernVM-FS | |
run: ls /cvmfs/dirac.egi.eu | |
- name: Retrieve the secret and decode it to a file | |
env: | |
HOSTCERT_BASE64: ${{ secrets.HOSTCERT_BASE64 }} | |
HOSTKEY_BASE64: ${{ secrets.HOSTKEY_BASE64 }} | |
run: | | |
cd Pilot | |
mkdir -p etc/grid-security | |
echo "$HOSTCERT_BASE64" | base64 --decode > etc/grid-security/hostcert.pem | |
echo "$HOSTKEY_BASE64" | base64 --decode > etc/grid-security/hostkey.pem | |
chmod 440 etc/grid-security/hostcert.pem | |
chmod 400 etc/grid-security/hostkey.pem | |
- name: tests | |
run: | | |
cd Pilot | |
cp ../tests/CI/pilot_newSchema.json pilot.json | |
sed -i "s/VAR_JENKINS_SITE/DIRAC.Jenkins.ch/g" pilot.json | |
sed -i "s/VAR_JENKINS_CE/jenkins-full.cern.ch/g" pilot.json | |
sed -i "s/VAR_JENKINS_QUEUE/jenkins-queue_not_important/g" pilot.json | |
sed -i "s/VAR_DIRAC_VERSION/${{ matrix.dirac_version }}/g" pilot.json | |
sed -i "s#VAR_CS#https://lbcertifdirac70.cern.ch:9135/Configuration/Server#g" pilot.json | |
sed -i "s#VAR_USERDN#/DC=ch/DC=cern/OU=Organic Units/OU=Users/CN=alboyer/CN=819281/CN=Alexandre Franck Boyer#g" pilot.json | |
sed -i "s#VAR_USERDN_GRIDPP#${DIRACUSERDN_GRIDPP}#g" pilot.json | |
g_job="testintegrationworkflow${GITHUB_JOB//-/}" | |
pilotUUID="${g_job//_/}""${GITHUB_RUN_ID}" | |
pilotUUID=$(echo $pilotUUID | rev | cut -c 1-32 | rev) | |
python dirac-pilot.py -M 1 -S DIRAC-Certification -N jenkins-full.cern.ch -Q jenkins-queue_not_important -n DIRAC.Jenkins.ch --preinstalledEnvPrefix=/cvmfs/dirac.egi.eu/dirac --cert --certLocation=${GITHUB_WORKSPACE}/Pilot/etc/grid-security --wnVO=${{ matrix.VO }} -o cvmfsOnly --pilotUUID="${pilotUUID}" --debug | |
################################## | |
### # extensions tests | |
################################## | |
################################## | |
#### LHCb | |
ext-lhcb_integration: | |
runs-on: ubuntu-latest | |
strategy: | |
matrix: | |
pilot_schema: | |
- pilot_oldSchema.json | |
- pilot_newSchema.json | |
pilot_version: | |
- master | |
- devel | |
steps: | |
- uses: actions/checkout@v4 | |
- uses: cvmfs-contrib/github-action-cvmfs@v3 | |
- name: Test CernVM-FS | |
run: ls /cvmfs/lhcb.cern.ch | |
- name: Retrieve the secret and decode it to a file | |
env: | |
HOSTCERT_BASE64: ${{ secrets.HOSTCERT_BASE64 }} | |
HOSTKEY_BASE64: ${{ secrets.HOSTKEY_BASE64 }} | |
run: | | |
cd Pilot | |
mkdir -p etc/grid-security/ | |
echo "$HOSTCERT_BASE64" | base64 --decode > etc/grid-security/hostcert.pem | |
echo "$HOSTKEY_BASE64" | base64 --decode > etc/grid-security/hostkey.pem | |
chmod 440 etc/grid-security/hostcert.pem | |
chmod 400 etc/grid-security/hostkey.pem | |
- name: tests | |
env: | |
X509_CERT_DIR: /cvmfs/lhcb.cern.ch/etc/grid-security/certificates | |
X509_VOMS_DIR: /cvmfs/lhcb.cern.ch/etc/grid-security/vomsdir | |
DIRAC_VOMSES: /cvmfs/lhcb.cern.ch/etc/grid-security/vomses | |
run: | | |
cd Pilot | |
export VO_LHCB_SW_DIR=${GITHUB_WORKSPACE}/Pilot | |
curl https://gitlab.cern.ch/lhcb-dirac/LHCbPilot/-/raw/${{ matrix.pilot_version }}/LHCbPilot/LHCbPilotCommands.py -o LHCbPilotCommands.py | |
cp ../tests/CI/${{ matrix.pilot_schema }} pilot.json | |
sed -i "s/VAR_JENKINS_SITE/DIRAC.Jenkins.ch/g" pilot.json | |
sed -i "s/VAR_JENKINS_CE/jenkins-lhcb.cern.ch/g" pilot.json | |
sed -i "s/VAR_JENKINS_QUEUE/jenkins-queue_not_important/g" pilot.json | |
sed -i "s/VAR_DIRAC_VERSION/v11.0.20/g" pilot.json | |
sed -i "s#VAR_CS#https://lbcertifdirac70.cern.ch:9135/Configuration/Server#g" pilot.json | |
sed -i "s#VAR_USERDN#/DC=ch/DC=cern/OU=Organic Units/OU=Users/CN=alboyer/CN=819281/CN=Alexandre Franck Boyer#g" pilot.json | |
sed -i "s#VAR_USERDN_GRIDPP#${DIRACUSERDN_GRIDPP}#g" pilot.json | |
g_job="testintegrationworkflow${GITHUB_JOB//-/}" | |
pilotUUID="${g_job//_/}""${GITHUB_RUN_ID}" | |
pilotUUID=$(echo $pilotUUID | rev | cut -c 1-32 | rev) | |
python dirac-pilot.py -M 1 -S DIRAC-Certification -N jenkins-lhcb.cern.ch -Q jenkins-queue_not_important -n DIRAC.Jenkins.ch -e LHCb -l LHCb -E LHCbPilot --cert --certLocation=${GITHUB_WORKSPACE}/Pilot/etc/grid-security --wnVO=LHCb -o lbRunOnly --pilotUUID="${pilotUUID}" --debug | |
ext-lhcb_integration_dirac_installer_no_env: | |
runs-on: ubuntu-latest | |
strategy: | |
matrix: | |
pilot_schema: | |
- pilot_oldSchema.json | |
- pilot_newSchema.json | |
ce: | |
- jenkins-lhcb-d.cern.ch | |
- jenkins-lhcb-dd.cern.ch | |
steps: | |
- uses: actions/checkout@v4 | |
- uses: cvmfs-contrib/github-action-cvmfs@v3 | |
- name: Test CernVM-FS | |
run: ls /cvmfs/lhcb.cern.ch | |
- name: Retrieve the secret and decode it to a file | |
env: | |
HOSTCERT_BASE64: ${{ secrets.HOSTCERT_BASE64 }} | |
HOSTKEY_BASE64: ${{ secrets.HOSTKEY_BASE64 }} | |
run: | | |
cd Pilot | |
mkdir -p etc/grid-security/ | |
echo "$HOSTCERT_BASE64" | base64 --decode > etc/grid-security/hostcert.pem | |
echo "$HOSTKEY_BASE64" | base64 --decode > etc/grid-security/hostkey.pem | |
chmod 440 etc/grid-security/hostcert.pem | |
chmod 400 etc/grid-security/hostkey.pem | |
- name: tests | |
run: | | |
cd Pilot | |
curl https://lhcbdirac.s3.cern.ch/Pilot3/LHCbPilotCommands.py -o LHCbPilotCommands.py | |
cp ../tests/CI/${{ matrix.pilot_schema }} pilot.json | |
sed -i "s/VAR_JENKINS_SITE/DIRAC.Jenkins.ch/g" pilot.json | |
sed -i "s/VAR_JENKINS_CE/${{ matrix.ce }}/g" pilot.json | |
sed -i "s/VAR_JENKINS_QUEUE/jenkins-queue_not_important/g" pilot.json | |
sed -i "s/VAR_DIRAC_VERSION/v11.0.20/g" pilot.json | |
sed -i "s#VAR_CS#https://lbcertifdirac70.cern.ch:9135/Configuration/Server#g" pilot.json | |
sed -i "s#VAR_USERDN#/DC=ch/DC=cern/OU=Organic Units/OU=Users/CN=alboyer/CN=819281/CN=Alexandre Franck Boyer#g" pilot.json | |
sed -i "s#VAR_USERDN_GRIDPP#${DIRACUSERDN_GRIDPP}#g" pilot.json | |
g_job="testintegrationworkflow${GITHUB_JOB//-/}" | |
pilotUUID="${g_job//_/}""${GITHUB_RUN_ID}" | |
pilotUUID=$(echo $pilotUUID | rev | cut -c 1-32 | rev) | |
python dirac-pilot.py -M 1 -S DIRAC-Certification -N ${{ matrix.ce }} -Q jenkins-queue_not_important -n DIRAC.Jenkins.ch -e LHCb -l LHCb -E LHCbPilot --preinstalledEnvPrefix=/cvmfs/lhcb.cern.ch/lhcbdirac/ --cert --certLocation=${GITHUB_WORKSPACE}/Pilot/etc/grid-security --wnVO=LHCb -o cvmfsOnly --pilotUUID="${pilotUUID}" --debug | |
ext-lhcb_integration_no_env: | |
runs-on: ubuntu-latest | |
strategy: | |
matrix: | |
pilot_schema: | |
- pilot_oldSchema.json | |
- pilot_newSchema.json | |
pilot_version: | |
- master | |
- devel | |
steps: | |
- uses: actions/checkout@v4 | |
- uses: cvmfs-contrib/github-action-cvmfs@v3 | |
- name: Test CernVM-FS | |
run: ls /cvmfs/lhcb.cern.ch | |
- name: Retrieve the secret and decode it to a file | |
env: | |
HOSTCERT_BASE64: ${{ secrets.HOSTCERT_BASE64 }} | |
HOSTKEY_BASE64: ${{ secrets.HOSTKEY_BASE64 }} | |
run: | | |
cd Pilot | |
mkdir -p etc/grid-security/ | |
echo "$HOSTCERT_BASE64" | base64 --decode > etc/grid-security/hostcert.pem | |
echo "$HOSTKEY_BASE64" | base64 --decode > etc/grid-security/hostkey.pem | |
chmod 440 etc/grid-security/hostcert.pem | |
chmod 400 etc/grid-security/hostkey.pem | |
- name: tests | |
run: | | |
cd Pilot | |
curl https://gitlab.cern.ch/lhcb-dirac/LHCbPilot/-/raw/${{ matrix.pilot_version }}/LHCbPilot/LHCbPilotCommands.py -o LHCbPilotCommands.py | |
cp ../tests/CI/${{ matrix.pilot_schema }} pilot.json | |
sed -i "s/VAR_JENKINS_SITE/DIRAC.Jenkins.ch/g" pilot.json | |
sed -i "s/VAR_JENKINS_CE/jenkins-lhcb-d.cern.ch/g" pilot.json | |
sed -i "s/VAR_JENKINS_QUEUE/jenkins-queue_not_important/g" pilot.json | |
sed -i "s/VAR_DIRAC_VERSION/v11.0.20/g" pilot.json | |
sed -i "s#VAR_CS#https://lbcertifdirac70.cern.ch:9135/Configuration/Server#g" pilot.json | |
sed -i "s#VAR_USERDN#/DC=ch/DC=cern/OU=Organic Units/OU=Users/CN=alboyer/CN=819281/CN=Alexandre Franck Boyer#g" pilot.json | |
sed -i "s#VAR_USERDN_GRIDPP#${DIRACUSERDN_GRIDPP}#g" pilot.json | |
g_job="testintegrationworkflow${GITHUB_JOB//-/}" | |
pilotUUID="${g_job//_/}""${GITHUB_RUN_ID}" | |
pilotUUID=$(echo $pilotUUID | rev | cut -c 1-32 | rev) | |
python dirac-pilot.py -M 1 -S DIRAC-Certification -N jenkins-lhcb.cern.ch -Q jenkins-queue_not_important -n DIRAC.Jenkins.ch -e LHCb -l LHCb -E LHCbPilot --preinstalledEnvPrefix=/cvmfs/lhcb.cern.ch/lhcbdirac/ --cert --certLocation=${GITHUB_WORKSPACE}/Pilot/etc/grid-security --wnVO=LHCb -o cvmfsOnly --pilotUUID="${pilotUUID}" --debug |