From 8bcd4341491bf980479f841743a217d0497ca0c7 Mon Sep 17 00:00:00 2001 From: Todor Ivanov Date: Fri, 24 May 2024 23:15:51 +0300 Subject: [PATCH] =?UTF-8?q?W=D0=9C=D0=90gent=20switch=20deployment=20model?= =?UTF-8?q?=20copy=20runtime=20fix=2011990=20(#1490)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Parsing the WMA_TAG as early as possible && Forcing runtime scripts checks && Creating mount points based on WMA_VER_RELEASE instead of WMA_TAG * Calling check_wmatag * Fix wrong initFlag cleared && _exec_sql call && typos --- docker/pypi/wmagent/Dockerfile | 12 +++++++- docker/pypi/wmagent/bin/manage-common.sh | 9 ++++++ docker/pypi/wmagent/init.sh | 34 +++++++++++++++++++++++ docker/pypi/wmagent/install.sh | 12 ++------ docker/pypi/wmagent/wmagent-docker-run.sh | 33 ++++++++++++++-------- 5 files changed, 78 insertions(+), 22 deletions(-) diff --git a/docker/pypi/wmagent/Dockerfile b/docker/pypi/wmagent/Dockerfile index 1b0e9bbcb..2912c8006 100644 --- a/docker/pypi/wmagent/Dockerfile +++ b/docker/pypi/wmagent/Dockerfile @@ -1,3 +1,4 @@ +# syntax=docker/dockerfile-upstream:master FROM registry.cern.ch/cmsweb/wmagent-base:pypi-20230705 MAINTAINER Valentin Kuznetsov vkuznet@gmail.com @@ -7,6 +8,15 @@ ARG WMA_TAG=$TAG ENV WMA_TAG=$WMA_TAG ENV WMA_ROOT_DIR=/data +# Parsing the WMA_TAG in parts step by step +ENV WMA_VER_MINOR=${WMA_TAG#*.*.} +ENV WMA_VER_MAJOR=${WMA_TAG%.$WMA_VER_MINOR} +ENV WMA_VER_MINOR=${WMA_VER_MINOR%rc*} +ENV WMA_VER_MINOR=${WMA_VER_MINOR%.*} +ENV WMA_VER_RELEASE=${WMA_VER_MAJOR}.${WMA_VER_MINOR} +ENV WMA_VER_PATCH=${WMA_TAG#$WMA_VER_RELEASE} +ENV WMA_VER_PATCH=${WMA_VER_PATCH#.} + # Basic WMAgent directory structure passed to all scripts through env variables: # NOTE: Those should be static and depend only on $WMA_BASE_DIR ENV WMA_BASE_DIR=$WMA_ROOT_DIR/srv/wmagent @@ -14,7 +24,7 @@ ENV WMA_ADMIN_DIR=$WMA_ROOT_DIR/admin/wmagent ENV WMA_CERTS_DIR=$WMA_ROOT_DIR/certs # ENV WMA_HOSTADMIN_DIR=$WMA_ADMIN_DIR/hostadmin -ENV WMA_CURRENT_DIR=$WMA_BASE_DIR/$WMA_TAG +ENV WMA_CURRENT_DIR=$WMA_BASE_DIR/$WMA_VER_RELEASE ENV WMA_AUTH_DIR=$WMA_CURRENT_DIR/auth/ ENV WMA_INSTALL_DIR=$WMA_CURRENT_DIR/install ENV WMA_STATE_DIR=$WMA_CURRENT_DIR/state diff --git a/docker/pypi/wmagent/bin/manage-common.sh b/docker/pypi/wmagent/bin/manage-common.sh index 07ce7cb8a..7d81b5a80 100644 --- a/docker/pypi/wmagent/bin/manage-common.sh +++ b/docker/pypi/wmagent/bin/manage-common.sh @@ -118,6 +118,15 @@ _exec_oracle() { fi } +_exec_sql() { + case $AGENT_FLAVOR in + 'mysql') + _exec_mysql "$@" $wmaDBName ;; + 'oracle') + _exec_oracle "$@" ;; + esac +} + _init_valid(){ # Auxiliary function to shorten repetitive compares of .init* files to the current WMA_BUILD_ID # :param $1: The full path to the .init* file to be checked. diff --git a/docker/pypi/wmagent/init.sh b/docker/pypi/wmagent/init.sh index f1b035667..de214600c 100755 --- a/docker/pypi/wmagent/init.sh +++ b/docker/pypi/wmagent/init.sh @@ -50,6 +50,7 @@ echo "=======================================================" echo "Starting WMAgent with the following initialisation data:" echo "-------------------------------------------------------" echo " - WMAgent Version : $WMA_TAG" +echo " - WMAgent Release Cycle : $WMA_VER_RELEASE" echo " - WMAgent User : $WMA_USER" echo " - WMAgent Root path : $WMA_ROOT_DIR" echo " - WMAgent Host : $HOSTNAME" @@ -118,6 +119,15 @@ _copy_runtime() { echo "$FUNCNAME: Copying content from: $WMA_DEPLOY_DIR/etc/ to: $WMA_CONFIG_DIR/" cp -ra $WMA_DEPLOY_DIR/etc/ $WMA_CONFIG_DIR/ echo $WMA_BUILD_ID > $wmaInitRuntime + + # NOTE: If the sql database have been previously initialized, then the + # current call of _copy_runtime has been due to a change of the + # $WMA_TAG - either a patch version or a release candidate change + # and we need to update the record at the database as well + _init_valid $wmaInitSqlDB && { + echo "$FUNCNAME: Preserving the current WMA_TAG at the wma_init database" + _exec_sql "update wma_init set init_value='$WMA_TAG' where init_param='wma_tag';" + } else echo "$FUNCNAME: ERROR: \$WMA_DEPLOY_DIR: $WMA_DEPLOY_DIR is not a root path for \$pythonLib: $pythonLib" echo "$FUNCNAME: ERROR: We cannot find the correct WMCore/WMRuntime source to copy at the current host!" @@ -331,6 +341,29 @@ EOF echo "-----------------------------------------------------------------------" } +check_wmatag() { + # A function to check if there have been any change in the WMA_TAG since last + # WMAgent initialization at the host. It does the check by comparing the whole + # WMA_TAG from the environment and the one previously recorded at relational + # database in the table `wma_init`. + # + # NOTE: Before we even start checking all the rest of the initialization flags, we should + # check if this agent has ever been used, and if there is any patch version + # or release candidate change between the current one and the one previously + # initialized at the host. If such discrepancy is confirmed, we should enforce Runtime + # code copy by deleting only the $wmaInitSqlDB and let the rest of the init process take over. + _init_valid $wmaInitSqlDB && { + echo "$FUNCNAME: This agent has been previously initialized. Checking for WMAgent version change since last run." + local wmaTagCurr=$WMA_TAG + local wmaTagSqlDB=$(_exec_sql "select init_value from wma_init where init_param = 'wma_tag';") + [[ $wmaTagCurr == $wmaTagSqlDB ]] || { + echo "$FUNCNAME: Found version change since last run: $wmaTagCurr vs. $wmaTagSqlDB" + echo "$FUNCNAME: Enforcing Runtime code check and copy if needed." + rm $wmaInitRuntime + } + } +} + check_wmagent_init() { # A function to check all previously populated */.init files # from all previous steps and compare them with the /data/.wmaBuildId @@ -602,6 +635,7 @@ start_agent() { main(){ basic_checks check_wmasecrets + check_wmatag check_wmagent_init || { (deploy_to_host) || { err=$?; echo "ERROR: deploy_to_host"; exit $err ;} (check_databases) || { err=$?; echo "ERROR: check_databases"; exit $err ;} diff --git a/docker/pypi/wmagent/install.sh b/docker/pypi/wmagent/install.sh index f765414ee..868d17085 100755 --- a/docker/pypi/wmagent/install.sh +++ b/docker/pypi/wmagent/install.sh @@ -48,19 +48,12 @@ done WMA_TAG_REG="^[0-9]+\.[0-9]+\.[0-9]{1,2}((\.|rc)[0-9]{1,2})?$" [[ $WMA_TAG =~ $WMA_TAG_REG ]] || { echo "WMA_TAG: $WMA_TAG does not match requered expression: $WMA_TAG_REG"; echo "EXIT with Error 1" ; exit 1 ;} -# Parsing the WMA_TAG -declare -A WMA_VER -WMA_VER[release]=$(echo $WMA_TAG| sed -nr 's/.*(^[0-9]+\.[0-9]+\.[0-9]+).*$/\1/p') -WMA_VER[patch]=$(echo $WMA_TAG| sed -nr 's/.*(^[0-9]+\.[0-9]+\.[0-9]+)//p') -WMA_VER[major]=$(echo $WMA_TAG| sed -nr 's/.*(^[0-9]+\.[0-9]+).*/\1/p') -WMA_VER[minor]=$(echo $WMA_TAG| sed -nr 's/^[0-9]+\.[0-9]+\.([0-9]+).*$/\1/p') - echo echo "=======================================================================" echo "Starting new WMAgent deployment with the following initialisation data:" echo "-----------------------------------------------------------------------" echo " - WMAgent Version : $WMA_TAG" -echo " - WMAgent Release Cycle : ${WMA_VER[release]}" +echo " - WMAgent Release Cycle : $WMA_VER_RELEASE" echo " - WMAgent User : $WMA_USER" echo " - WMAgent Root path : $WMA_ROOT_DIR" echo " - Python Version : $(python --version)" @@ -114,9 +107,10 @@ stepMsg="Generating and preserving current build id" echo "-----------------------------------------------------------------------" echo "Start $stepMsg" -echo ${WMA_VER[release]} | sha256sum | awk '{print $1}' > $WMA_ROOT_DIR/.wmaBuildId +echo $WMA_VER_RELEASE | sha256sum | awk '{print $1}' > $WMA_ROOT_DIR/.wmaBuildId echo "WMA_BUILD_ID:`cat $WMA_ROOT_DIR/.wmaBuildId`" echo "WMA_BUILD_ID preserved at: $WMA_ROOT_DIR/.wmaBuildId " + echo "Done $stepMsg!" && echo echo "-----------------------------------------------------------------------" diff --git a/docker/pypi/wmagent/wmagent-docker-run.sh b/docker/pypi/wmagent/wmagent-docker-run.sh index a38cf713f..68d3e2e74 100755 --- a/docker/pypi/wmagent/wmagent-docker-run.sh +++ b/docker/pypi/wmagent/wmagent-docker-run.sh @@ -43,6 +43,15 @@ while getopts ":t:hp" opt; do esac done +# Parsing the WMA_TAG in parts step by step +WMA_VER_MINOR=${WMA_TAG#*.*.} +WMA_VER_MAJOR=${WMA_TAG%.$WMA_VER_MINOR} +WMA_VER_MINOR=${WMA_VER_MINOR%rc*} +WMA_VER_MINOR=${WMA_VER_MINOR%.*} +WMA_VER_RELEASE=${WMA_VER_MAJOR}.${WMA_VER_MINOR} +WMA_VER_PATCH=${WMA_TAG#$WMA_VER_RELEASE} +WMA_VER_PATCH=${WMA_VER_PATCH#.} + wmaUser=$(id -un) wmaGroup=$(id -gn) @@ -73,9 +82,9 @@ fi # create regular mount points at runtime [[ -d $HOST_MOUNT_DIR/admin/wmagent ]] || (mkdir -p $HOST_MOUNT_DIR/admin/wmagent) || exit $? -[[ -d $HOST_MOUNT_DIR/srv/wmagent/$WMA_TAG/install ]] || (mkdir -p $HOST_MOUNT_DIR/srv/wmagent/$WMA_TAG/install) || exit $? -[[ -d $HOST_MOUNT_DIR/srv/wmagent/$WMA_TAG/config ]] || (mkdir -p $HOST_MOUNT_DIR/srv/wmagent/$WMA_TAG/config) || exit $? -[[ -d $HOST_MOUNT_DIR/srv/wmagent/$WMA_TAG/logs ]] || { mkdir -p $HOST_MOUNT_DIR/srv/wmagent/$WMA_TAG/logs ;} || exit $? +[[ -d $HOST_MOUNT_DIR/srv/wmagent/$WMA_VER_RELEASE/install ]] || (mkdir -p $HOST_MOUNT_DIR/srv/wmagent/$WMA_VER_RELEASE/install) || exit $? +[[ -d $HOST_MOUNT_DIR/srv/wmagent/$WMA_VER_RELEASE/config ]] || (mkdir -p $HOST_MOUNT_DIR/srv/wmagent/$WMA_VER_RELEASE/config) || exit $? +[[ -d $HOST_MOUNT_DIR/srv/wmagent/$WMA_VER_RELEASE/logs ]] || { mkdir -p $HOST_MOUNT_DIR/srv/wmagent/$WMA_VER_RELEASE/logs ;} || exit $? # NOTE: Before mounting /etc/tnsnames.ora we should check it exists, otherwise the run will fail on the FNAL agents tnsMount="" @@ -92,9 +101,9 @@ $tnsMount \ --mount type=bind,source=/etc/condor,target=/etc/condor,readonly \ --mount type=bind,source=/tmp,target=/tmp \ --mount type=bind,source=/data/certs,target=/data/certs \ ---mount type=bind,source=$HOST_MOUNT_DIR/srv/wmagent/$WMA_TAG/install,target=/data/srv/wmagent/current/install \ ---mount type=bind,source=$HOST_MOUNT_DIR/srv/wmagent/$WMA_TAG/config,target=/data/srv/wmagent/current/config \ ---mount type=bind,source=$HOST_MOUNT_DIR/srv/wmagent/$WMA_TAG/logs,target=/data/srv/wmagent/current/logs \ +--mount type=bind,source=$HOST_MOUNT_DIR/srv/wmagent/$WMA_VER_RELEASE/install,target=/data/srv/wmagent/current/install \ +--mount type=bind,source=$HOST_MOUNT_DIR/srv/wmagent/$WMA_VER_RELEASE/config,target=/data/srv/wmagent/current/config \ +--mount type=bind,source=$HOST_MOUNT_DIR/srv/wmagent/$WMA_VER_RELEASE/logs,target=/data/srv/wmagent/current/logs \ --mount type=bind,source=$HOST_MOUNT_DIR/admin/wmagent,target=/data/admin/wmagent \ --mount type=bind,source=$HOST_MOUNT_DIR/admin/etc/passwd,target=/etc/passwd,readonly \ --mount type=bind,source=$HOST_MOUNT_DIR/admin/etc/group,target=/etc/group,readonly \ @@ -103,17 +112,17 @@ $tnsMount \ " $PULL && { - echo "Pulling Docker image: registry.cern.ch/cmsweb/wmagent:$WMA_TAG" + echo "Pulling Docker image: registry.cern.ch/cmsweb/wmagent:$WMA_VER_RELEASE" docker login registry.cern.ch - docker pull registry.cern.ch/cmsweb/wmagent:$WMA_TAG - docker tag registry.cern.ch/cmsweb/wmagent:$WMA_TAG local/wmagent:$WMA_TAG - docker tag registry.cern.ch/cmsweb/wmagent:$WMA_TAG local/wmagent:latest + docker pull registry.cern.ch/cmsweb/wmagent:$WMA_VER_RELEASE + docker tag registry.cern.ch/cmsweb/wmagent:$WMA_VER_RELEASE local/wmagent:$WMA_VER_RELEASE + docker tag registry.cern.ch/cmsweb/wmagent:$WMA_VER_RELEASE local/wmagent:latest } -echo "Checking if there is no other wmagent container running and creating a link to the $WMA_TAG in the host mount area." +echo "Checking if there is no other wmagent container running and creating a link to the $WMA_VER_RELEASE in the host mount area." [[ $(docker container inspect -f '{{.State.Status}}' wmagent 2>/dev/null) == 'running' ]] || ( [[ -h $HOST_MOUNT_DIR/srv/wmagent/current ]] && rm -f $HOST_MOUNT_DIR/srv/wmagent/current - ln -s $HOST_MOUNT_DIR/srv/wmagent/$WMA_TAG $HOST_MOUNT_DIR/srv/wmagent/current ) + ln -s $HOST_MOUNT_DIR/srv/wmagent/$WMA_VER_RELEASE $HOST_MOUNT_DIR/srv/wmagent/current ) echo "Starting wmagent:$WMA_TAG docker container with user: $wmaUser:$wmaGroup" docker run $dockerOpts local/wmagent:$WMA_TAG