From a38af0d7bbb01a7feea9b306f190ad7f340aa194 Mon Sep 17 00:00:00 2001 From: Marcel Clausen Date: Tue, 6 Sep 2022 16:29:01 +0200 Subject: [PATCH 1/5] BGDIINF_SB-2530: add a first draft of readiness probe * readiness probe will be written to checker_ready.txt in shared volume or tmp folder * liveness and readiness probe will be activated after the bootstrapping of the container, during the initialisation phase, the container is considered as healthy --- scripts/checker.sh | 50 +++++++++++++++++++++++++++++++++++++++-- scripts/docker-cmd.sh | 20 ++++++++++++++++- scripts/docker-entry.sh | 5 +++++ 3 files changed, 72 insertions(+), 3 deletions(-) diff --git a/scripts/checker.sh b/scripts/checker.sh index 566a34d6..efce4e1c 100755 --- a/scripts/checker.sh +++ b/scripts/checker.sh @@ -3,5 +3,51 @@ set -e set -u set -o pipefail -# Do the lifeness check exit status 0 -> success | 1 -> fail -searchd --status 1> /dev/null && exit 0 || exit 1 \ No newline at end of file +# mountpoint with readiness probe file +MOUNT=$(realpath "${PROBE_MOUNTPOINT:-/tmp}") +READY_FILE="${MOUNT}/checker_ready.txt" +BOOTSTRAP_FILE="${MOUNT}/bootstrapped.txt" +PRECACHING_SECONDS=900 + +# get file age in seconds +function fileAge +{ + local fileMod + if fileMod=$(stat -c %Y -- "$1") + then + echo $(( $(date +%s) - fileMod )) + else + return $? + fi +} + +clean_probe_files() { + rm -f "${READY_FILE}" || : + rm -f "${BOOTSTRAP_FILE}" || : +} + +# source this stuff until here +[ "$0" = "${BASH_SOURCE[*]}" ] || return 0 + +# step 1 +# skip probes during the bootstrap of the container, initial rsync +[ -e "${BOOTSTRAP_FILE}" ] || exit 0 + +# step 2 +# if searchd is running, container is ready for connections and ready probe will be activated +if searchd --status 1> /dev/null; then + echo "READY" > "${READY_FILE}" + # change age of bootstrap file, readiness probe is activated now + touch -d "${PRECACHING_SECONDS} seconds ago" "${BOOTSTRAP_FILE}" + exit 0 +fi + +# step 3 +# do not raise an error during the first PRECACHING_SECONDS after bootstrap (pre-cache phase) +(( $(fileAge "${BOOTSTRAP_FILE}") < PRECACHING_SECONDS )) && exit 0 + +# final step +# after PRECACHING_SECONDS: +# fail if searchd is not running +echo "searchd not running" +exit 1 \ No newline at end of file diff --git a/scripts/docker-cmd.sh b/scripts/docker-cmd.sh index 711ea5c5..96f71528 100755 --- a/scripts/docker-cmd.sh +++ b/scripts/docker-cmd.sh @@ -1,6 +1,12 @@ #!/bin/bash set -euo pipefail +# shellcheck source=checker.sh +source checker.sh + +# Capture Exit Code +trap clean_probe_files EXIT + # fancy output green='\e[0;32m' red='\e[0;31m' @@ -38,7 +44,19 @@ rsync --update --delete -av --exclude "*.tmp.*" --stats ${SPHINXINDEX_EFS} ${SPH service cron start || exit 1 crontab < docker-crontab +# once the initial sync and index creation is done the docker volume is ready / bootstrapped +# from now on the liveness probe is checking the searchd status +touch "${BOOTSTRAP_FILE}" +echo "bootstrap finished $(date +"%F %T"), start precaching indexes" > "${READY_FILE}" + # starting the searchd service # will load the sphinx indexes from EFS --sync--> Volume --> into memory +# precaching can take up to 15m echo -e "${green}starting searchd service ...${NC}" -/usr/bin/searchd --nodetach "$@" \ No newline at end of file +/usr/bin/searchd "$@" + +# this inifite loop is necessary to trap docker signals SIGHUP SIGTERM +# and do the clean-up in the volume with the trap function +while true; do + sleep 10 +done \ No newline at end of file diff --git a/scripts/docker-entry.sh b/scripts/docker-entry.sh index 51f1a69d..5ef39986 100755 --- a/scripts/docker-entry.sh +++ b/scripts/docker-entry.sh @@ -1,6 +1,11 @@ #!/bin/bash set -euo pipefail +# shellcheck source=checker.sh +source checker.sh +# geodata has to have rw access on probe mountpoint +chown -R geodata:geodata "${MOUNT}" + # build sphinx config with current environment cat conf/*.part > conf/sphinx.conf.in envsubst < conf/sphinx.conf.in > conf/sphinx.conf From b2bd3e5e381b29a40a936619099c8b2261eb9de1 Mon Sep 17 00:00:00 2001 From: Marcel Clausen Date: Wed, 7 Sep 2022 07:40:51 +0200 Subject: [PATCH 2/5] shellcheck --- scripts/docker-cmd.sh | 6 +++--- scripts/docker-entry.sh | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/scripts/docker-cmd.sh b/scripts/docker-cmd.sh index 96f71528..0169786e 100755 --- a/scripts/docker-cmd.sh +++ b/scripts/docker-cmd.sh @@ -1,7 +1,7 @@ #!/bin/bash +# shellcheck disable=SC1091 set -euo pipefail -# shellcheck source=checker.sh source checker.sh # Capture Exit Code @@ -28,7 +28,7 @@ for index in "${array_orphaned[@]}"; do [[ -z ${index} ]] && continue # skip .new files, we need them to sighup searchd / rotate index updates if [[ ! $index == *.new ]]; then - echo -e "\t${red} deleting orphaned index ${index} from filesystem. ${NC}" + echo -e "\\t${red} deleting orphaned index ${index} from filesystem. ${NC}" rm -rf "${SPHINXINDEX_EFS}${index}".* fi done @@ -59,4 +59,4 @@ echo -e "${green}starting searchd service ...${NC}" # and do the clean-up in the volume with the trap function while true; do sleep 10 -done \ No newline at end of file +done diff --git a/scripts/docker-entry.sh b/scripts/docker-entry.sh index 5ef39986..fcce0e20 100755 --- a/scripts/docker-entry.sh +++ b/scripts/docker-entry.sh @@ -1,7 +1,7 @@ #!/bin/bash +# shellcheck disable=SC1091 set -euo pipefail -# shellcheck source=checker.sh source checker.sh # geodata has to have rw access on probe mountpoint chown -R geodata:geodata "${MOUNT}" @@ -17,4 +17,4 @@ cp -f conf/wordforms_main.txt /etc/sphinxsearch/wordforms_main.txt # always remove lock files from mounted shared storage rm -rf /var/lib/sphinxsearch/data/index/*.spl 2> /dev/null || : -exec gosu geodata "$@" \ No newline at end of file +exec gosu geodata "$@" From b5a57e8c3f63626eb6990429d60cd77a3104e54e Mon Sep 17 00:00:00 2001 From: Marcel Clausen Date: Wed, 7 Sep 2022 10:21:37 +0200 Subject: [PATCH 3/5] precaching can take up to 20m --- scripts/checker.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/checker.sh b/scripts/checker.sh index efce4e1c..62abdf7a 100755 --- a/scripts/checker.sh +++ b/scripts/checker.sh @@ -7,7 +7,7 @@ set -o pipefail MOUNT=$(realpath "${PROBE_MOUNTPOINT:-/tmp}") READY_FILE="${MOUNT}/checker_ready.txt" BOOTSTRAP_FILE="${MOUNT}/bootstrapped.txt" -PRECACHING_SECONDS=900 +PRECACHING_SECONDS=1800 # get file age in seconds function fileAge From d208367f8a92c4255fc4bfc6fbfc6ba42399807d Mon Sep 17 00:00:00 2001 From: Marcel Clausen Date: Wed, 7 Sep 2022 11:21:27 +0200 Subject: [PATCH 4/5] simplify readiness and healthcheck the start_period has to be increased to 1hour for this change. if the container is up and healty in less than that time, the probes will be activated. this will allow us to use docker-compose --wait and the container status will be the same for alb and docker. executing the searchd as main process is working fine with the trap, the volume is cleaned up with: * docker stop/kill * pkill searchd inside container * docker-compose down --- scripts/checker.sh | 37 +++---------------------------------- scripts/docker-cmd.sh | 14 +------------- 2 files changed, 4 insertions(+), 47 deletions(-) diff --git a/scripts/checker.sh b/scripts/checker.sh index 62abdf7a..7b6287ed 100755 --- a/scripts/checker.sh +++ b/scripts/checker.sh @@ -6,48 +6,17 @@ set -o pipefail # mountpoint with readiness probe file MOUNT=$(realpath "${PROBE_MOUNTPOINT:-/tmp}") READY_FILE="${MOUNT}/checker_ready.txt" -BOOTSTRAP_FILE="${MOUNT}/bootstrapped.txt" -PRECACHING_SECONDS=1800 - -# get file age in seconds -function fileAge -{ - local fileMod - if fileMod=$(stat -c %Y -- "$1") - then - echo $(( $(date +%s) - fileMod )) - else - return $? - fi -} clean_probe_files() { rm -f "${READY_FILE}" || : - rm -f "${BOOTSTRAP_FILE}" || : } # source this stuff until here [ "$0" = "${BASH_SOURCE[*]}" ] || return 0 -# step 1 -# skip probes during the bootstrap of the container, initial rsync -[ -e "${BOOTSTRAP_FILE}" ] || exit 0 - -# step 2 -# if searchd is running, container is ready for connections and ready probe will be activated if searchd --status 1> /dev/null; then echo "READY" > "${READY_FILE}" - # change age of bootstrap file, readiness probe is activated now - touch -d "${PRECACHING_SECONDS} seconds ago" "${BOOTSTRAP_FILE}" exit 0 -fi - -# step 3 -# do not raise an error during the first PRECACHING_SECONDS after bootstrap (pre-cache phase) -(( $(fileAge "${BOOTSTRAP_FILE}") < PRECACHING_SECONDS )) && exit 0 - -# final step -# after PRECACHING_SECONDS: -# fail if searchd is not running -echo "searchd not running" -exit 1 \ No newline at end of file +else + exit 1 +fi \ No newline at end of file diff --git a/scripts/docker-cmd.sh b/scripts/docker-cmd.sh index 0169786e..d04b5890 100755 --- a/scripts/docker-cmd.sh +++ b/scripts/docker-cmd.sh @@ -44,19 +44,7 @@ rsync --update --delete -av --exclude "*.tmp.*" --stats ${SPHINXINDEX_EFS} ${SPH service cron start || exit 1 crontab < docker-crontab -# once the initial sync and index creation is done the docker volume is ready / bootstrapped -# from now on the liveness probe is checking the searchd status -touch "${BOOTSTRAP_FILE}" -echo "bootstrap finished $(date +"%F %T"), start precaching indexes" > "${READY_FILE}" - # starting the searchd service # will load the sphinx indexes from EFS --sync--> Volume --> into memory -# precaching can take up to 15m echo -e "${green}starting searchd service ...${NC}" -/usr/bin/searchd "$@" - -# this inifite loop is necessary to trap docker signals SIGHUP SIGTERM -# and do the clean-up in the volume with the trap function -while true; do - sleep 10 -done +/usr/bin/searchd --nodetach "$@" \ No newline at end of file From 20b24de7417253d0ba2ae167fc55c8e56cb08e2f Mon Sep 17 00:00:00 2001 From: Marcel Clausen Date: Wed, 7 Sep 2022 15:15:55 +0200 Subject: [PATCH 5/5] feedback ltshb --- scripts/checker.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scripts/checker.sh b/scripts/checker.sh index 7b6287ed..69f838b2 100755 --- a/scripts/checker.sh +++ b/scripts/checker.sh @@ -8,7 +8,7 @@ MOUNT=$(realpath "${PROBE_MOUNTPOINT:-/tmp}") READY_FILE="${MOUNT}/checker_ready.txt" clean_probe_files() { - rm -f "${READY_FILE}" || : + rm -f "${READY_FILE}" || : } # source this stuff until here @@ -18,5 +18,6 @@ if searchd --status 1> /dev/null; then echo "READY" > "${READY_FILE}" exit 0 else + clean_probe_files exit 1 fi \ No newline at end of file