Allow restarting openqa-webui-daemon without downtime

dist/rpm/openQA.spec

@@ -17,7 +17,9 @@
 
 
 # can't use linebreaks here!
-%define openqa_services openqa-webui.service openqa-gru.service openqa-websockets.service openqa-scheduler.service openqa-enqueue-audit-event-cleanup.service openqa-enqueue-audit-event-cleanup.timer openqa-enqueue-asset-cleanup.service openqa-enqueue-asset-cleanup.timer openqa-enqueue-result-cleanup.service openqa-enqueue-result-cleanup.timer openqa-enqueue-bug-cleanup.service openqa-enqueue-bug-cleanup.timer
+%define openqa_main_service openqa-webui.service
+%define openqa_extra_services openqa-gru.service openqa-websockets.service openqa-scheduler.service openqa-enqueue-audit-event-cleanup.service openqa-enqueue-audit-event-cleanup.timer openqa-enqueue-asset-cleanup.service openqa-enqueue-asset-cleanup.timer openqa-enqueue-result-cleanup.service openqa-enqueue-result-cleanup.timer openqa-enqueue-bug-cleanup.service openqa-enqueue-bug-cleanup.timer
+%define openqa_services %{openqa_main_service} %{openqa_extra_services}
 %define openqa_worker_services openqa-worker.target openqa-slirpvde.service openqa-vde_switch.service openqa-worker-cacheservice.service openqa-worker-cacheservice-minion.service
 %if %{undefined tmpfiles_create}
 %define tmpfiles_create() \
@@ -504,7 +506,12 @@ fi
 %service_del_preun openqa-continuous-update.timer
 
 %postun
-%service_del_postun %{openqa_services}
+# reload main service (but do not restart it via service_del_postun to minimize downtimes)
+if [ -x /usr/bin/systemctl ] && [ $1 -ge 1 ]; then
+    /usr/bin/systemctl reload %{openqa_main_service} || :
+fi
+# restart other services
+%service_del_postun %{openqa_extra_services}
 %restart_on_update apparmor
 
 %postun worker

lib/OpenQA/Utils.pm

@@ -840,10 +840,10 @@ sub set_listen_address {
     my $port = shift;
 
     return if $ENV{MOJO_LISTEN};
-    my @listen_addresses = ("http://127.0.0.1:$port");
+    my @listen_addresses = ("http://127.0.0.1:$port?reuse=1");
 
     # Check for IPv6
-    push @listen_addresses, "http://[::1]:$port" if IO::Socket::IP->new(Listen => 5, LocalAddr => '::1');
+    push @listen_addresses, "http://[::1]:$port?reuse=1" if IO::Socket::IP->new(Listen => 5, LocalAddr => '::1');
 
     $ENV{MOJO_LISTEN} = join ',', @listen_addresses;
 }

script/openqa-webui-daemon

@@ -1,3 +1,38 @@
-#!/bin/sh -e
-# Our API commands are very expensive, so the default timeouts are too tight
-exec "$(dirname "$0")"/openqa prefork -m production --proxy -i 100 -H 900 -w 30 -c 1 -G 800 "$@"
+#!/bin/bash
+set -e
+
+pid=
+pid_idx=0
+pid_dir=${OPENQA_BASEDIR:-/var/lib}/openqa/webui
+openqa_args=("$@")
+openqa_dir=$(dirname "$0")
+
+function start_service {
+    # keep track of the previous and next PID
+    pid_last=$pid
+    pid_file=$pid_dir/prefork-$pid_idx.pid
+    pid_idx=$(((pid_idx + 1) % 2))
+    rm -f "$pid_file"
+
+    # start openQA in the background
+    # note: Our API commands are very expensive, so the default timeouts are too tight.
+    "$openqa_dir"/openqa prefork -m production --proxy -i 100 -H 900 -w 30 -c 1 -G 800 -P "$pid_file" "${openqa_args[@]}" &
+    pid=$!
+
+    # wait until openQA is ready to accept requests by waiting for its PID file
+    while [[ ! -e $pid_file ]] && [[ -e /proc/$pid ]]; do sleep 1; done
+
+    # terminate a previously started openQA instance
+    [[ $pid_last ]] && kill -s TERM "$pid_last"
+
+    # keep running until openQA terminates (with the "wait"-builtin so bash can handle SIGHUP)
+    wait "$pid"
+}
+
+# create directory for PID file but ignore errors at this point as the dir is not required by all sub commands
+mkdir -p "$pid_dir" || true
+
+# start service now and restart it gracefully when we receive SIGHUP
+# see https://docs.mojolicious.org/Mojolicious/Guides/Cookbook#Zero-downtime-software-upgrades
+trap start_service SIGHUP
+start_service

systemd/openqa-webui.service

@@ -1,12 +1,12 @@
 [Unit]
 Description=The openQA web UI
-Wants=openqa-setup-db.service
+Wants=openqa-setup-db.service openqa-livehandler.service openqa-websockets.service openqa-gru.service openqa-enqueue-asset-cleanup.timer openqa-enqueue-result-cleanup.timer openqa-enqueue-bug-cleanup.timer openqa-minion-restart.path
 After=postgresql.service openqa-setup-db.service openqa-scheduler.service nss-lookup.target remote-fs.target
-Requires=openqa-livehandler.service openqa-websockets.service openqa-gru.service openqa-enqueue-asset-cleanup.timer openqa-enqueue-result-cleanup.timer openqa-enqueue-bug-cleanup.timer openqa-minion-restart.path
 
 [Service]
 User=geekotest
 ExecStart=/usr/share/openqa/script/openqa-webui-daemon
+ExecReload=kill -HUP $MAINPID
 
 [Install]
 WantedBy=multi-user.target