Allow restarting openqa-webui-daemon without downtime

* Allow restarting `openqa-webui-daemon` without downtime by sending SIGHUP
  to the process or reloading the systemd unit `openqa-webui.service`
* Start the Mojolicious application with `reuse=1` as mentioned on
  https://docs.mojolicious.org/Mojolicious/Guides/Cookbook#Zero-downtime-software-upgrades
* Note that other services are not covered but those are also not user
  facing or retried and thus not required
* See https://progress.opensuse.org/issues/162533

lib/OpenQA/Utils.pm

@@ -840,10 +840,10 @@ sub set_listen_address {
     my $port = shift;
 
     return if $ENV{MOJO_LISTEN};
-    my @listen_addresses = ("http://127.0.0.1:$port");
+    my @listen_addresses = ("http://127.0.0.1:$port?reuse=1");
 
     # Check for IPv6
-    push @listen_addresses, "http://[::1]:$port" if IO::Socket::IP->new(Listen => 5, LocalAddr => '::1');
+    push @listen_addresses, "http://[::1]:$port?reuse=1" if IO::Socket::IP->new(Listen => 5, LocalAddr => '::1');
 
     $ENV{MOJO_LISTEN} = join ',', @listen_addresses;
 }

script/openqa-webui-daemon

@@ -1,3 +1,37 @@
-#!/bin/sh -e
-# Our API commands are very expensive, so the default timeouts are too tight
-exec "$(dirname "$0")"/openqa prefork -m production --proxy -i 100 -H 900 -w 30 -c 1 -G 800 "$@"
+#!/bin/bash
+set -e
+
+pid=
+pid_idx=0
+pid_dir=${OPENQA_BASEDIR:-/var/lib}/openqa/webui
+openqa_args=("$@")
+openqa_dir=$(dirname "$0")
+
+function start_service {
+    # keep track of the previous and next PID
+    pid_last=$pid
+    pid_file=$pid_dir/prefork-$pid_idx.pid
+    pid_idx=$(((pid_idx + 1) % 2))
+    rm -f "$pid_file"
+
+    # start openQA in the background
+    # note: Our API commands are very expensive, so the default timeouts are too tight.
+    "$openqa_dir"/openqa prefork -m production --proxy -i 100 -H 900 -w 30 -c 1 -G 800 -P "$pid_file" "${openqa_args[@]}" &
+    pid=$!
+
+    # wait until openQA is ready to accept requests by waiting for its PID file
+    while [[ ! -e $pid_file ]] && [[ -e /proc/$pid ]]; do sleep 1; done
+
+    # terminate a previously started openQA instance
+    [[ $pid_last ]] && kill -s TERM "$pid_last"
+
+    # keep running until openQA terminates (with the "wait"-builtin so bash can handle SIGHUP)
+    wait "$pid"
+}
+
+mkdir -p "$pid_dir"
+
+# start service now and restart it gracefully when we receive SIGHUP
+# see https://docs.mojolicious.org/Mojolicious/Guides/Cookbook#Zero-downtime-software-upgrades
+trap start_service SIGHUP
+start_service

systemd/openqa-webui.service

@@ -7,6 +7,7 @@ Requires=openqa-livehandler.service openqa-websockets.service openqa-gru.service
 [Service]
 User=geekotest
 ExecStart=/usr/share/openqa/script/openqa-webui-daemon
+ExecReload=kill -HUP $MAINPID
 
 [Install]
 WantedBy=multi-user.target