Added safety around stopping postgresql server during upgrade

Previously we executed pg_ctl stop (aka "Fast" stop) and assumed the server would be stopped immediately which is not guaranteed.

Now wait up to 5 seconds for the server to stop.
If it does not stop in that time, fail the package postinstall script.

Ticket: ENT-10647
Changelog: title

packaging/common/cfengine-hub/postinstall.sh

@@ -437,6 +437,12 @@ init_postgres_dir()
           # Started successfully, stop it again, the migration requires it to be not running.
           (cd /tmp && su cfpostgres -c "$PREFIX/bin/pg_ctl -w -D $PREFIX/state/pg/data -l /var/log/postgresql.log stop")
 
+          # Wait a while if we have to for the server to be stopped
+          if ! wait_for_cf_postgres_down; then
+            cf_console echo "Error: unable to shutdown postgresql server. Showing last of /var/log/postgresql.log for clues."
+            cf_console tail /var/log/postgresql.log
+            exit 1
+          fi
           # Copy over the new config as well, user should take at look at it.
           cf_console echo "Installing the $pgconfig_type postgresql.conf file as $PREFIX/state/pg/data/postgresql.conf.new."
           cf_console echo "Please review it and update $PREFIX/state/pg/data/postgresql.conf accordingly."

packaging/common/script-templates/script-common.sh

@@ -99,6 +99,24 @@ wait_for_cf_postgres() {
     $PREFIX/bin/psql cfsettings -c "SELECT 1;" >/dev/null 2>&1
 }
 
+wait_for_cf_postgres_down() {
+    # wait for CFEngine Postgresql service to be shutdown, up to 5 sec.
+    # Returns 0 if postgresql service is not running
+    # Returns non-0 otherwise (1 if exited by timeout)
+    for i in $(seq 1 5); do
+        true "checking if Postgresql is shutdown..."
+        if ! "$PREFIX"/bin/pg_isready >/dev/null 2>&1; then
+            true "Postgresql is shutdown, moving on"
+            return 0
+        fi
+        true "waiting 1 sec for Postgresql to shutdown..."
+        sleep 1
+    done
+    # Note: it is important that this is the last command of this function.
+    # Return code of `pg_isready` is the return code of whole function.
+    ! "$PREFIX"/bin/pg_isready >/dev/null 2>&1
+}
+
 safe_cp() {
     # "safe" alternative to `cp`. Tries `cp -al` first, and if it fails - `cp -a`.
     # Deletes partially-copied files if copy operation fails.