ENT-10647: Added safety for hub upgrade in packaging scriptlets

packaging/common/cfengine-hub/postinstall.sh

@@ -429,25 +429,37 @@ init_postgres_dir()
     if [ -f "$BACKUP_DIR/data/postgresql.conf.modified" ]; then
       # User-modified file from the previous old version of CFEngine exists, try to use it.
       cp -a "$BACKUP_DIR/data/postgresql.conf.modified" "$PREFIX/state/pg/data/postgresql.conf"
-      (cd /tmp && su cfpostgres -c "$PREFIX/bin/pg_ctl -w -D $PREFIX/state/pg/data -l /var/log/postgresql.log start")
-      if [ $? = 0 ]; then
-        # Started successfully, stop it again, the migration requires it to be not running.
-        (cd /tmp && su cfpostgres -c "$PREFIX/bin/pg_ctl -w -D $PREFIX/state/pg/data -l /var/log/postgresql.log stop")
-
-        # Copy over the new config as well, user should take at look at it.
-        cf_console echo "Installing the $pgconfig_type postgresql.conf file as $PREFIX/state/pg/data/postgresql.conf.new."
-        cf_console echo "Please review it and update $PREFIX/state/pg/data/postgresql.conf accordingly."
-        cp -a "$new_pgconfig_file" "$PREFIX/state/pg/data/postgresql.conf.new"
-        chown cfpostgres "$PREFIX/state/pg/data/postgresql.conf.new"
-      else
-        # Failed to start, move the old file aside and use the new one.
-        mv "$PREFIX/state/pg/data/postgresql.conf" "$PREFIX/state/pg/data/postgresql.conf.old"
-        cf_console echo "Warning: failed to use the old postgresql.conf file, using the $pgconfig_type one."
-        cf_console echo "Please review the $PREFIX/state/pg/data/postgresql.conf file and update it accordingly."
-        cf_console echo "The original file was saved as $PREFIX/state/pg/data/postgresql.conf.old"
-        cp -a "$new_pgconfig_file" "$PREFIX/state/pg/data/postgresql.conf"
-        chown cfpostgres "$PREFIX/state/pg/data/postgresql.conf"
-      fi
+      # start subshell to disable error handling temporarily
+      (
+        set +e
+        (cd /tmp && su cfpostgres -c "$PREFIX/bin/pg_ctl -w -D $PREFIX/state/pg/data -l /var/log/postgresql.log start")
+        if [ $? = 0 ]; then
+          # Started successfully, stop it again, the migration requires it to be not running.
+          (cd /tmp && su cfpostgres -c "$PREFIX/bin/pg_ctl -w -D $PREFIX/state/pg/data -l /var/log/postgresql.log stop")
+
+          # Wait a while if we have to for the server to be stopped
+          if ! wait_for_cf_postgres_down; then
+            cf_console echo "Error: unable to shutdown postgresql server. Showing last of /var/log/postgresql.log for clues."
+            cf_console tail /var/log/postgresql.log
+            exit 1
+          fi
+          # Copy over the new config as well, user should take at look at it.
+          cf_console echo "Installing the $pgconfig_type postgresql.conf file as $PREFIX/state/pg/data/postgresql.conf.new."
+          cf_console echo "Please review it and update $PREFIX/state/pg/data/postgresql.conf accordingly."
+          cp -a "$new_pgconfig_file" "$PREFIX/state/pg/data/postgresql.conf.new"
+          chown cfpostgres "$PREFIX/state/pg/data/postgresql.conf.new"
+        else
+          # Failed to start, move the old file aside and use the new one.
+          mv "$PREFIX/state/pg/data/postgresql.conf" "$PREFIX/state/pg/data/postgresql.conf.old"
+          cf_console echo "Warning: failed to use the old postgresql.conf file, using the $pgconfig_type one."
+          cf_console echo "Please review the $PREFIX/state/pg/data/postgresql.conf file and update it accordingly."
+          cf_console echo "The original file was saved as $PREFIX/state/pg/data/postgresql.conf.old"
+          cf_console echo "last 10 lines of /var/log/postgresql.log for determining cause of failure"
+          cf_console tail -10 /var/log/postgresql.log
+          cp -a "$new_pgconfig_file" "$PREFIX/state/pg/data/postgresql.conf"
+          chown cfpostgres "$PREFIX/state/pg/data/postgresql.conf"
+        fi
+      )
     else
       # No user-modified file, just use the new recommended or default config (see generate_new_postgres_conf())
       cp -a "$new_pgconfig_file" "$PREFIX/state/pg/data/postgresql.conf"
@@ -684,6 +696,11 @@ do_migration() {
       exit 0 # exits only from (...)
     fi
     cf_console echo "Migration using pg_upgrade failed."
+    # here pg_upgrade probably said something like
+    # Consult the last few lines of "/var/cfengine/state/pg/data/pg_upgrade_output.d/20230913T150025.959/log/pg_upgrade_server.log" for the probable cause of the failure.
+    cf_console echo "Showing last lines of any related log files:"
+    _daysearch=$(date +%Y%m%d)
+    find "$PREFIX"/state/pg/data/pg_upgrade_output.d -name *.log | grep "$_daysearch" | xargs tail
     cf_console echo
     check_disk_space # will abort if low on disk space
     init_postgres_dir "$new_pgconfig_file" "$pgconfig_type"

packaging/common/script-templates/script-common.sh

@@ -99,6 +99,24 @@ wait_for_cf_postgres() {
     $PREFIX/bin/psql cfsettings -c "SELECT 1;" >/dev/null 2>&1
 }
 
+wait_for_cf_postgres_down() {
+    # wait for CFEngine Postgresql service to be shutdown, up to 5 sec.
+    # Returns 0 if postgresql service is not running
+    # Returns non-0 otherwise (1 if exited by timeout)
+    for i in $(seq 1 5); do
+        true "checking if Postgresql is shutdown..."
+        if ! "$PREFIX"/bin/pg_isready >/dev/null 2>&1; then
+            true "Postgresql is shutdown, moving on"
+            return 0
+        fi
+        true "waiting 1 sec for Postgresql to shutdown..."
+        sleep 1
+    done
+    # Note: it is important that this is the last command of this function.
+    # Return code of `pg_isready` is the return code of whole function.
+    ! "$PREFIX"/bin/pg_isready >/dev/null 2>&1
+}
+
 safe_cp() {
     # "safe" alternative to `cp`. Tries `cp -al` first, and if it fails - `cp -a`.
     # Deletes partially-copied files if copy operation fails.