diff --git a/.circleci/config.yml b/.circleci/config.yml index e4aac72a0..58720a84a 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -53,7 +53,7 @@ workflows: parameters: dist: [rockylinux9] # Odd version of Postgres for RHEL - pgversion: ["15"] + pgversion: ["17"] - e2e: <<: *e2e matrix: @@ -311,7 +311,7 @@ jobs: pgversion: description: "PostgreSQL version for repository." type: enum - enum: ["16", "15", "14", "13", "12", "11", "10", "9.6", "9.5"] + enum: ["17", "16", "15", "14", "13", "12", "11", "10", "9.6", "9.5"] resource_class: large docker: - image: dalibo/buildpack-postgres:<< parameters.dist >> @@ -325,10 +325,10 @@ jobs: POSTGRES_USER: postgres POSTGRES_PASSWORD: *PGPASSWORD command: [ - -c, log_statement=all, - -c, log_connections=on, - -c, "log_line_prefix=%m [%p]: [%l-1] app=%a,db=%d,client=%h,user=%u ", - -c, cluster_name=repository, + -c, log_statement=all, + -c, log_connections=on, + -c, "log_line_prefix=%m [%p]: [%l-1] app=%a,db=%d,client=%h,user=%u ", + -c, cluster_name=repository, ] - image: selenium/standalone-firefox@sha256:b6d8279268b3183d0d33e667e82fec1824298902f77718764076de763673124f environment: diff --git a/.editorconfig b/.editorconfig index 79b8dfd02..b01925994 100644 --- a/.editorconfig +++ b/.editorconfig @@ -17,9 +17,6 @@ max_line_length = 88 [*.{css,html,js,json,scss,yml,yaml,sql,vue}] indent_size = 2 -[*.sql] -indent_style = tab - [{*.sh,Makefile}] indent_size = 8 indent_style = tab diff --git a/CHANGELOG.md b/CHANGELOG.md index 3f7ce3039..ae21004d9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,7 @@ Ensure you use consistent title format. **Other changes** +- Postgres 17 support. - ui: Fix deletion of host metrics when removing an instance of multi-instances host. - ui: Remove support for 7.x agents. - ui: Accept editing an offline instance. diff --git a/agent/share/auto_configure.sh b/agent/share/auto_configure.sh index 3c657c1d6..90431f5c6 100755 --- a/agent/share/auto_configure.sh +++ b/agent/share/auto_configure.sh @@ -18,115 +18,123 @@ SYSUSER=${SYSUSER-postgres} set -o pipefail catchall() { - local exit_code=$? + local exit_code=$? - trap - INT EXIT TERM + trap - INT EXIT TERM - # shellcheck disable=SC2181 - if [ $exit_code -gt 0 ] ; then - fatal "Failure. See ${LOGFILE} for details." - else - rm -f "${LOGFILE}" - fi + # shellcheck disable=SC2181 + if [ $exit_code -gt 0 ]; then + fatal "Failure. See ${LOGFILE} for details." + else + rm -f "${LOGFILE}" + fi } error() { - echo -e "\\e[1;31m$*\\e[0m" | tee -a /dev/fd/3 >&2 + echo -e "\\e[1;31m$*\\e[0m" | tee -a /dev/fd/3 >&2 } fatal() { - error "$@" - exit 1 + error "$@" + exit 1 } log() { - echo "$@" | tee -a /dev/fd/3 >&2 + echo "$@" | tee -a /dev/fd/3 >&2 } psql() { - local wrapper - wrapper=() + local wrapper + wrapper=() - if ! [ "$(whoami)" = "$SYSUSER" ] ; then - wrapper=(sudo -Eu "$SYSUSER") - fi + if ! [ "$(whoami)" = "$SYSUSER" ]; then + wrapper=(sudo -Eu "$SYSUSER") + fi - command "${wrapper[@]}" psql -AtX "$@" + command "${wrapper[@]}" psql -AtX "$@" } query_pgsettings() { - # Usage: query_pgsettings name [default] + # Usage: query_pgsettings name [default] - local name=$1; shift - local default=${1-}; shift - val=$(psql -c "SELECT setting FROM pg_settings WHERE name = '${name}';") + local name=$1 + shift + local default=${1-} + shift + val=$(psql -c "SELECT setting FROM pg_settings WHERE name = '${name}';") - echo "${val:-${default}}" + echo "${val:-${default}}" } find_next_free_port() { - local port - local used_a - local used - mapfile -t used_a < <(ss -ln4t '( sport >= 2345 and sport <= 3000 )' | grep -Po ':\K\d+') - # To mock ss output, use seq: - # mapfile -t used_a < <(seq 2345 3000) - used="${used_a[*]:-}" - for port in {2345..3000} ; do - if [[ " $used " =~ \ $port\ ]] ; then continue ; fi - echo "$port" - return - done - log "No free TCP port found between 2345 and 3000. Force with env TEMBOARD_PORT." - return 1 + local port + local used_a + local used + mapfile -t used_a < <(ss -ln4t '( sport >= 2345 and sport <= 3000 )' | grep -Po ':\K\d+') + # To mock ss output, use seq: + # mapfile -t used_a < <(seq 2345 3000) + used="${used_a[*]:-}" + for port in {2345..3000}; do + if [[ " $used " =~ \ $port\ ]]; then continue; fi + echo "$port" + return + done + log "No free TCP port found between 2345 and 3000. Force with env TEMBOARD_PORT." + return 1 } generate_configuration() { - # Usage: generate_configuration homedir sslcert sslkey cluster_name - - # Generates minimal configuration required to adapt default - # configuration to this cluster. - - local ui_url=$1; shift - local home=$1; shift - local sslcert=$1; shift - local sslkey=$1; shift - local instance=$1; shift - local has_statements=$1; shift - - local pg_ctl - local port - local version - - sudo -u "$SYSUSER" test -r "$sslkey" - sudo -u "$SYSUSER" test -r "$sslcert" - - version="$(temboard-agent --version | sed 's/^/# /')" - created="$(date)" - - port="${TEMBOARD_PORT-$(find_next_free_port)}" - test -n "$port" - log "Configuring temboard-agent to run on port ${port}." - if ! pg_ctl="$(command -v pg_ctl)" ; then - pg_ctl="/bin/false" - log "Can't find pg_ctl in PATH." - log "Please configure it manually to enable restart feature." - fi - - plugins=(administration dashboard maintenance monitoring pgconf) - if [ -n "$has_statements" ] ; then - plugins+=(statements) - fi - printf -v qplugins ', "%s"' "${plugins[@]}" # loose jsonify - - local usepeer - if [ -n "${PGPASSWORD-}" ] ; then - usepeer= - else - usepeer=1 - fi - - cat <<-EOF + # Usage: generate_configuration homedir sslcert sslkey cluster_name + + # Generates minimal configuration required to adapt default + # configuration to this cluster. + + local ui_url=$1 + shift + local home=$1 + shift + local sslcert=$1 + shift + local sslkey=$1 + shift + local instance=$1 + shift + local has_statements=$1 + shift + + local pg_ctl + local port + local version + + sudo -u "$SYSUSER" test -r "$sslkey" + sudo -u "$SYSUSER" test -r "$sslcert" + + version="$(temboard-agent --version | sed 's/^/# /')" + created="$(date)" + + port="${TEMBOARD_PORT-$(find_next_free_port)}" + test -n "$port" + log "Configuring temboard-agent to run on port ${port}." + if ! pg_ctl="$(command -v pg_ctl)"; then + pg_ctl="/bin/false" + log "Can't find pg_ctl in PATH." + log "Please configure it manually to enable restart feature." + fi + + plugins=(administration dashboard maintenance monitoring pgconf) + if [ -n "$has_statements" ]; then + plugins+=(statements) + fi + printf -v qplugins ', "%s"' "${plugins[@]}" # loose jsonify + + local usepeer + if [ -n "${PGPASSWORD-}" ]; then + usepeer= + else + usepeer=1 + fi + + cat <<-EOF # # T E M B O A R D A G E N T C O N F I G U R A T I O N # @@ -207,7 +215,7 @@ generate_configuration() { # Comma separated list of database names to monitor. * for all. dbnames = * # List of probes to run, comma separator, * for all. - # Available probes: bgwriter,blocks,btree_bloat,cpu,db_size,filesystems_size,heap_bloat,loadavg,locks,memory,process,replication_connection,replication_lag,sessions,tblspc_size,temp_files_size_delta,wal_files,xacts + # Available probes: bgwriter,blocks,btree_bloat,checkpointer,cpu,db_size,filesystems_size,heap_bloat,loadavg,locks,memory,process,replication_connection,replication_lag,sessions,tblspc_size,temp_files_size_delta,wal_files,xacts probes = * # Interval, in second, between each run of the process executing # the probes. Default: 60 @@ -225,98 +233,100 @@ generate_configuration() { } search_bindir() { - # Usage: search_bindir pgversion - - # Search for bin directory where pg_ctl is installed for this version. - - local pgversion=$1; shift - for d in /usr/lib/postgresql/$pgversion /usr/pgsql-$pgversion ; do - if [ -x "$d/bin/pg_ctl" ] ; then - echo "$d/bin" - return - fi - done - return 1 + # Usage: search_bindir pgversion + + # Search for bin directory where pg_ctl is installed for this version. + + local pgversion=$1 + shift + for d in /usr/lib/postgresql/$pgversion /usr/pgsql-$pgversion; do + if [ -x "$d/bin/pg_ctl" ]; then + echo "$d/bin" + return + fi + done + return 1 } setup_pq() { - # Ensure used libpq vars are defined for configuration template. - - export PGUSER=${PGUSER-postgres} - log "Configuring for PostgreSQL user ${PGUSER}." - export PGDATABASE=${PGDATABASE-${PGUSER}} - export PGPORT=${PGPORT-5432} - log "Configuring for cluster on port ${PGPORT}." - export PGHOST=${PGHOST-$(query_pgsettings unix_socket_directories)} - PGHOST=${PGHOST%%,*} - if ! psql -c "SELECT 'Postgres connection working.';" ; then - fatal "Can't connect to Postgres cluster." - fi - export PGDATA - PGDATA=$(query_pgsettings data_directory) - log "Configuring for cluster at ${PGDATA}." - - read -r PGVERSION < "${PGDATA}/PG_VERSION" - if ! command -v pg_ctl &>/dev/null ; then - if bindir=$(search_bindir "$PGVERSION") ; then - log "Using ${bindir}/pg_ctl." - export PATH=$bindir:$PATH - fi - fi - - # Instance name defaults to cluster_name. If unset (e.g. Postgres 9.4), - # use the tail of ${PGDATA} after ~postgres has been removed. If PGDATA - # is not in postgres home, compute a cluster name from version and port. - local home - home="$(eval readlink -e "~${SYSUSER}")" - if [ -z "${PGDATA##"${home}"/*}" ] ; then - default_cluster_name="${PGDATA##"${home}"/}" - else - default_cluster_name=$PGVERSION/pg${PGPORT} - fi - export PGCLUSTER_NAME - PGCLUSTER_NAME=$(query_pgsettings cluster_name "$default_cluster_name") - log "Cluster name is $PGCLUSTER_NAME." + # Ensure used libpq vars are defined for configuration template. + + export PGUSER=${PGUSER-postgres} + log "Configuring for PostgreSQL user ${PGUSER}." + export PGDATABASE=${PGDATABASE-${PGUSER}} + export PGPORT=${PGPORT-5432} + log "Configuring for cluster on port ${PGPORT}." + export PGHOST=${PGHOST-$(query_pgsettings unix_socket_directories)} + PGHOST=${PGHOST%%,*} + if ! psql -c "SELECT 'Postgres connection working.';"; then + fatal "Can't connect to Postgres cluster." + fi + export PGDATA + PGDATA=$(query_pgsettings data_directory) + log "Configuring for cluster at ${PGDATA}." + + read -r PGVERSION <"${PGDATA}/PG_VERSION" + if ! command -v pg_ctl &>/dev/null; then + if bindir=$(search_bindir "$PGVERSION"); then + log "Using ${bindir}/pg_ctl." + export PATH=$bindir:$PATH + fi + fi + + # Instance name defaults to cluster_name. If unset (e.g. Postgres 9.4), + # use the tail of ${PGDATA} after ~postgres has been removed. If PGDATA + # is not in postgres home, compute a cluster name from version and port. + local home + home="$(eval readlink -e "~${SYSUSER}")" + if [ -z "${PGDATA##"${home}"/*}" ]; then + default_cluster_name="${PGDATA##"${home}"/}" + else + default_cluster_name=$PGVERSION/pg${PGPORT} + fi + export PGCLUSTER_NAME + PGCLUSTER_NAME=$(query_pgsettings cluster_name "$default_cluster_name") + log "Cluster name is $PGCLUSTER_NAME." } setup_ssl() { - local name=${1//\//-}; shift - local pki - read -r pki < <(readlink -e /etc/pki/tls /etc/ssl "$ETCDIR/$name") - pki="${PKIDIR-$pki}" - - if [ -f "$pki/certs/ssl-cert-snakeoil.pem" ] && [ -f "$pki/private/ssl-cert-snakeoil.key" ] ; then - log "Using snake-oil SSL certificate." - sslcert=$pki/certs/ssl-cert-snakeoil.pem - sslkey=$pki/private/ssl-cert-snakeoil.key - else - sslcert=$pki/certs/temboard-agent-$name.pem - sslkey=$pki/private/temboard-agent-$name.key - openssl req -new -x509 -days 365 -nodes \ - -subj "/C=XX/ST= /L=Default/O=Default/OU= /CN= " \ - -out "$sslcert" -keyout "$sslkey" - chmod 0640 "$sslkey" - chgrp "$(id --group --name "$SYSUSER")" "$sslcert" "$sslkey" - fi - - readlink -e "$sslcert" "$sslkey" + local name=${1//\//-} + shift + local pki + read -r pki < <(readlink -e /etc/pki/tls /etc/ssl "$ETCDIR/$name") + pki="${PKIDIR-$pki}" + + if [ -f "$pki/certs/ssl-cert-snakeoil.pem" ] && [ -f "$pki/private/ssl-cert-snakeoil.key" ]; then + log "Using snake-oil SSL certificate." + sslcert=$pki/certs/ssl-cert-snakeoil.pem + sslkey=$pki/private/ssl-cert-snakeoil.key + else + sslcert=$pki/certs/temboard-agent-$name.pem + sslkey=$pki/private/temboard-agent-$name.key + openssl req -new -x509 -days 365 -nodes \ + -subj "/C=XX/ST= /L=Default/O=Default/OU= /CN= " \ + -out "$sslcert" -keyout "$sslkey" + chmod 0640 "$sslkey" + chgrp "$(id --group --name "$SYSUSER")" "$sslcert" "$sslkey" + fi + + readlink -e "$sslcert" "$sslkey" } -exec 0>&- # Close stdin. +exec 0>&- # Close stdin. -if [ -n "${DEBUG-}" ] ; then - exec 3>/dev/null +if [ -n "${DEBUG-}" ]; then + exec 3>/dev/null else - exec 3>&2 2>"${LOGFILE}" 1>&2 - chmod 0600 "${LOGFILE}" - trap 'catchall' INT EXIT TERM + exec 3>&2 2>"${LOGFILE}" 1>&2 + chmod 0600 "${LOGFILE}" + trap 'catchall' INT EXIT TERM fi # Now, log everything. set -x -if [ -z "${1-}" ] ; then - fatal "Missing temBoard UI URL as first parameter." +if [ -z "${1-}" ]; then + fatal "Missing temBoard UI URL as first parameter." fi umask 037 @@ -324,8 +334,8 @@ umask 037 cd "$(readlink -m "${BASH_SOURCE[0]}/..")" export TEMBOARD_HOSTNAME=${TEMBOARD_HOSTNAME-$(hostname --fqdn)} -if [ -n "${TEMBOARD_HOSTNAME##*.*}" ] ; then - fatal "FQDN is not properly configured. Set agent hostname with TEMBOARD_HOSTNAME env var.". +if [ -n "${TEMBOARD_HOSTNAME##*.*}" ]; then + fatal "FQDN is not properly configured. Set agent hostname with TEMBOARD_HOSTNAME env var.". fi log "Using hostname ${TEMBOARD_HOSTNAME}." @@ -334,19 +344,19 @@ setup_pq name=${PGCLUSTER_NAME} home=${VARDIR}/${name} -if [ -f "${ETCDIR}/${name}/temboard-agent.conf" ] ; then - error "${ETCDIR}/${name}/temboard-agent.conf already exists." - error "To clean previous installation, use" - error - error " ${0/auto_configure/purge} ${name}" - error - fatal "Refusing to overwrite existing configuration." +if [ -f "${ETCDIR}/${name}/temboard-agent.conf" ]; then + error "${ETCDIR}/${name}/temboard-agent.conf already exists." + error "To clean previous installation, use" + error + error " ${0/auto_configure/purge} ${name}" + error + fatal "Refusing to overwrite existing configuration." fi # Create directories mkdir --parents \ - "${ETCDIR}/${name}/temboard-agent.conf.d/" \ - "${LOGDIR}" "${home}" + "${ETCDIR}/${name}/temboard-agent.conf.d/" \ + "${LOGDIR}" "${home}" chown --recursive "${SYSUSER}:${SYSUSER}" "${ETCDIR}" "${VARDIR}" "${LOGDIR}" has_statements=$(psql -c "SELECT 'HAS_STATEMENTS' FROM information_schema.tables WHERE table_schema = 'public' AND table_name = 'pg_stat_statements';") @@ -354,7 +364,10 @@ has_statements=$(psql -c "SELECT 'HAS_STATEMENTS' FROM information_schema.tables # Start with default configuration log "Configuring temboard-agent in ${ETCDIR}/${name}/temboard-agent.conf ." -mapfile -t sslfiles < <(set -eu; setup_ssl "$name") +mapfile -t sslfiles < <( + set -eu + setup_ssl "$name" +) # Inject autoconfiguration in dedicated file. conf=${ETCDIR}/${name}/temboard-agent.conf @@ -365,20 +378,20 @@ chown "$SYSUSER:$SYSUSER" "$conf" sudo -Eu "${SYSUSER}" $(type -p temboard-agent) -c "$conf" discover >/dev/null # systemd -if readlink /proc/1/exe | grep -q systemd && [ -w /etc/systemd/system ] ; then - unit="temboard-agent@$(systemd-escape "${name}").service" - log "Configuring systemd unit ${unit}." - if [ "${SYSUSER}" != "postgres" ] ; then - mkdir -p "/etc/systemd/system/$unit.d/" - cat > "/etc/systemd/system/$unit.d/user.conf" <<-EOF +if readlink /proc/1/exe | grep -q systemd && [ -w /etc/systemd/system ]; then + unit="temboard-agent@$(systemd-escape "${name}").service" + log "Configuring systemd unit ${unit}." + if [ "${SYSUSER}" != "postgres" ]; then + mkdir -p "/etc/systemd/system/$unit.d/" + cat >"/etc/systemd/system/$unit.d/user.conf" <<-EOF [Service] User=${SYSUSER} Group=${SYSGROUP} EOF - fi - start_cmd="systemctl enable --now $unit" + fi + start_cmd="systemctl enable --now $unit" else - start_cmd="sudo -u ${SYSUSER} temboard-agent -c $conf" + start_cmd="sudo -u ${SYSUSER} temboard-agent -c $conf" fi log diff --git a/agent/temboardagent/plugins/monitoring/openmetrics.py b/agent/temboardagent/plugins/monitoring/openmetrics.py index 2edc8033a..c4012bcbe 100644 --- a/agent/temboardagent/plugins/monitoring/openmetrics.py +++ b/agent/temboardagent/plugins/monitoring/openmetrics.py @@ -506,9 +506,68 @@ def generate_bgwriter_samples(data, hostinfo): "maxwritten_clean", ) for stat in bgwriter_stats: + if stat not in data[0]: + continue yield Sample("pg_stat_bgwriter_" + stat, value=data[0][stat]) +METADATAS.update( + { + m.name: m + for m in [ + MetricMetadata( + "pg_stat_checkpointer_buffers_written", + help_="Number of buffers written during checkpoints", + type_=MetricMetadata.COUNTER, + ), + MetricMetadata( + "pg_stat_checkpointer_sync_time", + help_=( + "Total amount of time that has been spent in the portion of " + "checkpoint processing where files are synchronized to disk, " + "in milliseconds" + ), + type_=MetricMetadata.COUNTER, + ), + MetricMetadata( + "pg_stat_checkpointer_write_time", + help_=( + "Total amount of time that has been spent in the portion of " + "checkpoint processing where files are written to disk, in " + "milliseconds" + ), + type_=MetricMetadata.COUNTER, + ), + MetricMetadata( + "pg_stat_checkpointer_num_requested", + help_="Number of requested checkpoints that have been performed", + type_=MetricMetadata.COUNTER, + ), + MetricMetadata( + "pg_stat_checkpointer_num_timed", + help_="Number of scheduled checkpoints that have been performed", + type_=MetricMetadata.COUNTER, + ), + MetricMetadata( + "pg_stat_checkpointer_stats_reset", + help_="Time at which these statistics were last reset", + type_=MetricMetadata.COUNTER, + ), + ] + } +) + + +def generate_checkpointer_samples(data, hostinfo): + yield Sample( + "pg_stat_checkpointer_stats_reset", value=fromisoformat(data[0]["stats_reset"]) + ) + + stats = ("buffers_written", "sync_time", "write_time", "num_timed", "num_requested") + for stat in stats: + yield Sample("pg_stat_checkpointer_" + stat, value=data[0][stat]) + + METADATAS.update( { m.name: m diff --git a/agent/temboardagent/plugins/monitoring/probes.py b/agent/temboardagent/plugins/monitoring/probes.py index fc5893f98..c631a7d71 100644 --- a/agent/temboardagent/plugins/monitoring/probes.py +++ b/agent/temboardagent/plugins/monitoring/probes.py @@ -11,6 +11,7 @@ from ...inventory import SysInfo from ...plugins.maintenance.functions import INDEX_BTREE_BLOAT_SQL +from ...queries import QUERIES from ...toolkit.utils import utcnow from . import db @@ -497,10 +498,23 @@ class probe_blocks(SqlProbe): delta_key = "dbname" +class probe_checkpointer(SqlProbe): + level = "instance" + min_version = 170000 + sql = """select * from pg_stat_checkpointer""" + delta_columns = ["num_timed", "num_requested", "buffers_written"] + + class probe_bgwriter(SqlProbe): level = "instance" min_version = 80300 - sql = """select * from pg_stat_bgwriter""" + + def check(self, version): + if version < 170000: + self.sql = """select * from pg_stat_bgwriter""" + else: + self.sql = QUERIES["monitoring-bgwriter"] + delta_columns = [ "checkpoints_timed", "checkpoints_req", diff --git a/agent/temboardagent/queries/monitoring-bgwriter.sql b/agent/temboardagent/queries/monitoring-bgwriter.sql new file mode 100644 index 000000000..88577dc6c --- /dev/null +++ b/agent/temboardagent/queries/monitoring-bgwriter.sql @@ -0,0 +1,19 @@ +WITH backends AS ( + SELECT SUM(writes) AS buffers_backend, + SUM(fsyncs) AS buffers_backend_fsync + FROM pg_stat_io + WHERE backend_type = 'client backend' +) +SELECT cp.num_timed AS checkpoints_timed, + cp.num_requested AS checkpoints_req, + cp.write_time AS checkpoint_write_time, + cp.sync_time AS checkpoint_sync_time, + cp.buffers_written AS buffers_checkpoint, + bg.buffers_clean, + bg.maxwritten_clean, + backends.buffers_backend, + backends.buffers_backend_fsync, + bg.buffers_alloc + FROM pg_stat_bgwriter AS bg, + pg_stat_checkpointer AS cp, + backends; diff --git a/dev/docker-compose.massagent.yml b/dev/docker-compose.massagent.yml index 180b09ec8..5205a85ad 100644 --- a/dev/docker-compose.massagent.yml +++ b/dev/docker-compose.massagent.yml @@ -14,7 +14,7 @@ volumes: services: postgres: - image: postgres:${PGVERSION-16}-alpine + image: postgres:${PGVERSION-17}-alpine environment: POSTGRES_PASSWORD: confinment command: [ diff --git a/dev/postgres-ha-entrypoint.sh b/dev/postgres-ha-entrypoint.sh index 52dec1705..163446de4 100755 --- a/dev/postgres-ha-entrypoint.sh +++ b/dev/postgres-ha-entrypoint.sh @@ -15,115 +15,114 @@ # If a postgres is down, restart it manually with docker compose. # If you need to reset the setup, trash with make clean. - # shellcheck source=/dev/null . /usr/local/bin/docker-entrypoint.sh _ha_setup() { - docker_setup_env - - chown postgres:postgres /var/lib/postgresql/archive - - echo "Waiting for $PEER_HOST to have network." - if ! peerhost="$(_retry getent hosts "$PEER_HOST")" ; then - echo "$PEER_HOST down. Can't elect primary." - exit 1 - fi - - if [ "$DATABASE_ALREADY_EXISTS" = "true" ] ; then - # We are in restarting mode. - # Check if the other node is primary. - # Or guess based on IP. - if is_in_recovery=$(_retry env PGPASSWORD="$POSTGRES_PASSWORD" psql -h "$PEER_HOST" -U "$POSTGRES_USER" -Aqt -c 'SELECT pg_is_in_recovery();') ; then - if [ "$is_in_recovery" = "t" ] ; then - echo "$PEER_HOST restarted as secondary. Restarting as primary." - rm -f "$PGDATA/standby.signal" - else - echo "$PEER_HOST restared as primary. Failback as secondary." - _ha_failback - fi - else - # If other node does not respond he may be waiting for us too. - # To release this deadlock, fallback to initial election base on _ha_have_i_precedence. - if _ha_have_i_precedence "$peerhost" ; then - echo "$PEER_HOST does not respond. Restarting as primary from precedence." - # Offline pg_promote() - rm -f "$PGDATA/standby.signal" - else - echo "$PEER_HOST does not respond. Restarting as secondary from precedence." - _ha_failback - fi - fi - else - if _ha_have_i_precedence "$peerhost" ; then - echo "Elected as primary." - # replication is configured in - # postgres-setup-primary.sh - else - echo "Elected as secondary." - sleep 3 - _ha_init_secondary - fi - fi + docker_setup_env + + chown postgres:postgres /var/lib/postgresql/archive + + echo "Waiting for $PEER_HOST to have network." + if ! peerhost="$(_retry getent hosts "$PEER_HOST")"; then + echo "$PEER_HOST down. Can't elect primary." + exit 1 + fi + + if [ "$DATABASE_ALREADY_EXISTS" = "true" ]; then + # We are in restarting mode. + # Check if the other node is primary. + # Or guess based on IP. + if is_in_recovery=$(_retry env PGPASSWORD="$POSTGRES_PASSWORD" psql -h "$PEER_HOST" -U "$POSTGRES_USER" -Aqt -c 'SELECT pg_is_in_recovery();'); then + if [ "$is_in_recovery" = "t" ]; then + echo "$PEER_HOST restarted as secondary. Restarting as primary." + rm -f "$PGDATA/standby.signal" + else + echo "$PEER_HOST restared as primary. Failback as secondary." + _ha_failback + fi + else + # If other node does not respond he may be waiting for us too. + # To release this deadlock, fallback to initial election base on _ha_have_i_precedence. + if _ha_have_i_precedence "$peerhost"; then + echo "$PEER_HOST does not respond. Restarting as primary from precedence." + # Offline pg_promote() + rm -f "$PGDATA/standby.signal" + else + echo "$PEER_HOST does not respond. Restarting as secondary from precedence." + _ha_failback + fi + fi + else + if _ha_have_i_precedence "$peerhost"; then + echo "Elected as primary." + # replication is configured in + # postgres-setup-primary.sh + else + echo "Elected as secondary." + sleep 3 + _ha_init_secondary + fi + fi } - _ha_have_i_precedence() { - # Compute precedence based on IP. - IFS=" " read -r _ winner _ < <( ( getent hosts "$HOSTNAME" ; echo "$1"; ) | sort | head -1) - echo "$winner has precedence." - test "${winner}" = "$HOSTNAME" + # Compute precedence based on IP. + IFS=" " read -r _ winner _ < <(( + getent hosts "$HOSTNAME" + echo "$1" + ) | sort | head -1) + echo "$winner has precedence." + test "${winner}" = "$HOSTNAME" } - _ha_init_secondary() { - echo "Waiting for $PEER_HOST to serve." - export PGUSER="$POSTGRES_USER" - export PGPASSWORD="$POSTGRES_PASSWORD" - - _retry psql -Aqt -h "$PEER_HOST" -c 'SELECT NULL' - - echo "Initializing PGDATA with pg_basebackup." - pg_basebackup \ - -h "$PEER_HOST" -p 5432 -U $POSTGRES_USER \ - -D "$PGDATA" \ - --format=p \ - --write-recovery-conf \ - --wal-method=stream \ - --checkpoint=fast + echo "Waiting for $PEER_HOST to serve." + export PGUSER="$POSTGRES_USER" + export PGPASSWORD="$POSTGRES_PASSWORD" + + _retry psql -Aqt -h "$PEER_HOST" -c 'SELECT NULL' + + echo "Initializing PGDATA with pg_basebackup." + pg_basebackup \ + -h "$PEER_HOST" -p 5432 -U $POSTGRES_USER \ + -D "$PGDATA" \ + --format=p \ + --write-recovery-conf \ + --wal-method=stream \ + --checkpoint=fast } _ha_failback() { - # offline pg_demote. - touch "$PGADATA/standby.signal" - echo "Waiting for primary to come up." - _retry env PGPASSWORD="$POSTGRES_PASSWORD" psql -h "$PEER_HOST" -U "$POSTGRES_USER" -Aqt -c "SELECT pg_switch_wal();" - echo "Rewind pgdata to failback." - su-exec postgres pg_rewind \ - --source-server="postgresql://$POSTGRES_USER:$POSTGRES_PASSWORD@$PEER_HOST:5432/" \ - --target-pgdata="$PGDATA" \ - --write-recovery-conf \ - --no-ensure-shutdown + # offline pg_demote. + touch "$PGADATA/standby.signal" + echo "Waiting for primary to come up." + _retry env PGPASSWORD="$POSTGRES_PASSWORD" psql -h "$PEER_HOST" -U "$POSTGRES_USER" -Aqt -c "SELECT pg_switch_wal();" + echo "Rewind pgdata to failback." + gosu postgres pg_rewind \ + --source-server="postgresql://$POSTGRES_USER:$POSTGRES_PASSWORD@$PEER_HOST:5432/" \ + --target-pgdata="$PGDATA" \ + --write-recovery-conf \ + --no-ensure-shutdown } - _retry() { - for i in {2..7} ; do - if "$@" ; then - return - else - echo "Retrying in one second, attempt #$i." - sleep 1 - fi - done - - "$@" + for i in {2..7}; do + if "$@"; then + return + else + echo "Retrying in one second, attempt #$i." + sleep 1 + fi + done + + "$@" } -if [ -v PEER_HOST ] ; then - PGAPP="dev-docker-ha-entrypoint" _ha_setup +if [ -v PEER_HOST ]; then + PGAPP="dev-docker-ha-entrypoint" _ha_setup else - echo 'PEER_HOST undefined. No HA setup.' + echo 'PEER_HOST undefined. No HA setup.' fi # trigger docker-entrypoin.sh main diff --git a/docker-compose.yml b/docker-compose.yml index febc6d80b..05b2f09dc 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -6,7 +6,7 @@ version: '3.8' services: # The Postgres server to store temBoard data. repository: - image: postgres:16-alpine + image: postgres:17-alpine ports: ["5432:5432"] environment: POSTGRES_USER: postgres @@ -28,7 +28,7 @@ services: # First instance. postgres0: - image: &postgres_image postgres:16rc1-alpine + image: &postgres_image postgres:17-alpine volumes: - data0:/var/lib/postgresql/data - run0:/var/run/postgresql diff --git a/tests/fixtures/postgres.py b/tests/fixtures/postgres.py index f761cfe7d..a490d0af6 100644 --- a/tests/fixtures/postgres.py +++ b/tests/fixtures/postgres.py @@ -18,7 +18,19 @@ class PostgreSQLVersions(dict): # A mapping from major version -> bindir. # List of agent supported PostgreSQL versions. - SUPPORTED_VERSIONS = ["16", "15", "14", "13", "12", "11", "10", "9.6", "9.5", "9.4"] + SUPPORTED_VERSIONS = [ + "17", + "16", + "15", + "14", + "13", + "12", + "11", + "10", + "9.6", + "9.5", + "9.4", + ] def search_installed_versions(self): patterns = [ diff --git a/ui/temboardui/plugins/statements/__init__.py b/ui/temboardui/plugins/statements/__init__.py index a7e70d592..6e78e2243 100644 --- a/ui/temboardui/plugins/statements/__init__.py +++ b/ui/temboardui/plugins/statements/__init__.py @@ -764,8 +764,14 @@ def add_statement(session, instance, data): statement["local_blks_written"], statement["temp_blks_read"], statement["temp_blks_written"], - statement["blk_read_time"], - statement["blk_write_time"], + # DEPRECATED: Column renamed in Postgres 17. + statement["shared_blk_read_time"] + if "shared_blk_read_time" in statement + else statement["blk_read_time"], + statement["shared_blk_write_time"] + # DEPRECATED: Column renamed in Postgres 17. + if "shared_blk_write_time" in statement + else statement["blk_write_time"], statement["total_plan_time"] if "total_plan_time" in statement else None, diff --git a/ui/temboardui/static/src/views/Statements.vue b/ui/temboardui/static/src/views/Statements.vue index 307727970..17daee60c 100644 --- a/ui/temboardui/static/src/views/Statements.vue +++ b/ui/temboardui/static/src/views/Statements.vue @@ -496,7 +496,8 @@ function onFromToUpdate(from_, to_) {