PMM-13154 PG17 support in QAN (#3159)

* PMM-13154 Intermediate progress * PMM-12154 Some progress * PMM-12154 Some progress * PMM-12154 Fix linter warn * PMM-12154 Cleanup * PMM-12154 Fix * PMM-12154 Update PG version in agent CI * PMM-12154 Update PG tests * PMM-12154 Update PG tests * PMM-12154 Fix PG17 tag * PMM-12154 Update PG tests * PMM-12154 Try fix * PMM-12154 Try fix * PMM-12154 Try fix * PMM-12154 Try fix * PMM-13154 Update PG tests * PMM-12154 Update PG tests * PMM-12154 Update PG tests * PMM-12154 Update PG tests * PMM-13154 Update PG version matrix in CI * PMM-13154 Use dev image of percona distribution for pg17 * PMM-13154 Update PG tests * PMM-13154 Update PG tests * PMM-12154 Cleanup * PMM-13154 Add local block read/write timers to QAN, update QAN API * PMM-13154 Fixed * PMM-13154 Update test data * PMM-13154 Fixes * PMM-13154 Fixes * PMM-13154 Fixes * Rewrite pg_stat_statments view reading * PMM-13154 Fix * PMM-13154 Fix * PMM-13154 Fix * PMM-13154 Fix * PMM-13154 Fix * PMM-13154 Fix * PMM-13154 Fix * PMM-13154 Add licence header * PMM-13154 Fix tests * PMM-13154 Update tests * PMM-13154 Fix style * PMM-13154 Fix CH migration * PMM-13154 Remove redundant fields backfill * Use PG in CI 17rc1 * PMM-13154 Format sources * PMM-13154 Update test --------- Co-authored-by: Jiří Čtvrtka <[email protected]> Co-authored-by: Nurlan Moldomurov <[email protected]> (cherry picked from commit 500f640)
percona · Oct 11, 2024 · a0db063 · a0db063
1 parent c8d4286
commit a0db063
Show file tree

Hide file tree

Showing 36 changed files with 2,270 additions and 910 deletions.
diff --git a/.github/workflows/agent.yml b/.github/workflows/agent.yml
@@ -26,15 +26,15 @@ jobs:
     strategy:
       matrix:
         images:
-          - { mysql: 'mysql:5.6', mongo: 'mongo:4.2', postgres: 'postgres:10', pmm_server: 'percona/pmm-server:2.0.0' }
-          - { mysql: 'mysql:5.7', mongo: 'mongo:4.4', postgres: 'postgres:11', pmm_server: 'percona/pmm-server:2.0.1' }
-          - { mysql: 'mysql:8.0', mongo: 'mongo:4.4', postgres: 'postgres:12', pmm_server: 'percona/pmm-server:2.1.0' }
-          - { mysql: 'mysql:8.0', mongo: 'mongo:4.4', postgres: 'postgres:13', pmm_server: 'percona/pmm-server:2.2.0' }
+          - { mysql: 'mysql:5.6', mongo: 'mongo:4.2', postgres: 'postgres:14', pmm_server: 'percona/pmm-server:2.0.0' }
+          - { mysql: 'mysql:5.7', mongo: 'mongo:4.4', postgres: 'postgres:15', pmm_server: 'percona/pmm-server:2.0.1' }
+          - { mysql: 'mysql:8.0', mongo: 'mongo:4.4', postgres: 'postgres:16', pmm_server: 'percona/pmm-server:2.1.0' }
+          - { mysql: 'mysql:8.0', mongo: 'mongo:4.4', postgres: 'postgres:17rc1', pmm_server: 'percona/pmm-server:2.2.0' }
 
           # Percona + latest PMM Server release
-          - { mysql: 'percona:5.6', mongo: 'percona/percona-server-mongodb:4.2', postgres: 'perconalab/percona-distribution-postgresql:11', pg_libs: 'pg_stat_statements,pg_stat_monitor', pmm_server: 'percona/pmm-server:2' }
-          - { mysql: 'percona:5.7', mongo: 'percona/percona-server-mongodb:4.4', postgres: 'perconalab/percona-distribution-postgresql:12.8-pg_stat', pg_libs: 'pg_stat_statements,pg_stat_monitor', pmm_server: 'percona/pmm-server:2' }
-          - { mysql: 'percona:8.0', mongo: 'percona/percona-server-mongodb:4.4', postgres: 'perconalab/percona-distribution-postgresql:13.5-pg_stat', pg_libs: 'pg_stat_statements,pg_stat_monitor', pmm_server: 'percona/pmm-server:2' }
+          - { mysql: 'percona:5.6', mongo: 'percona/percona-server-mongodb:4.2', postgres: 'perconalab/percona-distribution-postgresql:15', pg_libs: 'pg_stat_statements,pg_stat_monitor', pmm_server: 'percona/pmm-server:2' }
+          - { mysql: 'percona:5.7', mongo: 'percona/percona-server-mongodb:4.4', postgres: 'perconalab/percona-distribution-postgresql:16', pg_libs: 'pg_stat_statements,pg_stat_monitor', pmm_server: 'percona/pmm-server:2' }
+          - { mysql: 'percona:8.0', mongo: 'percona/percona-server-mongodb:4.4', postgres: 'perconalab/percona-distribution-postgresql:17-dev', pg_libs: 'pg_stat_statements,pg_stat_monitor', pmm_server: 'percona/pmm-server:2' }
 
           # MariaDB (only 3 latest GA versions)
           #   + older supported MongoDB versions
@@ -44,9 +44,9 @@ jobs:
           # - MYSQL_IMAGE=mariadb:10.0
           # - MYSQL_IMAGE=mariadb:10.1
 
-          - { mysql: 'mariadb:10.2', mongo: 'mongo:4.4',                          postgres: 'postgres:9.4', pmm_server: 'perconalab/pmm-server:dev-latest' }
-          - { mysql: 'mariadb:10.3', mongo: 'percona/percona-server-mongodb:4.4', postgres: 'postgres:9.5', pmm_server: 'perconalab/pmm-server:dev-latest' }
-          - { mysql: 'mariadb:10.4',                                              postgres: 'postgres:9.6', pmm_server: 'perconalab/pmm-server:dev-latest' }
+          - { mysql: 'mariadb:10.2', mongo: 'mongo:4.4',                          postgres: 'postgres:12', pmm_server: 'perconalab/pmm-server:dev-latest' }
+          - { mysql: 'mariadb:10.3', mongo: 'percona/percona-server-mongodb:4.4', postgres: 'postgres:13', pmm_server: 'perconalab/pmm-server:dev-latest' }
+          - { mysql: 'mariadb:10.4',                                              postgres: 'postgres:14', pmm_server: 'perconalab/pmm-server:dev-latest' }
 
     continue-on-error: true
 

diff --git a/agent/agents/postgres/pgstatmonitor/pgstatmonitor.go b/agent/agents/postgres/pgstatmonitor/pgstatmonitor.go
@@ -85,6 +85,13 @@ const (
 	pgStatMonitorVersion20PG13
 	pgStatMonitorVersion20PG14
 	pgStatMonitorVersion20PG15
+	pgStatMonitorVersion20PG16
+	pgStatMonitorVersion21PG12
+	pgStatMonitorVersion21PG13
+	pgStatMonitorVersion21PG14
+	pgStatMonitorVersion21PG15
+	pgStatMonitorVersion21PG16
+	pgStatMonitorVersion21PG17
 )
 
 const (
@@ -184,40 +191,54 @@ func getPGMonitorVersion(q *reform.Querier) (pgStatMonitorVersion, pgStatMonitor
 
 	var version pgStatMonitorVersion
 	switch {
+	case vPGSM.Core().GreaterThanOrEqual(v21):
+		switch {
+		case vPG >= 17:
+			version = pgStatMonitorVersion21PG17
+		case vPG >= 16:
+			version = pgStatMonitorVersion21PG16
+		case vPG >= 15:
+			version = pgStatMonitorVersion21PG15
+		case vPG >= 14:
+			version = pgStatMonitorVersion21PG14
+		case vPG >= 13:
+			version = pgStatMonitorVersion21PG13
+		default:
+			version = pgStatMonitorVersion21PG12
+		}
 	case vPGSM.Core().GreaterThanOrEqual(v20):
-		if vPG >= 15 {
+		switch {
+		case vPG >= 16:
+			version = pgStatMonitorVersion20PG16
+		case vPG >= 15:
 			version = pgStatMonitorVersion20PG15
-			break
-		}
-		if vPG >= 14 {
+		case vPG >= 14:
 			version = pgStatMonitorVersion20PG14
-			break
-		}
-		if vPG >= 13 {
+		case vPG >= 13:
 			version = pgStatMonitorVersion20PG13
-			break
+		default:
+			version = pgStatMonitorVersion20PG12
 		}
-		version = pgStatMonitorVersion20PG12
 	case vPGSM.Core().GreaterThanOrEqual(v11):
-		if vPG >= 14 {
+		switch {
+		case vPG >= 14:
 			version = pgStatMonitorVersion11PG14
-			break
-		}
-		if vPG >= 13 {
+		case vPG >= 13:
 			version = pgStatMonitorVersion11PG13
-			break
+		default:
+			version = pgStatMonitorVersion11PG12
 		}
-		version = pgStatMonitorVersion11PG12
+
 	case vPGSM.Core().GreaterThanOrEqual(v10):
-		if vPG >= 14 {
+		switch {
+		case vPG >= 14:
 			version = pgStatMonitorVersion10PG14
-			break
-		}
-		if vPG >= 13 {
+		case vPG >= 13:
 			version = pgStatMonitorVersion10PG13
-			break
+		default:
+			version = pgStatMonitorVersion10PG12
 		}
-		version = pgStatMonitorVersion10PG12
+
 	case vPGSM.GreaterThanOrEqual(v09):
 		version = pgStatMonitorVersion09
 	case vPGSM.GreaterThanOrEqual(v08):
@@ -563,10 +584,10 @@ func (m *PGStatMonitorQAN) makeBuckets(current, cache map[time.Time]map[string]*
 				mb.Postgresql.HistogramItems = histogram
 			}
 
-			if (currentPSM.PlanTotalTime - prevPSM.PlanTotalTime) != 0 {
-				mb.Postgresql.MPlanTimeSum = float32(currentPSM.PlanTotalTime-prevPSM.PlanTotalTime) / 1000
-				mb.Postgresql.MPlanTimeMin = float32(currentPSM.PlanMinTime) / 1000
-				mb.Postgresql.MPlanTimeMax = float32(currentPSM.PlanMaxTime) / 1000
+			if (currentPSM.TotalPlanTime - prevPSM.TotalPlanTime) != 0 {
+				mb.Postgresql.MPlanTimeSum = float32(currentPSM.TotalPlanTime-prevPSM.TotalPlanTime) / 1000
+				mb.Postgresql.MPlanTimeMin = float32(currentPSM.MinPlanTime) / 1000
+				mb.Postgresql.MPlanTimeMax = float32(currentPSM.MaxPlanTime) / 1000
 				mb.Postgresql.MPlanTimeCnt = count
 			}
 
@@ -618,9 +639,11 @@ func (m *PGStatMonitorQAN) makeBuckets(current, cache map[time.Time]map[string]*
 				{float32(currentPSM.WalBytes - prevPSM.WalBytes), &mb.Postgresql.MWalBytesSum, &mb.Postgresql.MWalBytesCnt},
 
 				// convert milliseconds to seconds
-				{float32(currentPSM.TotalTime-prevPSM.TotalTime) / 1000, &mb.Common.MQueryTimeSum, &mb.Common.MQueryTimeCnt},
-				{float32(currentPSM.BlkReadTime-prevPSM.BlkReadTime) / 1000, &mb.Postgresql.MBlkReadTimeSum, &mb.Postgresql.MBlkReadTimeCnt},
-				{float32(currentPSM.BlkWriteTime-prevPSM.BlkWriteTime) / 1000, &mb.Postgresql.MBlkWriteTimeSum, &mb.Postgresql.MBlkWriteTimeCnt},
+				{float32(currentPSM.TotalExecTime-prevPSM.TotalExecTime) / 1000, &mb.Common.MQueryTimeSum, &mb.Common.MQueryTimeCnt},
+				{float32(currentPSM.SharedBlkReadTime-prevPSM.SharedBlkReadTime) / 1000, &mb.Postgresql.MSharedBlkReadTimeSum, &mb.Postgresql.MSharedBlkReadTimeCnt},
+				{float32(currentPSM.SharedBlkWriteTime-prevPSM.SharedBlkWriteTime) / 1000, &mb.Postgresql.MSharedBlkWriteTimeSum, &mb.Postgresql.MSharedBlkWriteTimeCnt},
+				{float32(currentPSM.LocalBlkReadTime-prevPSM.LocalBlkReadTime) / 1000, &mb.Postgresql.MLocalBlkReadTimeSum, &mb.Postgresql.MLocalBlkReadTimeCnt},
+				{float32(currentPSM.LocalBlkWriteTime-prevPSM.LocalBlkWriteTime) / 1000, &mb.Postgresql.MLocalBlkWriteTimeSum, &mb.Postgresql.MLocalBlkWriteTimeCnt},
 
 				// convert microseconds to seconds
 				{float32(cpuSysTime) / 1000000, &mb.Postgresql.MCpuSysTimeSum, &mb.Postgresql.MCpuSysTimeCnt},