Skip to content

Commit

Permalink
PMM-13154 PG17 support in QAN (#3159)
Browse files Browse the repository at this point in the history 
* PMM-13154 Intermediate progress

* PMM-12154 Some progress

* PMM-12154 Some progress

* PMM-12154 Fix linter warn

* PMM-12154 Cleanup

* PMM-12154 Fix

* PMM-12154 Update PG version in agent CI

* PMM-12154 Update PG tests

* PMM-12154 Update PG tests

* PMM-12154 Fix PG17 tag

* PMM-12154 Update PG tests

* PMM-12154 Try fix

* PMM-12154 Try fix

* PMM-12154 Try fix

* PMM-12154 Try fix

* PMM-13154 Update PG tests

* PMM-12154 Update PG tests

* PMM-12154 Update PG tests

* PMM-12154 Update PG tests

* PMM-13154 Update PG version matrix in CI

* PMM-13154 Use dev image of percona distribution for pg17

* PMM-13154 Update PG tests

* PMM-13154 Update PG tests

* PMM-12154 Cleanup

* PMM-13154 Add local block read/write timers to QAN, update QAN API

* PMM-13154 Fixed

* PMM-13154 Update test data

* PMM-13154 Fixes

* PMM-13154 Fixes

* PMM-13154 Fixes

* Rewrite pg_stat_statments view reading

* PMM-13154 Fix

* PMM-13154 Fix

* PMM-13154 Fix

* PMM-13154 Fix

* PMM-13154 Fix

* PMM-13154 Fix

* PMM-13154 Fix

* PMM-13154 Add licence header

* PMM-13154 Fix tests

* PMM-13154 Update tests

* PMM-13154 Fix style

* PMM-13154 Fix CH migration

* PMM-13154 Remove redundant fields backfill

* Use PG in CI 17rc1

* PMM-13154 Format sources

* PMM-13154 Update test

---------

Co-authored-by: Jiří Čtvrtka <[email protected]>
Co-authored-by: Nurlan Moldomurov <[email protected]>
  • Loading branch information
@artemgavrilov @JiriCtvrtka @BupycHuk
3 people authored Oct 11, 2024
1 parent 2df2bdc commit 500f640
Show file tree
Hide file tree
Showing 36 changed files with 2,270 additions and 910 deletions.
20 changes: 10 additions & 10 deletions .github/workflows/agent.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,15 +26,15 @@ jobs:
strategy:
matrix:
images:
- { mysql: 'mysql:5.6', mongo: 'mongo:4.2', postgres: 'postgres:10', pmm_server: 'percona/pmm-server:2.0.0' }
- { mysql: 'mysql:5.7', mongo: 'mongo:4.4', postgres: 'postgres:11', pmm_server: 'percona/pmm-server:2.0.1' }
- { mysql: 'mysql:8.0', mongo: 'mongo:4.4', postgres: 'postgres:12', pmm_server: 'percona/pmm-server:2.1.0' }
- { mysql: 'mysql:8.0', mongo: 'mongo:4.4', postgres: 'postgres:13', pmm_server: 'percona/pmm-server:2.2.0' }
- { mysql: 'mysql:5.6', mongo: 'mongo:4.2', postgres: 'postgres:14', pmm_server: 'percona/pmm-server:2.0.0' }
- { mysql: 'mysql:5.7', mongo: 'mongo:4.4', postgres: 'postgres:15', pmm_server: 'percona/pmm-server:2.0.1' }
- { mysql: 'mysql:8.0', mongo: 'mongo:4.4', postgres: 'postgres:16', pmm_server: 'percona/pmm-server:2.1.0' }
- { mysql: 'mysql:8.0', mongo: 'mongo:4.4', postgres: 'postgres:17rc1', pmm_server: 'percona/pmm-server:2.2.0' }

# Percona + latest PMM Server release
- { mysql: 'percona:5.6', mongo: 'percona/percona-server-mongodb:4.2', postgres: 'perconalab/percona-distribution-postgresql:11', pg_libs: 'pg_stat_statements,pg_stat_monitor', pmm_server: 'percona/pmm-server:2' }
- { mysql: 'percona:5.7', mongo: 'percona/percona-server-mongodb:4.4', postgres: 'perconalab/percona-distribution-postgresql:12.8-pg_stat', pg_libs: 'pg_stat_statements,pg_stat_monitor', pmm_server: 'percona/pmm-server:2' }
- { mysql: 'percona:8.0', mongo: 'percona/percona-server-mongodb:4.4', postgres: 'perconalab/percona-distribution-postgresql:13.5-pg_stat', pg_libs: 'pg_stat_statements,pg_stat_monitor', pmm_server: 'percona/pmm-server:2' }
- { mysql: 'percona:5.6', mongo: 'percona/percona-server-mongodb:4.2', postgres: 'perconalab/percona-distribution-postgresql:15', pg_libs: 'pg_stat_statements,pg_stat_monitor', pmm_server: 'percona/pmm-server:2' }
- { mysql: 'percona:5.7', mongo: 'percona/percona-server-mongodb:4.4', postgres: 'perconalab/percona-distribution-postgresql:16', pg_libs: 'pg_stat_statements,pg_stat_monitor', pmm_server: 'percona/pmm-server:2' }
- { mysql: 'percona:8.0', mongo: 'percona/percona-server-mongodb:4.4', postgres: 'perconalab/percona-distribution-postgresql:17-dev', pg_libs: 'pg_stat_statements,pg_stat_monitor', pmm_server: 'percona/pmm-server:2' }

# MariaDB (only 3 latest GA versions)
# + older supported MongoDB versions
Expand All @@ -44,9 +44,9 @@ jobs:
# - MYSQL_IMAGE=mariadb:10.0
# - MYSQL_IMAGE=mariadb:10.1

- { mysql: 'mariadb:10.2', mongo: 'mongo:4.4', postgres: 'postgres:9.4', pmm_server: 'perconalab/pmm-server:dev-latest' }
- { mysql: 'mariadb:10.3', mongo: 'percona/percona-server-mongodb:4.4', postgres: 'postgres:9.5', pmm_server: 'perconalab/pmm-server:dev-latest' }
- { mysql: 'mariadb:10.4', postgres: 'postgres:9.6', pmm_server: 'perconalab/pmm-server:dev-latest' }
- { mysql: 'mariadb:10.2', mongo: 'mongo:4.4', postgres: 'postgres:12', pmm_server: 'perconalab/pmm-server:dev-latest' }
- { mysql: 'mariadb:10.3', mongo: 'percona/percona-server-mongodb:4.4', postgres: 'postgres:13', pmm_server: 'perconalab/pmm-server:dev-latest' }
- { mysql: 'mariadb:10.4', postgres: 'postgres:14', pmm_server: 'perconalab/pmm-server:dev-latest' }

continue-on-error: true

Expand Down
79 changes: 51 additions & 28 deletions agent/agents/postgres/pgstatmonitor/pgstatmonitor.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -85,6 +85,13 @@ const (
pgStatMonitorVersion20PG13
pgStatMonitorVersion20PG14
pgStatMonitorVersion20PG15
pgStatMonitorVersion20PG16
pgStatMonitorVersion21PG12
pgStatMonitorVersion21PG13
pgStatMonitorVersion21PG14
pgStatMonitorVersion21PG15
pgStatMonitorVersion21PG16
pgStatMonitorVersion21PG17
)

const (
Expand Down Expand Up @@ -184,40 +191,54 @@ func getPGMonitorVersion(q *reform.Querier) (pgStatMonitorVersion, pgStatMonitor

var version pgStatMonitorVersion
switch {
case vPGSM.Core().GreaterThanOrEqual(v21):
switch {
case vPG >= 17:
version = pgStatMonitorVersion21PG17
case vPG >= 16:
version = pgStatMonitorVersion21PG16
case vPG >= 15:
version = pgStatMonitorVersion21PG15
case vPG >= 14:
version = pgStatMonitorVersion21PG14
case vPG >= 13:
version = pgStatMonitorVersion21PG13
default:
version = pgStatMonitorVersion21PG12
}
case vPGSM.Core().GreaterThanOrEqual(v20):
if vPG >= 15 {
switch {
case vPG >= 16:
version = pgStatMonitorVersion20PG16
case vPG >= 15:
version = pgStatMonitorVersion20PG15
break
}
if vPG >= 14 {
case vPG >= 14:
version = pgStatMonitorVersion20PG14
break
}
if vPG >= 13 {
case vPG >= 13:
version = pgStatMonitorVersion20PG13
break
default:
version = pgStatMonitorVersion20PG12
}
version = pgStatMonitorVersion20PG12
case vPGSM.Core().GreaterThanOrEqual(v11):
if vPG >= 14 {
switch {
case vPG >= 14:
version = pgStatMonitorVersion11PG14
break
}
if vPG >= 13 {
case vPG >= 13:
version = pgStatMonitorVersion11PG13
break
default:
version = pgStatMonitorVersion11PG12
}
version = pgStatMonitorVersion11PG12

case vPGSM.Core().GreaterThanOrEqual(v10):
if vPG >= 14 {
switch {
case vPG >= 14:
version = pgStatMonitorVersion10PG14
break
}
if vPG >= 13 {
case vPG >= 13:
version = pgStatMonitorVersion10PG13
break
default:
version = pgStatMonitorVersion10PG12
}
version = pgStatMonitorVersion10PG12

case vPGSM.GreaterThanOrEqual(v09):
version = pgStatMonitorVersion09
case vPGSM.GreaterThanOrEqual(v08):
Expand Down Expand Up @@ -563,10 +584,10 @@ func (m *PGStatMonitorQAN) makeBuckets(current, cache map[time.Time]map[string]*
mb.Postgresql.HistogramItems = histogram
}

if (currentPSM.PlanTotalTime - prevPSM.PlanTotalTime) != 0 {
mb.Postgresql.MPlanTimeSum = float32(currentPSM.PlanTotalTime-prevPSM.PlanTotalTime) / 1000
mb.Postgresql.MPlanTimeMin = float32(currentPSM.PlanMinTime) / 1000
mb.Postgresql.MPlanTimeMax = float32(currentPSM.PlanMaxTime) / 1000
if (currentPSM.TotalPlanTime - prevPSM.TotalPlanTime) != 0 {
mb.Postgresql.MPlanTimeSum = float32(currentPSM.TotalPlanTime-prevPSM.TotalPlanTime) / 1000
mb.Postgresql.MPlanTimeMin = float32(currentPSM.MinPlanTime) / 1000
mb.Postgresql.MPlanTimeMax = float32(currentPSM.MaxPlanTime) / 1000
mb.Postgresql.MPlanTimeCnt = count
}

Expand Down Expand Up @@ -618,9 +639,11 @@ func (m *PGStatMonitorQAN) makeBuckets(current, cache map[time.Time]map[string]*
{float32(currentPSM.WalBytes - prevPSM.WalBytes), &mb.Postgresql.MWalBytesSum, &mb.Postgresql.MWalBytesCnt},

// convert milliseconds to seconds
{float32(currentPSM.TotalTime-prevPSM.TotalTime) / 1000, &mb.Common.MQueryTimeSum, &mb.Common.MQueryTimeCnt},
{float32(currentPSM.BlkReadTime-prevPSM.BlkReadTime) / 1000, &mb.Postgresql.MBlkReadTimeSum, &mb.Postgresql.MBlkReadTimeCnt},
{float32(currentPSM.BlkWriteTime-prevPSM.BlkWriteTime) / 1000, &mb.Postgresql.MBlkWriteTimeSum, &mb.Postgresql.MBlkWriteTimeCnt},
{float32(currentPSM.TotalExecTime-prevPSM.TotalExecTime) / 1000, &mb.Common.MQueryTimeSum, &mb.Common.MQueryTimeCnt},
{float32(currentPSM.SharedBlkReadTime-prevPSM.SharedBlkReadTime) / 1000, &mb.Postgresql.MSharedBlkReadTimeSum, &mb.Postgresql.MSharedBlkReadTimeCnt},
{float32(currentPSM.SharedBlkWriteTime-prevPSM.SharedBlkWriteTime) / 1000, &mb.Postgresql.MSharedBlkWriteTimeSum, &mb.Postgresql.MSharedBlkWriteTimeCnt},
{float32(currentPSM.LocalBlkReadTime-prevPSM.LocalBlkReadTime) / 1000, &mb.Postgresql.MLocalBlkReadTimeSum, &mb.Postgresql.MLocalBlkReadTimeCnt},
{float32(currentPSM.LocalBlkWriteTime-prevPSM.LocalBlkWriteTime) / 1000, &mb.Postgresql.MLocalBlkWriteTimeSum, &mb.Postgresql.MLocalBlkWriteTimeCnt},

// convert microseconds to seconds
{float32(cpuSysTime) / 1000000, &mb.Postgresql.MCpuSysTimeSum, &mb.Postgresql.MCpuSysTimeCnt},
Expand Down
Loading

0 comments on commit 500f640

Please sign in to comment.