From 49a0e81b0c8c99268e5b912752f67af1fa4b1d12 Mon Sep 17 00:00:00 2001 From: Nurlan Moldomurov Date: Mon, 19 Aug 2024 15:39:47 +0600 Subject: [PATCH] PMM-12848 Enable currentopmetrics collector in mongodb_exporter. (#3118) * PMM-12848 Enable currentopmetrics collector in mongodb_exporter. * PMM-7 fix golint. * PMM-12848 Add currentopmetrics to scrape config. * PMM-12848 collect currentopmetrics in low resolution. * PMM-12848 Fix tests. --- managed/services/agents/mongodb.go | 216 +++++------------- managed/services/agents/mongodb_test.go | 29 +++ .../victoriametrics/scrape_configs.go | 3 + .../victoriametrics/scrape_configs_test.go | 4 +- managed/utils/collectors/collectors.go | 17 ++ 5 files changed, 102 insertions(+), 167 deletions(-) diff --git a/managed/services/agents/mongodb.go b/managed/services/agents/mongodb.go index 0e043e3904..3b1fa5f68d 100644 --- a/managed/services/agents/mongodb.go +++ b/managed/services/agents/mongodb.go @@ -30,17 +30,13 @@ import ( "github.com/percona/pmm/version" ) -type collectorArgs struct { - enabled bool - enableParam string -} - var ( // New MongoDB Exporter will be released with PMM agent v2.10.0. newMongoExporterPMMVersion = version.MustParse("2.9.99") v2_25_0 = version.MustParse("2.25.0-0") v2_26_0 = version.MustParse("2.26.0-0") v2_41_1 = version.MustParse("2.41.1-0") + v2_42_0 = version.MustParse("2.42.0-0") ) // mongodbExporterConfig returns desired configuration of mongodb_exporter process. @@ -50,23 +46,10 @@ func mongodbExporterConfig(node *models.Node, service *models.Service, exporter listenAddress := getExporterListenAddress(node, exporter) tdp := exporter.TemplateDelimiters(service) - var args []string - // Starting with PMM 2.10.0, we are shipping the new mongodb_exporter - // Starting with PMM 2.25.0, we change the discovering-mode making it to discover all databases. - // Until now, discovering mode was not working properly and was enabled only if mongodb.collstats-colls= - // was specified in the command line. - // Starting with PMM 2.26.0, we disabled all collectors by default and added flags to enable them. - // Starting with PMM 2.41.1 we added shards collector. - if pmmAgentVersion.Less(v2_26_0) { - args = oldPMMAgentArgs(exporter, tdp, listenAddress, pmmAgentVersion) - } else { - args = v226Args(exporter, tdp, listenAddress, pmmAgentVersion) - } - - args = collectors.FilterOutCollectors("--collect.", args, exporter.DisabledCollectors) + args := getArgs(exporter, tdp, listenAddress, pmmAgentVersion) if pointer.GetString(exporter.MetricsPath) != "" { - args = append(args, "--web.telemetry-path="+*exporter.MetricsPath) + args = append(args, "--web.telemetry-path="+*exporter.MetricsPath) //nolint:goconst } args = withLogLevel(args, exporter.LogLevel, pmmAgentVersion, true) @@ -101,171 +84,74 @@ func mongodbExporterConfig(node *models.Node, service *models.Service, exporter return res, nil } -func oldPMMAgentArgs(exporter *models.Agent, tdp *models.DelimiterPair, listenAddress string, pmmAgentVersion *version.Parsed) []string { +// getArgs returns the appropriate arguments based on the PMM agent version. +func getArgs(exporter *models.Agent, tdp *models.DelimiterPair, listenAddress string, pmmAgentVersion *version.Parsed) []string { var args []string + switch { - case !pmmAgentVersion.Less(v2_25_0): // >= 2.25 - args = v225Args(exporter, tdp, listenAddress) - case !pmmAgentVersion.Less(newMongoExporterPMMVersion): // >= 2.10 - args = []string{ - "--mongodb.global-conn-pool", - "--compatible-mode", - "--web.listen-address=" + listenAddress + ":" + tdp.Left + " .listen_port " + tdp.Right, //nolint:goconst + case !pmmAgentVersion.Less(v2_25_0): // >= 2.26.0 + args = buildBaseArgs(listenAddress, tdp) + args = append(args, "--discovering-mode") + + defaultEnabledCollectors := []string{"diagnosticdata", "replicasetstatus"} + collectAll := exporter.MongoDBOptions != nil && exporter.MongoDBOptions.EnableAllCollectors + + if !pmmAgentVersion.Less(v2_26_0) { + defaultEnabledCollectors = []string{} + args = append(args, "--collector.diagnosticdata", "--collector.replicasetstatus") + if collectAll { + args = append(args, "--collector.collstats", "--collector.dbstats", "--collector.indexstats", "--collector.topmetrics") + } } - default: - args = []string{ - "--collect.collection", - "--collect.database", - "--collect.topmetrics", - "--no-collect.connpoolstats", - "--no-collect.indexusage", - "--web.listen-address=" + listenAddress + ":" + tdp.Left + " .listen_port " + tdp.Right, + if !pmmAgentVersion.Less(v2_41_1) && collectAll { // >= 2.41.1 + args = append(args, "--collector.shards") + } + if !pmmAgentVersion.Less(v2_42_0) && collectAll { // >= 2.42.0 + args = append(args, "--collector.currentopmetrics") } - } - return args -} - -func v226Args(exporter *models.Agent, tdp *models.DelimiterPair, listenAddress string, pmmAgentVersion *version.Parsed) []string { - collectAll := false - if exporter.MongoDBOptions != nil { - collectAll = exporter.MongoDBOptions.EnableAllCollectors - } - - collstatsLimit := int32(200) - if exporter.MongoDBOptions != nil && exporter.MongoDBOptions.CollectionsLimit != -1 { - collstatsLimit = exporter.MongoDBOptions.CollectionsLimit - } - collectors := defaultCollectors(collectAll) + args = collectors.FilterOutCollectors("--collector.", args, exporter.DisabledCollectors) + args = append(args, collectors.DisableDefaultEnabledCollectors("--no-collector.", defaultEnabledCollectors, exporter.DisabledCollectors)...) - if !pmmAgentVersion.Less(v2_41_1) { // >= 2.41.1 - collectors["shards"] = collectorArgs{ - enabled: collectAll, - enableParam: "--collector.shards", + if exporter.MongoDBOptions != nil && len(exporter.MongoDBOptions.StatsCollections) != 0 { + args = append(args, "--mongodb.collstats-colls="+strings.Join(exporter.MongoDBOptions.StatsCollections, ",")) + if !pmmAgentVersion.Less(v2_26_0) { + args = append(args, "--mongodb.indexstats-colls="+strings.Join(exporter.MongoDBOptions.StatsCollections, ",")) + } } - } - for _, collector := range exporter.DisabledCollectors { - col, ok := collectors[strings.ToLower(collector)] - if !ok { - continue + if exporter.MongoDBOptions != nil { + collstatsLimit := int32(200) + if exporter.MongoDBOptions.CollectionsLimit != -1 { + collstatsLimit = exporter.MongoDBOptions.CollectionsLimit + } + args = append(args, fmt.Sprintf("--collector.collstats-limit=%d", collstatsLimit)) } - col.enabled = false - collectors[strings.ToLower(collector)] = col - } - - args := []string{ - "--mongodb.global-conn-pool", - "--compatible-mode", - "--web.listen-address=" + listenAddress + ":" + tdp.Left + " .listen_port " + tdp.Right, - "--discovering-mode", - } - if exporter.MongoDBOptions != nil && len(exporter.MongoDBOptions.StatsCollections) != 0 { - args = append(args, "--mongodb.collstats-colls="+strings.Join(exporter.MongoDBOptions.StatsCollections, ",")) - args = append(args, "--mongodb.indexstats-colls="+strings.Join(exporter.MongoDBOptions.StatsCollections, ",")) - } + case !pmmAgentVersion.Less(newMongoExporterPMMVersion): // >= 2.10.0 + args = buildBaseArgs(listenAddress, tdp) - if exporter.MongoDBOptions != nil { - args = append(args, fmt.Sprintf("--collector.collstats-limit=%d", collstatsLimit)) - } - - for _, collector := range collectors { - if collector.enabled && collector.enableParam != "" { - args = append(args, collector.enableParam) + default: // < 2.10.0 + args = []string{ + "--collect.collection", + "--collect.database", + "--collect.topmetrics", + "--no-collect.connpoolstats", + "--no-collect.indexusage", + "--web.listen-address=" + listenAddress + ":" + tdp.Left + " .listen_port " + tdp.Right, //nolint:goconst } + + args = collectors.FilterOutCollectors("--collect.", args, exporter.DisabledCollectors) } return args } -func v225Args(exporter *models.Agent, tdp *models.DelimiterPair, listenAddress string) []string { - type collectorArgs struct { - enabled bool - enableParam string - disableParam string - } - - collectors := map[string]collectorArgs{ - "diagnosticdata": { - enabled: true, - disableParam: "--no-collector.diagnosticdata", - }, - "replicasetstatus": { - enabled: true, - disableParam: "--no-collector.replicasetstatus", - }, - // disabled until we have better information on the resources usage impact - "dbstats": { - enabled: false, - enableParam: "--collector.dbstats", - }, - // disabled until we have better information on the resources usage impact - "topmetrics": { - enabled: false, - enableParam: "--collector.topmetrics", - }, - } - - for _, collector := range exporter.DisabledCollectors { - col := collectors[strings.ToLower(collector)] - col.enabled = false - collectors[strings.ToLower(collector)] = col - } - - args := []string{ +func buildBaseArgs(listenAddress string, tdp *models.DelimiterPair) []string { + return []string{ "--mongodb.global-conn-pool", "--compatible-mode", "--web.listen-address=" + listenAddress + ":" + tdp.Left + " .listen_port " + tdp.Right, - "--discovering-mode", - } - - if exporter.MongoDBOptions != nil && len(exporter.MongoDBOptions.StatsCollections) != 0 { - args = append(args, "--mongodb.collstats-colls="+strings.Join(exporter.MongoDBOptions.StatsCollections, ",")) - } - - if exporter.MongoDBOptions != nil && exporter.MongoDBOptions.CollectionsLimit != 0 { - args = append(args, fmt.Sprintf("--collector.collstats-limit=%d", exporter.MongoDBOptions.CollectionsLimit)) - } - - for _, collector := range collectors { - if collector.enabled && collector.enableParam != "" { - args = append(args, collector.enableParam) - } - if !collector.enabled && collector.disableParam != "" { - args = append(args, collector.disableParam) - } - } - - return args -} - -func defaultCollectors(collectAll bool) map[string]collectorArgs { - return map[string]collectorArgs{ - "diagnosticdata": { - enabled: true, - enableParam: "--collector.diagnosticdata", - }, - "replicasetstatus": { - enabled: true, - enableParam: "--collector.replicasetstatus", - }, - "collstats": { - enabled: collectAll, - enableParam: "--collector.collstats", - }, - "dbstats": { - enabled: collectAll, - enableParam: "--collector.dbstats", - }, - "indexstats": { - enabled: collectAll, - enableParam: "--collector.indexstats", - }, - "topmetrics": { - enabled: collectAll, - enableParam: "--collector.topmetrics", - }, } } diff --git a/managed/services/agents/mongodb_test.go b/managed/services/agents/mongodb_test.go index 8accc04e9e..0864effc51 100644 --- a/managed/services/agents/mongodb_test.go +++ b/managed/services/agents/mongodb_test.go @@ -384,6 +384,35 @@ func TestMongodbExporterConfig2411(t *testing.T) { require.NoError(t, err) require.Equal(t, expected.Args, actual.Args) }) + + t.Run("Enable all collectors and disable some", func(t *testing.T) { + exporter.MongoDBOptions = &models.MongoDBOptions{ + EnableAllCollectors: true, + StatsCollections: []string{"db1.col1.one", "db2.col2", "db3"}, + } + exporter.DisabledCollectors = []string{"dbstats", "topmetrics"} + + expected.Args = []string{ + "--collector.collstats", + "--collector.collstats-limit=0", + "--collector.diagnosticdata", + "--collector.indexstats", + "--collector.replicasetstatus", + "--collector.shards", + "--compatible-mode", + "--discovering-mode", + // this should be here even if limit=0 because it could be used to filter dbstats + // since dbstats is not depending the number of collections present in the db. + "--mongodb.collstats-colls=db1.col1.one,db2.col2,db3", + "--mongodb.global-conn-pool", + "--mongodb.indexstats-colls=db1.col1.one,db2.col2,db3", + "--web.listen-address=0.0.0.0:{{ .listen_port }}", + "--web.config={{ .TextFiles.webConfigPlaceholder }}", + } + actual, err := mongodbExporterConfig(node, mongodb, exporter, exposeSecrets, pmmAgentVersion) + require.NoError(t, err) + require.Equal(t, expected.Args, actual.Args) + }) } func TestMongodbExporterConfig(t *testing.T) { diff --git a/managed/services/victoriametrics/scrape_configs.go b/managed/services/victoriametrics/scrape_configs.go index 9dc595a871..d2a3104fcd 100644 --- a/managed/services/victoriametrics/scrape_configs.go +++ b/managed/services/victoriametrics/scrape_configs.go @@ -422,6 +422,9 @@ func scrapeConfigsForMongoDBExporter(params *scrapeConfigParams) ([]*config.Scra if !params.pmmAgentVersion.Less(version.MustParse("2.41.1-0")) { defaultCollectors = append(defaultCollectors, "shards") } + if !params.pmmAgentVersion.Less(version.MustParse("2.42.0-0")) { + defaultCollectors = append(defaultCollectors, "currentopmetrics") + } lr, err := scrapeConfigForStandardExporter("lr", params.metricsResolution.LR, params, defaultCollectors) if err != nil { diff --git a/managed/services/victoriametrics/scrape_configs_test.go b/managed/services/victoriametrics/scrape_configs_test.go index d09afa1988..c327bbac9d 100644 --- a/managed/services/victoriametrics/scrape_configs_test.go +++ b/managed/services/victoriametrics/scrape_configs_test.go @@ -758,7 +758,7 @@ func TestScrapeConfig(t *testing.T) { ScrapeTimeout: scrapeTimeout(s.LR), MetricsPath: "/metrics", Params: map[string][]string{ - "collect[]": {"collstats", "dbstats", "indexstats", "shards"}, + "collect[]": {"collstats", "currentopmetrics", "dbstats", "indexstats", "shards"}, }, HTTPClientConfig: config.HTTPClientConfig{ BasicAuth: &config.BasicAuth{ @@ -790,7 +790,7 @@ func TestScrapeConfig(t *testing.T) { node: node, service: service, agent: agent, - pmmAgentVersion: version.MustParse("2.41.1"), + pmmAgentVersion: version.MustParse("2.42.0"), metricsResolution: s, }) require.NoError(t, err) diff --git a/managed/utils/collectors/collectors.go b/managed/utils/collectors/collectors.go index 66e464ff40..5286e61518 100644 --- a/managed/utils/collectors/collectors.go +++ b/managed/utils/collectors/collectors.go @@ -42,3 +42,20 @@ func FilterOutCollectors(prefix string, args, disabledCollectors []string) []str } return enabledArgs } + +// DisableDefaultEnabledCollectors returns CLI arguments to disable default enabled collectors based on input. +// DefaultCollectors and disabledCollectors should be collector names without prefix. +// Result will be returned with prefix. +func DisableDefaultEnabledCollectors(prefix string, defaultCollectors []string, disabledCollectors []string) []string { + defaultCollectorsMap := make(map[string]struct{}) + for _, defaultCollector := range defaultCollectors { + defaultCollectorsMap[defaultCollector] = struct{}{} + } + args := []string{} + for _, collector := range disabledCollectors { + if _, ok := defaultCollectorsMap[collector]; ok { + args = append(args, fmt.Sprintf("%s%s", prefix, collector)) + } + } + return args +}