PMM-13141 enable fcv collector

managed/services/agents/mongodb.go

@@ -30,17 +30,14 @@ import (
 	"github.com/percona/pmm/version"
 )
 
-type collectorArgs struct {
-	enabled     bool
-	enableParam string
-}
-
 var (
 	// New MongoDB Exporter will be released with PMM agent v2.10.0.
 	newMongoExporterPMMVersion = version.MustParse("2.9.99")
 	v2_25_0                    = version.MustParse("2.25.0-0")
 	v2_26_0                    = version.MustParse("2.26.0-0")
 	v2_41_1                    = version.MustParse("2.41.1-0")
+	v2_42_0                    = version.MustParse("2.42.0-0")
+	v2_43_0                    = version.MustParse("2.43.0-0")
 )
 
 // mongodbExporterConfig returns desired configuration of mongodb_exporter process.
@@ -50,23 +47,10 @@ func mongodbExporterConfig(node *models.Node, service *models.Service, exporter
 	listenAddress := getExporterListenAddress(node, exporter)
 	tdp := exporter.TemplateDelimiters(service)
 
-	var args []string
-	// Starting with PMM 2.10.0, we are shipping the new mongodb_exporter
-	// Starting with PMM 2.25.0, we change the discovering-mode making it to discover all databases.
-	// Until now, discovering mode was not working properly and was enabled only if mongodb.collstats-colls=
-	// was specified in the command line.
-	// Starting with PMM 2.26.0, we disabled all collectors by default and added flags to enable them.
-	// Starting with PMM 2.41.1 we added shards collector.
-	if pmmAgentVersion.Less(v2_26_0) {
-		args = oldPMMAgentArgs(exporter, tdp, listenAddress, pmmAgentVersion)
-	} else {
-		args = v226Args(exporter, tdp, listenAddress, pmmAgentVersion)
-	}
-
-	args = collectors.FilterOutCollectors("--collect.", args, exporter.DisabledCollectors)
+	args := getArgs(exporter, tdp, listenAddress, pmmAgentVersion)
 
 	if pointer.GetString(exporter.MetricsPath) != "" {
-		args = append(args, "--web.telemetry-path="+*exporter.MetricsPath)
+		args = append(args, "--web.telemetry-path="+*exporter.MetricsPath) //nolint:goconst
 	}
 
 	args = withLogLevel(args, exporter.LogLevel, pmmAgentVersion, true)
@@ -101,171 +85,77 @@ func mongodbExporterConfig(node *models.Node, service *models.Service, exporter
 	return res, nil
 }
 
-func oldPMMAgentArgs(exporter *models.Agent, tdp *models.DelimiterPair, listenAddress string, pmmAgentVersion *version.Parsed) []string {
+// getArgs returns the appropriate arguments based on the PMM agent version.
+func getArgs(exporter *models.Agent, tdp *models.DelimiterPair, listenAddress string, pmmAgentVersion *version.Parsed) []string {
 	var args []string
+
 	switch {
-	case !pmmAgentVersion.Less(v2_25_0): // >= 2.25
-		args = v225Args(exporter, tdp, listenAddress)
-	case !pmmAgentVersion.Less(newMongoExporterPMMVersion): // >= 2.10
-		args = []string{
-			"--mongodb.global-conn-pool",
-			"--compatible-mode",
-			"--web.listen-address=" + listenAddress + ":" + tdp.Left + " .listen_port " + tdp.Right, //nolint:goconst
+	case !pmmAgentVersion.Less(v2_25_0): // >= 2.26.0
+		args = buildBaseArgs(listenAddress, tdp)
+		args = append(args, "--discovering-mode")
+
+		defaultEnabledCollectors := []string{"diagnosticdata", "replicasetstatus"}
+		collectAll := exporter.MongoDBOptions != nil && exporter.MongoDBOptions.EnableAllCollectors
+
+		if !pmmAgentVersion.Less(v2_26_0) {
+			defaultEnabledCollectors = []string{}
+			args = append(args, "--collector.diagnosticdata", "--collector.replicasetstatus")
+			if collectAll {
+				args = append(args, "--collector.collstats", "--collector.dbstats", "--collector.indexstats", "--collector.topmetrics")
+			}
 		}
-	default:
-		args = []string{
-			"--collect.collection",
-			"--collect.database",
-			"--collect.topmetrics",
-			"--no-collect.connpoolstats",
-			"--no-collect.indexusage",
-			"--web.listen-address=" + listenAddress + ":" + tdp.Left + " .listen_port " + tdp.Right,
+		if !pmmAgentVersion.Less(v2_41_1) && collectAll { // >= 2.41.1
+			args = append(args, "--collector.shards")
+		}
+		if !pmmAgentVersion.Less(v2_42_0) && collectAll { // >= 2.42.0
+			args = append(args, "--collector.currentopmetrics")
+		}
+		if !pmmAgentVersion.Less(v2_43_0) && collectAll { // >= 2.43.0
+			args = append(args, "--collector.fcv")
 		}
-	}
-	return args
-}
-
-func v226Args(exporter *models.Agent, tdp *models.DelimiterPair, listenAddress string, pmmAgentVersion *version.Parsed) []string {
-	collectAll := false
-	if exporter.MongoDBOptions != nil {
-		collectAll = exporter.MongoDBOptions.EnableAllCollectors
-	}
-
-	collstatsLimit := int32(200)
-	if exporter.MongoDBOptions != nil && exporter.MongoDBOptions.CollectionsLimit != -1 {
-		collstatsLimit = exporter.MongoDBOptions.CollectionsLimit
-	}
 
-	collectors := defaultCollectors(collectAll)
+		args = collectors.FilterOutCollectors("--collector.", args, exporter.DisabledCollectors)
+		args = append(args, collectors.DisableDefaultEnabledCollectors("--no-collector.", defaultEnabledCollectors, exporter.DisabledCollectors)...)
 
-	if !pmmAgentVersion.Less(v2_41_1) { // >= 2.41.1
-		collectors["shards"] = collectorArgs{
-			enabled:     collectAll,
-			enableParam: "--collector.shards",
+		if exporter.MongoDBOptions != nil && len(exporter.MongoDBOptions.StatsCollections) != 0 {
+			args = append(args, "--mongodb.collstats-colls="+strings.Join(exporter.MongoDBOptions.StatsCollections, ","))
+			if !pmmAgentVersion.Less(v2_26_0) {
+				args = append(args, "--mongodb.indexstats-colls="+strings.Join(exporter.MongoDBOptions.StatsCollections, ","))
+			}
 		}
-	}
 
-	for _, collector := range exporter.DisabledCollectors {
-		col, ok := collectors[strings.ToLower(collector)]
-		if !ok {
-			continue
+		if exporter.MongoDBOptions != nil {
+			collstatsLimit := int32(200)
+			if exporter.MongoDBOptions.CollectionsLimit != -1 {
+				collstatsLimit = exporter.MongoDBOptions.CollectionsLimit
+			}
+			args = append(args, fmt.Sprintf("--collector.collstats-limit=%d", collstatsLimit))
 		}
-		col.enabled = false
-		collectors[strings.ToLower(collector)] = col
-	}
-
-	args := []string{
-		"--mongodb.global-conn-pool",
-		"--compatible-mode",
-		"--web.listen-address=" + listenAddress + ":" + tdp.Left + " .listen_port " + tdp.Right,
-		"--discovering-mode",
-	}
 
-	if exporter.MongoDBOptions != nil && len(exporter.MongoDBOptions.StatsCollections) != 0 {
-		args = append(args, "--mongodb.collstats-colls="+strings.Join(exporter.MongoDBOptions.StatsCollections, ","))
-		args = append(args, "--mongodb.indexstats-colls="+strings.Join(exporter.MongoDBOptions.StatsCollections, ","))
-	}
+	case !pmmAgentVersion.Less(newMongoExporterPMMVersion): // >= 2.10.0
+		args = buildBaseArgs(listenAddress, tdp)
 
-	if exporter.MongoDBOptions != nil {
-		args = append(args, fmt.Sprintf("--collector.collstats-limit=%d", collstatsLimit))
-	}
-
-	for _, collector := range collectors {
-		if collector.enabled && collector.enableParam != "" {
-			args = append(args, collector.enableParam)
+	default: // < 2.10.0
+		args = []string{
+			"--collect.collection",
+			"--collect.database",
+			"--collect.topmetrics",
+			"--no-collect.connpoolstats",
+			"--no-collect.indexusage",
+			"--web.listen-address=" + listenAddress + ":" + tdp.Left + " .listen_port " + tdp.Right, //nolint:goconst
 		}
+
+		args = collectors.FilterOutCollectors("--collect.", args, exporter.DisabledCollectors)
 	}
 
 	return args
 }
 
-func v225Args(exporter *models.Agent, tdp *models.DelimiterPair, listenAddress string) []string {
-	type collectorArgs struct {
-		enabled      bool
-		enableParam  string
-		disableParam string
-	}
-
-	collectors := map[string]collectorArgs{
-		"diagnosticdata": {
-			enabled:      true,
-			disableParam: "--no-collector.diagnosticdata",
-		},
-		"replicasetstatus": {
-			enabled:      true,
-			disableParam: "--no-collector.replicasetstatus",
-		},
-		// disabled until we have better information on the resources usage impact
-		"dbstats": {
-			enabled:     false,
-			enableParam: "--collector.dbstats",
-		},
-		// disabled until we have better information on the resources usage impact
-		"topmetrics": {
-			enabled:     false,
-			enableParam: "--collector.topmetrics",
-		},
-	}
-
-	for _, collector := range exporter.DisabledCollectors {
-		col := collectors[strings.ToLower(collector)]
-		col.enabled = false
-		collectors[strings.ToLower(collector)] = col
-	}
-
-	args := []string{
+func buildBaseArgs(listenAddress string, tdp *models.DelimiterPair) []string {
+	return []string{
 		"--mongodb.global-conn-pool",
 		"--compatible-mode",
 		"--web.listen-address=" + listenAddress + ":" + tdp.Left + " .listen_port " + tdp.Right,
-		"--discovering-mode",
-	}
-
-	if exporter.MongoDBOptions != nil && len(exporter.MongoDBOptions.StatsCollections) != 0 {
-		args = append(args, "--mongodb.collstats-colls="+strings.Join(exporter.MongoDBOptions.StatsCollections, ","))
-	}
-
-	if exporter.MongoDBOptions != nil && exporter.MongoDBOptions.CollectionsLimit != 0 {
-		args = append(args, fmt.Sprintf("--collector.collstats-limit=%d", exporter.MongoDBOptions.CollectionsLimit))
-	}
-
-	for _, collector := range collectors {
-		if collector.enabled && collector.enableParam != "" {
-			args = append(args, collector.enableParam)
-		}
-		if !collector.enabled && collector.disableParam != "" {
-			args = append(args, collector.disableParam)
-		}
-	}
-
-	return args
-}
-
-func defaultCollectors(collectAll bool) map[string]collectorArgs {
-	return map[string]collectorArgs{
-		"diagnosticdata": {
-			enabled:     true,
-			enableParam: "--collector.diagnosticdata",
-		},
-		"replicasetstatus": {
-			enabled:     true,
-			enableParam: "--collector.replicasetstatus",
-		},
-		"collstats": {
-			enabled:     collectAll,
-			enableParam: "--collector.collstats",
-		},
-		"dbstats": {
-			enabled:     collectAll,
-			enableParam: "--collector.dbstats",
-		},
-		"indexstats": {
-			enabled:     collectAll,
-			enableParam: "--collector.indexstats",
-		},
-		"topmetrics": {
-			enabled:     collectAll,
-			enableParam: "--collector.topmetrics",
-		},
 	}
 }
 

managed/services/agents/mongodb_test.go

@@ -384,6 +384,35 @@ func TestMongodbExporterConfig2411(t *testing.T) {
 		require.NoError(t, err)
 		require.Equal(t, expected.Args, actual.Args)
 	})
+
+	t.Run("Enable all collectors and disable some", func(t *testing.T) {
+		exporter.MongoDBOptions = &models.MongoDBOptions{
+			EnableAllCollectors: true,
+			StatsCollections:    []string{"db1.col1.one", "db2.col2", "db3"},
+		}
+		exporter.DisabledCollectors = []string{"dbstats", "topmetrics"}
+
+		expected.Args = []string{
+			"--collector.collstats",
+			"--collector.collstats-limit=0",
+			"--collector.diagnosticdata",
+			"--collector.indexstats",
+			"--collector.replicasetstatus",
+			"--collector.shards",
+			"--compatible-mode",
+			"--discovering-mode",
+			// this should be here even if limit=0 because it could be used to filter dbstats
+			// since dbstats is not depending the number of collections present in the db.
+			"--mongodb.collstats-colls=db1.col1.one,db2.col2,db3",
+			"--mongodb.global-conn-pool",
+			"--mongodb.indexstats-colls=db1.col1.one,db2.col2,db3",
+			"--web.listen-address=0.0.0.0:{{ .listen_port }}",
+			"--web.config={{ .TextFiles.webConfigPlaceholder }}",
+		}
+		actual, err := mongodbExporterConfig(node, mongodb, exporter, exposeSecrets, pmmAgentVersion)
+		require.NoError(t, err)
+		require.Equal(t, expected.Args, actual.Args)
+	})
 }
 
 func TestMongodbExporterConfig(t *testing.T) {

managed/services/victoriametrics/scrape_configs.go

@@ -422,6 +422,12 @@ func scrapeConfigsForMongoDBExporter(params *scrapeConfigParams) ([]*config.Scra
 		if !params.pmmAgentVersion.Less(version.MustParse("2.41.1-0")) {
 			defaultCollectors = append(defaultCollectors, "shards")
 		}
+		if !params.pmmAgentVersion.Less(version.MustParse("2.42.0-0")) {
+			defaultCollectors = append(defaultCollectors, "currentopmetrics")
+		}
+		if !params.pmmAgentVersion.Less(version.MustParse("2.43.0-0")) {
+			defaultCollectors = append(defaultCollectors, "fcv")
+		}
 
 		lr, err := scrapeConfigForStandardExporter("lr", params.metricsResolution.LR, params, defaultCollectors)
 		if err != nil {

managed/services/victoriametrics/scrape_configs_test.go

@@ -758,7 +758,7 @@ func TestScrapeConfig(t *testing.T) {
 					ScrapeTimeout:  scrapeTimeout(s.LR),
 					MetricsPath:    "/metrics",
 					Params: map[string][]string{
-						"collect[]": {"collstats", "dbstats", "indexstats", "shards"},
+						"collect[]": {"collstats", "currentopmetrics", "dbstats", "indexstats", "shards"},
 					},
 					HTTPClientConfig: config.HTTPClientConfig{
 						BasicAuth: &config.BasicAuth{
@@ -790,7 +790,7 @@ func TestScrapeConfig(t *testing.T) {
 				node:              node,
 				service:           service,
 				agent:             agent,
-				pmmAgentVersion:   version.MustParse("2.41.1"),
+				pmmAgentVersion:   version.MustParse("2.42.0"),
 				metricsResolution: s,
 			})
 			require.NoError(t, err)

managed/utils/collectors/collectors.go

@@ -42,3 +42,20 @@ func FilterOutCollectors(prefix string, args, disabledCollectors []string) []str
 	}
 	return enabledArgs
 }
+
+// DisableDefaultEnabledCollectors returns CLI arguments to disable default enabled collectors based on input.
+// DefaultCollectors and disabledCollectors should be collector names without prefix.
+// Result will be returned with prefix.
+func DisableDefaultEnabledCollectors(prefix string, defaultCollectors []string, disabledCollectors []string) []string {
+	defaultCollectorsMap := make(map[string]struct{})
+	for _, defaultCollector := range defaultCollectors {
+		defaultCollectorsMap[defaultCollector] = struct{}{}
+	}
+	args := []string{}
+	for _, collector := range disabledCollectors {
+		if _, ok := defaultCollectorsMap[collector]; ok {
+			args = append(args, fmt.Sprintf("%s%s", prefix, collector))
+		}
+	}
+	return args
+}