PMM-12463 Add telemetry envvar datasource (#2532)

* PMM-12463 allow ireturn on DataSources

* PMM-12463 fix some command hints

* PMM-12463 add ENV_VAR dataSource

* PMM-12463 add a provisionary envvar metrics extraction

* PMM-12463 add a test for EnvVar datasource

* PMM-12463 revert staticckeck linter

* PMM-12463 fix linter warnings

* PMM-12463 revert changes to main.go

* PMM-12463 refactor the output format

* PMM-12463 add a transform to strip values

* PMM-12463 fix linter warnings

* PMM-12463 refactor transform, add tests

* PMM-12463 run format

* PMM-12463 follow up on review comments

* PMM-12185 remove the debug statement

* PMM-12185 don't be too noisy when a ds is not initialized

.golangci.yml

@@ -51,6 +51,7 @@ linters-settings:
       - github.com/charmbracelet/bubbletea.Model
       - github.com/percona/pmm/admin/commands.Result
       - github.com/percona/pmm/agent/runner/actions.Action
+      - github.com/percona/pmm/managed/services/telemetry.DataSource
 
   lll:
     line-length: 170

managed/services/config/config.go

@@ -73,7 +73,7 @@ func (s *Service) Load() error {
 	var cfg Config
 
 	if _, err := os.Stat(configPath); err == nil {
-		s.l.Trace("config exist, reading file")
+		s.l.Trace("config exists, reading file")
 		buf, err := os.ReadFile(configPath) //nolint:gosec
 		if err != nil {
 			return errors.Wrapf(err, "error while reading config [%s]", configPath)

managed/services/config/pmm-managed.yaml

@@ -20,6 +20,8 @@ services:
         enabled: true
         timeout: 5s
         db_file: /srv/grafana/grafana.db
+      ENV_VARS:
+        enabled: true
     reporting:
       send: true
       send_on_start: false

managed/services/telemetry/config.default.yml

@@ -936,3 +936,36 @@ telemetry:
     data:
       - metric_name: "postgresql_db_count"
         value: 1
+
+  # Note: use these for testing and PMM-12462
+  # - id: PMMServerFeatureToggles
+  #   source: ENV_VARS
+  #   summary: "Use of feature toggles in PMM Server"
+  #   data:
+  #     - metric_name: "pmm_server_disable_telemetry"
+  #       column: "DISABLE_TELEMETRY"
+  #     - metric_name: "pmm_server_enable_alerting"
+  #       column: "ENABLE_ALERTING"
+  #     - metric_name: "pmm_server_enable_backup_management"
+  #       column: "ENABLE_BACKUP_MANAGEMENT"
+  #     - metric_name: "pmm_server_enable_debug"
+  #       column: "ENABLE_DEBUG"
+  #     - metric_name: "pmm_server_enable_rbac"
+  #       column: "ENABLE_RBAC"
+
+  # - id: PMMServerFeatureTogglesStripValues
+  #   source: ENV_VARS
+  #   summary: "Use of feature toggles in PMM Server"
+  #   transform:
+  #     type: StripValues
+  #   data:
+  #     - metric_name: "pmm_server_disable_telemetry"
+  #       column: "DISABLE_TELEMETRY"
+  #     - metric_name: "pmm_server_enable_alerting"
+  #       column: "ENABLE_ALERTING"
+  #     - metric_name: "pmm_server_enable_backup_management"
+  #       column: "ENABLE_BACKUP_MANAGEMENT"
+  #     - metric_name: "pmm_server_enable_debug"
+  #       column: "ENABLE_DEBUG"
+  #     - metric_name: "pmm_server_enable_rbac"
+  #       column: "ENABLE_RBAC"

managed/services/telemetry/config.go

@@ -37,19 +37,31 @@ const (
 	envReportingRetryBackoff = "PERCONA_TEST_TELEMETRY_RETRY_BACKOFF"
 )
 
+const (
+	dsVM              = DataSourceName("VM")
+	dsQANDBSelect     = DataSourceName("QANDB_SELECT")
+	dsPMMDBSelect     = DataSourceName("PMMDB_SELECT")
+	dsGrafanaDBSelect = DataSourceName("GRAFANADB_SELECT")
+	dsEnvVars         = DataSourceName("ENV_VARS")
+)
+
+// DataSources holds all possible data source types.
+type DataSources struct {
+	VM              *DataSourceVictoriaMetrics `yaml:"VM"`
+	QanDBSelect     *DSConfigQAN               `yaml:"QANDB_SELECT"`
+	PmmDBSelect     *DSConfigPMMDB             `yaml:"PMMDB_SELECT"`
+	GrafanaDBSelect *DSGrafanaSqliteDB         `yaml:"GRAFANADB_SELECT"`
+	EnvVars         *DSConfigEnvVars           `yaml:"ENV_VARS"`
+}
+
 // ServiceConfig telemetry config.
 type ServiceConfig struct {
 	l            *logrus.Entry
-	Enabled      bool     `yaml:"enabled"`
-	telemetry    []Config `yaml:"-"`
-	SaasHostname string   `yaml:"saas_hostname"`
-	DataSources  struct {
-		VM              *DataSourceVictoriaMetrics `yaml:"VM"`
-		QanDBSelect     *DSConfigQAN               `yaml:"QANDB_SELECT"`
-		PmmDBSelect     *DSConfigPMMDB             `yaml:"PMMDB_SELECT"`
-		GrafanaDBSelect *DSGrafanaSqliteDB         `yaml:"GRAFANADB_SELECT"`
-	} `yaml:"datasources"`
-	Reporting ReportingConfig `yaml:"reporting"`
+	Enabled      bool            `yaml:"enabled"`
+	telemetry    []Config        `yaml:"-"`
+	SaasHostname string          `yaml:"saas_hostname"`
+	DataSources  DataSources     `yaml:"datasources"`
+	Reporting    ReportingConfig `yaml:"reporting"`
 }
 
 // FileConfig top level telemetry config element.
@@ -100,7 +112,11 @@ type DSConfigPMMDB struct { //nolint:musttag
 	} `yaml:"separate_credentials"`
 }
 
-// Config telemetry config.
+type DSConfigEnvVars struct {
+	Enabled bool `yaml:"enabled"`
+}
+
+// Config is a telemetry config.
 type Config struct {
 	ID        string           `yaml:"id"`
 	Source    string           `yaml:"source"`
@@ -111,21 +127,23 @@ type Config struct {
 	Data      []ConfigData
 }
 
-// ConfigTransform telemetry config transformation.
+// ConfigTransform is a telemetry config transformation.
 type ConfigTransform struct {
 	Type   ConfigTransformType `yaml:"type"`
 	Metric string              `yaml:"metric"`
 }
 
-// ConfigTransformType config transform type.
+// ConfigTransformType is a config transform type.
 type ConfigTransformType string
 
 const (
-	// JSONTransformType JSON type.
-	JSONTransformType = ConfigTransformType("JSON")
+	// JSONTransform converts multiple metrics in one formatted as JSON.
+	JSONTransform = ConfigTransformType("JSON")
+	// StripValuesTransform strips values from metrics, replacing them with 0/1 depending on presence.
+	StripValuesTransform = ConfigTransformType("StripValues")
 )
 
-// ConfigData telemetry config.
+// ConfigData is a telemetry data config.
 type ConfigData struct {
 	MetricName string `yaml:"metric_name"`
 	Label      string `yaml:"label"`

managed/services/telemetry/config_test.go

@@ -48,6 +48,8 @@ datasources:
     enabled: true
     timeout: 2s
     db_file: /srv/grafana/grafana.db
+  ENV_VARS:
+    enabled: true
 
 reporting:
   send: true
@@ -71,12 +73,7 @@ reporting:
 			RetryCount:   2,
 			SendTimeout:  time.Second * 10,
 		},
-		DataSources: struct {
-			VM              *DataSourceVictoriaMetrics `yaml:"VM"`
-			QanDBSelect     *DSConfigQAN               `yaml:"QANDB_SELECT"`
-			PmmDBSelect     *DSConfigPMMDB             `yaml:"PMMDB_SELECT"`
-			GrafanaDBSelect *DSGrafanaSqliteDB         `yaml:"GRAFANADB_SELECT"`
-		}{
+		DataSources: DataSources{
 			VM: &DataSourceVictoriaMetrics{
 				Enabled: true,
 				Timeout: time.Second * 2,
@@ -103,6 +100,9 @@ reporting:
 				Timeout: time.Second * 2,
 				DBFile:  "/srv/grafana/grafana.db",
 			},
+			EnvVars: &DSConfigEnvVars{
+				Enabled: true,
+			},
 		},
 	}
 	assert.Equal(t, actual, expected)

managed/services/telemetry/datasource_envvars.go

@@ -0,0 +1,84 @@
+// Copyright (C) 2023 Percona LLC
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+// Package telemetry provides telemetry functionality.
+package telemetry
+
+import (
+	"context"
+	"os"
+
+	pmmv1 "github.com/percona-platform/saas/gen/telemetry/events/pmm"
+	"github.com/sirupsen/logrus"
+)
+
+type dsEnvvars struct {
+	l      *logrus.Entry
+	config DSConfigEnvVars
+}
+
+// check interfaces.
+var (
+	_ DataSource = (*dsEnvvars)(nil)
+)
+
+// NewDataSourceEnvVars makes a new data source for collecting envvars.
+func NewDataSourceEnvVars(config DSConfigEnvVars, l *logrus.Entry) DataSource {
+	return &dsEnvvars{
+		l:      l,
+		config: config,
+	}
+}
+
+// Enabled flag that determines if data source is enabled.
+func (d *dsEnvvars) Enabled() bool {
+	return d.config.Enabled
+}
+
+func (d *dsEnvvars) Init(_ context.Context) error {
+	return nil
+}
+
+func (d *dsEnvvars) FetchMetrics(_ context.Context, config Config) ([]*pmmv1.ServerMetric_Metric, error) {
+	var metrics []*pmmv1.ServerMetric_Metric
+
+	check := make(map[string]bool, len(config.Data))
+
+	for _, col := range config.Data {
+		if col.Column == "" {
+			d.l.Warnf("no column defined or empty column name in config %s", config.ID)
+			continue
+		}
+		if value, ok := os.LookupEnv(col.Column); ok && value != "" {
+			if _, alreadyHasItem := check[col.MetricName]; alreadyHasItem {
+				d.l.Warnf("repeated metric key %s found in config %s, the last will win", col.MetricName, config.ID)
+				continue
+			}
+
+			check[col.MetricName] = true
+
+			metrics = append(metrics, &pmmv1.ServerMetric_Metric{
+				Key:   col.MetricName,
+				Value: value,
+			})
+		}
+	}
+
+	return metrics, nil
+}
+
+func (d *dsEnvvars) Dispose(_ context.Context) error {
+	return nil
+}