diff --git a/.golangci.yml b/.golangci.yml
index 8b64eff852..98c4492936 100644
--- a/.golangci.yml
+++ b/.golangci.yml
@@ -51,6 +51,7 @@ linters-settings:
- github.com/charmbracelet/bubbletea.Model
- github.com/percona/pmm/admin/commands.Result
- github.com/percona/pmm/agent/runner/actions.Action
+ - github.com/percona/pmm/managed/services/telemetry.DataSource
lll:
line-length: 170
diff --git a/managed/services/config/config.go b/managed/services/config/config.go
index da63fa58e7..fd4cedf7b5 100644
--- a/managed/services/config/config.go
+++ b/managed/services/config/config.go
@@ -73,7 +73,7 @@ func (s *Service) Load() error {
var cfg Config
if _, err := os.Stat(configPath); err == nil {
- s.l.Trace("config exist, reading file")
+ s.l.Trace("config exists, reading file")
buf, err := os.ReadFile(configPath) //nolint:gosec
if err != nil {
return errors.Wrapf(err, "error while reading config [%s]", configPath)
diff --git a/managed/services/config/pmm-managed.yaml b/managed/services/config/pmm-managed.yaml
index d9341b020f..c9dee75d74 100644
--- a/managed/services/config/pmm-managed.yaml
+++ b/managed/services/config/pmm-managed.yaml
@@ -20,6 +20,8 @@ services:
enabled: true
timeout: 5s
db_file: /srv/grafana/grafana.db
+ ENV_VARS:
+ enabled: true
reporting:
send: true
send_on_start: false
diff --git a/managed/services/telemetry/config.default.yml b/managed/services/telemetry/config.default.yml
index 74924104e0..a7803f8109 100644
--- a/managed/services/telemetry/config.default.yml
+++ b/managed/services/telemetry/config.default.yml
@@ -936,3 +936,36 @@ telemetry:
data:
- metric_name: "postgresql_db_count"
value: 1
+
+ # Note: use these for testing and PMM-12462
+ # - id: PMMServerFeatureToggles
+ # source: ENV_VARS
+ # summary: "Use of feature toggles in PMM Server"
+ # data:
+ # - metric_name: "pmm_server_disable_telemetry"
+ # column: "DISABLE_TELEMETRY"
+ # - metric_name: "pmm_server_enable_alerting"
+ # column: "ENABLE_ALERTING"
+ # - metric_name: "pmm_server_enable_backup_management"
+ # column: "ENABLE_BACKUP_MANAGEMENT"
+ # - metric_name: "pmm_server_enable_debug"
+ # column: "ENABLE_DEBUG"
+ # - metric_name: "pmm_server_enable_rbac"
+ # column: "ENABLE_RBAC"
+
+ # - id: PMMServerFeatureTogglesStripValues
+ # source: ENV_VARS
+ # summary: "Use of feature toggles in PMM Server"
+ # transform:
+ # type: StripValues
+ # data:
+ # - metric_name: "pmm_server_disable_telemetry"
+ # column: "DISABLE_TELEMETRY"
+ # - metric_name: "pmm_server_enable_alerting"
+ # column: "ENABLE_ALERTING"
+ # - metric_name: "pmm_server_enable_backup_management"
+ # column: "ENABLE_BACKUP_MANAGEMENT"
+ # - metric_name: "pmm_server_enable_debug"
+ # column: "ENABLE_DEBUG"
+ # - metric_name: "pmm_server_enable_rbac"
+ # column: "ENABLE_RBAC"
diff --git a/managed/services/telemetry/config.go b/managed/services/telemetry/config.go
index 40372b297f..97ea351a8c 100644
--- a/managed/services/telemetry/config.go
+++ b/managed/services/telemetry/config.go
@@ -37,19 +37,31 @@ const (
envReportingRetryBackoff = "PERCONA_TEST_TELEMETRY_RETRY_BACKOFF"
)
+const (
+ dsVM = DataSourceName("VM")
+ dsQANDBSelect = DataSourceName("QANDB_SELECT")
+ dsPMMDBSelect = DataSourceName("PMMDB_SELECT")
+ dsGrafanaDBSelect = DataSourceName("GRAFANADB_SELECT")
+ dsEnvVars = DataSourceName("ENV_VARS")
+)
+
+// DataSources holds all possible data source types.
+type DataSources struct {
+ VM *DataSourceVictoriaMetrics `yaml:"VM"`
+ QanDBSelect *DSConfigQAN `yaml:"QANDB_SELECT"`
+ PmmDBSelect *DSConfigPMMDB `yaml:"PMMDB_SELECT"`
+ GrafanaDBSelect *DSGrafanaSqliteDB `yaml:"GRAFANADB_SELECT"`
+ EnvVars *DSConfigEnvVars `yaml:"ENV_VARS"`
+}
+
// ServiceConfig telemetry config.
type ServiceConfig struct {
l *logrus.Entry
- Enabled bool `yaml:"enabled"`
- telemetry []Config `yaml:"-"`
- SaasHostname string `yaml:"saas_hostname"`
- DataSources struct {
- VM *DataSourceVictoriaMetrics `yaml:"VM"`
- QanDBSelect *DSConfigQAN `yaml:"QANDB_SELECT"`
- PmmDBSelect *DSConfigPMMDB `yaml:"PMMDB_SELECT"`
- GrafanaDBSelect *DSGrafanaSqliteDB `yaml:"GRAFANADB_SELECT"`
- } `yaml:"datasources"`
- Reporting ReportingConfig `yaml:"reporting"`
+ Enabled bool `yaml:"enabled"`
+ telemetry []Config `yaml:"-"`
+ SaasHostname string `yaml:"saas_hostname"`
+ DataSources DataSources `yaml:"datasources"`
+ Reporting ReportingConfig `yaml:"reporting"`
}
// FileConfig top level telemetry config element.
@@ -100,7 +112,11 @@ type DSConfigPMMDB struct { //nolint:musttag
} `yaml:"separate_credentials"`
}
-// Config telemetry config.
+type DSConfigEnvVars struct {
+ Enabled bool `yaml:"enabled"`
+}
+
+// Config is a telemetry config.
type Config struct {
ID string `yaml:"id"`
Source string `yaml:"source"`
@@ -111,21 +127,23 @@ type Config struct {
Data []ConfigData
}
-// ConfigTransform telemetry config transformation.
+// ConfigTransform is a telemetry config transformation.
type ConfigTransform struct {
Type ConfigTransformType `yaml:"type"`
Metric string `yaml:"metric"`
}
-// ConfigTransformType config transform type.
+// ConfigTransformType is a config transform type.
type ConfigTransformType string
const (
- // JSONTransformType JSON type.
- JSONTransformType = ConfigTransformType("JSON")
+ // JSONTransform converts multiple metrics in one formatted as JSON.
+ JSONTransform = ConfigTransformType("JSON")
+ // StripValuesTransform strips values from metrics, replacing them with 0/1 depending on presence.
+ StripValuesTransform = ConfigTransformType("StripValues")
)
-// ConfigData telemetry config.
+// ConfigData is a telemetry data config.
type ConfigData struct {
MetricName string `yaml:"metric_name"`
Label string `yaml:"label"`
diff --git a/managed/services/telemetry/config_test.go b/managed/services/telemetry/config_test.go
index 5d497688d0..7a7da4ab7e 100644
--- a/managed/services/telemetry/config_test.go
+++ b/managed/services/telemetry/config_test.go
@@ -48,6 +48,8 @@ datasources:
enabled: true
timeout: 2s
db_file: /srv/grafana/grafana.db
+ ENV_VARS:
+ enabled: true
reporting:
send: true
@@ -71,12 +73,7 @@ reporting:
RetryCount: 2,
SendTimeout: time.Second * 10,
},
- DataSources: struct {
- VM *DataSourceVictoriaMetrics `yaml:"VM"`
- QanDBSelect *DSConfigQAN `yaml:"QANDB_SELECT"`
- PmmDBSelect *DSConfigPMMDB `yaml:"PMMDB_SELECT"`
- GrafanaDBSelect *DSGrafanaSqliteDB `yaml:"GRAFANADB_SELECT"`
- }{
+ DataSources: DataSources{
VM: &DataSourceVictoriaMetrics{
Enabled: true,
Timeout: time.Second * 2,
@@ -103,6 +100,9 @@ reporting:
Timeout: time.Second * 2,
DBFile: "/srv/grafana/grafana.db",
},
+ EnvVars: &DSConfigEnvVars{
+ Enabled: true,
+ },
},
}
assert.Equal(t, actual, expected)
diff --git a/managed/services/telemetry/datasource_envvars.go b/managed/services/telemetry/datasource_envvars.go
new file mode 100644
index 0000000000..4dc320a51c
--- /dev/null
+++ b/managed/services/telemetry/datasource_envvars.go
@@ -0,0 +1,84 @@
+// Copyright (C) 2023 Percona LLC
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see .
+
+// Package telemetry provides telemetry functionality.
+package telemetry
+
+import (
+ "context"
+ "os"
+
+ pmmv1 "github.com/percona-platform/saas/gen/telemetry/events/pmm"
+ "github.com/sirupsen/logrus"
+)
+
+type dsEnvvars struct {
+ l *logrus.Entry
+ config DSConfigEnvVars
+}
+
+// check interfaces.
+var (
+ _ DataSource = (*dsEnvvars)(nil)
+)
+
+// NewDataSourceEnvVars makes a new data source for collecting envvars.
+func NewDataSourceEnvVars(config DSConfigEnvVars, l *logrus.Entry) DataSource {
+ return &dsEnvvars{
+ l: l,
+ config: config,
+ }
+}
+
+// Enabled flag that determines if data source is enabled.
+func (d *dsEnvvars) Enabled() bool {
+ return d.config.Enabled
+}
+
+func (d *dsEnvvars) Init(_ context.Context) error {
+ return nil
+}
+
+func (d *dsEnvvars) FetchMetrics(_ context.Context, config Config) ([]*pmmv1.ServerMetric_Metric, error) {
+ var metrics []*pmmv1.ServerMetric_Metric
+
+ check := make(map[string]bool, len(config.Data))
+
+ for _, col := range config.Data {
+ if col.Column == "" {
+ d.l.Warnf("no column defined or empty column name in config %s", config.ID)
+ continue
+ }
+ if value, ok := os.LookupEnv(col.Column); ok && value != "" {
+ if _, alreadyHasItem := check[col.MetricName]; alreadyHasItem {
+ d.l.Warnf("repeated metric key %s found in config %s, the last will win", col.MetricName, config.ID)
+ continue
+ }
+
+ check[col.MetricName] = true
+
+ metrics = append(metrics, &pmmv1.ServerMetric_Metric{
+ Key: col.MetricName,
+ Value: value,
+ })
+ }
+ }
+
+ return metrics, nil
+}
+
+func (d *dsEnvvars) Dispose(_ context.Context) error {
+ return nil
+}
diff --git a/managed/services/telemetry/datasource_envvars_test.go b/managed/services/telemetry/datasource_envvars_test.go
new file mode 100644
index 0000000000..5139d68c48
--- /dev/null
+++ b/managed/services/telemetry/datasource_envvars_test.go
@@ -0,0 +1,151 @@
+// Copyright (C) 2023 Percona LLC
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see .
+
+// Package telemetry provides telemetry functionality.
+package telemetry
+
+import (
+ "context"
+ "os"
+ "testing"
+
+ pmmv1 "github.com/percona-platform/saas/gen/telemetry/events/pmm"
+ "github.com/sirupsen/logrus"
+ "github.com/stretchr/testify/assert"
+ "github.com/stretchr/testify/require"
+)
+
+func TestEnvVarsDatasource(t *testing.T) {
+ // NOTE: t.Parallel() is not possible when using a different set of envvars for each test.
+ t.Parallel()
+
+ type testEnvVars map[string]string
+
+ ctx, cancel := context.WithCancel(context.Background())
+ logger := logrus.StandardLogger()
+ logger.SetLevel(logrus.DebugLevel)
+ logEntry := logrus.NewEntry(logger)
+
+ setup := func(t *testing.T, envVars testEnvVars) (DataSource, func()) {
+ t.Helper()
+ for key, val := range envVars {
+ os.Setenv(key, val) //nolint:errcheck
+ }
+
+ evConf := &DSConfigEnvVars{
+ Enabled: true,
+ }
+ dsEnvVars := NewDataSourceEnvVars(*evConf, logEntry)
+
+ return dsEnvVars, func() {
+ for key := range envVars {
+ os.Unsetenv(key) //nolint:errcheck
+ }
+ err := dsEnvVars.Dispose(ctx)
+ require.NoError(t, err)
+ }
+ }
+
+ t.Cleanup(func() {
+ cancel()
+ })
+
+ t.Run("Basic", func(t *testing.T) {
+ t.Parallel()
+
+ envVars := testEnvVars{
+ "TEST_ENV_VAR1": "1",
+ "TEST_ENV_VAR2": "test",
+ "TEST_ENV_VAR3": "true",
+ "TEST_ENV_VAR4": "1.1",
+ "TEST_ENV_VAR5": "",
+ }
+ config := &Config{
+ ID: "test",
+ Source: "ENV_VARS",
+ Summary: "EnvVar test query",
+ Data: []ConfigData{
+ {MetricName: "test_env_var1", Column: "TEST_ENV_VAR1"},
+ {MetricName: "test_env_var2", Column: "TEST_ENV_VAR2"},
+ {MetricName: "test_env_var3", Column: "TEST_ENV_VAR3"},
+ {MetricName: "test_env_var4", Column: "TEST_ENV_VAR4"},
+ {MetricName: "test_env_var5", Column: "TEST_ENV_VAR5"},
+ },
+ }
+
+ dsEnvVars, dispose := setup(t, envVars)
+ t.Cleanup(func() { dispose() })
+
+ err := dsEnvVars.Init(ctx)
+ require.NoError(t, err)
+
+ metrics, err := dsEnvVars.FetchMetrics(ctx, *config)
+ require.NoError(t, err)
+
+ expected := []*pmmv1.ServerMetric_Metric{
+ {Key: "test_env_var1", Value: "1"},
+ {Key: "test_env_var2", Value: "test"},
+ {Key: "test_env_var3", Value: "true"},
+ {Key: "test_env_var4", Value: "1.1"},
+ }
+ assert.Equal(t, expected, metrics)
+ })
+
+ t.Run("StripValues", func(t *testing.T) {
+ t.Parallel()
+
+ envVars := testEnvVars{
+ "TEST_ENV_VAR6": "1",
+ "TEST_ENV_VAR7": "test",
+ "TEST_ENV_VAR8": "true",
+ "TEST_ENV_VAR9": "1.1",
+ "TEST_ENV_VAR10": "",
+ }
+ config := &Config{
+ ID: "test",
+ Source: "ENV_VARS",
+ Summary: "EnvVar test query",
+ Transform: &ConfigTransform{
+ Type: "StripValues",
+ },
+ Data: []ConfigData{
+ {MetricName: "test_env_var6", Column: "TEST_ENV_VAR6"},
+ {MetricName: "test_env_var7", Column: "TEST_ENV_VAR7"},
+ {MetricName: "test_env_var8", Column: "TEST_ENV_VAR8"},
+ {MetricName: "test_env_var9", Column: "TEST_ENV_VAR9"},
+ {MetricName: "test_env_var10", Column: "TEST_ENV_VAR10"},
+ },
+ }
+
+ dsEnvVars, dispose := setup(t, envVars)
+ t.Cleanup(func() { dispose() })
+
+ err := dsEnvVars.Init(ctx)
+ require.NoError(t, err)
+
+ metrics, err := dsEnvVars.FetchMetrics(ctx, *config)
+ require.NoError(t, err)
+
+ expected := []*pmmv1.ServerMetric_Metric{
+ {Key: "test_env_var6", Value: "1"},
+ {Key: "test_env_var7", Value: "1"},
+ {Key: "test_env_var8", Value: "1"},
+ {Key: "test_env_var9", Value: "1"},
+ }
+ metrics, err = transformExportValues(config, metrics)
+ require.NoError(t, err)
+ assert.Equal(t, expected, metrics)
+ })
+}
diff --git a/managed/services/telemetry/datasource_grafana_sqlitedb.go b/managed/services/telemetry/datasource_grafana_sqlitedb.go
index 689a3decfd..1c45141088 100644
--- a/managed/services/telemetry/datasource_grafana_sqlitedb.go
+++ b/managed/services/telemetry/datasource_grafana_sqlitedb.go
@@ -47,7 +47,7 @@ func (d *dsGrafanaSelect) Enabled() bool {
}
// NewDataSourceGrafanaSqliteDB makes new data source for grafana sqlite database metrics.
-func NewDataSourceGrafanaSqliteDB(config DSGrafanaSqliteDB, l *logrus.Entry) DataSource { //nolint:ireturn
+func NewDataSourceGrafanaSqliteDB(config DSGrafanaSqliteDB, l *logrus.Entry) DataSource {
return &dsGrafanaSelect{
l: l,
config: config,
diff --git a/managed/services/telemetry/datasource_grafana_sqlitedb_test.go b/managed/services/telemetry/datasource_grafana_sqlitedb_test.go
index 4877c9263d..ec140ce550 100644
--- a/managed/services/telemetry/datasource_grafana_sqlitedb_test.go
+++ b/managed/services/telemetry/datasource_grafana_sqlitedb_test.go
@@ -27,7 +27,7 @@ import (
"github.com/stretchr/testify/require"
)
-func TestDatasource(t *testing.T) {
+func TestGrafanaSqliteDatasource(t *testing.T) {
t.Parallel()
logger := logrus.StandardLogger()
logger.SetLevel(logrus.DebugLevel)
diff --git a/managed/services/telemetry/datasource_pmmdb_select.go b/managed/services/telemetry/datasource_pmmdb_select.go
index 08e6485e36..8633784e7c 100644
--- a/managed/services/telemetry/datasource_pmmdb_select.go
+++ b/managed/services/telemetry/datasource_pmmdb_select.go
@@ -44,7 +44,7 @@ func (d *dsPmmDBSelect) Enabled() bool {
}
// NewDsPmmDBSelect make new PMM DB Select data source.
-func NewDsPmmDBSelect(config DSConfigPMMDB, l *logrus.Entry) (DataSource, error) { //nolint:ireturn
+func NewDsPmmDBSelect(config DSConfigPMMDB, l *logrus.Entry) (DataSource, error) {
db, err := openPMMDBConnection(config, l)
if err != nil {
return nil, err
diff --git a/managed/services/telemetry/datasource_qandb_select.go b/managed/services/telemetry/datasource_qandb_select.go
index 3f80053282..d0e9049415 100644
--- a/managed/services/telemetry/datasource_qandb_select.go
+++ b/managed/services/telemetry/datasource_qandb_select.go
@@ -42,7 +42,7 @@ func (d *dsQanDBSelect) Enabled() bool {
}
// NewDsQanDBSelect make new QAN DB Select data source.
-func NewDsQanDBSelect(config DSConfigQAN, l *logrus.Entry) (DataSource, error) { //nolint:ireturn
+func NewDsQanDBSelect(config DSConfigQAN, l *logrus.Entry) (DataSource, error) {
db, err := openQANDBConnection(config.DSN, config.Enabled, l)
if err != nil {
return nil, err
diff --git a/managed/services/telemetry/datasources.go b/managed/services/telemetry/datasources.go
index 8eefc3c38e..af9a0c36d7 100644
--- a/managed/services/telemetry/datasources.go
+++ b/managed/services/telemetry/datasources.go
@@ -54,13 +54,16 @@ func NewDataSourceRegistry(config ServiceConfig, l *logrus.Entry) (DataSourceLoc
grafanaDB := NewDataSourceGrafanaSqliteDB(*config.DataSources.GrafanaDBSelect, l)
+ envVars := NewDataSourceEnvVars(*config.DataSources.EnvVars, l)
+
return &dataSourceRegistry{
l: l,
dataSources: map[DataSourceName]DataSource{
- "VM": vmDB,
- "PMMDB_SELECT": pmmDB,
- "QANDB_SELECT": qanDB,
- "GRAFANADB_SELECT": grafanaDB,
+ dsVM: vmDB,
+ dsPMMDBSelect: pmmDB,
+ dsQANDBSelect: qanDB,
+ dsGrafanaDBSelect: grafanaDB,
+ dsEnvVars: envVars,
},
}, nil
}
diff --git a/managed/services/telemetry/telemetry.go b/managed/services/telemetry/telemetry.go
index 99b491b0e9..94c17d8a99 100644
--- a/managed/services/telemetry/telemetry.go
+++ b/managed/services/telemetry/telemetry.go
@@ -56,7 +56,7 @@ type Service struct {
sDistributionMethod serverpb.DistributionMethod
tDistributionMethod pmmv1.DistributionMethod
sendCh chan *pmmv1.ServerMetric
- dataSourcesMap map[string]DataSource
+ dataSourcesMap map[DataSourceName]DataSource
extensions map[ExtensionType]Extension
@@ -101,7 +101,7 @@ func NewService(db *reform.DB, portalClient *platform.Client, pmmVersion string,
}
// LocateTelemetryDataSource retrieves DataSource by name.
-func (s *Service) LocateTelemetryDataSource(name string) (DataSource, error) { //nolint:ireturn
+func (s *Service) LocateTelemetryDataSource(name string) (DataSource, error) {
return s.dsRegistry.LocateTelemetryDataSource(name)
}
@@ -117,7 +117,7 @@ func (s *Service) Run(ctx context.Context) {
doSend := func() {
var settings *models.Settings
- err := s.db.InTransaction(func(tx *reform.TX) error {
+ err := s.db.InTransactionContext(ctx, nil, func(tx *reform.TX) error {
var e error
if settings, e = models.GetSettings(tx); e != nil {
return e
@@ -140,12 +140,12 @@ func (s *Service) Run(ctx context.Context) {
if s.config.Reporting.Send {
s.sendCh <- report
} else {
- s.l.Info("Telemetry sent is disabled.")
+ s.l.Info("Sending telemetry is disabled.")
}
}
if s.config.Reporting.SendOnStart {
- s.l.Debug("Telemetry on start is enabled, sending...")
+ s.l.Debug("Sending telemetry on start is enabled, in progress...")
doSend()
}
@@ -213,7 +213,7 @@ func (s *Service) processSendCh(ctx context.Context) {
}
func (s *Service) prepareReport(ctx context.Context) *pmmv1.ServerMetric {
- initializedDataSources := make(map[string]DataSource)
+ initializedDataSources := make(map[DataSourceName]DataSource)
telemetryMetric, _ := s.makeMetric(ctx)
var totalTime time.Duration
@@ -246,7 +246,7 @@ func (s *Service) prepareReport(ctx context.Context) *pmmv1.ServerMetric {
}
// locate DS in initialized state
- ds := initializedDataSources[telemetry.Source]
+ ds := initializedDataSources[DataSourceName(telemetry.Source)]
if ds == nil {
s.l.Debugf("cannot find initialized telemetry datasource: %s", telemetry.Source)
continue
@@ -268,15 +268,23 @@ func (s *Service) prepareReport(ctx context.Context) *pmmv1.ServerMetric {
}
if telemetry.Transform != nil {
- if telemetry.Transform.Type == JSONTransformType {
+ switch telemetry.Transform.Type {
+ case JSONTransform:
telemetryCopy := telemetry // G601: Implicit memory aliasing in for loop. (gosec)
metrics, err = transformToJSON(&telemetryCopy, metrics)
if err != nil {
s.l.Debugf("failed to transform to JSON: %s", err)
continue
}
- } else {
- s.l.Errorf("Unsupported transform type: %s", telemetry.Transform.Type)
+ case StripValuesTransform:
+ telemetryCopy := telemetry // G601: Implicit memory aliasing in for loop. (gosec)
+ metrics, err = transformExportValues(&telemetryCopy, metrics)
+ if err != nil {
+ s.l.Debugf("failed to strip values: %s", err)
+ continue
+ }
+ default:
+ s.l.Errorf("unsupported transform type: %s", telemetry.Transform.Type)
}
}
@@ -287,7 +295,7 @@ func (s *Service) prepareReport(ctx context.Context) *pmmv1.ServerMetric {
for sourceName, dataSource := range initializedDataSources {
err := dataSource.Dispose(ctx)
if err != nil {
- s.l.Debugf("Dispose of %s datasource failed: %v", sourceName, err)
+ s.l.Debugf("Disposing of %s datasource failed: %v", sourceName, err)
continue
}
}
@@ -299,15 +307,15 @@ func (s *Service) prepareReport(ctx context.Context) *pmmv1.ServerMetric {
return telemetryMetric
}
-func (s *Service) locateDataSources(telemetryConfig []Config) map[string]DataSource {
- dataSources := make(map[string]DataSource)
+func (s *Service) locateDataSources(telemetryConfig []Config) map[DataSourceName]DataSource {
+ dataSources := make(map[DataSourceName]DataSource)
for _, telemetry := range telemetryConfig {
ds, err := s.LocateTelemetryDataSource(telemetry.Source)
if err != nil {
s.l.Debugf("failed to lookup telemetry datasource for [%s]:[%s]", telemetry.Source, telemetry.ID)
continue
}
- dataSources[telemetry.Source] = ds
+ dataSources[DataSourceName(telemetry.Source)] = ds
}
return dataSources
diff --git a/managed/services/telemetry/telemetry_test.go b/managed/services/telemetry/telemetry_test.go
index 726e80cbaa..501147eb66 100644
--- a/managed/services/telemetry/telemetry_test.go
+++ b/managed/services/telemetry/telemetry_test.go
@@ -193,12 +193,7 @@ func getServiceConfig(pgPortHost string, qanDSN string, vmDSN string) ServiceCon
RetryCount: 2,
SendTimeout: time.Second * 10,
},
- DataSources: struct {
- VM *DataSourceVictoriaMetrics `yaml:"VM"`
- QanDBSelect *DSConfigQAN `yaml:"QANDB_SELECT"`
- PmmDBSelect *DSConfigPMMDB `yaml:"PMMDB_SELECT"`
- GrafanaDBSelect *DSGrafanaSqliteDB `yaml:"GRAFANADB_SELECT"`
- }{
+ DataSources: DataSources{
VM: &DataSourceVictoriaMetrics{
Enabled: true,
Timeout: time.Second * 2,
@@ -237,6 +232,9 @@ func getServiceConfig(pgPortHost string, qanDSN string, vmDSN string) ServiceCon
Timeout: time.Second * 2,
DBFile: "/srv/grafana/grafana.db",
},
+ EnvVars: &DSConfigEnvVars{
+ Enabled: true,
+ },
},
}
return serviceConfig
@@ -302,12 +300,7 @@ func getTestConfig(sendOnStart bool, testSourceName string, reportingInterval ti
},
},
SaasHostname: "",
- DataSources: struct {
- VM *DataSourceVictoriaMetrics `yaml:"VM"`
- QanDBSelect *DSConfigQAN `yaml:"QANDB_SELECT"`
- PmmDBSelect *DSConfigPMMDB `yaml:"PMMDB_SELECT"`
- GrafanaDBSelect *DSGrafanaSqliteDB `yaml:"GRAFANADB_SELECT"`
- }{},
+ DataSources: DataSources{},
Reporting: ReportingConfig{
Send: true,
SendOnStart: sendOnStart,
diff --git a/managed/services/telemetry/transform.go b/managed/services/telemetry/transform.go
index b9ab4c7ac6..2616c7317b 100644
--- a/managed/services/telemetry/transform.go
+++ b/managed/services/telemetry/transform.go
@@ -33,8 +33,8 @@ func transformToJSON(config *Config, metrics []*pmmv1.ServerMetric_Metric) ([]*p
return nil, errors.Errorf("no transformation config is set")
}
- if config.Transform.Type != JSONTransformType {
- return nil, errors.Errorf("not supported transformation type [%s], it must be [%s]", config.Transform.Type, JSONTransformType)
+ if config.Transform.Type != JSONTransform {
+ return nil, errors.Errorf("unsupported transformation type [%s], it must be [%s]", config.Transform.Type, JSONTransform)
}
if len(config.Data) == 0 || config.Data[0].MetricName == "" {
@@ -86,6 +86,27 @@ func transformToJSON(config *Config, metrics []*pmmv1.ServerMetric_Metric) ([]*p
}, nil
}
+func transformExportValues(config *Config, metrics []*pmmv1.ServerMetric_Metric) ([]*pmmv1.ServerMetric_Metric, error) {
+ if len(metrics) == 0 {
+ return metrics, nil
+ }
+
+ if config.Transform.Type != StripValuesTransform {
+ return nil, errors.Errorf("unspported transformation type [%s], it must be [%s]", config.Transform.Type, StripValuesTransform)
+ }
+
+ if config.Source != string(dsEnvVars) {
+ return nil, errors.Errorf("this transform can only be used for %s data source", dsEnvVars)
+ }
+
+ for _, metric := range metrics {
+ // Here we replace the metric value with "1", which stands for "present".
+ metric.Value = "1"
+ }
+
+ return metrics, nil
+}
+
func removeEmpty(metrics []*pmmv1.ServerMetric_Metric) []*pmmv1.ServerMetric_Metric {
result := make([]*pmmv1.ServerMetric_Metric, 0, len(metrics))
diff --git a/managed/services/telemetry/transform_test.go b/managed/services/telemetry/transform_test.go
index 6e4e8f2cbc..65a59fe6bd 100644
--- a/managed/services/telemetry/transform_test.go
+++ b/managed/services/telemetry/transform_test.go
@@ -16,18 +16,18 @@
package telemetry
import (
- "fmt"
"testing"
pmmv1 "github.com/percona-platform/saas/gen/telemetry/events/pmm"
"github.com/stretchr/testify/assert"
)
-func Test_transformToJSON(t *testing.T) {
+func TestTransformToJSON(t *testing.T) {
type args struct {
config *Config
metrics []*pmmv1.ServerMetric_Metric
}
+
noMetrics := []*pmmv1.ServerMetric_Metric{}
tests := []struct {
@@ -39,7 +39,7 @@ func Test_transformToJSON(t *testing.T) {
{
name: "nil metrics",
args: args{
- config: config(),
+ config: configJSON(),
metrics: nil,
},
want: nil,
@@ -48,7 +48,7 @@ func Test_transformToJSON(t *testing.T) {
{
name: "empty metrics",
args: args{
- config: config(),
+ config: configJSON(),
metrics: noMetrics,
},
want: noMetrics,
@@ -57,7 +57,7 @@ func Test_transformToJSON(t *testing.T) {
{
name: "no Transform in config",
args: args{
- config: config().noTransform(),
+ config: configJSON().noTransform(),
metrics: noMetrics,
},
want: noMetrics,
@@ -66,7 +66,7 @@ func Test_transformToJSON(t *testing.T) {
{
name: "no Metrics config",
args: args{
- config: config().noFirstMetricConfig(),
+ config: configJSON().noFirstMetricConfig(),
metrics: noMetrics,
},
want: noMetrics,
@@ -75,7 +75,7 @@ func Test_transformToJSON(t *testing.T) {
{
name: "no Metric Name config",
args: args{
- config: config().noFirstMetricNameConfig(),
+ config: configJSON().noFirstMetricNameConfig(),
metrics: noMetrics,
},
want: noMetrics,
@@ -84,7 +84,7 @@ func Test_transformToJSON(t *testing.T) {
{
name: "invalid seq",
args: args{
- config: config(),
+ config: configJSON(),
metrics: []*pmmv1.ServerMetric_Metric{
{Key: "my-metric", Value: "v1"},
{Key: "b", Value: "v1"},
@@ -98,7 +98,7 @@ func Test_transformToJSON(t *testing.T) {
{
name: "correct seq",
args: args{
- config: config(),
+ config: configJSON(),
metrics: []*pmmv1.ServerMetric_Metric{
{Key: "my-metric", Value: "v1"},
{Key: "b", Value: "v1"},
@@ -107,29 +107,32 @@ func Test_transformToJSON(t *testing.T) {
},
},
want: []*pmmv1.ServerMetric_Metric{
- {Key: config().Transform.Metric, Value: `{"v":[{"b":"v1","my-metric":"v1"},{"b":"v1","my-metric":"v1"}]}`},
+ {Key: configJSON().Transform.Metric, Value: `{"v":[{"b":"v1","my-metric":"v1"},{"b":"v1","my-metric":"v1"}]}`},
},
wantErr: assert.NoError,
},
{
name: "happy path",
args: args{
- config: config(),
+ config: configJSON(),
metrics: []*pmmv1.ServerMetric_Metric{
- {Key: config().Data[0].MetricName, Value: "v1"},
- {Key: config().Data[0].MetricName, Value: "v2"},
+ {Key: configJSON().Data[0].MetricName, Value: "v1"},
+ {Key: configJSON().Data[0].MetricName, Value: "v2"},
},
},
want: []*pmmv1.ServerMetric_Metric{
- {Key: config().Transform.Metric, Value: `{"v":[{"my-metric":"v1"},{"my-metric":"v2"}]}`},
+ {Key: configJSON().Transform.Metric, Value: `{"v":[{"my-metric":"v1"},{"my-metric":"v2"}]}`},
},
wantErr: assert.NoError,
},
}
+
for _, tt := range tests {
+ tt := tt
t.Run(tt.name, func(t *testing.T) {
got, err := transformToJSON(tt.args.config, tt.args.metrics)
- if !tt.wantErr(t, err, fmt.Sprintf("transformToJSON(%v, %v)", tt.args.config, tt.args.metrics)) {
+ if !tt.wantErr(t, err) {
+ t.Logf("config: %v", tt.args.config)
return
}
assert.Equalf(t, tt.want, got, "transformToJSON(%v, %v)", tt.args.config, tt.args.metrics)
@@ -137,11 +140,103 @@ func Test_transformToJSON(t *testing.T) {
}
}
-func config() *Config {
+func TestTransformExportValues(t *testing.T) {
+ type args struct {
+ config *Config
+ metrics []*pmmv1.ServerMetric_Metric
+ }
+
+ noMetrics := []*pmmv1.ServerMetric_Metric{}
+
+ tests := []struct {
+ name string
+ args args
+ want []*pmmv1.ServerMetric_Metric
+ wantErr assert.ErrorAssertionFunc
+ }{
+ {
+ name: "nil metrics",
+ args: args{
+ config: configEnvVars(),
+ metrics: nil,
+ },
+ want: nil,
+ wantErr: assert.NoError,
+ },
+ {
+ name: "empty metrics",
+ args: args{
+ config: configEnvVars(),
+ metrics: noMetrics,
+ },
+ want: noMetrics,
+ wantErr: assert.NoError,
+ },
+ {
+ name: "no Transform in config",
+ args: args{
+ config: configEnvVars().noTransform(),
+ metrics: noMetrics,
+ },
+ want: noMetrics,
+ wantErr: assert.NoError,
+ },
+ {
+ name: "no Metrics config",
+ args: args{
+ config: configEnvVars().noFirstMetricConfig(),
+ metrics: noMetrics,
+ },
+ want: noMetrics,
+ wantErr: assert.NoError,
+ },
+ {
+ name: "invalid data source",
+ args: args{
+ config: configEnvVars().changeDataSource(dsPMMDBSelect),
+ metrics: []*pmmv1.ServerMetric_Metric{
+ {Key: "metric-a", Value: "v1"},
+ {Key: "metric-b", Value: "v2"},
+ },
+ },
+ want: nil,
+ wantErr: assert.Error,
+ },
+ {
+ name: "happy path",
+ args: args{
+ config: configEnvVars(),
+ metrics: []*pmmv1.ServerMetric_Metric{
+ {Key: "metric-a", Value: "v1"},
+ {Key: "metric-b", Value: "v2"},
+ },
+ },
+ want: []*pmmv1.ServerMetric_Metric{
+ {Key: "metric-a", Value: "1"},
+ {Key: "metric-b", Value: "1"},
+ },
+ wantErr: assert.NoError,
+ },
+ }
+
+ for _, tt := range tests {
+ tt := tt
+ t.Run(tt.name, func(t *testing.T) {
+ got, err := transformExportValues(tt.args.config, tt.args.metrics)
+ if !tt.wantErr(t, err) {
+ t.Logf("config: %v", tt.args.config)
+ return
+ }
+ assert.Equalf(t, tt.want, got, "transformExportValues(%v, %v)", tt.args.config, tt.args.metrics)
+ })
+ }
+}
+
+func configJSON() *Config {
return &Config{
Transform: &ConfigTransform{
Metric: "metric",
- Type: JSONTransformType,
+ Type: JSONTransform,
},
Data: []ConfigData{
{MetricName: "my-metric", Label: "label"},
@@ -149,6 +244,15 @@ func config() *Config {
}
}
+func configEnvVars() *Config {
+ return &Config{
+ Source: "ENV_VARS",
+ Transform: &ConfigTransform{
+ Type: StripValuesTransform,
+ },
+ }
+}
+
func (c *Config) noTransform() *Config {
c.Transform = nil
return c
@@ -164,7 +268,12 @@ func (c *Config) noFirstMetricNameConfig() *Config {
return c
}
-func Test_removeEmpty(t *testing.T) {
+func (c *Config) changeDataSource(s DataSourceName) *Config {
+ c.Source = string(s)
+ return c
+}
+
+func TestRemoveEmpty(t *testing.T) {
type args struct {
metrics []*pmmv1.ServerMetric_Metric
}