initial wiring agent integration for integration-next (grafana#5638)

* initial wiring agent integration for integration-next Signed-off-by: erikbaranowski <[email protected]> --------- Signed-off-by: erikbaranowski <[email protected]>
hainenber · Nov 1, 2023 · fdf7dcb · fdf7dcb
1 parent c0a75ec
commit fdf7dcb
Show file tree

Hide file tree

Showing 8 changed files with 172 additions and 21 deletions.
diff --git a/converter/internal/staticconvert/internal/build/agent_exporter.go b/converter/internal/staticconvert/internal/build/agent_exporter.go
@@ -1,26 +1,26 @@
 package build
 
 import (
-	"fmt"
-
 	"github.com/grafana/agent/component/discovery"
 	"github.com/grafana/agent/component/prometheus/exporter/agent"
-	"github.com/grafana/agent/converter/internal/common"
 	agent_exporter "github.com/grafana/agent/pkg/integrations/agent"
+	agent_exporter_v2 "github.com/grafana/agent/pkg/integrations/v2/agent"
 )
 
 func (b *IntegrationsConfigBuilder) appendAgentExporter(config *agent_exporter.Config) discovery.Exports {
 	args := toAgentExporter(config)
-	compLabel := common.LabelForParts(b.globalCtx.LabelPrefix, config.Name())
-	b.f.Body().AppendBlock(common.NewBlockWithOverride(
-		[]string{"prometheus", "exporter", "agent"},
-		compLabel,
-		args,
-	))
-
-	return common.NewDiscoveryExports(fmt.Sprintf("prometheus.exporter.agent.%s.targets", compLabel))
+	return b.appendExporterBlock(args, config.Name(), "agent")
 }
 
 func toAgentExporter(config *agent_exporter.Config) *agent.Arguments {
 	return &agent.Arguments{}
 }
+
+func (b *IntegrationsConfigBuilder) appendAgentExporterV2(config *agent_exporter_v2.Config) discovery.Exports {
+	args := toAgentExporterV2(config)
+	return b.appendExporterBlock(args, config.Name(), "agent")
+}
+
+func toAgentExporterV2(config *agent_exporter_v2.Config) *agent.Arguments {
+	return &agent.Arguments{}
+}
diff --git a/converter/internal/staticconvert/internal/build/builder.go b/converter/internal/staticconvert/internal/build/builder.go
@@ -4,8 +4,11 @@ import (
 	"fmt"
 	"strings"
 
+	"github.com/grafana/agent/component"
 	"github.com/grafana/agent/component/discovery"
+	"github.com/grafana/agent/component/prometheus/remotewrite"
 	"github.com/grafana/agent/converter/diag"
+	"github.com/grafana/agent/converter/internal/common"
 	"github.com/grafana/agent/converter/internal/prometheusconvert"
 	"github.com/grafana/agent/pkg/config"
 	agent_exporter "github.com/grafana/agent/pkg/integrations/agent"
@@ -34,10 +37,14 @@ import (
 	"github.com/grafana/agent/pkg/integrations/snowflake_exporter"
 	"github.com/grafana/agent/pkg/integrations/squid_exporter"
 	"github.com/grafana/agent/pkg/integrations/statsd_exporter"
+	agent_exporter_v2 "github.com/grafana/agent/pkg/integrations/v2/agent"
+	common_v2 "github.com/grafana/agent/pkg/integrations/v2/common"
 	"github.com/grafana/agent/pkg/integrations/windows_exporter"
+	"github.com/grafana/river/scanner"
 	"github.com/grafana/river/token/builder"
 	"github.com/prometheus/common/model"
 	prom_config "github.com/prometheus/prometheus/config"
+	"github.com/prometheus/prometheus/model/relabel"
 )
 
 type IntegrationsConfigBuilder struct {
@@ -151,10 +158,6 @@ func (b *IntegrationsConfigBuilder) appendV1Integrations() {
 	}
 }
 
-func (b *IntegrationsConfigBuilder) appendV2Integrations() {
-
-}
-
 func (b *IntegrationsConfigBuilder) appendExporter(commonConfig *int_config.Common, name string, extraTargets []discovery.Target) {
 	scrapeConfig := prom_config.DefaultScrapeConfig
 	scrapeConfig.JobName = fmt.Sprintf("integrations/%s", name)
@@ -193,10 +196,100 @@ func (b *IntegrationsConfigBuilder) appendExporter(commonConfig *int_config.Comm
 	b.globalCtx.InitializeRemoteWriteExports()
 }
 
+func (b *IntegrationsConfigBuilder) appendV2Integrations() {
+	for _, integration := range b.cfg.Integrations.ConfigV2.Configs {
+		var exports discovery.Exports
+		var commonConfig common_v2.MetricsConfig
+
+		switch itg := integration.(type) {
+		case *agent_exporter_v2.Config:
+			exports = b.appendAgentExporterV2(itg)
+			commonConfig = itg.Common
+		}
+
+		if len(exports.Targets) > 0 {
+			b.appendExporterV2(&commonConfig, integration.Name(), exports.Targets)
+		}
+	}
+}
+
+func (b *IntegrationsConfigBuilder) appendExporterV2(commonConfig *common_v2.MetricsConfig, name string, extraTargets []discovery.Target) {
+	var relabelConfigs []*relabel.Config
+
+	for _, extraLabel := range commonConfig.ExtraLabels {
+		defaultConfig := relabel.DefaultRelabelConfig
+		relabelConfig := &defaultConfig
+		relabelConfig.SourceLabels = []model.LabelName{"__address__"}
+		relabelConfig.TargetLabel = extraLabel.Name
+		relabelConfig.Replacement = extraLabel.Value
+
+		relabelConfigs = append(relabelConfigs, relabelConfig)
+	}
+
+	commonConfig.ApplyDefaults(b.cfg.Integrations.ConfigV2.Metrics.Autoscrape)
+	scrapeConfig := prom_config.DefaultScrapeConfig
+	scrapeConfig.JobName = fmt.Sprintf("integrations/%s", name)
+	scrapeConfig.RelabelConfigs = commonConfig.Autoscrape.RelabelConfigs
+	scrapeConfig.MetricRelabelConfigs = commonConfig.Autoscrape.MetricRelabelConfigs
+	scrapeConfig.ScrapeInterval = commonConfig.Autoscrape.ScrapeInterval
+	scrapeConfig.ScrapeTimeout = commonConfig.Autoscrape.ScrapeTimeout
+	scrapeConfig.RelabelConfigs = relabelConfigs
+
+	scrapeConfigs := []*prom_config.ScrapeConfig{&scrapeConfig}
+
+	var remoteWriteExports *remotewrite.Exports
+	for _, metrics := range b.cfg.Metrics.Configs {
+		if metrics.Name == commonConfig.Autoscrape.MetricsInstance {
+			// This must match the name of the existing remote write config in the metrics config:
+			label, err := scanner.SanitizeIdentifier("metrics_" + metrics.Name)
+			if err != nil {
+				b.diags.Add(diag.SeverityLevelCritical, fmt.Sprintf("failed to sanitize job name: %s", err))
+			}
+
+			remoteWriteExports = &remotewrite.Exports{
+				Receiver: common.ConvertAppendable{Expr: "prometheus.remote_write." + label + ".receiver"},
+			}
+			break
+		}
+	}
+
+	if remoteWriteExports == nil {
+		b.diags.Add(diag.SeverityLevelCritical, fmt.Sprintf("integration %s is looking for an undefined metrics config: %s", name, commonConfig.Autoscrape.MetricsInstance))
+	}
+
+	promConfig := &prom_config.Config{
+		GlobalConfig:  b.cfg.Metrics.Global.Prometheus,
+		ScrapeConfigs: scrapeConfigs,
+	}
+
+	jobNameToCompLabelsFunc := func(jobName string) string {
+		labelSuffix := strings.TrimPrefix(jobName, "integrations/")
+		if labelSuffix == "" {
+			return b.globalCtx.LabelPrefix
+		}
+
+		return fmt.Sprintf("%s_%s", b.globalCtx.LabelPrefix, labelSuffix)
+	}
+
+	// Need to pass in the remote write reference from the metrics config here:
+	b.diags.AddAll(prometheusconvert.AppendAllNested(b.f, promConfig, jobNameToCompLabelsFunc, extraTargets, remoteWriteExports))
+}
+
 func splitByCommaNullOnEmpty(s string) []string {
 	if s == "" {
 		return nil
 	}
 
 	return strings.Split(s, ",")
 }
+
+func (b *IntegrationsConfigBuilder) appendExporterBlock(args component.Arguments, name string, exporterName string) discovery.Exports {
+	compLabel := common.LabelForParts(b.globalCtx.LabelPrefix, name)
+	b.f.Body().AppendBlock(common.NewBlockWithOverride(
+		[]string{"prometheus", "exporter", exporterName},
+		compLabel,
+		args,
+	))
+
+	return common.NewDiscoveryExports(fmt.Sprintf("prometheus.exporter.%s.%s.targets", exporterName, compLabel))
+}
diff --git a/converter/internal/staticconvert/staticconvert.go b/converter/internal/staticconvert/staticconvert.go
@@ -29,10 +29,6 @@ import (
 // as enabling integrations-next.
 func Convert(in []byte, extraArgs []string) ([]byte, diag.Diagnostics) {
 	var diags diag.Diagnostics
-	// diags := validateExtraArgs(extraArgs)
-	// if len(diags) > 0 {
-	// 	return nil, diags
-	// }
 
 	fs := flag.NewFlagSet("convert", flag.ContinueOnError)
 	args := []string{"-config.file", "convert", "-config.expand-env"}

diff --git a/converter/internal/staticconvert/testdata-v2/integrations_v2.river b/converter/internal/staticconvert/testdata-v2/integrations_v2.river
@@ -0,0 +1,34 @@
+prometheus.remote_write "metrics_default" {
+	endpoint {
+		name = "default-8be96f"
+		url  = "http://localhost:9009/api/prom/push"
+
+		queue_config { }
+
+		metadata_config { }
+	}
+}
+
+prometheus.exporter.agent "integrations_agent" { }
+
+discovery.relabel "integrations_agent" {
+	targets = prometheus.exporter.agent.integrations_agent.targets
+
+	rule {
+		source_labels = ["__address__"]
+		target_label  = "test_label"
+		replacement   = "test_label_value"
+	}
+
+	rule {
+		source_labels = ["__address__"]
+		target_label  = "test_label_2"
+		replacement   = "test_label_value_2"
+	}
+}
+
+prometheus.scrape "integrations_agent" {
+	targets    = discovery.relabel.integrations_agent.output
+	forward_to = [prometheus.remote_write.metrics_default.receiver]
+	job_name   = "integrations/agent"
+}
diff --git a/converter/internal/staticconvert/testdata-v2/integrations_v2.yaml b/converter/internal/staticconvert/testdata-v2/integrations_v2.yaml
@@ -2,7 +2,14 @@ metrics:
   global:
     remote_write:
       - url: http://localhost:9009/api/prom/push
+  configs:
+    - name: default
 
 integrations:
   agent:
-    instance: "agent"
+    instance: "default"
+    autoscrape:
+      metrics_instance: "default"
+    extra_labels:
+      test_label: test_label_value
+      test_label_2: test_label_value_2
diff --git a/converter/internal/staticconvert/testdata-v2/missing_metrics_config.diags b/converter/internal/staticconvert/testdata-v2/missing_metrics_config.diags
@@ -0,0 +1,2 @@
+(Critical) integration agent is looking for an undefined metrics config: not_default
+(Warning) Please review your agent command line flags and ensure they are set in your Flow mode config file where necessary.
diff --git a/converter/internal/staticconvert/testdata-v2/missing_metrics_config.yaml b/converter/internal/staticconvert/testdata-v2/missing_metrics_config.yaml
@@ -0,0 +1,12 @@
+metrics:
+  global:
+    remote_write:
+      - url: http://localhost:9009/api/prom/push
+  configs:
+    - name: default
+
+integrations:
+  agent:
+    instance: "default"
+    autoscrape:
+      metrics_instance: "not_default"
diff --git a/converter/internal/staticconvert/validate.go b/converter/internal/staticconvert/validate.go
@@ -33,6 +33,7 @@ import (
 	"github.com/grafana/agent/pkg/integrations/squid_exporter"
 	"github.com/grafana/agent/pkg/integrations/statsd_exporter"
 	v2 "github.com/grafana/agent/pkg/integrations/v2"
+	agent_exporter_v2 "github.com/grafana/agent/pkg/integrations/v2/agent"
 	"github.com/grafana/agent/pkg/integrations/windows_exporter"
 	"github.com/grafana/agent/pkg/logs"
 	"github.com/grafana/agent/pkg/metrics"
@@ -157,7 +158,13 @@ func validateIntegrationsV1(integrationsConfig *v1.ManagerConfig) diag.Diagnosti
 func validateIntegrationsV2(integrationsConfig *v2.SubsystemOptions) diag.Diagnostics {
 	var diags diag.Diagnostics
 
-	// TODO
+	for _, integration := range integrationsConfig.Configs {
+		switch itg := integration.(type) {
+		case *agent_exporter_v2.Config:
+		default:
+			diags.Add(diag.SeverityLevelError, fmt.Sprintf("The converter does not support converting the provided %s integration.", itg.Name()))
+		}
+	}
 
 	return diags
 }