From fda642483ada466d7ce0f457cf60dc81f5e3e8ba Mon Sep 17 00:00:00 2001 From: Jacob Aronoff Date: Tue, 17 Sep 2024 11:24:54 -0400 Subject: [PATCH] add tests --- Makefile | 2 +- cmd/otel-allocator/config/config.go | 74 ++++++++ cmd/otel-allocator/watcher/promOperator.go | 18 +- .../watcher/promOperator_test.go | 172 ++++++++++++++++-- config/manager/kustomization.yaml | 19 ++ .../00-assert.yaml | 11 +- .../00-install.yaml | 2 +- 7 files changed, 259 insertions(+), 39 deletions(-) diff --git a/Makefile b/Makefile index 3a8396009a..d217a2d784 100644 --- a/Makefile +++ b/Makefile @@ -304,7 +304,7 @@ e2e-prometheuscr: chainsaw # Target allocator end-to-tests .PHONY: e2e-targetallocator e2e-targetallocator: chainsaw - $(CHAINSAW) test --test-dir ./tests/e2e-targetallocator + $(CHAINSAW) test --apply-timeout 45s --assert-timeout 45s --cleanup-timeout 45s --delete-timeout 45s --error-timeout 45s --exec-timeout 45s --test-dir ./tests/e2e-targetallocator # end-to-end-test for Annotations/Labels Filters .PHONY: e2e-metadata-filters diff --git a/cmd/otel-allocator/config/config.go b/cmd/otel-allocator/config/config.go index 3e3fd389c7..b03f4ce354 100644 --- a/cmd/otel-allocator/config/config.go +++ b/cmd/otel-allocator/config/config.go @@ -24,6 +24,7 @@ import ( "time" "github.com/go-logr/logr" + monitoringv1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1" "github.com/prometheus/common/model" promconfig "github.com/prometheus/prometheus/config" _ "github.com/prometheus/prometheus/discovery/install" @@ -58,6 +59,79 @@ type Config struct { HTTPS HTTPSServerConfig `yaml:"https,omitempty"` } +func (cfg *Config) GetPrometheus() *monitoringv1.Prometheus { + // we want to use endpointslices by default + serviceDiscoveryRole := monitoringv1.ServiceDiscoveryRole("EndpointSlice") + prom := &monitoringv1.Prometheus{ + Spec: monitoringv1.PrometheusSpec{ + CommonPrometheusFields: monitoringv1.CommonPrometheusFields{ + ScrapeInterval: monitoringv1.Duration(cfg.PrometheusCR.ScrapeInterval.String()), + ServiceMonitorSelector: cfg.PrometheusCR.ServiceMonitorSelector, + PodMonitorSelector: cfg.PrometheusCR.PodMonitorSelector, + ServiceMonitorNamespaceSelector: cfg.PrometheusCR.ServiceMonitorNamespaceSelector, + PodMonitorNamespaceSelector: cfg.PrometheusCR.PodMonitorNamespaceSelector, + ServiceDiscoveryRole: &serviceDiscoveryRole, + }, + }, + } + + // if there's no prom config set, just use the above defaults. + if cfg.PromConfig == nil { + return prom + } + + // the prometheus-operator provides no automatic conversion for global fields + if len(cfg.PromConfig.GlobalConfig.ScrapeProtocols) > 0 { + var scrapeProtocols []monitoringv1.ScrapeProtocol + for _, protocol := range cfg.PromConfig.GlobalConfig.ScrapeProtocols { + scrapeProtocols = append(scrapeProtocols, monitoringv1.ScrapeProtocol(protocol)) + } + prom.Spec.CommonPrometheusFields.ScrapeProtocols = scrapeProtocols + } + scrapeInterval := monitoringv1.Duration(cfg.PromConfig.GlobalConfig.ScrapeInterval.String()) + if len(scrapeInterval) > 0 { + prom.Spec.CommonPrometheusFields.ScrapeInterval = scrapeInterval + } + prom.Spec.CommonPrometheusFields.ScrapeTimeout = monitoringv1.Duration(cfg.PromConfig.GlobalConfig.ScrapeTimeout.String()) + if len(cfg.PromConfig.GlobalConfig.ExternalLabels) > 0 { + labels := map[string]string{} + for _, label := range cfg.PromConfig.GlobalConfig.ExternalLabels { + labels[label.Name] = label.Value + } + prom.Spec.CommonPrometheusFields.ExternalLabels = labels + } + if cfg.PromConfig.GlobalConfig.BodySizeLimit > 0 { + bodySizeLimit := monitoringv1.ByteSize(cfg.PromConfig.GlobalConfig.BodySizeLimit.String()) + prom.Spec.CommonPrometheusFields.BodySizeLimit = &bodySizeLimit + } + if cfg.PromConfig.GlobalConfig.SampleLimit > 0 { + sampleLimit := uint64(cfg.PromConfig.GlobalConfig.SampleLimit) + prom.Spec.CommonPrometheusFields.SampleLimit = &sampleLimit + } + if cfg.PromConfig.GlobalConfig.TargetLimit > 0 { + targetLimit := uint64(cfg.PromConfig.GlobalConfig.TargetLimit) + prom.Spec.CommonPrometheusFields.TargetLimit = &targetLimit + } + if cfg.PromConfig.GlobalConfig.LabelLimit > 0 { + labelLimit := uint64(cfg.PromConfig.GlobalConfig.LabelLimit) + prom.Spec.CommonPrometheusFields.LabelLimit = &labelLimit + } + if cfg.PromConfig.GlobalConfig.LabelNameLengthLimit > 0 { + labelNameLengthLimit := uint64(cfg.PromConfig.GlobalConfig.LabelNameLengthLimit) + prom.Spec.CommonPrometheusFields.LabelNameLengthLimit = &labelNameLengthLimit + } + if cfg.PromConfig.GlobalConfig.LabelValueLengthLimit > 0 { + labelValueLengthLimit := uint64(cfg.PromConfig.GlobalConfig.LabelValueLengthLimit) + prom.Spec.CommonPrometheusFields.LabelValueLengthLimit = &labelValueLengthLimit + } + if cfg.PromConfig.GlobalConfig.KeepDroppedTargets > 0 { + keepDroppedTargets := uint64(cfg.PromConfig.GlobalConfig.KeepDroppedTargets) + prom.Spec.CommonPrometheusFields.KeepDroppedTargets = &keepDroppedTargets + } + + return prom +} + type PrometheusCRConfig struct { Enabled bool `yaml:"enabled,omitempty"` PodMonitorSelector *metav1.LabelSelector `yaml:"pod_monitor_selector,omitempty"` diff --git a/cmd/otel-allocator/watcher/promOperator.go b/cmd/otel-allocator/watcher/promOperator.go index ae2ddcb68e..38ee32ddde 100644 --- a/cmd/otel-allocator/watcher/promOperator.go +++ b/cmd/otel-allocator/watcher/promOperator.go @@ -71,23 +71,7 @@ func NewPrometheusCRWatcher(ctx context.Context, logger logr.Logger, cfg allocat return nil, err } - // we want to use endpointslices by default - serviceDiscoveryRole := monitoringv1.ServiceDiscoveryRole("EndpointSlice") - - // TODO: We should make these durations configurable - prom := &monitoringv1.Prometheus{ - Spec: monitoringv1.PrometheusSpec{ - CommonPrometheusFields: monitoringv1.CommonPrometheusFields{ - ScrapeInterval: monitoringv1.Duration(cfg.PrometheusCR.ScrapeInterval.String()), - ServiceMonitorSelector: cfg.PrometheusCR.ServiceMonitorSelector, - PodMonitorSelector: cfg.PrometheusCR.PodMonitorSelector, - ServiceMonitorNamespaceSelector: cfg.PrometheusCR.ServiceMonitorNamespaceSelector, - PodMonitorNamespaceSelector: cfg.PrometheusCR.PodMonitorNamespaceSelector, - ServiceDiscoveryRole: &serviceDiscoveryRole, - }, - }, - } - + prom := cfg.GetPrometheus() promOperatorLogger := level.NewFilter(log.NewLogfmtLogger(os.Stderr), level.AllowWarn()) promOperatorSlogLogger := slog.New(slog.NewTextHandler(os.Stderr, &slog.HandlerOptions{Level: slog.LevelWarn})) generator, err := prometheus.NewConfigGenerator(promOperatorLogger, prom, true) diff --git a/cmd/otel-allocator/watcher/promOperator_test.go b/cmd/otel-allocator/watcher/promOperator_test.go index 7bd3f0f443..db317d3d25 100644 --- a/cmd/otel-allocator/watcher/promOperator_test.go +++ b/cmd/otel-allocator/watcher/promOperator_test.go @@ -21,6 +21,7 @@ import ( "testing" "time" + "github.com/alecthomas/units" "github.com/go-kit/log" "github.com/go-kit/log/level" monitoringv1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1" @@ -35,6 +36,7 @@ import ( promconfig "github.com/prometheus/prometheus/config" "github.com/prometheus/prometheus/discovery" kubeDiscovery "github.com/prometheus/prometheus/discovery/kubernetes" + "github.com/prometheus/prometheus/model/labels" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" v1 "k8s.io/api/core/v1" @@ -99,6 +101,7 @@ func TestLoadConfig(t *testing.T) { }, cfg: allocatorconfig.Config{ PrometheusCR: allocatorconfig.PrometheusCRConfig{ + ScrapeInterval: model.Duration(30 * time.Second), ServiceMonitorSelector: &metav1.LabelSelector{}, PodMonitorSelector: &metav1.LabelSelector{}, }, @@ -191,6 +194,7 @@ func TestLoadConfig(t *testing.T) { }, cfg: allocatorconfig.Config{ PrometheusCR: allocatorconfig.PrometheusCRConfig{ + ScrapeInterval: model.Duration(30 * time.Second), ServiceMonitorSelector: &metav1.LabelSelector{}, PodMonitorSelector: &metav1.LabelSelector{}, }, @@ -259,6 +263,7 @@ func TestLoadConfig(t *testing.T) { }, cfg: allocatorconfig.Config{ PrometheusCR: allocatorconfig.PrometheusCRConfig{ + ScrapeInterval: model.Duration(30 * time.Second), ServiceMonitorSelector: &metav1.LabelSelector{}, PodMonitorSelector: &metav1.LabelSelector{}, }, @@ -356,6 +361,7 @@ func TestLoadConfig(t *testing.T) { }, cfg: allocatorconfig.Config{ PrometheusCR: allocatorconfig.PrometheusCRConfig{ + ScrapeInterval: model.Duration(30 * time.Second), ServiceMonitorSelector: &metav1.LabelSelector{}, PodMonitorSelector: &metav1.LabelSelector{}, }, @@ -467,6 +473,7 @@ func TestLoadConfig(t *testing.T) { }, cfg: allocatorconfig.Config{ PrometheusCR: allocatorconfig.PrometheusCRConfig{ + ScrapeInterval: model.Duration(30 * time.Second), ServiceMonitorSelector: &metav1.LabelSelector{}, PodMonitorSelector: &metav1.LabelSelector{}, }, @@ -557,6 +564,7 @@ func TestLoadConfig(t *testing.T) { }, cfg: allocatorconfig.Config{ PrometheusCR: allocatorconfig.PrometheusCRConfig{ + ScrapeInterval: model.Duration(30 * time.Second), ServiceMonitorSelector: &metav1.LabelSelector{ MatchLabels: map[string]string{ "testsvc": "testsvc", @@ -628,6 +636,7 @@ func TestLoadConfig(t *testing.T) { }, cfg: allocatorconfig.Config{ PrometheusCR: allocatorconfig.PrometheusCRConfig{ + ScrapeInterval: model.Duration(30 * time.Second), PodMonitorSelector: &metav1.LabelSelector{ MatchLabels: map[string]string{ "testpod": "testpod", @@ -696,6 +705,7 @@ func TestLoadConfig(t *testing.T) { }, cfg: allocatorconfig.Config{ PrometheusCR: allocatorconfig.PrometheusCRConfig{ + ScrapeInterval: model.Duration(30 * time.Second), ServiceMonitorSelector: &metav1.LabelSelector{}, PodMonitorSelector: &metav1.LabelSelector{}, ServiceMonitorNamespaceSelector: &metav1.LabelSelector{ @@ -766,6 +776,7 @@ func TestLoadConfig(t *testing.T) { }, cfg: allocatorconfig.Config{ PrometheusCR: allocatorconfig.PrometheusCRConfig{ + ScrapeInterval: model.Duration(30 * time.Second), ServiceMonitorSelector: &metav1.LabelSelector{}, PodMonitorSelector: &metav1.LabelSelector{}, PodMonitorNamespaceSelector: &metav1.LabelSelector{ @@ -802,6 +813,145 @@ func TestLoadConfig(t *testing.T) { }, }, }, + { + name: "simple test (global config)", + serviceMonitors: []*monitoringv1.ServiceMonitor{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: "simple", + Namespace: "test", + }, + Spec: monitoringv1.ServiceMonitorSpec{ + JobLabel: "test", + Endpoints: []monitoringv1.Endpoint{ + { + Port: "web", + }, + }, + }, + }, + }, + podMonitors: []*monitoringv1.PodMonitor{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: "simple", + Namespace: "test", + }, + Spec: monitoringv1.PodMonitorSpec{ + JobLabel: "test", + PodMetricsEndpoints: []monitoringv1.PodMetricsEndpoint{ + { + Port: "web", + }, + }, + }, + }, + }, + cfg: allocatorconfig.Config{ + PromConfig: &promconfig.Config{ + GlobalConfig: promconfig.GlobalConfig{ + ScrapeInterval: model.Duration(60 * time.Second), + ScrapeTimeout: model.Duration(20 * time.Second), + ScrapeProtocols: []promconfig.ScrapeProtocol{ + promconfig.OpenMetricsText1_0_0, + promconfig.OpenMetricsText0_0_1, + promconfig.PrometheusText0_0_4, + promconfig.PrometheusProto, + }, + ExternalLabels: []labels.Label{ + { + Name: "example", + Value: "value", + }, + }, + BodySizeLimit: units.Kibibyte, + SampleLimit: 100, + TargetLimit: 100, + LabelLimit: 100, + LabelNameLengthLimit: 100, + LabelValueLengthLimit: 100, + KeepDroppedTargets: 100, + }, + }, + PrometheusCR: allocatorconfig.PrometheusCRConfig{ + ScrapeInterval: model.Duration(30 * time.Second), + ServiceMonitorSelector: &metav1.LabelSelector{}, + PodMonitorSelector: &metav1.LabelSelector{}, + }, + }, + want: &promconfig.Config{ + ScrapeConfigs: []*promconfig.ScrapeConfig{ + { + JobName: "serviceMonitor/test/simple/0", + ScrapeInterval: model.Duration(60 * time.Second), + ScrapeProtocols: []promconfig.ScrapeProtocol{ + promconfig.OpenMetricsText1_0_0, + promconfig.OpenMetricsText0_0_1, + promconfig.PrometheusText0_0_4, + promconfig.PrometheusProto, + }, + BodySizeLimit: 1024, + SampleLimit: 100, + TargetLimit: 100, + LabelLimit: 100, + LabelNameLengthLimit: 100, + LabelValueLengthLimit: 100, + KeepDroppedTargets: 100, + ScrapeTimeout: model.Duration(20 * time.Second), + HonorTimestamps: true, + HonorLabels: false, + Scheme: "http", + MetricsPath: "/metrics", + ServiceDiscoveryConfigs: []discovery.Config{ + &kubeDiscovery.SDConfig{ + Role: "endpointslice", + NamespaceDiscovery: kubeDiscovery.NamespaceDiscovery{ + Names: []string{"test"}, + IncludeOwnNamespace: false, + }, + HTTPClientConfig: config.DefaultHTTPClientConfig, + }, + }, + HTTPClientConfig: config.DefaultHTTPClientConfig, + EnableCompression: true, + }, + { + JobName: "podMonitor/test/simple/0", + ScrapeInterval: model.Duration(60 * time.Second), + ScrapeProtocols: []promconfig.ScrapeProtocol{ + promconfig.OpenMetricsText1_0_0, + promconfig.OpenMetricsText0_0_1, + promconfig.PrometheusText0_0_4, + promconfig.PrometheusProto, + }, + BodySizeLimit: 1024, + SampleLimit: 100, + TargetLimit: 100, + LabelLimit: 100, + LabelNameLengthLimit: 100, + LabelValueLengthLimit: 100, + KeepDroppedTargets: 100, + ScrapeTimeout: model.Duration(20 * time.Second), + HonorTimestamps: true, + HonorLabels: false, + Scheme: "http", + MetricsPath: "/metrics", + ServiceDiscoveryConfigs: []discovery.Config{ + &kubeDiscovery.SDConfig{ + Role: "pod", + NamespaceDiscovery: kubeDiscovery.NamespaceDiscovery{ + Names: []string{"test"}, + IncludeOwnNamespace: false, + }, + HTTPClientConfig: config.DefaultHTTPClientConfig, + }, + }, + HTTPClientConfig: config.DefaultHTTPClientConfig, + EnableCompression: true, + }, + }, + }, + }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { @@ -869,6 +1019,7 @@ func TestNamespaceLabelUpdate(t *testing.T) { cfg := allocatorconfig.Config{ PrometheusCR: allocatorconfig.PrometheusCRConfig{ + ScrapeInterval: model.Duration(30 * time.Second), ServiceMonitorSelector: &metav1.LabelSelector{}, PodMonitorSelector: &metav1.LabelSelector{}, PodMonitorNamespaceSelector: &metav1.LabelSelector{ @@ -1081,26 +1232,11 @@ func getTestPrometheusCRWatcher(t *testing.T, svcMonitors []*monitoringv1.Servic } factory := informers.NewMonitoringInformerFactories(map[string]struct{}{v1.NamespaceAll: {}}, map[string]struct{}{}, mClient, 0, nil) - informers, err := getInformers(factory) + informerMap, err := getInformers(factory) if err != nil { t.Fatal(t, err) } - - serviceDiscoveryRole := monitoringv1.ServiceDiscoveryRole("EndpointSlice") - - prom := &monitoringv1.Prometheus{ - Spec: monitoringv1.PrometheusSpec{ - CommonPrometheusFields: monitoringv1.CommonPrometheusFields{ - ScrapeInterval: monitoringv1.Duration("30s"), - ServiceMonitorSelector: cfg.PrometheusCR.ServiceMonitorSelector, - PodMonitorSelector: cfg.PrometheusCR.PodMonitorSelector, - ServiceMonitorNamespaceSelector: cfg.PrometheusCR.ServiceMonitorNamespaceSelector, - PodMonitorNamespaceSelector: cfg.PrometheusCR.PodMonitorNamespaceSelector, - ServiceDiscoveryRole: &serviceDiscoveryRole, - }, - }, - } - + prom := cfg.GetPrometheus() promOperatorLogger := level.NewFilter(log.NewLogfmtLogger(os.Stderr), level.AllowWarn()) promOperatorSlogLogger := slog.New(slog.NewTextHandler(os.Stderr, &slog.HandlerOptions{Level: slog.LevelWarn})) @@ -1132,7 +1268,7 @@ func getTestPrometheusCRWatcher(t *testing.T, svcMonitors []*monitoringv1.Servic return &PrometheusCRWatcher{ kubeMonitoringClient: mClient, k8sClient: k8sClient, - informers: informers, + informers: informerMap, nsInformer: nsMonInf, stopChannel: make(chan struct{}), configGenerator: generator, diff --git a/config/manager/kustomization.yaml b/config/manager/kustomization.yaml index 5c5f0b84cb..c90abd3b0b 100644 --- a/config/manager/kustomization.yaml +++ b/config/manager/kustomization.yaml @@ -1,2 +1,21 @@ resources: - manager.yaml +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +images: +- name: controller + newName: ghcr.io/open-telemetry/opentelemetry-operator/opentelemetry-operator + newTag: 0.107.0-47-g8719954d +patches: +- patch: '[{"op":"add","path":"/spec/template/spec/containers/0/args/-","value":"--target-allocator-image=ghcr.io/open-telemetry/opentelemetry-operator/target-allocator:ve2e-targetallocator"}]' + target: + kind: Deployment +- patch: '[{"op":"add","path":"/spec/template/spec/containers/0/args/-","value":"--operator-opamp-bridge-image=ghcr.io/open-telemetry/opentelemetry-operator/operator-opamp-bridge:ve2e-targetallocator"}]' + target: + kind: Deployment +- patch: '[{"op":"add","path":"/spec/template/spec/containers/0/args/-","value":"--target-allocator-image=ghcr.io/open-telemetry/opentelemetry-operator/target-allocator:v0.107.0-47-g8719954d"}]' + target: + kind: Deployment +- patch: '[{"op":"add","path":"/spec/template/spec/containers/0/args/-","value":"--operator-opamp-bridge-image=ghcr.io/open-telemetry/opentelemetry-operator/operator-opamp-bridge:v0.107.0-47-g8719954d"}]' + target: + kind: Deployment diff --git a/tests/e2e-targetallocator/targetallocator-prometheuscr/00-assert.yaml b/tests/e2e-targetallocator/targetallocator-prometheuscr/00-assert.yaml index dd705e927b..c3e3cc1f28 100644 --- a/tests/e2e-targetallocator/targetallocator-prometheuscr/00-assert.yaml +++ b/tests/e2e-targetallocator/targetallocator-prometheuscr/00-assert.yaml @@ -28,7 +28,14 @@ data: endpoint: 0.0.0.0:9090 receivers: prometheus: - config: {} + config: + global: + scrape_interval: 30s + scrape_protocols: + - PrometheusProto + - OpenMetricsText1.0.0 + - OpenMetricsText0.0.1 + - PrometheusText0.0.4 target_allocator: collector_id: ${POD_NAME} endpoint: http://prometheus-cr-targetallocator:80 @@ -42,4 +49,4 @@ data: - prometheus kind: ConfigMap metadata: - name: prometheus-cr-collector-52e1d2ae + name: prometheus-cr-collector-837a5cfe diff --git a/tests/e2e-targetallocator/targetallocator-prometheuscr/00-install.yaml b/tests/e2e-targetallocator/targetallocator-prometheuscr/00-install.yaml index bf55421f5f..edcc760aff 100644 --- a/tests/e2e-targetallocator/targetallocator-prometheuscr/00-install.yaml +++ b/tests/e2e-targetallocator/targetallocator-prometheuscr/00-install.yaml @@ -137,7 +137,7 @@ spec: prometheus: config: global: - scrape_interval: 30s + scrape_timeout: 1s scrape_protocols: ['PrometheusProto','OpenMetricsText1.0.0','OpenMetricsText0.0.1','PrometheusText0.0.4'] scrape_configs: []