From b18609d75e33d809de7fe2ab4d001409382fa7ac Mon Sep 17 00:00:00 2001 From: Helder Costa <54836608+htcosta@users.noreply.github.com> Date: Fri, 24 Nov 2023 22:06:55 +0100 Subject: [PATCH 01/12] Fixed the url links (#5850) Backticks should be used inside the link text, to properly be displayed as inline code style, while still preserving the url link. --- docs/sources/operator/helm-getting-started.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/sources/operator/helm-getting-started.md b/docs/sources/operator/helm-getting-started.md index bc2f516e10fa..78245505d859 100644 --- a/docs/sources/operator/helm-getting-started.md +++ b/docs/sources/operator/helm-getting-started.md @@ -20,8 +20,8 @@ In this guide, you'll learn how to deploy [Grafana Agent Operator]({{< relref ". To deploy Agent Operator with Helm, make sure that you have the following: - A Kubernetes cluster -- The `[kubectl](https://kubernetes.io/docs/tasks/tools/#kubectl)` command-line client installed and configured on your machine -- The `[helm](https://helm.sh/docs/intro/install/)` command-line client installed and configured on your machine +- The [`kubectl`](https://kubernetes.io/docs/tasks/tools/#kubectl) command-line client installed and configured on your machine +- The [`helm`](https://helm.sh/docs/intro/install/) command-line client installed and configured on your machine > **Note:** Agent Operator is currently in beta and its custom resources are subject to change. From b0de873affc95d4d0170ea5e4a10a1690a6c884e Mon Sep 17 00:00:00 2001 From: Tolya Korniltsev Date: Mon, 27 Nov 2023 15:53:42 +0700 Subject: [PATCH 02/12] fix(pyroscope.ebpf): add python arm64 support (#5856) * fix(pyroscope.ebpf): add python arm64 support, add musl libc arm64 support * changelog --- CHANGELOG.md | 2 ++ go.mod | 14 +++++++------- go.sum | 12 ++++++------ 3 files changed, 15 insertions(+), 13 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b394bce2533c..cbe4952c8291 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -31,6 +31,8 @@ Main (unreleased) Previously, only `remote.*` and `local.*` components could be referenced without a circular dependency. (@rfratto) +- `pyroscope.ebpf` support python on arm64 platforms. (@korniltsev) + ### Bugfixes - Permit `X-Faro-Session-ID` header in CORS requests for the `faro.receiver` diff --git a/go.mod b/go.mod index 9362ef24beb5..76f5ba8c6e08 100644 --- a/go.mod +++ b/go.mod @@ -47,7 +47,7 @@ require ( github.com/google/dnsmasq_exporter v0.2.1-0.20230620100026-44b14480804a github.com/google/go-cmp v0.5.9 github.com/google/go-jsonnet v0.18.0 - github.com/google/pprof v0.0.0-20230705174524-200ffdc848b8 + github.com/google/pprof v0.0.0-20230926050212-f7f687d19a98 github.com/google/renameio/v2 v2.0.0 github.com/google/uuid v1.3.1 github.com/gorilla/mux v1.8.0 @@ -59,7 +59,7 @@ require ( github.com/grafana/loki v1.6.2-0.20231004111112-07cbef92268a github.com/grafana/pyroscope-go/godeltaprof v0.1.3 github.com/grafana/pyroscope/api v0.2.0 - github.com/grafana/pyroscope/ebpf v0.3.2 + github.com/grafana/pyroscope/ebpf v0.3.3 github.com/grafana/regexp v0.0.0-20221123153739-15dc172cd2db github.com/grafana/river v0.3.0 github.com/grafana/snowflake-prometheus-exporter v0.0.0-20221213150626-862cad8e9538 @@ -86,7 +86,7 @@ require ( github.com/jaegertracing/jaeger v1.50.0 github.com/jmespath/go-jmespath v0.4.0 github.com/json-iterator/go v1.1.12 - github.com/klauspost/compress v1.17.0 + github.com/klauspost/compress v1.17.1 github.com/lib/pq v1.10.7 github.com/mackerelio/go-osstat v0.2.3 github.com/miekg/dns v1.1.55 @@ -582,7 +582,7 @@ require ( go.opencensus.io v0.24.0 // indirect go.opentelemetry.io/collector/config/internal v0.87.0 // indirect go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.45.0 // indirect - go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.45.0 + go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.45.0 // indirect go.opentelemetry.io/contrib/propagators/b3 v1.19.0 // indirect go.opentelemetry.io/otel/bridge/opencensus v0.42.0 // indirect go4.org/netipx v0.0.0-20230125063823-8449b0a6169f // indirect @@ -616,8 +616,7 @@ require ( github.com/natefinch/atomic v1.0.1 github.com/open-telemetry/opentelemetry-collector-contrib/processor/filterprocessor v0.87.0 github.com/open-telemetry/opentelemetry-collector-contrib/receiver/vcenterreceiver v0.87.0 - go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v0.42.0 - go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.19.0 + go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v0.42.0 ) require ( @@ -641,7 +640,8 @@ require ( github.com/sercand/kuberesolver/v4 v4.0.0 // indirect github.com/sony/gobreaker v0.5.0 // indirect go.opentelemetry.io/otel/exporters/otlp/otlpmetric v0.42.0 // indirect - go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v0.42.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v0.42.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.19.0 // indirect go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.19.0 // indirect go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v0.42.0 // indirect go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.19.0 // indirect diff --git a/go.sum b/go.sum index 3678215219aa..c80cb90ef4ca 100644 --- a/go.sum +++ b/go.sum @@ -985,8 +985,8 @@ github.com/google/pprof v0.0.0-20210601050228-01bbb1931b22/go.mod h1:kpwsk12EmLe github.com/google/pprof v0.0.0-20210609004039-a478d1d731e9/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20211214055906-6f57359322fd/go.mod h1:KgnwoLYCZ8IQu3XUZ8Nc/bM9CCZFOyjUNOSygVozoDg= -github.com/google/pprof v0.0.0-20230705174524-200ffdc848b8 h1:n6vlPhxsA+BW/XsS5+uqi7GyzaLa5MH7qlSLBZtRdiA= -github.com/google/pprof v0.0.0-20230705174524-200ffdc848b8/go.mod h1:Jh3hGz2jkYak8qXPD19ryItVnUgpgeqzdkY/D0EaeuA= +github.com/google/pprof v0.0.0-20230926050212-f7f687d19a98 h1:pUa4ghanp6q4IJHwE9RwLgmVFfReJN+KbQ8ExNEUUoQ= +github.com/google/pprof v0.0.0-20230926050212-f7f687d19a98/go.mod h1:czg5+yv1E0ZGTi6S6vVK1mke0fV+FaUhNGcd6VRS9Ik= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/renameio/v2 v2.0.0 h1:UifI23ZTGY8Tt29JbYFiuyIU3eX+RNFtUwefq9qAhxg= github.com/google/renameio/v2 v2.0.0/go.mod h1:BtmJXm5YlszgC+TD4HOEEUFgkJP3nLxehU6hfe7jRt4= @@ -1071,8 +1071,8 @@ github.com/grafana/pyroscope-go/godeltaprof v0.1.3 h1:eunWpv1B3Z7ZK9o4499EmQGlY+ github.com/grafana/pyroscope-go/godeltaprof v0.1.3/go.mod h1:1HSPtjU8vLG0jE9JrTdzjgFqdJ/VgN7fvxBNq3luJko= github.com/grafana/pyroscope/api v0.2.0 h1:TzOxL0s6SiaLEy944ZAKgHcx/JDRJXu4O8ObwkqR6p4= github.com/grafana/pyroscope/api v0.2.0/go.mod h1:nhH+xai9cYFgs6lMy/+L0pKj0d5yCMwji/QAiQFCP+U= -github.com/grafana/pyroscope/ebpf v0.3.2 h1:o2Ib5Syot/GeYchPpr4bu091WML70g2L+D5LHWjZZT4= -github.com/grafana/pyroscope/ebpf v0.3.2/go.mod h1:n6KfoTGMietvw4+vhhU1X8WiBOVxn5p8YxDy9BORUvY= +github.com/grafana/pyroscope/ebpf v0.3.3 h1:BAdrZfxGn5QuGrVOTK6oIsjRGHoIguBlWehAWaCAshM= +github.com/grafana/pyroscope/ebpf v0.3.3/go.mod h1:WK4a+9QTRDOh6imvPgglbwblfxkvyS2cYGWF9jlKTI0= github.com/grafana/regexp v0.0.0-20221123153739-15dc172cd2db h1:7aN5cccjIqCLTzedH7MZzRZt5/lsAHch6Z3L2ZGn5FA= github.com/grafana/regexp v0.0.0-20221123153739-15dc172cd2db/go.mod h1:M5qHK+eWfAv8VR/265dIuEpL3fNfeC21tXXp9itM24A= github.com/grafana/river v0.3.0 h1:6TsaR/vkkcppUM9I0muGbPIUedCtpPu6OWreE5+CE6g= @@ -1441,8 +1441,8 @@ github.com/klauspost/asmfmt v1.3.2/go.mod h1:AG8TuvYojzulgDAMCnYn50l/5QV3Bs/tp6j github.com/klauspost/compress v1.11.0/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= github.com/klauspost/compress v1.11.12/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= -github.com/klauspost/compress v1.17.0 h1:Rnbp4K9EjcDuVuHtd0dgA4qNuv9yKDYKK1ulpJwgrqM= -github.com/klauspost/compress v1.17.0/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= +github.com/klauspost/compress v1.17.1 h1:NE3C767s2ak2bweCZo3+rdP4U/HoyVXLv/X9f2gPS5g= +github.com/klauspost/compress v1.17.1/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= github.com/klauspost/cpuid/v2 v2.2.5 h1:0E5MSMDEoAulmXNFquVs//DdoomxaoTY1kUhbc/qbZg= github.com/klauspost/cpuid/v2 v2.2.5/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws= github.com/knadh/koanf v1.5.0 h1:q2TSd/3Pyc/5yP9ldIrSdIz26MCcyNQzW0pEAugLPNs= From 9fdb57cb0d37158efd3784b4734b181e7a2f625a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C4=90=E1=BB=97=20Tr=E1=BB=8Dng=20H=E1=BA=A3i?= <41283691+hainenber@users.noreply.github.com> Date: Mon, 27 Nov 2023 17:21:03 +0700 Subject: [PATCH 03/12] feat(otelcol/exp/prom): add support for resource_to_telemetry_conversion (#5807) * feat(otelcol/exp/prom): add support for resource_to_telemetry_conversion Signed-off-by: hainenber * Update otelcol.exporter.prometheus.md Co-authored-by: Clayton Cornell <131809008+clayton-cornell@users.noreply.github.com> * Update docs/sources/flow/reference/components/otelcol.exporter.prometheus.md Co-authored-by: Paulin Todev * feat(otelcol/exp/prom): test case when resourceToTelemetryConversion is false Signed-off-by: hainenber * Apply suggestions from code review --------- Signed-off-by: hainenber Co-authored-by: Clayton Cornell <131809008+clayton-cornell@users.noreply.github.com> Co-authored-by: Paulin Todev --- CHANGELOG.md | 5 +- .../prometheus/internal/convert/convert.go | 50 ++- .../internal/convert/convert_test.go | 290 +++++++++++++++++- .../otelcol/exporter/prometheus/prometheus.go | 31 +- .../exporter/prometheus/prometheus_test.go | 27 +- .../components/otelcol.exporter.prometheus.md | 1 + 6 files changed, 355 insertions(+), 49 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index cbe4952c8291..162ec02a8d88 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -31,7 +31,10 @@ Main (unreleased) Previously, only `remote.*` and `local.*` components could be referenced without a circular dependency. (@rfratto) -- `pyroscope.ebpf` support python on arm64 platforms. (@korniltsev) +- Add a `resource_to_telemetry_conversion` argument to `otelcol.exporter.prometheus` + for converting resource attributes to Prometheus labels. (@hainenber) + +- `pyroscope.ebpf` support python on arm64 platforms. (@korniltsev) ### Bugfixes diff --git a/component/otelcol/exporter/prometheus/internal/convert/convert.go b/component/otelcol/exporter/prometheus/internal/convert/convert.go index 3e2a2578c5a4..0a7039e7195e 100644 --- a/component/otelcol/exporter/prometheus/internal/convert/convert.go +++ b/component/otelcol/exporter/prometheus/internal/convert/convert.go @@ -65,6 +65,8 @@ type Options struct { IncludeScopeLabels bool // AddMetricSuffixes controls whether suffixes are added to metric names. Defaults to true. AddMetricSuffixes bool + // ResourceToTelemetryConversion controls whether to convert resource attributes to Prometheus-compatible datapoint attributes + ResourceToTelemetryConversion bool } var _ consumer.Metrics = (*Converter)(nil) @@ -131,6 +133,7 @@ func (conv *Converter) consumeResourceMetrics(app storage.Appender, rm pmetric.R Type: textparse.MetricTypeGauge, Help: "Target metadata", }) + resAttrs := rm.Resource().Attributes() memResource := conv.getOrCreateResource(rm.Resource()) if conv.getOpts().IncludeTargetInfo { @@ -144,7 +147,7 @@ func (conv *Converter) consumeResourceMetrics(app storage.Appender, rm pmetric.R for smcount := 0; smcount < rm.ScopeMetrics().Len(); smcount++ { sm := rm.ScopeMetrics().At(smcount) - conv.consumeScopeMetrics(app, memResource, sm) + conv.consumeScopeMetrics(app, memResource, sm, resAttrs) } } @@ -219,7 +222,7 @@ func (conv *Converter) getOrCreateResource(res pcommon.Resource) *memorySeries { return entry } -func (conv *Converter) consumeScopeMetrics(app storage.Appender, memResource *memorySeries, sm pmetric.ScopeMetrics) { +func (conv *Converter) consumeScopeMetrics(app storage.Appender, memResource *memorySeries, sm pmetric.ScopeMetrics, resAttrs pcommon.Map) { scopeMD := conv.createOrUpdateMetadata("otel_scope_info", metadata.Metadata{ Type: textparse.MetricTypeGauge, }) @@ -236,7 +239,7 @@ func (conv *Converter) consumeScopeMetrics(app storage.Appender, memResource *me for mcount := 0; mcount < sm.Metrics().Len(); mcount++ { m := sm.Metrics().At(mcount) - conv.consumeMetric(app, memResource, memScope, m) + conv.consumeMetric(app, memResource, memScope, m, resAttrs) } } @@ -274,20 +277,27 @@ func (conv *Converter) getOrCreateScope(res *memorySeries, scope pcommon.Instrum return entry } -func (conv *Converter) consumeMetric(app storage.Appender, memResource *memorySeries, memScope *memorySeries, m pmetric.Metric) { +func (conv *Converter) consumeMetric(app storage.Appender, memResource *memorySeries, memScope *memorySeries, m pmetric.Metric, resAttrs pcommon.Map) { switch m.Type() { case pmetric.MetricTypeGauge: - conv.consumeGauge(app, memResource, memScope, m) + conv.consumeGauge(app, memResource, memScope, m, resAttrs) case pmetric.MetricTypeSum: - conv.consumeSum(app, memResource, memScope, m) + conv.consumeSum(app, memResource, memScope, m, resAttrs) case pmetric.MetricTypeHistogram: - conv.consumeHistogram(app, memResource, memScope, m) + conv.consumeHistogram(app, memResource, memScope, m, resAttrs) case pmetric.MetricTypeSummary: - conv.consumeSummary(app, memResource, memScope, m) + conv.consumeSummary(app, memResource, memScope, m, resAttrs) } } -func (conv *Converter) consumeGauge(app storage.Appender, memResource *memorySeries, memScope *memorySeries, m pmetric.Metric) { +func joinAttributeMaps(from, to pcommon.Map) { + from.Range(func(k string, v pcommon.Value) bool { + v.CopyTo(to.PutEmpty(k)) + return true + }) +} + +func (conv *Converter) consumeGauge(app storage.Appender, memResource *memorySeries, memScope *memorySeries, m pmetric.Metric, resAttrs pcommon.Map) { metricName := prometheus.BuildCompliantName(m, "", conv.opts.AddMetricSuffixes) metricMD := conv.createOrUpdateMetadata(metricName, metadata.Metadata{ @@ -302,6 +312,10 @@ func (conv *Converter) consumeGauge(app storage.Appender, memResource *memorySer for dpcount := 0; dpcount < m.Gauge().DataPoints().Len(); dpcount++ { dp := m.Gauge().DataPoints().At(dpcount) + if conv.getOpts().ResourceToTelemetryConversion { + joinAttributeMaps(resAttrs, dp.Attributes()) + } + memSeries := conv.getOrCreateSeries(memResource, memScope, metricName, dp.Attributes()) if err := writeSeries(app, memSeries, dp, getNumberDataPointValue(dp)); err != nil { level.Error(conv.log).Log("msg", "failed to write metric sample", metricName, "err", err) @@ -389,7 +403,7 @@ func getNumberDataPointValue(dp pmetric.NumberDataPoint) float64 { return 0 } -func (conv *Converter) consumeSum(app storage.Appender, memResource *memorySeries, memScope *memorySeries, m pmetric.Metric) { +func (conv *Converter) consumeSum(app storage.Appender, memResource *memorySeries, memScope *memorySeries, m pmetric.Metric, resAttrs pcommon.Map) { metricName := prometheus.BuildCompliantName(m, "", conv.opts.AddMetricSuffixes) // Excerpt from the spec: @@ -430,6 +444,10 @@ func (conv *Converter) consumeSum(app storage.Appender, memResource *memorySerie for dpcount := 0; dpcount < m.Sum().DataPoints().Len(); dpcount++ { dp := m.Sum().DataPoints().At(dpcount) + if conv.getOpts().ResourceToTelemetryConversion { + joinAttributeMaps(resAttrs, dp.Attributes()) + } + memSeries := conv.getOrCreateSeries(memResource, memScope, metricName, dp.Attributes()) val := getNumberDataPointValue(dp) @@ -447,7 +465,7 @@ func (conv *Converter) consumeSum(app storage.Appender, memResource *memorySerie } } -func (conv *Converter) consumeHistogram(app storage.Appender, memResource *memorySeries, memScope *memorySeries, m pmetric.Metric) { +func (conv *Converter) consumeHistogram(app storage.Appender, memResource *memorySeries, memScope *memorySeries, m pmetric.Metric, resAttrs pcommon.Map) { metricName := prometheus.BuildCompliantName(m, "", conv.opts.AddMetricSuffixes) if m.Histogram().AggregationTemporality() != pmetric.AggregationTemporalityCumulative { @@ -469,6 +487,10 @@ func (conv *Converter) consumeHistogram(app storage.Appender, memResource *memor for dpcount := 0; dpcount < m.Histogram().DataPoints().Len(); dpcount++ { dp := m.Histogram().DataPoints().At(dpcount) + if conv.getOpts().ResourceToTelemetryConversion { + joinAttributeMaps(resAttrs, dp.Attributes()) + } + // Sum metric if dp.HasSum() { sumMetric := conv.getOrCreateSeries(memResource, memScope, metricName+"_sum", dp.Attributes()) @@ -606,7 +628,7 @@ func (conv *Converter) convertExemplar(otelExemplar pmetric.Exemplar, ts time.Ti } } -func (conv *Converter) consumeSummary(app storage.Appender, memResource *memorySeries, memScope *memorySeries, m pmetric.Metric) { +func (conv *Converter) consumeSummary(app storage.Appender, memResource *memorySeries, memScope *memorySeries, m pmetric.Metric, resAttrs pcommon.Map) { metricName := prometheus.BuildCompliantName(m, "", conv.opts.AddMetricSuffixes) metricMD := conv.createOrUpdateMetadata(metricName, metadata.Metadata{ @@ -621,6 +643,10 @@ func (conv *Converter) consumeSummary(app storage.Appender, memResource *memoryS for dpcount := 0; dpcount < m.Summary().DataPoints().Len(); dpcount++ { dp := m.Summary().DataPoints().At(dpcount) + if conv.getOpts().ResourceToTelemetryConversion { + joinAttributeMaps(resAttrs, dp.Attributes()) + } + // Sum metric { sumMetric := conv.getOrCreateSeries(memResource, memScope, metricName+"_sum", dp.Attributes()) diff --git a/component/otelcol/exporter/prometheus/internal/convert/convert_test.go b/component/otelcol/exporter/prometheus/internal/convert/convert_test.go index 80a6bce1a55b..dcace6574b6c 100644 --- a/component/otelcol/exporter/prometheus/internal/convert/convert_test.go +++ b/component/otelcol/exporter/prometheus/internal/convert/convert_test.go @@ -18,12 +18,13 @@ func TestConverter(t *testing.T) { input string expect string - showTimestamps bool - includeTargetInfo bool - includeScopeInfo bool - includeScopeLabels bool - addMetricSuffixes bool - enableOpenMetrics bool + showTimestamps bool + includeTargetInfo bool + includeScopeInfo bool + includeScopeLabels bool + addMetricSuffixes bool + enableOpenMetrics bool + resourceToTelemetryConversion bool }{ { name: "Gauge", @@ -838,6 +839,274 @@ func TestConverter(t *testing.T) { addMetricSuffixes: true, enableOpenMetrics: true, }, + { + name: "Gauge: convert resource attributes to metric label", + input: `{ + "resource_metrics": [{ + "resource": { + "attributes": [{ + "key": "service.name", + "value": { "stringValue": "myservice" } + }, { + "key": "service.instance.id", + "value": { "stringValue": "instance" } + }, { + "key": "raw", + "value": { "stringValue": "test" } + },{ + "key": "foo.one", + "value": { "stringValue": "foo" } + }, { + "key": "bar.one", + "value": { "stringValue": "bar" } + }] + }, + "scope_metrics": [{ + "metrics": [{ + "name": "test_metric_gauge", + "gauge": { + "data_points": [{ + "as_double": 1234.56 + }] + } + }] + }] + }] + }`, + expect: ` + # TYPE test_metric_gauge gauge + test_metric_gauge{bar_one="bar",foo_one="foo",instance="instance",service_instance_id="instance",job="myservice",service_name="myservice",raw="test"} 1234.56 + `, + enableOpenMetrics: true, + resourceToTelemetryConversion: true, + }, + { + name: "Gauge: NOT convert resource attributes to metric label", + input: `{ + "resource_metrics": [{ + "resource": { + "attributes": [{ + "key": "service.name", + "value": { "stringValue": "myservice" } + }, { + "key": "service.instance.id", + "value": { "stringValue": "instance" } + }, { + "key": "raw", + "value": { "stringValue": "test" } + },{ + "key": "foo.one", + "value": { "stringValue": "foo" } + }, { + "key": "bar.one", + "value": { "stringValue": "bar" } + }] + }, + "scope_metrics": [{ + "metrics": [{ + "name": "test_metric_gauge", + "gauge": { + "data_points": [{ + "as_double": 1234.56 + }] + } + }] + }] + }] + }`, + expect: ` + # TYPE test_metric_gauge gauge + test_metric_gauge{instance="instance",job="myservice"} 1234.56 + `, + enableOpenMetrics: true, + resourceToTelemetryConversion: false, + }, + { + name: "Summary: convert resource attributes to metric label", + input: `{ + "resource_metrics": [{ + "resource": { + "attributes": [{ + "key": "service.name", + "value": { "stringValue": "myservice" } + }, { + "key": "service.instance.id", + "value": { "stringValue": "instance" } + }, { + "key": "raw", + "value": { "stringValue": "test" } + },{ + "key": "foo.one", + "value": { "stringValue": "foo" } + }, { + "key": "bar.one", + "value": { "stringValue": "bar" } + }] + }, + "scope_metrics": [{ + "metrics": [{ + "name": "test_metric_summary", + "unit": "seconds", + "summary": { + "data_points": [{ + "start_time_unix_nano": 1000000000, + "time_unix_nano": 1000000000, + "count": 333, + "sum": 100, + "quantile_values": [ + { "quantile": 0, "value": 100 }, + { "quantile": 0.5, "value": 400 }, + { "quantile": 1, "value": 500 } + ] + }] + } + }] + }] + }] + }`, + expect: ` + # TYPE test_metric_summary summary + test_metric_summary{bar_one="bar",foo_one="foo",instance="instance",service_instance_id="instance",job="myservice",service_name="myservice",raw="test",quantile="0.0"} 100.0 + test_metric_summary{bar_one="bar",foo_one="foo",instance="instance",service_instance_id="instance",job="myservice",service_name="myservice",raw="test",quantile="0.5"} 400.0 + test_metric_summary{bar_one="bar",foo_one="foo",instance="instance",service_instance_id="instance",job="myservice",service_name="myservice",raw="test",quantile="1.0"} 500.0 + test_metric_summary_sum{bar_one="bar",foo_one="foo",instance="instance",service_instance_id="instance",job="myservice",service_name="myservice",raw="test"} 100.0 + test_metric_summary_count{bar_one="bar",foo_one="foo",instance="instance",service_instance_id="instance",job="myservice",service_name="myservice",raw="test"} 333 + `, + enableOpenMetrics: true, + resourceToTelemetryConversion: true, + }, + { + name: "Histogram: convert resource attributes to metric label", + input: `{ + "resource_metrics": [{ + "resource": { + "attributes": [{ + "key": "service.name", + "value": { "stringValue": "myservice" } + }, { + "key": "service.instance.id", + "value": { "stringValue": "instance" } + }, { + "key": "raw", + "value": { "stringValue": "test" } + },{ + "key": "foo.one", + "value": { "stringValue": "foo" } + }, { + "key": "bar.one", + "value": { "stringValue": "bar" } + }] + }, + "scope_metrics": [{ + "metrics": [ + { + "name": "test_metric_histogram", + "unit": "seconds", + "histogram": { + "aggregation_temporality": 2, + "data_points": [{ + "start_time_unix_nano": 1000000000, + "time_unix_nano": 1000000000, + "count": 333, + "sum": 100, + "bucket_counts": [0, 111, 0, 222], + "explicit_bounds": [0.25, 0.5, 0.75, 1.0], + "exemplars":[ + { + "time_unix_nano": 1000000001, + "as_double": 0.3, + "span_id": "aaaaaaaaaaaaaaaa", + "trace_id": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + }, + { + "time_unix_nano": 1000000003, + "as_double": 1.5, + "span_id": "cccccccccccccccc", + "trace_id": "cccccccccccccccccccccccccccccccc" + }, + { + "time_unix_nano": 1000000002, + "as_double": 0.5, + "span_id": "bbbbbbbbbbbbbbbb", + "trace_id": "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" + } + ] + }] + } + } + ] + }] + }] + }`, + expect: ` + # TYPE test_metric_histogram histogram + test_metric_histogram_bucket{bar_one="bar",foo_one="foo",instance="instance",service_instance_id="instance",job="myservice",service_name="myservice",raw="test",le="0.25"} 0 + test_metric_histogram_bucket{bar_one="bar",foo_one="foo",instance="instance",service_instance_id="instance",job="myservice",service_name="myservice",raw="test",le="0.5"} 111 # {span_id="aaaaaaaaaaaaaaaa",trace_id="aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"} 0.3 + test_metric_histogram_bucket{bar_one="bar",foo_one="foo",instance="instance",service_instance_id="instance",job="myservice",service_name="myservice",raw="test",le="0.75"} 111 # {span_id="bbbbbbbbbbbbbbbb",trace_id="bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"} 0.5 + test_metric_histogram_bucket{bar_one="bar",foo_one="foo",instance="instance",service_instance_id="instance",job="myservice",service_name="myservice",raw="test",le="1.0"} 333 + test_metric_histogram_bucket{bar_one="bar",foo_one="foo",instance="instance",service_instance_id="instance",job="myservice",service_name="myservice",raw="test",le="+Inf"} 333 # {span_id="cccccccccccccccc",trace_id="cccccccccccccccccccccccccccccccc"} 1.5 + test_metric_histogram_sum{bar_one="bar",foo_one="foo",instance="instance",service_instance_id="instance",job="myservice",service_name="myservice",raw="test"} 100.0 + test_metric_histogram_count{bar_one="bar",foo_one="foo",instance="instance",service_instance_id="instance",job="myservice",service_name="myservice",raw="test"} 333 + `, + enableOpenMetrics: true, + resourceToTelemetryConversion: true, + }, + { + name: "Monotonic sum: convert resource attributes to metric label", + input: `{ + "resource_metrics": [{ + "resource": { + "attributes": [{ + "key": "service.name", + "value": { "stringValue": "myservice" } + }, { + "key": "service.instance.id", + "value": { "stringValue": "instance" } + }, { + "key": "raw", + "value": { "stringValue": "test" } + },{ + "key": "foo.one", + "value": { "stringValue": "foo" } + }, { + "key": "bar.one", + "value": { "stringValue": "bar" } + }] + }, + "scope_metrics": [{ + "metrics": [ + { + "name": "test_metric_mono_sum_total", + "unit": "seconds", + "sum": { + "aggregation_temporality": 2, + "is_monotonic": true, + "data_points": [{ + "start_time_unix_nano": 1000000000, + "time_unix_nano": 1000000000, + "as_double": 15, + "exemplars":[ + { + "time_unix_nano": 1000000001, + "as_double": 0.3, + "span_id": "aaaaaaaaaaaaaaaa", + "trace_id": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + } + ] + }] + } + } + ] + }] + }] + }`, + expect: ` + # TYPE test_metric_mono_sum counter + test_metric_mono_sum_total{bar_one="bar",foo_one="foo",instance="instance",service_instance_id="instance",job="myservice",service_name="myservice",raw="test"} 15.0 # {span_id="aaaaaaaaaaaaaaaa",trace_id="aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"} 0.3 + `, + enableOpenMetrics: true, + resourceToTelemetryConversion: true, + }, } decoder := &pmetric.JSONUnmarshaler{} @@ -851,10 +1120,11 @@ func TestConverter(t *testing.T) { l := util.TestLogger(t) conv := convert.New(l, appenderAppendable{Inner: &app}, convert.Options{ - IncludeTargetInfo: tc.includeTargetInfo, - IncludeScopeInfo: tc.includeScopeInfo, - IncludeScopeLabels: tc.includeScopeLabels, - AddMetricSuffixes: tc.addMetricSuffixes, + IncludeTargetInfo: tc.includeTargetInfo, + IncludeScopeInfo: tc.includeScopeInfo, + IncludeScopeLabels: tc.includeScopeLabels, + AddMetricSuffixes: tc.addMetricSuffixes, + ResourceToTelemetryConversion: tc.resourceToTelemetryConversion, }) require.NoError(t, conv.ConsumeMetrics(context.Background(), payload)) diff --git a/component/otelcol/exporter/prometheus/prometheus.go b/component/otelcol/exporter/prometheus/prometheus.go index 77e894dc6321..7da1c03868ea 100644 --- a/component/otelcol/exporter/prometheus/prometheus.go +++ b/component/otelcol/exporter/prometheus/prometheus.go @@ -31,21 +31,23 @@ func init() { // Arguments configures the otelcol.exporter.prometheus component. type Arguments struct { - IncludeTargetInfo bool `river:"include_target_info,attr,optional"` - IncludeScopeInfo bool `river:"include_scope_info,attr,optional"` - IncludeScopeLabels bool `river:"include_scope_labels,attr,optional"` - GCFrequency time.Duration `river:"gc_frequency,attr,optional"` - ForwardTo []storage.Appendable `river:"forward_to,attr"` - AddMetricSuffixes bool `river:"add_metric_suffixes,attr,optional"` + IncludeTargetInfo bool `river:"include_target_info,attr,optional"` + IncludeScopeInfo bool `river:"include_scope_info,attr,optional"` + IncludeScopeLabels bool `river:"include_scope_labels,attr,optional"` + GCFrequency time.Duration `river:"gc_frequency,attr,optional"` + ForwardTo []storage.Appendable `river:"forward_to,attr"` + AddMetricSuffixes bool `river:"add_metric_suffixes,attr,optional"` + ResourceToTelemetryConversion bool `river:"resource_to_telemetry_conversion,attr,optional"` } // DefaultArguments holds defaults values. var DefaultArguments = Arguments{ - IncludeTargetInfo: true, - IncludeScopeInfo: false, - IncludeScopeLabels: true, - GCFrequency: 5 * time.Minute, - AddMetricSuffixes: true, + IncludeTargetInfo: true, + IncludeScopeInfo: false, + IncludeScopeLabels: true, + GCFrequency: 5 * time.Minute, + AddMetricSuffixes: true, + ResourceToTelemetryConversion: false, } // SetToDefault implements river.Defaulter. @@ -151,8 +153,9 @@ func (c *Component) Update(newConfig component.Arguments) error { func convertArgumentsToConvertOptions(args Arguments) convert.Options { return convert.Options{ - IncludeTargetInfo: args.IncludeTargetInfo, - IncludeScopeInfo: args.IncludeScopeInfo, - AddMetricSuffixes: args.AddMetricSuffixes, + IncludeTargetInfo: args.IncludeTargetInfo, + IncludeScopeInfo: args.IncludeScopeInfo, + AddMetricSuffixes: args.AddMetricSuffixes, + ResourceToTelemetryConversion: args.ResourceToTelemetryConversion, } } diff --git a/component/otelcol/exporter/prometheus/prometheus_test.go b/component/otelcol/exporter/prometheus/prometheus_test.go index 2939c8962346..7e642ff9b585 100644 --- a/component/otelcol/exporter/prometheus/prometheus_test.go +++ b/component/otelcol/exporter/prometheus/prometheus_test.go @@ -23,12 +23,13 @@ func TestArguments_UnmarshalRiver(t *testing.T) { forward_to = [] `, expected: prometheus.Arguments{ - IncludeTargetInfo: true, - IncludeScopeInfo: false, - IncludeScopeLabels: true, - GCFrequency: 5 * time.Minute, - AddMetricSuffixes: true, - ForwardTo: []storage.Appendable{}, + IncludeTargetInfo: true, + IncludeScopeInfo: false, + IncludeScopeLabels: true, + GCFrequency: 5 * time.Minute, + AddMetricSuffixes: true, + ForwardTo: []storage.Appendable{}, + ResourceToTelemetryConversion: false, }, }, { @@ -39,15 +40,17 @@ func TestArguments_UnmarshalRiver(t *testing.T) { include_scope_labels = false gc_frequency = "1s" add_metric_suffixes = false + resource_to_telemetry_conversion = true forward_to = [] `, expected: prometheus.Arguments{ - IncludeTargetInfo: false, - IncludeScopeInfo: true, - IncludeScopeLabels: false, - GCFrequency: 1 * time.Second, - AddMetricSuffixes: false, - ForwardTo: []storage.Appendable{}, + IncludeTargetInfo: false, + IncludeScopeInfo: true, + IncludeScopeLabels: false, + GCFrequency: 1 * time.Second, + AddMetricSuffixes: false, + ForwardTo: []storage.Appendable{}, + ResourceToTelemetryConversion: true, }, }, { diff --git a/docs/sources/flow/reference/components/otelcol.exporter.prometheus.md b/docs/sources/flow/reference/components/otelcol.exporter.prometheus.md index 8c8f81d133d1..584b30bd97e4 100644 --- a/docs/sources/flow/reference/components/otelcol.exporter.prometheus.md +++ b/docs/sources/flow/reference/components/otelcol.exporter.prometheus.md @@ -46,6 +46,7 @@ Name | Type | Description | Defaul `add_metric_suffixes` | `boolean` | Whether to add type and unit suffixes to metrics names. | `true` | no `gc_frequency` | `duration` | How often to clean up stale metrics from memory. | `"5m"` | no `forward_to` | `list(receiver)` | Where to forward converted Prometheus metrics. | | yes +`resource_to_telemetry_conversion` | `boolean` | Whether to convert OTel resource attributes to Prometheus labels. | `false` | no By default, OpenTelemetry resources are converted into `target_info` metrics. OpenTelemetry instrumentation scopes are converted into `otel_scope_info` From a2348a0f4b928a32720b279a425f6223835d5038 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C4=90=E1=BB=97=20Tr=E1=BB=8Dng=20H=E1=BA=A3i?= <41283691+hainenber@users.noreply.github.com> Date: Mon, 27 Nov 2023 17:45:00 +0700 Subject: [PATCH 04/12] Bump otelgrpc to fix CVE 2023 47108 (#5806) * Fix CVE-2023-47108 by updating `otelgrpc` from v0.45.0 to v0.46.0. * Stop using the deprecated trace.NewNoopTracerProvider. Using noop.NewTracerProvider from "go.opentelemetry.io/otel/trace/noop" instead. * Reorder changelog Comply with the ordering in: https://github.com/grafana/agent/blob/main/docs/developer/contributing.md#updating-the-changelog --------- Signed-off-by: hainenber Co-authored-by: Paulin Todev --- CHANGELOG.md | 4 +++ .../otelcol/receiver/prometheus/prometheus.go | 8 +++--- go.mod | 14 +++++----- go.sum | 27 ++++++++++--------- pkg/flow/componenttest/componenttest.go | 4 +-- pkg/flow/internal/controller/loader_test.go | 6 ++--- pkg/flow/tracing/tracing.go | 1 + pkg/flow/tracing/wrap_tracer.go | 22 +++++++-------- pkg/traces/instance.go | 4 +-- pkg/traces/traceutils/server.go | 4 +-- service/cluster/cluster.go | 3 ++- service/http/http.go | 3 ++- service/http/http_test.go | 4 +-- 13 files changed, 56 insertions(+), 48 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 162ec02a8d88..048ba98797bb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,10 @@ internal API changes are not present. Main (unreleased) ----------------- +### Security fixes + +- Fix CVE-2023-47108 by updating `otelgrpc` from v0.45.0 to v0.46.0. (@hainenber) + ### Features - Agent Management: Introduce support for templated configuration. (@jcreixell) diff --git a/component/otelcol/receiver/prometheus/prometheus.go b/component/otelcol/receiver/prometheus/prometheus.go index 5a54de7337b8..b0493f4d6982 100644 --- a/component/otelcol/receiver/prometheus/prometheus.go +++ b/component/otelcol/receiver/prometheus/prometheus.go @@ -19,8 +19,8 @@ import ( "github.com/prometheus/prometheus/storage" otelcomponent "go.opentelemetry.io/collector/component" otelreceiver "go.opentelemetry.io/collector/receiver" - "go.opentelemetry.io/otel/metric/noop" - "go.opentelemetry.io/otel/trace" + metricNoop "go.opentelemetry.io/otel/metric/noop" + traceNoop "go.opentelemetry.io/otel/trace/noop" ) func init() { @@ -107,8 +107,8 @@ func (c *Component) Update(newConfig component.Arguments) error { Logger: zapadapter.New(c.opts.Logger), // TODO(tpaschalis): expose tracing and logging statistics. - TracerProvider: trace.NewNoopTracerProvider(), - MeterProvider: noop.NewMeterProvider(), + TracerProvider: traceNoop.NewTracerProvider(), + MeterProvider: metricNoop.NewMeterProvider(), ReportComponentStatus: func(*otelcomponent.StatusEvent) error { return nil diff --git a/go.mod b/go.mod index 76f5ba8c6e08..198f271eb81e 100644 --- a/go.mod +++ b/go.mod @@ -37,7 +37,7 @@ require ( github.com/go-git/go-git/v5 v5.4.2 github.com/go-kit/log v0.2.1 github.com/go-logfmt/logfmt v0.6.0 - github.com/go-logr/logr v1.2.4 + github.com/go-logr/logr v1.3.0 github.com/go-sourcemap/sourcemap v2.1.3+incompatible github.com/go-sql-driver/mysql v1.7.1 github.com/gogo/protobuf v1.3.2 @@ -45,7 +45,7 @@ require ( github.com/golang/snappy v0.0.4 github.com/google/cadvisor v0.47.0 github.com/google/dnsmasq_exporter v0.2.1-0.20230620100026-44b14480804a - github.com/google/go-cmp v0.5.9 + github.com/google/go-cmp v0.6.0 github.com/google/go-jsonnet v0.18.0 github.com/google/pprof v0.0.0-20230926050212-f7f687d19a98 github.com/google/renameio/v2 v2.0.0 @@ -201,13 +201,13 @@ require ( go.opentelemetry.io/collector/semconv v0.87.0 go.opentelemetry.io/collector/service v0.87.0 go.opentelemetry.io/contrib/instrumentation/github.com/gorilla/mux/otelmux v0.45.0 - go.opentelemetry.io/otel v1.19.0 + go.opentelemetry.io/otel v1.21.0 go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.19.0 go.opentelemetry.io/otel/exporters/prometheus v0.42.0 - go.opentelemetry.io/otel/metric v1.19.0 - go.opentelemetry.io/otel/sdk v1.19.0 + go.opentelemetry.io/otel/metric v1.21.0 + go.opentelemetry.io/otel/sdk v1.21.0 go.opentelemetry.io/otel/sdk/metric v1.19.0 - go.opentelemetry.io/otel/trace v1.19.0 + go.opentelemetry.io/otel/trace v1.21.0 go.opentelemetry.io/proto/otlp v1.0.0 go.uber.org/atomic v1.11.0 go.uber.org/goleak v1.2.1 @@ -581,7 +581,7 @@ require ( go.mongodb.org/mongo-driver v1.12.0 // indirect go.opencensus.io v0.24.0 // indirect go.opentelemetry.io/collector/config/internal v0.87.0 // indirect - go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.45.0 // indirect + go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.46.0 // indirect go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.45.0 // indirect go.opentelemetry.io/contrib/propagators/b3 v1.19.0 // indirect go.opentelemetry.io/otel/bridge/opencensus v0.42.0 // indirect diff --git a/go.sum b/go.sum index c80cb90ef4ca..1c65e8cf4249 100644 --- a/go.sum +++ b/go.sum @@ -732,8 +732,8 @@ github.com/go-logfmt/logfmt v0.5.1/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KE github.com/go-logfmt/logfmt v0.6.0 h1:wGYYu3uicYdqXVgoYbvnkrPVXkuLM1p1ifugDMEdRi4= github.com/go-logfmt/logfmt v0.6.0/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= -github.com/go-logr/logr v1.2.4 h1:g01GSCwiDw2xSZfjJ2/T9M+S6pFdcNtFYsp+Y43HYDQ= -github.com/go-logr/logr v1.2.4/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.3.0 h1:2y3SDp0ZXuc6/cjLSZ+Q3ir+QB9T/iG5yYRXqsagWSY= +github.com/go-logr/logr v1.3.0/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/go-logr/zapr v1.2.4 h1:QHVo+6stLbfJmYGkQ7uGHUCu5hnAFAj6mDe6Ea0SeOo= @@ -949,8 +949,9 @@ github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8/DtOE= github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= -github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-github v17.0.0+incompatible/go.mod h1:zLgOLi98H3fifZn+44m+umXrS52loVEgC2AApnigrVQ= github.com/google/go-github/v32 v32.1.0/go.mod h1:rIEpZD9CTDQwDK9GDrtMTycQNA4JU3qBsCizh3q2WCI= github.com/google/go-jsonnet v0.18.0 h1:/6pTy6g+Jh1a1I2UMoAODkqELFiVIdOxbNwv0DDzoOg= @@ -2378,16 +2379,16 @@ go.opentelemetry.io/collector/semconv v0.87.0 h1:BsG1jdLLRCBRlvUujk4QA86af7r/ZXn go.opentelemetry.io/collector/semconv v0.87.0/go.mod h1:j/8THcqVxFna1FpvA2zYIsUperEtOaRaqoLYIN4doWw= go.opentelemetry.io/contrib/instrumentation/github.com/gorilla/mux/otelmux v0.45.0 h1:CaagQrotQLgtDlHU6u9pE/Mf4mAwiLD8wrReIVt06lY= go.opentelemetry.io/contrib/instrumentation/github.com/gorilla/mux/otelmux v0.45.0/go.mod h1:LOjFy00/ZMyMYfKFPta6kZe2cDUc1sNo/qtv1pSORWA= -go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.45.0 h1:RsQi0qJ2imFfCvZabqzM9cNXBG8k6gXMv1A0cXRmH6A= -go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.45.0/go.mod h1:vsh3ySueQCiKPxFLvjWC4Z135gIa34TQ/NSqkDTZYUM= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.46.0 h1:PzIubN4/sjByhDRHLviCjJuweBXWFZWhghjg7cS28+M= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.46.0/go.mod h1:Ct6zzQEuGK3WpJs2n4dn+wfJYzd/+hNnxMRTWjGn30M= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.45.0 h1:x8Z78aZx8cOF0+Kkazoc7lwUNMGy0LrzEMxTm4BbTxg= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.45.0/go.mod h1:62CPTSry9QZtOaSsE3tOzhx6LzDhHnXJ6xHeMNNiM6Q= go.opentelemetry.io/contrib/propagators/b3 v1.19.0 h1:ulz44cpm6V5oAeg5Aw9HyqGFMS6XM7untlMEhD7YzzA= go.opentelemetry.io/contrib/propagators/b3 v1.19.0/go.mod h1:OzCmE2IVS+asTI+odXQstRGVfXQ4bXv9nMBRK0nNyqQ= go.opentelemetry.io/contrib/zpages v0.45.0 h1:jIwHHGoWzJoZdbIUtWdErjL85Gni6BignnAFqDtMRL4= go.opentelemetry.io/contrib/zpages v0.45.0/go.mod h1:4mIdA5hqH6hEx9sZgV50qKfQO8aIYolUZboHmz+G7vw= -go.opentelemetry.io/otel v1.19.0 h1:MuS/TNf4/j4IXsZuJegVzI1cwut7Qc00344rgH7p8bs= -go.opentelemetry.io/otel v1.19.0/go.mod h1:i0QyjOq3UPoTzff0PJB2N66fb4S0+rSbSB15/oyH9fY= +go.opentelemetry.io/otel v1.21.0 h1:hzLeKBZEL7Okw2mGzZ0cc4k/A7Fta0uoPgaJCr8fsFc= +go.opentelemetry.io/otel v1.21.0/go.mod h1:QZzNPQPm1zLX4gZK4cMi+71eaorMSGT3A4znnUvNNEo= go.opentelemetry.io/otel/bridge/opencensus v0.42.0 h1:QvC+bcZkWMphWPiVqRQygMj6M0/3TOuJEO+erRA7kI8= go.opentelemetry.io/otel/bridge/opencensus v0.42.0/go.mod h1:XJojP7g5DqYdiyArix/H9i1XzPPlIUc9dGLKtF9copI= go.opentelemetry.io/otel/exporters/otlp/otlpmetric v0.42.0 h1:ZtfnDL+tUrs1F0Pzfwbg2d59Gru9NCH3bgSHBM6LDwU= @@ -2408,14 +2409,14 @@ go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v0.42.0 h1:4jJuoeOo9W6hZn go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v0.42.0/go.mod h1:/MtYTE1SfC2QIcE0bDot6fIX+h+WvXjgTqgn9P0LNPE= go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.19.0 h1:Nw7Dv4lwvGrI68+wULbcq7su9K2cebeCUrDjVrUJHxM= go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.19.0/go.mod h1:1MsF6Y7gTqosgoZvHlzcaaM8DIMNZgJh87ykokoNH7Y= -go.opentelemetry.io/otel/metric v1.19.0 h1:aTzpGtV0ar9wlV4Sna9sdJyII5jTVJEvKETPiOKwvpE= -go.opentelemetry.io/otel/metric v1.19.0/go.mod h1:L5rUsV9kM1IxCj1MmSdS+JQAcVm319EUrDVLrt7jqt8= -go.opentelemetry.io/otel/sdk v1.19.0 h1:6USY6zH+L8uMH8L3t1enZPR3WFEmSTADlqldyHtJi3o= -go.opentelemetry.io/otel/sdk v1.19.0/go.mod h1:NedEbbS4w3C6zElbLdPJKOpJQOrGUJ+GfzpjUvI0v1A= +go.opentelemetry.io/otel/metric v1.21.0 h1:tlYWfeo+Bocx5kLEloTjbcDwBuELRrIFxwdQ36PlJu4= +go.opentelemetry.io/otel/metric v1.21.0/go.mod h1:o1p3CA8nNHW8j5yuQLdc1eeqEaPfzug24uvsyIEJRWM= +go.opentelemetry.io/otel/sdk v1.21.0 h1:FTt8qirL1EysG6sTQRZ5TokkU8d0ugCj8htOgThZXQ8= +go.opentelemetry.io/otel/sdk v1.21.0/go.mod h1:Nna6Yv7PWTdgJHVRD9hIYywQBRx7pbox6nwBnZIxl/E= go.opentelemetry.io/otel/sdk/metric v1.19.0 h1:EJoTO5qysMsYCa+w4UghwFV/ptQgqSL/8Ni+hx+8i1k= go.opentelemetry.io/otel/sdk/metric v1.19.0/go.mod h1:XjG0jQyFJrv2PbMvwND7LwCEhsJzCzV5210euduKcKY= -go.opentelemetry.io/otel/trace v1.19.0 h1:DFVQmlVbfVeOuBRrwdtaehRrWiL1JoVs9CPIQ1Dzxpg= -go.opentelemetry.io/otel/trace v1.19.0/go.mod h1:mfaSyvGyEJEI0nyV2I4qhNQnbBOUUmYZpYojqMnX2vo= +go.opentelemetry.io/otel/trace v1.21.0 h1:WD9i5gzvoUPuXIXH24ZNBudiarZDKuekPqi/E8fpfLc= +go.opentelemetry.io/otel/trace v1.21.0/go.mod h1:LGbsEB0f9LGjN+OZaQQ26sohbOmiMR+BaslueVtS/qQ= go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= go.opentelemetry.io/proto/otlp v1.0.0 h1:T0TX0tmXU8a3CbNXzEKGeU5mIVOdf0oykP+u2lIVU/I= go.opentelemetry.io/proto/otlp v1.0.0/go.mod h1:Sy6pihPLfYHkr3NkUbEhGHFhINUSI/v80hjKIs5JXpM= diff --git a/pkg/flow/componenttest/componenttest.go b/pkg/flow/componenttest/componenttest.go index b545db4bf3f4..f8af382a70df 100644 --- a/pkg/flow/componenttest/componenttest.go +++ b/pkg/flow/componenttest/componenttest.go @@ -11,12 +11,12 @@ import ( "github.com/grafana/agent/service/labelstore" "github.com/prometheus/client_golang/prometheus" - "go.opentelemetry.io/otel/trace" "go.uber.org/atomic" "github.com/go-kit/log" "github.com/grafana/agent/component" "github.com/grafana/agent/pkg/flow/logging" + "go.opentelemetry.io/otel/trace/noop" ) // A Controller is a testing controller which controls a single component. @@ -155,7 +155,7 @@ func (c *Controller) buildComponent(dataPath string, args component.Arguments) ( opts := component.Options{ ID: c.reg.Name + ".test", Logger: l, - Tracer: trace.NewNoopTracerProvider(), + Tracer: noop.NewTracerProvider(), DataPath: dataPath, OnStateChange: c.onStateChange, Registerer: prometheus.NewRegistry(), diff --git a/pkg/flow/internal/controller/loader_test.go b/pkg/flow/internal/controller/loader_test.go index fee35ec587b5..e93f757b1a2f 100644 --- a/pkg/flow/internal/controller/loader_test.go +++ b/pkg/flow/internal/controller/loader_test.go @@ -15,7 +15,7 @@ import ( "github.com/grafana/river/parser" "github.com/prometheus/client_golang/prometheus" "github.com/stretchr/testify/require" - "go.opentelemetry.io/otel/trace" + "go.opentelemetry.io/otel/trace/noop" _ "github.com/grafana/agent/pkg/flow/internal/testcomponents" // Include test components ) @@ -71,7 +71,7 @@ func TestLoader(t *testing.T) { return controller.LoaderOptions{ ComponentGlobals: controller.ComponentGlobals{ Logger: l, - TraceProvider: trace.NewNoopTracerProvider(), + TraceProvider: noop.NewTracerProvider(), DataPath: t.TempDir(), OnComponentUpdate: func(cn *controller.ComponentNode) { /* no-op */ }, Registerer: prometheus.NewRegistry(), @@ -205,7 +205,7 @@ func TestScopeWithFailingComponent(t *testing.T) { return controller.LoaderOptions{ ComponentGlobals: controller.ComponentGlobals{ Logger: l, - TraceProvider: trace.NewNoopTracerProvider(), + TraceProvider: noop.NewTracerProvider(), DataPath: t.TempDir(), OnComponentUpdate: func(cn *controller.ComponentNode) { /* no-op */ }, Registerer: prometheus.NewRegistry(), diff --git a/pkg/flow/tracing/tracing.go b/pkg/flow/tracing/tracing.go index 9cf5466dab29..12247d477a40 100644 --- a/pkg/flow/tracing/tracing.go +++ b/pkg/flow/tracing/tracing.go @@ -76,6 +76,7 @@ func (opts *JaegerRemoteSamplerOptions) SetToDefault() { // [trace.TracerProvider] and can be used to forward internally generated // traces to a OpenTelemetry Collector-compatible Flow component. type Tracer struct { + trace.TracerProvider sampler *lazySampler client *client exp *otlptrace.Exporter diff --git a/pkg/flow/tracing/wrap_tracer.go b/pkg/flow/tracing/wrap_tracer.go index 197e7ce3200b..5f166d874a9c 100644 --- a/pkg/flow/tracing/wrap_tracer.go +++ b/pkg/flow/tracing/wrap_tracer.go @@ -18,9 +18,9 @@ var ( // componentID as an attribute to each span. func WrapTracer(inner trace.TracerProvider, componentID string) trace.TracerProvider { return &wrappedProvider{ - inner: inner, - id: componentID, - spanName: componentIDAttributeKey, + TracerProvider: inner, + id: componentID, + spanName: componentIDAttributeKey, } } @@ -28,14 +28,14 @@ func WrapTracer(inner trace.TracerProvider, componentID string) trace.TracerProv // controller id as an attribute to each span. func WrapTracerForLoader(inner trace.TracerProvider, componentID string) trace.TracerProvider { return &wrappedProvider{ - inner: inner, - id: componentID, - spanName: controllerIDAttributeKey, + TracerProvider: inner, + id: componentID, + spanName: controllerIDAttributeKey, } } type wrappedProvider struct { - inner trace.TracerProvider + trace.TracerProvider id string spanName string } @@ -49,16 +49,16 @@ func (wp *wrappedProvider) Tracer(name string, options ...trace.TracerOption) tr otelComponentName := strings.TrimSuffix(wp.id, filepath.Ext(wp.id)) options = append(options, trace.WithInstrumentationAttributes(attribute.String(wp.spanName, otelComponentName))) } - innerTracer := wp.inner.Tracer(name, options...) + innerTracer := wp.TracerProvider.Tracer(name, options...) return &wrappedTracer{ - inner: innerTracer, + Tracer: innerTracer, id: wp.id, spanName: wp.spanName, } } type wrappedTracer struct { - inner trace.Tracer + trace.Tracer id string spanName string } @@ -66,7 +66,7 @@ type wrappedTracer struct { var _ trace.Tracer = (*wrappedTracer)(nil) func (tp *wrappedTracer) Start(ctx context.Context, spanName string, opts ...trace.SpanStartOption) (context.Context, trace.Span) { - ctx, span := tp.inner.Start(ctx, spanName, opts...) + ctx, span := tp.Tracer.Start(ctx, spanName, opts...) if tp.id != "" { span.SetAttributes( attribute.String(tp.spanName, tp.id), diff --git a/pkg/traces/instance.go b/pkg/traces/instance.go index 855b5b81a133..bf35e31ccb14 100644 --- a/pkg/traces/instance.go +++ b/pkg/traces/instance.go @@ -26,7 +26,7 @@ import ( "github.com/grafana/agent/pkg/traces/traceutils" "github.com/grafana/agent/pkg/util" prom_client "github.com/prometheus/client_golang/prometheus" - "go.opentelemetry.io/otel/trace" + "go.opentelemetry.io/otel/trace/noop" ) // Instance wraps the OpenTelemetry collector to enable tracing pipelines @@ -153,7 +153,7 @@ func (i *Instance) buildAndStartPipeline(ctx context.Context, cfg InstanceConfig OtelMetricReader: promExporter, DisableProcessMetrics: true, UseExternalMetricsServer: true, - TracerProvider: trace.NewNoopTracerProvider(), + TracerProvider: noop.NewTracerProvider(), //TODO: Plug in an AsyncErrorChannel to shut down the Agent in case of a fatal event LoggingOptions: []zap.Option{ zap.WrapCore(func(zapcore.Core) zapcore.Core { diff --git a/pkg/traces/traceutils/server.go b/pkg/traces/traceutils/server.go index 4889e873df42..2c484e3c086b 100644 --- a/pkg/traces/traceutils/server.go +++ b/pkg/traces/traceutils/server.go @@ -23,7 +23,7 @@ import ( "go.opentelemetry.io/collector/receiver" "go.opentelemetry.io/collector/receiver/otlpreceiver" "go.opentelemetry.io/collector/service" - "go.opentelemetry.io/otel/trace" + "go.opentelemetry.io/otel/trace/noop" "gopkg.in/yaml.v3" ) @@ -153,7 +153,7 @@ func newServer(addr string, callback func(ptrace.Traces)) (*server, error) { Connectors: connector.NewBuilder(otelCfg.Connectors, factories.Connectors), Extensions: extension.NewBuilder(otelCfg.Extensions, factories.Extensions), UseExternalMetricsServer: false, - TracerProvider: trace.NewNoopTracerProvider(), + TracerProvider: noop.NewTracerProvider(), }, otelCfg.Service) if err != nil { return nil, fmt.Errorf("failed to create Otel service: %w", err) diff --git a/service/cluster/cluster.go b/service/cluster/cluster.go index 23b8ea4f293a..2ea23e26017d 100644 --- a/service/cluster/cluster.go +++ b/service/cluster/cluster.go @@ -24,6 +24,7 @@ import ( "github.com/prometheus/client_golang/prometheus" "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/trace" + "go.opentelemetry.io/otel/trace/noop" "golang.org/x/net/http2" ) @@ -94,7 +95,7 @@ func New(opts Options) (*Service, error) { l = log.NewNopLogger() } if t == nil { - t = trace.NewNoopTracerProvider() + t = noop.NewTracerProvider() } ckitConfig := ckit.Config{ diff --git a/service/http/http.go b/service/http/http.go index cf703f942474..a8608f4dfcb5 100644 --- a/service/http/http.go +++ b/service/http/http.go @@ -26,6 +26,7 @@ import ( "github.com/prometheus/client_golang/prometheus/promhttp" "go.opentelemetry.io/contrib/instrumentation/github.com/gorilla/mux/otelmux" "go.opentelemetry.io/otel/trace" + "go.opentelemetry.io/otel/trace/noop" "golang.org/x/net/http2" "golang.org/x/net/http2/h2c" ) @@ -93,7 +94,7 @@ func New(opts Options) *Service { l = log.NewNopLogger() } if t == nil { - t = trace.NewNoopTracerProvider() + t = noop.NewTracerProvider() } if r == nil { r = prometheus.NewRegistry() diff --git a/service/http/http_test.go b/service/http/http_test.go index b341a677f129..660e0deee43a 100644 --- a/service/http/http_test.go +++ b/service/http/http_test.go @@ -16,7 +16,7 @@ import ( "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/common/config" "github.com/stretchr/testify/require" - "go.opentelemetry.io/otel/trace" + "go.opentelemetry.io/otel/trace/noop" ) func TestHTTP(t *testing.T) { @@ -168,7 +168,7 @@ func newTestEnvironment(t *testing.T) (*testEnvironment, error) { svc := New(Options{ Logger: util.TestLogger(t), - Tracer: trace.NewNoopTracerProvider(), + Tracer: noop.NewTracerProvider(), Gatherer: prometheus.NewRegistry(), ReadyFunc: func() bool { return true }, From 84344fb1a273ed1c25423e46a654f9af273636f8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C4=90=E1=BB=97=20Tr=E1=BB=8Dng=20H=E1=BA=A3i?= <41283691+hainenber@users.noreply.github.com> Date: Mon, 27 Nov 2023 21:22:08 +0700 Subject: [PATCH 05/12] fix(otelcol/fanoutconsumer): nil check during fanout consumer creation (#5854) Signed-off-by: hainenber --- CHANGELOG.md | 2 ++ component/otelcol/internal/fanoutconsumer/logs.go | 14 ++++++++++---- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 048ba98797bb..1f2e8bcd2680 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -48,6 +48,8 @@ Main (unreleased) - Fix issue with windows_exporter defaults not being set correctly. (@mattdurham) +- Fix agent crash when process null OTel's fan out consumers. (@hainenber) + v0.38.0 (2023-11-21) -------------------- diff --git a/component/otelcol/internal/fanoutconsumer/logs.go b/component/otelcol/internal/fanoutconsumer/logs.go index a01202686e01..a8ee4df45b7f 100644 --- a/component/otelcol/internal/fanoutconsumer/logs.go +++ b/component/otelcol/internal/fanoutconsumer/logs.go @@ -29,6 +29,10 @@ func Logs(in []otelcol.Consumer) otelconsumer.Logs { for i := 0; i < len(in)-1; i++ { consumer := in[i] + if consumer == nil { + continue + } + if consumer.Capabilities().MutatesData { clone = append(clone, consumer) } else { @@ -40,10 +44,12 @@ func Logs(in []otelcol.Consumer) otelconsumer.Logs { // The final consumer can be given to the passthrough list regardless of // whether it mutates as long as there's no other read-only consumers. - if len(passthrough) == 0 || !last.Capabilities().MutatesData { - passthrough = append(passthrough, last) - } else { - clone = append(clone, last) + if last != nil { + if len(passthrough) == 0 || !last.Capabilities().MutatesData { + passthrough = append(passthrough, last) + } else { + clone = append(clone, last) + } } return &logsFanout{ From f232fb476c84cee7c2c04a2f3b7cc28b5907b429 Mon Sep 17 00:00:00 2001 From: Craig Peterson <192540+captncraig@users.noreply.github.com> Date: Mon, 27 Nov 2023 14:15:30 -0500 Subject: [PATCH 06/12] prometheus.operator.* - Fix issue with missing targets when one monitor's name is a prefix of another (#5862) Co-authored-by: Paul Bormans <1399350+Paul424@users.noreply.github.com> --- CHANGELOG.md | 2 + .../prometheus/operator/common/crdmanager.go | 44 +++-- .../operator/common/crdmanager_test.go | 168 ++++++++++++++++++ .../prometheus/operator/common/interfaces.go | 23 +++ 4 files changed, 224 insertions(+), 13 deletions(-) create mode 100644 component/prometheus/operator/common/crdmanager_test.go create mode 100644 component/prometheus/operator/common/interfaces.go diff --git a/CHANGELOG.md b/CHANGELOG.md index 1f2e8bcd2680..a60a93219a85 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -50,6 +50,8 @@ Main (unreleased) - Fix agent crash when process null OTel's fan out consumers. (@hainenber) +- Fix issue in `prometheus.operator.*` where targets would be dropped if two crds share a common prefix in their names. (@Paul424, @captncraig) + v0.38.0 (2023-11-21) -------------------- diff --git a/component/prometheus/operator/common/crdmanager.go b/component/prometheus/operator/common/crdmanager.go index 9f8bd55f79f6..85f13719e970 100644 --- a/component/prometheus/operator/common/crdmanager.go +++ b/component/prometheus/operator/common/crdmanager.go @@ -42,12 +42,19 @@ const informerSyncTimeout = 10 * time.Second // crdManager is all of the fields required to run a crd based component. // on update, this entire thing should be recreated and restarted type crdManager struct { - mut sync.Mutex - discoveryConfigs map[string]discovery.Configs - scrapeConfigs map[string]*config.ScrapeConfig - debugInfo map[string]*operator.DiscoveredResource - discoveryManager *discovery.Manager - scrapeManager *scrape.Manager + mut sync.Mutex + + // these maps are keyed by job name + discoveryConfigs map[string]discovery.Configs + scrapeConfigs map[string]*config.ScrapeConfig + + // list of keys to the above maps for a given resource by `ns/name` + crdsToMapKeys map[string][]string + // debug info by `kind/ns/name` + debugInfo map[string]*operator.DiscoveredResource + + discoveryManager discoveryManager + scrapeManager scrapeManager clusteringUpdated chan struct{} ls labelstore.LabelStore @@ -80,6 +87,7 @@ func newCrdManager(opts component.Options, cluster cluster.Cluster, logger log.L cluster: cluster, discoveryConfigs: map[string]discovery.Configs{}, scrapeConfigs: map[string]*config.ScrapeConfig{}, + crdsToMapKeys: map[string][]string{}, debugInfo: map[string]*operator.DiscoveredResource{}, kind: kind, clusteringUpdated: make(chan struct{}, 1), @@ -392,6 +400,7 @@ func (c *crdManager) addPodMonitor(pm *promopv1.PodMonitor) { AdditionalRelabelConfigs: c.args.RelabelConfigs, ScrapeOptions: c.args.Scrape, } + mapKeys := []string{} for i, ep := range pm.Spec.PodMetricsEndpoints { var scrapeConfig *config.ScrapeConfig scrapeConfig, err = gen.GeneratePodMonitorConfig(pm, ep, i) @@ -400,6 +409,7 @@ func (c *crdManager) addPodMonitor(pm *promopv1.PodMonitor) { level.Error(c.logger).Log("name", pm.Name, "err", err, "msg", "error generating scrapeconfig from podmonitor") break } + mapKeys = append(mapKeys, scrapeConfig.JobName) c.mut.Lock() c.discoveryConfigs[scrapeConfig.JobName] = scrapeConfig.ServiceDiscoveryConfigs c.scrapeConfigs[scrapeConfig.JobName] = scrapeConfig @@ -409,6 +419,9 @@ func (c *crdManager) addPodMonitor(pm *promopv1.PodMonitor) { c.addDebugInfo(pm.Namespace, pm.Name, err) return } + c.mut.Lock() + c.crdsToMapKeys[fmt.Sprintf("%s/%s", pm.Namespace, pm.Name)] = mapKeys + c.mut.Unlock() if err = c.apply(); err != nil { level.Error(c.logger).Log("name", pm.Name, "err", err, "msg", "error applying scrape configs from "+c.kind) } @@ -442,6 +455,8 @@ func (c *crdManager) addServiceMonitor(sm *promopv1.ServiceMonitor) { AdditionalRelabelConfigs: c.args.RelabelConfigs, ScrapeOptions: c.args.Scrape, } + + mapKeys := []string{} for i, ep := range sm.Spec.Endpoints { var scrapeConfig *config.ScrapeConfig scrapeConfig, err = gen.GenerateServiceMonitorConfig(sm, ep, i) @@ -450,6 +465,7 @@ func (c *crdManager) addServiceMonitor(sm *promopv1.ServiceMonitor) { level.Error(c.logger).Log("name", sm.Name, "err", err, "msg", "error generating scrapeconfig from serviceMonitor") break } + mapKeys = append(mapKeys, scrapeConfig.JobName) c.mut.Lock() c.discoveryConfigs[scrapeConfig.JobName] = scrapeConfig.ServiceDiscoveryConfigs c.scrapeConfigs[scrapeConfig.JobName] = scrapeConfig @@ -459,6 +475,9 @@ func (c *crdManager) addServiceMonitor(sm *promopv1.ServiceMonitor) { c.addDebugInfo(sm.Namespace, sm.Name, err) return } + c.mut.Lock() + c.crdsToMapKeys[fmt.Sprintf("%s/%s", sm.Namespace, sm.Name)] = mapKeys + c.mut.Unlock() if err = c.apply(); err != nil { level.Error(c.logger).Log("name", sm.Name, "err", err, "msg", "error applying scrape configs from "+c.kind) } @@ -503,6 +522,7 @@ func (c *crdManager) addProbe(p *promopv1.Probe) { c.mut.Lock() c.discoveryConfigs[pmc.JobName] = pmc.ServiceDiscoveryConfigs c.scrapeConfigs[pmc.JobName] = pmc + c.crdsToMapKeys[fmt.Sprintf("%s/%s", p.Namespace, p.Name)] = []string{pmc.JobName} c.mut.Unlock() if err = c.apply(); err != nil { @@ -533,12 +553,10 @@ func (c *crdManager) onDeleteProbe(obj interface{}) { func (c *crdManager) clearConfigs(ns, name string) { c.mut.Lock() defer c.mut.Unlock() - prefix := fmt.Sprintf("%s/%s/%s", c.kind, ns, name) - for k := range c.discoveryConfigs { - if strings.HasPrefix(k, prefix) { - delete(c.discoveryConfigs, k) - delete(c.scrapeConfigs, k) - } + + for _, k := range c.crdsToMapKeys[fmt.Sprintf("%s/%s", ns, name)] { + delete(c.discoveryConfigs, k) + delete(c.scrapeConfigs, k) } - delete(c.debugInfo, prefix) + delete(c.debugInfo, fmt.Sprintf("%s/%s/%s", c.kind, ns, name)) } diff --git a/component/prometheus/operator/common/crdmanager_test.go b/component/prometheus/operator/common/crdmanager_test.go new file mode 100644 index 000000000000..7e3cd75fbd37 --- /dev/null +++ b/component/prometheus/operator/common/crdmanager_test.go @@ -0,0 +1,168 @@ +package common + +import ( + "testing" + + "golang.org/x/exp/maps" + + "github.com/go-kit/log" + "github.com/grafana/agent/component" + "github.com/grafana/agent/component/prometheus/operator" + "github.com/grafana/agent/service/cluster" + "github.com/grafana/agent/service/labelstore" + "github.com/prometheus/prometheus/config" + "github.com/prometheus/prometheus/discovery" + "github.com/prometheus/prometheus/discovery/targetgroup" + "github.com/prometheus/prometheus/scrape" + "k8s.io/apimachinery/pkg/util/intstr" + + promopv1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + "github.com/stretchr/testify/require" +) + +func TestClearConfigsSameNsSamePrefix(t *testing.T) { + logger := log.NewNopLogger() + m := newCrdManager( + component.Options{ + Logger: logger, + GetServiceData: func(name string) (interface{}, error) { return nil, nil }, + }, + cluster.Mock(), + logger, + &operator.DefaultArguments, + KindServiceMonitor, + labelstore.New(logger), + ) + + m.discoveryManager = newMockDiscoveryManager() + m.scrapeManager = newMockScrapeManager() + + targetPort := intstr.FromInt(9090) + m.onAddServiceMonitor(&promopv1.ServiceMonitor{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "monitoring", + Name: "svcmonitor", + }, + Spec: promopv1.ServiceMonitorSpec{ + Selector: metav1.LabelSelector{ + MatchLabels: map[string]string{ + "group": "my-group", + }, + }, + Endpoints: []promopv1.Endpoint{ + { + TargetPort: &targetPort, + ScrapeTimeout: "5s", + Interval: "10s", + }, + }, + }, + }) + m.onAddServiceMonitor(&promopv1.ServiceMonitor{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "monitoring", + Name: "svcmonitor-another", + }, + Spec: promopv1.ServiceMonitorSpec{ + Selector: metav1.LabelSelector{ + MatchLabels: map[string]string{ + "group": "my-group", + }, + }, + Endpoints: []promopv1.Endpoint{ + { + TargetPort: &targetPort, + ScrapeTimeout: "5s", + Interval: "10s", + }, + }, + }}) + + require.ElementsMatch(t, []string{"serviceMonitor/monitoring/svcmonitor-another/0", "serviceMonitor/monitoring/svcmonitor/0"}, maps.Keys(m.discoveryConfigs)) + m.clearConfigs("monitoring", "svcmonitor") + require.ElementsMatch(t, []string{"monitoring/svcmonitor", "monitoring/svcmonitor-another"}, maps.Keys(m.crdsToMapKeys)) + require.ElementsMatch(t, []string{"serviceMonitor/monitoring/svcmonitor-another/0"}, maps.Keys(m.discoveryConfigs)) + require.ElementsMatch(t, []string{"serviceMonitor/monitoring/svcmonitor-another"}, maps.Keys(m.debugInfo)) +} + +func TestClearConfigsProbe(t *testing.T) { + logger := log.NewNopLogger() + m := newCrdManager( + component.Options{ + Logger: logger, + GetServiceData: func(name string) (interface{}, error) { return nil, nil }, + }, + cluster.Mock(), + logger, + &operator.DefaultArguments, + KindProbe, + labelstore.New(logger), + ) + + m.discoveryManager = newMockDiscoveryManager() + m.scrapeManager = newMockScrapeManager() + + m.onAddProbe(&promopv1.Probe{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "monitoring", + Name: "probe", + }, + Spec: promopv1.ProbeSpec{}, + }) + m.onAddProbe(&promopv1.Probe{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "monitoring", + Name: "probe-another", + }, + Spec: promopv1.ProbeSpec{}}) + + require.ElementsMatch(t, []string{"probe/monitoring/probe-another", "probe/monitoring/probe"}, maps.Keys(m.discoveryConfigs)) + m.clearConfigs("monitoring", "probe") + require.ElementsMatch(t, []string{"monitoring/probe", "monitoring/probe-another"}, maps.Keys(m.crdsToMapKeys)) + require.ElementsMatch(t, []string{"probe/monitoring/probe-another"}, maps.Keys(m.discoveryConfigs)) + require.ElementsMatch(t, []string{"probe/monitoring/probe-another"}, maps.Keys(m.debugInfo)) +} + +type mockDiscoveryManager struct { +} + +func newMockDiscoveryManager() *mockDiscoveryManager { + return &mockDiscoveryManager{} +} + +func (m *mockDiscoveryManager) Run() error { + return nil +} + +func (m *mockDiscoveryManager) SyncCh() <-chan map[string][]*targetgroup.Group { + return nil +} + +func (m *mockDiscoveryManager) ApplyConfig(cfg map[string]discovery.Configs) error { + return nil +} + +type mockScrapeManager struct { +} + +func newMockScrapeManager() *mockScrapeManager { + return &mockScrapeManager{} +} + +func (m *mockScrapeManager) Run(tsets <-chan map[string][]*targetgroup.Group) error { + return nil +} + +func (m *mockScrapeManager) Stop() { + +} + +func (m *mockScrapeManager) TargetsActive() map[string][]*scrape.Target { + return nil +} + +func (m *mockScrapeManager) ApplyConfig(cfg *config.Config) error { + return nil +} diff --git a/component/prometheus/operator/common/interfaces.go b/component/prometheus/operator/common/interfaces.go new file mode 100644 index 000000000000..4652154f6dc6 --- /dev/null +++ b/component/prometheus/operator/common/interfaces.go @@ -0,0 +1,23 @@ +package common + +import ( + "github.com/prometheus/prometheus/config" + "github.com/prometheus/prometheus/discovery" + "github.com/prometheus/prometheus/discovery/targetgroup" + "github.com/prometheus/prometheus/scrape" +) + +// discoveryManager is an interface around discovery.Manager +type discoveryManager interface { + Run() error + SyncCh() <-chan map[string][]*targetgroup.Group + ApplyConfig(cfg map[string]discovery.Configs) error +} + +// scrapeManager is an interface around scrape.Manager +type scrapeManager interface { + Run(tsets <-chan map[string][]*targetgroup.Group) error + Stop() + TargetsActive() map[string][]*scrape.Target + ApplyConfig(cfg *config.Config) error +} From dcfdce56a167beba9840251313804f392ef299c1 Mon Sep 17 00:00:00 2001 From: Clayton Cornell <131809008+clayton-cornell@users.noreply.github.com> Date: Mon, 27 Nov 2023 11:46:56 -0800 Subject: [PATCH 07/12] Update shared content topics (#5845) * Update layout and syntax in shared topics * Updates to language passive voice use etc * General updates to words syntax and voice * Fix sorting in table * Fix table spacing * Correct capitalization of boolean * Apply suggestions from code review Co-authored-by: Robert Fratto * Update docs/sources/shared/flow/reference/components/otelcol-debug-metrics-block.md --------- Co-authored-by: Robert Fratto --- .../components/authorization-block.md | 13 +++--- .../reference/components/azuread-block.md | 6 +-- .../reference/components/basic-auth-block.md | 13 +++--- .../components/exporter-component-exports.md | 9 ++-- .../components/extract-field-block.md | 33 ++++++------- .../components/field-filter-block.md | 18 ++++---- .../components/http-client-config-block.md | 18 ++++---- .../components/local-file-arguments-text.md | 25 ++++------ .../reference/components/loki-server-grpc.md | 25 +++++----- .../reference/components/loki-server-http.md | 19 ++++---- .../components/managed_identity-block.md | 10 ++-- .../components/match-properties-block.md | 18 ++++---- .../flow/reference/components/oauth2-block.md | 23 +++++----- .../components/otelcol-compression-field.md | 7 ++- .../components/otelcol-debug-metrics-block.md | 12 ++--- .../otelcol-filter-attribute-block.md | 10 ++-- .../otelcol-filter-library-block.md | 12 ++--- .../otelcol-filter-log-severity-block.md | 13 +++--- .../components/otelcol-filter-regexp-block.md | 8 ++-- .../otelcol-filter-resource-block.md | 12 ++--- .../components/otelcol-grpc-authority.md | 7 ++- .../components/otelcol-grpc-balancer-name.md | 8 ++-- .../components/otelcol-queue-block.md | 29 +++++------- .../components/otelcol-retry-block.md | 34 ++++++-------- .../components/otelcol-tls-config-block.md | 41 ++++++++--------- .../reference/components/output-block-logs.md | 15 +++--- .../components/output-block-metrics.md | 9 ++-- .../components/output-block-traces.md | 15 +++--- .../flow/reference/components/output-block.md | 19 ++++---- .../components/prom-operator-scrape.md | 8 ++-- .../reference/components/rule-block-logs.md | 46 +++++++++---------- .../flow/reference/components/rule-block.md | 42 ++++++++--------- .../flow/reference/components/sigv4-block.md | 14 +++--- .../reference/components/tls-config-block.md | 25 +++++----- 34 files changed, 286 insertions(+), 330 deletions(-) diff --git a/docs/sources/shared/flow/reference/components/authorization-block.md b/docs/sources/shared/flow/reference/components/authorization-block.md index 190cd11f8bb9..11a74326f997 100644 --- a/docs/sources/shared/flow/reference/components/authorization-block.md +++ b/docs/sources/shared/flow/reference/components/authorization-block.md @@ -10,11 +10,10 @@ description: Shared content, authorization block headless: true --- -Name | Type | Description | Default | Required ----- | ---- | ----------- | ------- | -------- -`type` | `string` | Authorization type, for example, "Bearer". | | no -`credentials` | `secret` | Secret value. | | no -`credentials_file` | `string` | File containing the secret value. | | no +Name | Type | Description | Default | Required +-------------------|----------|--------------------------------------------|---------|--------- +`credentials_file` | `string` | File containing the secret value. | | no +`credentials` | `secret` | Secret value. | | no +`type` | `string` | Authorization type, for example, "Bearer". | | no -`credential` and `credentials_file` are mutually exclusive and only one can be -provided inside of an `authorization` block. +`credential` and `credentials_file` are mutually exclusive, and only one can be provided inside an `authorization` block. diff --git a/docs/sources/shared/flow/reference/components/azuread-block.md b/docs/sources/shared/flow/reference/components/azuread-block.md index ebdf436d02fe..07d974385134 100644 --- a/docs/sources/shared/flow/reference/components/azuread-block.md +++ b/docs/sources/shared/flow/reference/components/azuread-block.md @@ -10,11 +10,11 @@ description: Shared content, azuread block headless: true --- -Name | Type | Description | Default | Required ----- | ---- | ----------- | ------- | -------- +Name | Type | Description | Default | Required +--------|----------|------------------|-----------------|--------- `cloud` | `string` | The Azure Cloud. | `"AzurePublic"` | no The supported values for `cloud` are: * `"AzurePublic"` * `"AzureChina"` -* `"AzureGovernment"` \ No newline at end of file +* `"AzureGovernment"` diff --git a/docs/sources/shared/flow/reference/components/basic-auth-block.md b/docs/sources/shared/flow/reference/components/basic-auth-block.md index 06c81f660e3c..62f7e0a25d61 100644 --- a/docs/sources/shared/flow/reference/components/basic-auth-block.md +++ b/docs/sources/shared/flow/reference/components/basic-auth-block.md @@ -10,11 +10,10 @@ description: Shared content, basic auth block headless: true --- -Name | Type | Description | Default | Required ----- | ---- | ----------- | ------- | -------- -`username` | `string` | Basic auth username. | | no -`password` | `secret` | Basic auth password. | | no -`password_file` | `string` | File containing the basic auth password. | | no +Name | Type | Description | Default | Required +----------------|----------|------------------------------------------|---------|--------- +`password_file` | `string` | File containing the basic auth password. | | no +`password` | `secret` | Basic auth password. | | no +`username` | `string` | Basic auth username. | | no -`password` and `password_file` are mutually exclusive and only one can be -provided inside of a `basic_auth` block. +`password` and `password_file` are mutually exclusive, and only one can be provided inside a `basic_auth` block. diff --git a/docs/sources/shared/flow/reference/components/exporter-component-exports.md b/docs/sources/shared/flow/reference/components/exporter-component-exports.md index beb717a13fae..f1a8ca440cd9 100644 --- a/docs/sources/shared/flow/reference/components/exporter-component-exports.md +++ b/docs/sources/shared/flow/reference/components/exporter-component-exports.md @@ -13,15 +13,12 @@ headless: true The following fields are exported and can be referenced by other components. Name | Type | Description ---------- | ------------------- | ----------- +----------|---------------------|---------------------------------------------------------- `targets` | `list(map(string))` | The targets that can be used to collect exporter metrics. -For example, the `targets` can either be passed to a `discovery.relabel` -component to rewrite the targets' label sets, or to a `prometheus.scrape` -component that collects the exposed metrics. +For example, the `targets` can either be passed to a `discovery.relabel` component to rewrite the targets' label sets or to a `prometheus.scrape` component that collects the exposed metrics. -The exported targets will use the configured [in-memory traffic][] address -specified by the [run command][]. +The exported targets use the configured [in-memory traffic][] address specified by the [run command][]. [in-memory traffic]: {{< relref "../../../../flow/concepts/component_controller.md#in-memory-traffic" >}} [run command]: {{< relref "../../../../flow/reference/cli/run.md" >}} diff --git a/docs/sources/shared/flow/reference/components/extract-field-block.md b/docs/sources/shared/flow/reference/components/extract-field-block.md index 5036097d155f..207f2bc6053d 100644 --- a/docs/sources/shared/flow/reference/components/extract-field-block.md +++ b/docs/sources/shared/flow/reference/components/extract-field-block.md @@ -12,31 +12,28 @@ headless: true The following attributes are supported: -Name | Type | Description | Default | Required ----- |----------------|----------------------------------------------------------------------------------------------------------|---------| -------- -`tag_name` | `string` | The name of the resource attribute that will be added to logs, metrics, or spans. | `""` | no -`key` | `string` | The annotation (or label) name. This must exactly match an annotation (or label) name. | `""` | no -`key_regex` | `string` | A regular expression used to extract a key that matches the regex. | `""` | no -`regex` | `string` | An optional field used to extract a sub-string from a complex field value. | `""` | no -`from` | `string` | The source of the labels or annotations. Allowed values are `pod` and `namespace`. | `pod` | no - -When `tag_name` is not specified, a default tag name will be used with the format: +Name | Type | Description | Default | Required +------------|----------|----------------------------------------------------------------------------------------|---------|--------- +`from` | `string` | The source of the labels or annotations. Allowed values are `pod` and `namespace`. | `pod` | no +`key_regex` | `string` | A regular expression used to extract a key that matches the regular expression. | `""` | no +`key` | `string` | The annotation or label name. This key must exactly match an annotation or label name. | `""` | no +`regex` | `string` | An optional field used to extract a sub-string from a complex field value. | `""` | no +`tag_name` | `string` | The name of the resource attribute added to logs, metrics, or spans. | `""` | no + +When you don't specify the `tag_name`, a default tag name is used with the format: * `k8s.pod.annotations.` * `k8s.pod.labels.