From 59af36427c0a3188c64013e4e482519a79e7a071 Mon Sep 17 00:00:00 2001 From: William Dumont Date: Thu, 14 Dec 2023 15:12:45 +0100 Subject: [PATCH] Update otelcol.receiver.prometheus with v0.87.0 opentelemetry collector contrib version (#5799) * copy pasted code from opentelemetry, no manual change * Update docs/sources/flow/release-notes.md --------- Co-authored-by: Paulin Todev Co-authored-by: Clayton Cornell <131809008+clayton-cornell@users.noreply.github.com> --- CHANGELOG.md | 9 + .../prometheus/internal/appendable.go | 28 +- .../receiver/prometheus/internal/doc.go | 2 +- .../receiver/prometheus/internal/logger.go | 13 +- .../prometheus/internal/logger_test.go | 13 +- .../receiver/prometheus/internal/metadata.go | 13 +- .../prometheus/internal/metricfamily.go | 193 +++-- .../prometheus/internal/metricfamily_test.go | 209 +++++- .../prometheus/internal/metrics_adjuster.go | 84 ++- .../internal/metrics_adjuster_test.go | 77 +- .../prometheus/internal/metricsutil_test.go | 13 +- .../prometheus/internal/prom_to_otlp.go | 13 +- .../prometheus/internal/prom_to_otlp_test.go | 14 +- .../internal/staleness_end_to_end_test.go | 241 ++++++ .../internal/starttimemetricadjuster.go | 18 +- .../internal/starttimemetricadjuster_test.go | 14 +- .../prometheus/internal/transaction.go | 226 ++++-- .../prometheus/internal/transaction_test.go | 708 ++++++++++++++---- .../receiver/prometheus/internal/util.go | 28 +- .../receiver/prometheus/internal/util_test.go | 15 +- .../otelcol/receiver/prometheus/prometheus.go | 14 +- .../receiver/prometheus/prometheus_test.go | 26 + docs/sources/flow/release-notes.md | 8 +- go.mod | 9 + go.sum | 15 +- 25 files changed, 1533 insertions(+), 470 deletions(-) create mode 100644 component/otelcol/receiver/prometheus/internal/staleness_end_to_end_test.go diff --git a/CHANGELOG.md b/CHANGELOG.md index 18682feae07f..4ff616c25480 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,12 @@ Main (unreleased) ### Breaking changes +- `otelcol.receiver.prometheus` will drop all `otel_scope_info` metrics when converting them to OTLP. (@wildum) + - If the `otel_scope_info` metric has labels `otel_scope_name` and `otel_scope_version`, + their values will be used to set OTLP Instrumentation Scope name and version respectively. + - Labels of `otel_scope_info` metrics other than `otel_scope_name` and `otel_scope_version` + are added as scope attributes with the matching name and version. + - The `target` block in `prometheus.exporter.blackbox` requires a mandatory `name` argument instead of a block label. (@hainenber) @@ -37,6 +43,9 @@ Main (unreleased) - `pyroscope.ebpf` support python on arm64 platforms. (@korniltsev) +- `otelcol.receiver.prometheus` does not drop histograms without buckets anymore. (@wildum) + +- Added exemplars support to `otelcol.receiver.prometheus`. (@wildum) - `mimir.rules.kubernetes` may now retry its startup on failure. (@hainenber) - Added links between compatible components in the documentation to make it diff --git a/component/otelcol/receiver/prometheus/internal/appendable.go b/component/otelcol/receiver/prometheus/internal/appendable.go index bbcf6b9ab055..d8b26a2900bc 100644 --- a/component/otelcol/receiver/prometheus/internal/appendable.go +++ b/component/otelcol/receiver/prometheus/internal/appendable.go @@ -1,16 +1,5 @@ // Copyright The OpenTelemetry Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. +// SPDX-License-Identifier: Apache-2.0 package internal // import "github.com/open-telemetry/opentelemetry-collector-contrib/receiver/prometheusreceiver/internal" @@ -21,17 +10,18 @@ import ( "github.com/prometheus/prometheus/model/labels" "github.com/prometheus/prometheus/storage" - "go.opentelemetry.io/collector/component" "go.opentelemetry.io/collector/consumer" "go.opentelemetry.io/collector/receiver" "go.opentelemetry.io/collector/receiver/receiverhelper" ) // appendable translates Prometheus scraping diffs into OpenTelemetry format. + type appendable struct { sink consumer.Metrics metricAdjuster MetricsAdjuster useStartTimeMetric bool + trimSuffixes bool startTimeMetricRegex *regexp.Regexp externalLabels labels.Labels @@ -46,17 +36,18 @@ func NewAppendable( gcInterval time.Duration, useStartTimeMetric bool, startTimeMetricRegex *regexp.Regexp, - receiverID component.ID, - externalLabels labels.Labels) (storage.Appendable, error) { + useCreatedMetric bool, + externalLabels labels.Labels, + trimSuffixes bool) (storage.Appendable, error) { var metricAdjuster MetricsAdjuster if !useStartTimeMetric { - metricAdjuster = NewInitialPointAdjuster(set.Logger, gcInterval) + metricAdjuster = NewInitialPointAdjuster(set.Logger, gcInterval, useCreatedMetric) } else { metricAdjuster = NewStartTimeMetricAdjuster(set.Logger, startTimeMetricRegex) } - obsrecv, err := receiverhelper.NewObsReport(receiverhelper.ObsReportSettings{ReceiverID: receiverID, Transport: transport, ReceiverCreateSettings: set}) + obsrecv, err := receiverhelper.NewObsReport(receiverhelper.ObsReportSettings{ReceiverID: set.ID, Transport: transport, ReceiverCreateSettings: set}) if err != nil { return nil, err } @@ -69,9 +60,10 @@ func NewAppendable( startTimeMetricRegex: startTimeMetricRegex, externalLabels: externalLabels, obsrecv: obsrecv, + trimSuffixes: trimSuffixes, }, nil } func (o *appendable) Appender(ctx context.Context) storage.Appender { - return newTransaction(ctx, o.metricAdjuster, o.sink, o.externalLabels, o.settings, o.obsrecv) + return newTransaction(ctx, o.metricAdjuster, o.sink, o.externalLabels, o.settings, o.obsrecv, o.trimSuffixes) } diff --git a/component/otelcol/receiver/prometheus/internal/doc.go b/component/otelcol/receiver/prometheus/internal/doc.go index c3e2d1419364..f4b8aefe77ad 100644 --- a/component/otelcol/receiver/prometheus/internal/doc.go +++ b/component/otelcol/receiver/prometheus/internal/doc.go @@ -1,5 +1,5 @@ // Package internal is a near copy of -// https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/v0.61.0/receiver/prometheusreceiver/internal +// https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/v0.87.0/receiver/prometheusreceiver/internal // A copy was made because the upstream package is internal. If it is ever made // public, our copy can be removed. // diff --git a/component/otelcol/receiver/prometheus/internal/logger.go b/component/otelcol/receiver/prometheus/internal/logger.go index 5cfb210742be..726d236574df 100644 --- a/component/otelcol/receiver/prometheus/internal/logger.go +++ b/component/otelcol/receiver/prometheus/internal/logger.go @@ -1,16 +1,5 @@ // Copyright The OpenTelemetry Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. +// SPDX-License-Identifier: Apache-2.0 package internal // import "github.com/open-telemetry/opentelemetry-collector-contrib/receiver/prometheusreceiver/internal" diff --git a/component/otelcol/receiver/prometheus/internal/logger_test.go b/component/otelcol/receiver/prometheus/internal/logger_test.go index 9913080bf0c8..5a17fd051a27 100644 --- a/component/otelcol/receiver/prometheus/internal/logger_test.go +++ b/component/otelcol/receiver/prometheus/internal/logger_test.go @@ -1,16 +1,5 @@ // Copyright The OpenTelemetry Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. +// SPDX-License-Identifier: Apache-2.0 package internal diff --git a/component/otelcol/receiver/prometheus/internal/metadata.go b/component/otelcol/receiver/prometheus/internal/metadata.go index 4cd3a012bdea..cea58a0e1dc9 100644 --- a/component/otelcol/receiver/prometheus/internal/metadata.go +++ b/component/otelcol/receiver/prometheus/internal/metadata.go @@ -1,16 +1,5 @@ // Copyright The OpenTelemetry Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. +// SPDX-License-Identifier: Apache-2.0 package internal // import "github.com/open-telemetry/opentelemetry-collector-contrib/receiver/prometheusreceiver/internal" diff --git a/component/otelcol/receiver/prometheus/internal/metricfamily.go b/component/otelcol/receiver/prometheus/internal/metricfamily.go index 461f8c9253fa..e501ee5da384 100644 --- a/component/otelcol/receiver/prometheus/internal/metricfamily.go +++ b/component/otelcol/receiver/prometheus/internal/metricfamily.go @@ -1,30 +1,29 @@ // Copyright The OpenTelemetry Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. +// SPDX-License-Identifier: Apache-2.0 package internal // import "github.com/open-telemetry/opentelemetry-collector-contrib/receiver/prometheusreceiver/internal" import ( + "encoding/hex" "fmt" + "math" "sort" "strings" + "github.com/prometheus/prometheus/model/exemplar" "github.com/prometheus/prometheus/model/labels" "github.com/prometheus/prometheus/model/value" "github.com/prometheus/prometheus/scrape" "go.opentelemetry.io/collector/pdata/pcommon" "go.opentelemetry.io/collector/pdata/pmetric" "go.uber.org/zap" + + "github.com/open-telemetry/opentelemetry-collector-contrib/pkg/translator/prometheus" +) + +const ( + traceIDKey = "trace_id" + spanIDKey = "span_id" ) type metricFamily struct { @@ -41,15 +40,17 @@ type metricFamily struct { // a couple data complexValue (buckets and count/sum), a group of a metric family always share a same set of tags. for // simple types like counter and gauge, each data point is a group of itself type metricGroup struct { - family *metricFamily + mtype pmetric.MetricType ts int64 ls labels.Labels count float64 hasCount bool sum float64 hasSum bool + created float64 value float64 complexValue []*dataPoint + exemplars pmetric.ExemplarSlice } func newMetricFamily(metricName string, mc scrape.MetricMetadataStore, logger *zap.Logger) *metricFamily { @@ -79,12 +80,6 @@ func (mf *metricFamily) includesMetric(metricName string) bool { return metricName == mf.name } -func (mf *metricFamily) getGroupKey(ls labels.Labels) uint64 { - bytes := make([]byte, 0, 2048) - hash, _ := ls.HashWithoutLabels(bytes, getSortedNotUsefulLabels(mf.mtype)...) - return hash -} - func (mg *metricGroup) sortPoints() { sort.Slice(mg.complexValue, func(i, j int) bool { return mg.complexValue[i].boundary < mg.complexValue[j].boundary @@ -92,25 +87,28 @@ func (mg *metricGroup) sortPoints() { } func (mg *metricGroup) toDistributionPoint(dest pmetric.HistogramDataPointSlice) { - if !mg.hasCount || len(mg.complexValue) == 0 { + if !mg.hasCount { return } mg.sortPoints() - // for OCAgent Proto, the bounds won't include +inf - // TODO: (@odeke-em) should we also check OpenTelemetry Pdata for bucket bounds? - bounds := make([]float64, len(mg.complexValue)-1) - bucketCounts := make([]uint64, len(mg.complexValue)) + bucketCount := len(mg.complexValue) + 1 + // if the final bucket is +Inf, we ignore it + if bucketCount > 1 && mg.complexValue[bucketCount-2].boundary == math.Inf(1) { + bucketCount-- + } + + // for OTLP the bounds won't include +inf + bounds := make([]float64, bucketCount-1) + bucketCounts := make([]uint64, bucketCount) + var adjustedCount float64 pointIsStale := value.IsStaleNaN(mg.sum) || value.IsStaleNaN(mg.count) + for i := 0; i < bucketCount-1; i++ { + bounds[i] = mg.complexValue[i].boundary + adjustedCount = mg.complexValue[i].value - for i := 0; i < len(mg.complexValue); i++ { - if i != len(mg.complexValue)-1 { - // not need to add +inf as bound to oc proto - bounds[i] = mg.complexValue[i].boundary - } - adjustedCount := mg.complexValue[i].value // Buckets still need to be sent to know to set them as stale, // but a staleness NaN converted to uint64 would be an extremely large number. // Setting to 0 instead. @@ -122,6 +120,15 @@ func (mg *metricGroup) toDistributionPoint(dest pmetric.HistogramDataPointSlice) bucketCounts[i] = uint64(adjustedCount) } + // Add the final bucket based on the total count + adjustedCount = mg.count + if pointIsStale { + adjustedCount = 0 + } else if bucketCount > 1 { + adjustedCount -= mg.complexValue[bucketCount-2].value + } + bucketCounts[bucketCount-1] = uint64(adjustedCount) + point := dest.AppendEmpty() if pointIsStale { @@ -138,9 +145,24 @@ func (mg *metricGroup) toDistributionPoint(dest pmetric.HistogramDataPointSlice) // The timestamp MUST be in retrieved from milliseconds and converted to nanoseconds. tsNanos := timestampFromMs(mg.ts) - point.SetStartTimestamp(tsNanos) // metrics_adjuster adjusts the startTimestamp to the initial scrape timestamp + if mg.created != 0 { + point.SetStartTimestamp(timestampFromFloat64(mg.created)) + } else { + // metrics_adjuster adjusts the startTimestamp to the initial scrape timestamp + point.SetStartTimestamp(tsNanos) + } point.SetTimestamp(tsNanos) populateAttributes(pmetric.MetricTypeHistogram, mg.ls, point.Attributes()) + mg.setExemplars(point.Exemplars()) +} + +func (mg *metricGroup) setExemplars(exemplars pmetric.ExemplarSlice) { + if mg == nil { + return + } + if mg.exemplars.Len() > 0 { + mg.exemplars.MoveAndAppendTo(exemplars) + } } func (mg *metricGroup) toSummaryPoint(dest pmetric.SummaryDataPointSlice) { @@ -183,7 +205,12 @@ func (mg *metricGroup) toSummaryPoint(dest pmetric.SummaryDataPointSlice) { // The timestamp MUST be in retrieved from milliseconds and converted to nanoseconds. tsNanos := timestampFromMs(mg.ts) point.SetTimestamp(tsNanos) - point.SetStartTimestamp(tsNanos) // metrics_adjuster adjusts the startTimestamp to the initial scrape timestamp + if mg.created != 0 { + point.SetStartTimestamp(timestampFromFloat64(mg.created)) + } else { + // metrics_adjuster adjusts the startTimestamp to the initial scrape timestamp + point.SetStartTimestamp(tsNanos) + } populateAttributes(pmetric.MetricTypeSummary, mg.ls, point.Attributes()) } @@ -191,8 +218,13 @@ func (mg *metricGroup) toNumberDataPoint(dest pmetric.NumberDataPointSlice) { tsNanos := timestampFromMs(mg.ts) point := dest.AppendEmpty() // gauge/undefined types have no start time. - if mg.family.mtype == pmetric.MetricTypeSum { - point.SetStartTimestamp(tsNanos) // metrics_adjuster adjusts the startTimestamp to the initial scrape timestamp + if mg.mtype == pmetric.MetricTypeSum { + if mg.created != 0 { + point.SetStartTimestamp(timestampFromFloat64(mg.created)) + } else { + // metrics_adjuster adjusts the startTimestamp to the initial scrape timestamp + point.SetStartTimestamp(tsNanos) + } } point.SetTimestamp(tsNanos) if value.IsStaleNaN(mg.value) { @@ -201,6 +233,7 @@ func (mg *metricGroup) toNumberDataPoint(dest pmetric.NumberDataPointSlice) { point.SetDoubleValue(mg.value) } populateAttributes(pmetric.MetricTypeGauge, mg.ls, point.Attributes()) + mg.setExemplars(point.Exemplars()) } func populateAttributes(mType pmetric.MetricType, ls labels.Labels, dest pcommon.Map) { @@ -226,9 +259,10 @@ func (mf *metricFamily) loadMetricGroupOrCreate(groupKey uint64, ls labels.Label mg, ok := mf.groups[groupKey] if !ok { mg = &metricGroup{ - family: mf, - ts: ts, - ls: ls, + mtype: mf.mtype, + ts: ts, + ls: ls, + exemplars: pmetric.NewExemplarSlice(), } mf.groups[groupKey] = mg // maintaining data insertion order is helpful to generate stable/reproducible metric output @@ -237,9 +271,8 @@ func (mf *metricFamily) loadMetricGroupOrCreate(groupKey uint64, ls labels.Label return mg } -func (mf *metricFamily) Add(metricName string, ls labels.Labels, t int64, v float64) error { - groupKey := mf.getGroupKey(ls) - mg := mf.loadMetricGroupOrCreate(groupKey, ls, t) +func (mf *metricFamily) addSeries(seriesRef uint64, metricName string, ls labels.Labels, t int64, v float64) error { + mg := mf.loadMetricGroupOrCreate(seriesRef, ls, t) if mg.ts != t { return fmt.Errorf("inconsistent timestamps on metric points for metric %v", metricName) } @@ -254,6 +287,8 @@ func (mf *metricFamily) Add(metricName string, ls labels.Labels, t int64, v floa mg.ts = t mg.count = v mg.hasCount = true + case strings.HasSuffix(metricName, metricSuffixCreated): + mg.created = v default: boundary, err := getBoundary(mf.mtype, ls) if err != nil { @@ -261,6 +296,14 @@ func (mf *metricFamily) Add(metricName string, ls labels.Labels, t int64, v floa } mg.complexValue = append(mg.complexValue, &dataPoint{value: v, boundary: boundary}) } + case pmetric.MetricTypeSum: + if strings.HasSuffix(metricName, metricSuffixCreated) { + mg.created = v + } else { + mg.value = v + } + case pmetric.MetricTypeEmpty, pmetric.MetricTypeGauge, pmetric.MetricTypeExponentialHistogram: + fallthrough default: mg.value = v } @@ -268,13 +311,18 @@ func (mf *metricFamily) Add(metricName string, ls labels.Labels, t int64, v floa return nil } -func (mf *metricFamily) appendMetric(metrics pmetric.MetricSlice) { +func (mf *metricFamily) appendMetric(metrics pmetric.MetricSlice, trimSuffixes bool) { metric := pmetric.NewMetric() - metric.SetName(mf.name) + // Trims type and unit suffixes from metric name + name := mf.name + if trimSuffixes { + name = prometheus.TrimPromSuffixes(name, mf.mtype, mf.metadata.Unit) + } + metric.SetName(name) metric.SetDescription(mf.metadata.Help) - metric.SetUnit(mf.metadata.Unit) + metric.SetUnit(prometheus.UnitWordToUCUM(mf.metadata.Unit)) - pointCount := 0 + var pointCount int switch mf.mtype { case pmetric.MetricTypeHistogram: @@ -304,6 +352,8 @@ func (mf *metricFamily) appendMetric(metrics pmetric.MetricSlice) { } pointCount = sdpL.Len() + case pmetric.MetricTypeEmpty, pmetric.MetricTypeGauge, pmetric.MetricTypeExponentialHistogram: + fallthrough default: // Everything else should be set to a Gauge. gauge := metric.SetEmptyGauge() gdpL := gauge.DataPoints() @@ -319,3 +369,58 @@ func (mf *metricFamily) appendMetric(metrics pmetric.MetricSlice) { metric.MoveTo(metrics.AppendEmpty()) } + +func (mf *metricFamily) addExemplar(seriesRef uint64, e exemplar.Exemplar) { + mg := mf.groups[seriesRef] + if mg == nil { + return + } + es := mg.exemplars + convertExemplar(e, es.AppendEmpty()) +} + +func convertExemplar(pe exemplar.Exemplar, e pmetric.Exemplar) { + e.SetTimestamp(timestampFromMs(pe.Ts)) + e.SetDoubleValue(pe.Value) + e.FilteredAttributes().EnsureCapacity(len(pe.Labels)) + for _, lb := range pe.Labels { + switch strings.ToLower(lb.Name) { + case traceIDKey: + var tid [16]byte + err := decodeAndCopyToLowerBytes(tid[:], []byte(lb.Value)) + if err == nil { + e.SetTraceID(tid) + } else { + e.FilteredAttributes().PutStr(lb.Name, lb.Value) + } + case spanIDKey: + var sid [8]byte + err := decodeAndCopyToLowerBytes(sid[:], []byte(lb.Value)) + if err == nil { + e.SetSpanID(sid) + } else { + e.FilteredAttributes().PutStr(lb.Name, lb.Value) + } + default: + e.FilteredAttributes().PutStr(lb.Name, lb.Value) + } + } +} + +/* + decodeAndCopyToLowerBytes copies src to dst on lower bytes instead of higher + +1. If len(src) > len(dst) -> copy first len(dst) bytes as it is. Example -> src = []byte{0xab,0xcd,0xef,0xgh,0xij}, dst = [2]byte, result dst = [2]byte{0xab, 0xcd} +2. If len(src) = len(dst) -> copy src to dst as it is +3. If len(src) < len(dst) -> prepend required 0s and then add src to dst. Example -> src = []byte{0xab, 0xcd}, dst = [8]byte, result dst = [8]byte{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xab, 0xcd} +*/ +func decodeAndCopyToLowerBytes(dst []byte, src []byte) error { + var err error + decodedLen := hex.DecodedLen(len(src)) + if decodedLen >= len(dst) { + _, err = hex.Decode(dst, src[:hex.EncodedLen(len(dst))]) + } else { + _, err = hex.Decode(dst[len(dst)-decodedLen:], src) + } + return err +} diff --git a/component/otelcol/receiver/prometheus/internal/metricfamily_test.go b/component/otelcol/receiver/prometheus/internal/metricfamily_test.go index 4dc336c5c4fa..10c0f9579480 100644 --- a/component/otelcol/receiver/prometheus/internal/metricfamily_test.go +++ b/component/otelcol/receiver/prometheus/internal/metricfamily_test.go @@ -1,16 +1,5 @@ // Copyright The OpenTelemetry Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. +// SPDX-License-Identifier: Apache-2.0 package internal @@ -69,6 +58,12 @@ var mc = testMetadataStore{ Help: "This is some help for a histogram", Unit: "ms", }, + "histogram_with_created": scrape.MetricMetadata{ + Metric: "hg", + Type: textparse.MetricTypeHistogram, + Help: "This is some help for a histogram", + Unit: "ms", + }, "histogram_stale": scrape.MetricMetadata{ Metric: "hg_stale", Type: textparse.MetricTypeHistogram, @@ -81,6 +76,12 @@ var mc = testMetadataStore{ Help: "This is some help for a summary", Unit: "ms", }, + "summary_with_created": scrape.MetricMetadata{ + Metric: "s", + Type: textparse.MetricTypeSummary, + Help: "This is some help for a summary", + Unit: "ms", + }, "summary_stale": scrape.MetricMetadata{ Metric: "s_stale", Type: textparse.MetricTypeSummary, @@ -137,6 +138,49 @@ func TestMetricGroupData_toDistributionUnitTest(t *testing.T) { return point }, }, + { + name: "histogram with startTimestamp from _created", + metricName: "histogram_with_created", + intervalStartTimeMs: 11, + labels: labels.FromMap(map[string]string{"a": "A"}), + scrapes: []*scrape{ + {at: 11, value: 66, metric: "histogram_with_created_count"}, + {at: 11, value: 1004.78, metric: "histogram_with_created_sum"}, + {at: 11, value: 600.78, metric: "histogram_with_created_created"}, + { + at: 11, + value: 33, + metric: "histogram_with_created_bucket", + extraLabel: labels.Label{Name: "le", Value: "0.75"}, + }, + { + at: 11, + value: 55, + metric: "histogram_with_created_bucket", + extraLabel: labels.Label{Name: "le", Value: "2.75"}, + }, + { + at: 11, + value: 66, + metric: "histogram_with_created_bucket", + extraLabel: labels.Label{Name: "le", Value: "+Inf"}}, + }, + want: func() pmetric.HistogramDataPoint { + point := pmetric.NewHistogramDataPoint() + point.SetCount(66) + point.SetSum(1004.78) + + // the time in milliseconds -> nanoseconds. + point.SetTimestamp(pcommon.Timestamp(11 * time.Millisecond)) + point.SetStartTimestamp(timestampFromFloat64(600.78)) + + point.ExplicitBounds().FromRaw([]float64{0.75, 2.75}) + point.BucketCounts().FromRaw([]uint64{33, 22, 11}) + attributes := point.Attributes() + attributes.PutStr("a", "A") + return point + }, + }, { name: "histogram that is stale", metricName: "histogram_stale", @@ -174,6 +218,28 @@ func TestMetricGroupData_toDistributionUnitTest(t *testing.T) { }, wantErr: true, }, + { + name: "histogram without buckets", + metricName: "histogram", + intervalStartTimeMs: 11, + labels: labels.FromMap(map[string]string{"a": "A", "b": "B"}), + scrapes: []*scrape{ + {at: 11, value: 66, metric: "histogram_count"}, + {at: 11, value: 1004.78, metric: "histogram_sum"}, + }, + want: func() pmetric.HistogramDataPoint { + point := pmetric.NewHistogramDataPoint() + point.SetCount(66) + point.SetSum(1004.78) + point.SetTimestamp(pcommon.Timestamp(11 * time.Millisecond)) // the time in milliseconds -> nanoseconds. + point.SetStartTimestamp(pcommon.Timestamp(11 * time.Millisecond)) // the time in milliseconds -> nanoseconds. + point.BucketCounts().FromRaw([]uint64{66}) + attributes := point.Attributes() + attributes.PutStr("a", "A") + attributes.PutStr("b", "B") + return point + }, + }, } for _, tt := range tests { @@ -187,7 +253,8 @@ func TestMetricGroupData_toDistributionUnitTest(t *testing.T) { } else { lbls = tt.labels.Copy() } - err := mp.Add(tv.metric, lbls, tv.at, tv.value) + sRef, _ := getSeriesRef(nil, lbls, mp.mtype) + err := mp.addSeries(sRef, tv.metric, lbls, tv.at, tv.value) if tt.wantErr { if i != 0 { require.Error(t, err) @@ -202,11 +269,9 @@ func TestMetricGroupData_toDistributionUnitTest(t *testing.T) { } require.Len(t, mp.groups, 1) - groupKey := mp.getGroupKey(tt.labels.Copy()) - require.NotNil(t, mp.groups[groupKey]) sl := pmetric.NewMetricSlice() - mp.appendMetric(sl) + mp.appendMetric(sl, false) require.Equal(t, 1, sl.Len(), "Exactly one metric expected") metric := sl.At(0) @@ -308,6 +373,79 @@ func TestMetricGroupData_toSummaryUnitTest(t *testing.T) { return point }, }, + { + name: "summary_with_created", + labelsScrapes: []*labelsScrapes{ + { + labels: labels.FromMap(map[string]string{"a": "A", "b": "B"}), + scrapes: []*scrape{ + {at: 14, value: 10, metric: "summary_with_created_count"}, + {at: 14, value: 15, metric: "summary_with_created_sum"}, + {at: 14, value: 150, metric: "summary_with_created_created"}, + }, + }, + { + labels: labels.FromMap(map[string]string{"a": "A", "quantile": "0.0", "b": "B"}), + scrapes: []*scrape{ + {at: 14, value: 8, metric: "value"}, + }, + }, + { + labels: labels.FromMap(map[string]string{"a": "A", "quantile": "0.75", "b": "B"}), + scrapes: []*scrape{ + {at: 14, value: 33.7, metric: "value"}, + }, + }, + { + labels: labels.FromMap(map[string]string{"a": "A", "quantile": "0.50", "b": "B"}), + scrapes: []*scrape{ + {at: 14, value: 27, metric: "value"}, + }, + }, + { + labels: labels.FromMap(map[string]string{"a": "A", "quantile": "0.90", "b": "B"}), + scrapes: []*scrape{ + {at: 14, value: 56, metric: "value"}, + }, + }, + { + labels: labels.FromMap(map[string]string{"a": "A", "quantile": "0.99", "b": "B"}), + scrapes: []*scrape{ + {at: 14, value: 82, metric: "value"}, + }, + }, + }, + want: func() pmetric.SummaryDataPoint { + point := pmetric.NewSummaryDataPoint() + point.SetCount(10) + point.SetSum(15) + qtL := point.QuantileValues() + qn0 := qtL.AppendEmpty() + qn0.SetQuantile(0) + qn0.SetValue(8) + qn50 := qtL.AppendEmpty() + qn50.SetQuantile(.5) + qn50.SetValue(27) + qn75 := qtL.AppendEmpty() + qn75.SetQuantile(.75) + qn75.SetValue(33.7) + qn90 := qtL.AppendEmpty() + qn90.SetQuantile(.9) + qn90.SetValue(56) + qn99 := qtL.AppendEmpty() + qn99.SetQuantile(.99) + qn99.SetValue(82) + + // the time in milliseconds -> nanoseconds. + point.SetTimestamp(pcommon.Timestamp(14 * time.Millisecond)) + point.SetStartTimestamp(timestampFromFloat64(150)) + + attributes := point.Attributes() + attributes.PutStr("a", "A") + attributes.PutStr("b", "B") + return point + }, + }, { name: "summary_stale", labelsScrapes: []*labelsScrapes{ @@ -400,7 +538,9 @@ func TestMetricGroupData_toSummaryUnitTest(t *testing.T) { mp := newMetricFamily(tt.name, mc, zap.NewNop()) for _, lbs := range tt.labelsScrapes { for i, scrape := range lbs.scrapes { - err := mp.Add(scrape.metric, lbs.labels.Copy(), scrape.at, scrape.value) + lb := lbs.labels.Copy() + sRef, _ := getSeriesRef(nil, lb, mp.mtype) + err := mp.addSeries(sRef, scrape.metric, lb, scrape.at, scrape.value) if tt.wantErr { // The first scrape won't have an error if i != 0 { @@ -417,11 +557,9 @@ func TestMetricGroupData_toSummaryUnitTest(t *testing.T) { } require.Len(t, mp.groups, 1) - groupKey := mp.getGroupKey(tt.labelsScrapes[0].labels.Copy()) - require.NotNil(t, mp.groups[groupKey]) sl := pmetric.NewMetricSlice() - mp.appendMetric(sl) + mp.appendMetric(sl, false) require.Equal(t, 1, sl.Len(), "Exactly one metric expected") metric := sl.At(0) @@ -451,6 +589,29 @@ func TestMetricGroupData_toNumberDataUnitTest(t *testing.T) { intervalStartTimestampMs int64 want func() pmetric.NumberDataPoint }{ + { + metricKind: "counter", + name: "counter:: startTimestampMs from _created", + intervalStartTimestampMs: 11, + labels: labels.FromMap(map[string]string{"a": "A", "b": "B"}), + scrapes: []*scrape{ + {at: 13, value: 33.7, metric: "value"}, + {at: 13, value: 150, metric: "value_created"}, + }, + want: func() pmetric.NumberDataPoint { + point := pmetric.NewNumberDataPoint() + point.SetDoubleValue(33.7) + + // the time in milliseconds -> nanoseconds. + point.SetTimestamp(pcommon.Timestamp(13 * time.Millisecond)) + point.SetStartTimestamp(timestampFromFloat64(150)) + + attributes := point.Attributes() + attributes.PutStr("a", "A") + attributes.PutStr("b", "B") + return point + }, + }, { metricKind: "counter", name: "counter:: startTimestampMs of 11", @@ -496,15 +657,15 @@ func TestMetricGroupData_toNumberDataUnitTest(t *testing.T) { t.Run(tt.name, func(t *testing.T) { mp := newMetricFamily(tt.metricKind, mc, zap.NewNop()) for _, tv := range tt.scrapes { - require.NoError(t, mp.Add(tv.metric, tt.labels.Copy(), tv.at, tv.value)) + lb := tt.labels.Copy() + sRef, _ := getSeriesRef(nil, lb, mp.mtype) + require.NoError(t, mp.addSeries(sRef, tv.metric, lb, tv.at, tv.value)) } require.Len(t, mp.groups, 1) - groupKey := mp.getGroupKey(tt.labels.Copy()) - require.NotNil(t, mp.groups[groupKey]) sl := pmetric.NewMetricSlice() - mp.appendMetric(sl) + mp.appendMetric(sl, false) require.Equal(t, 1, sl.Len(), "Exactly one metric expected") metric := sl.At(0) diff --git a/component/otelcol/receiver/prometheus/internal/metrics_adjuster.go b/component/otelcol/receiver/prometheus/internal/metrics_adjuster.go index 1c0df5817a5b..a483b9588cff 100644 --- a/component/otelcol/receiver/prometheus/internal/metrics_adjuster.go +++ b/component/otelcol/receiver/prometheus/internal/metrics_adjuster.go @@ -1,16 +1,5 @@ // Copyright The OpenTelemetry Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. +// SPDX-License-Identifier: Apache-2.0 package internal // import "github.com/open-telemetry/opentelemetry-collector-contrib/receiver/prometheusreceiver/internal" @@ -19,16 +8,13 @@ import ( "sync" "time" - "github.com/open-telemetry/opentelemetry-collector-contrib/pkg/pdatautil" "go.opentelemetry.io/collector/pdata/pcommon" "go.opentelemetry.io/collector/pdata/pmetric" semconv "go.opentelemetry.io/collector/semconv/v1.6.1" "go.uber.org/zap" -) -// The code in this file has been heavily inspired by Otel Collector: -// https://github.com/open-telemetry/opentelemetry-collector-contrib/blob/main/receiver/prometheusreceiver/internal/metrics_adjuster.go -// In case of issues or changes check the file against the Collector to see if it was also updated. + "github.com/open-telemetry/opentelemetry-collector-contrib/pkg/pdatautil" +) // Notes on garbage collection (gc): // @@ -133,7 +119,7 @@ func (tsm *timeseriesMap) get(metric pmetric.Metric, kv pcommon.Map) (*timeserie return tsi, ok } -// Create a unique timeseries signature consisting of the metric name and label values. +// Create a unique string signature for attributes values sorted by attribute keys. func getAttributesSignature(m pcommon.Map) [16]byte { clearedMap := pcommon.NewMap() m.Range(func(k string, attrValue pcommon.Value) bool { @@ -238,8 +224,6 @@ func (jm *JobsMap) get(job, instance string) *timeseriesMap { return tsm2 } -// MetricsAdjuster adjusts the start time of metrics when converting between -// Prometheus and OTel. type MetricsAdjuster interface { AdjustMetrics(metrics pmetric.Metrics) error } @@ -248,21 +232,23 @@ type MetricsAdjuster interface { // and provides AdjustMetricSlice, which takes a sequence of metrics and adjust their start times based on // the initial points. type initialPointAdjuster struct { - jobsMap *JobsMap - logger *zap.Logger + jobsMap *JobsMap + logger *zap.Logger + useCreatedMetric bool } // NewInitialPointAdjuster returns a new MetricsAdjuster that adjust metrics' start times based on the initial received points. -func NewInitialPointAdjuster(logger *zap.Logger, gcInterval time.Duration) MetricsAdjuster { +func NewInitialPointAdjuster(logger *zap.Logger, gcInterval time.Duration, useCreatedMetric bool) MetricsAdjuster { return &initialPointAdjuster{ - jobsMap: NewJobsMap(gcInterval), - logger: logger, + jobsMap: NewJobsMap(gcInterval), + logger: logger, + useCreatedMetric: useCreatedMetric, } } // AdjustMetrics takes a sequence of metrics and adjust their start times based on the initial and // previous points in the timeseriesMap. -func (ma *initialPointAdjuster) AdjustMetrics(metrics pmetric.Metrics) error { +func (a *initialPointAdjuster) AdjustMetrics(metrics pmetric.Metrics) error { // By contract metrics will have at least 1 data point, so for sure will have at least one ResourceMetrics. job, found := metrics.ResourceMetrics().At(0).Resource().Attributes().Get(semconv.AttributeServiceName) @@ -274,7 +260,7 @@ func (ma *initialPointAdjuster) AdjustMetrics(metrics pmetric.Metrics) error { if !found { return errors.New("adjusting metrics without instance") } - tsm := ma.jobsMap.get(job.Str(), instance.Str()) + tsm := a.jobsMap.get(job.Str(), instance.Str()) // The lock on the relevant timeseriesMap is held throughout the adjustment process to ensure that // nothing else can modify the data used for adjustment. @@ -291,17 +277,20 @@ func (ma *initialPointAdjuster) AdjustMetrics(metrics pmetric.Metrics) error { // gauges don't need to be adjusted so no additional processing is necessary case pmetric.MetricTypeHistogram: - adjustMetricHistogram(tsm, metric) + a.adjustMetricHistogram(tsm, metric) case pmetric.MetricTypeSummary: - adjustMetricSummary(tsm, metric) + a.adjustMetricSummary(tsm, metric) case pmetric.MetricTypeSum: - adjustMetricSum(tsm, metric) + a.adjustMetricSum(tsm, metric) + + case pmetric.MetricTypeEmpty, pmetric.MetricTypeExponentialHistogram: + fallthrough default: // this shouldn't happen - ma.logger.Info("Adjust - skipping unexpected point", zap.String("type", dataType.String())) + a.logger.Info("Adjust - skipping unexpected point", zap.String("type", dataType.String())) } } } @@ -309,7 +298,7 @@ func (ma *initialPointAdjuster) AdjustMetrics(metrics pmetric.Metrics) error { return nil } -func adjustMetricHistogram(tsm *timeseriesMap, current pmetric.Metric) { +func (a *initialPointAdjuster) adjustMetricHistogram(tsm *timeseriesMap, current pmetric.Metric) { histogram := current.Histogram() if histogram.AggregationTemporality() != pmetric.AggregationTemporalityCumulative { // Only dealing with CumulativeDistributions. @@ -319,6 +308,15 @@ func adjustMetricHistogram(tsm *timeseriesMap, current pmetric.Metric) { currentPoints := histogram.DataPoints() for i := 0; i < currentPoints.Len(); i++ { currentDist := currentPoints.At(i) + + // start timestamp was set from _created + if a.useCreatedMetric && + !currentDist.Flags().NoRecordedValue() && + currentDist.StartTimestamp() < currentDist.Timestamp() { + + continue + } + tsi, found := tsm.get(current, currentDist.Attributes()) if !found { // initialize everything. @@ -349,10 +347,19 @@ func adjustMetricHistogram(tsm *timeseriesMap, current pmetric.Metric) { } } -func adjustMetricSum(tsm *timeseriesMap, current pmetric.Metric) { +func (a *initialPointAdjuster) adjustMetricSum(tsm *timeseriesMap, current pmetric.Metric) { currentPoints := current.Sum().DataPoints() for i := 0; i < currentPoints.Len(); i++ { currentSum := currentPoints.At(i) + + // start timestamp was set from _created + if a.useCreatedMetric && + !currentSum.Flags().NoRecordedValue() && + currentSum.StartTimestamp() < currentSum.Timestamp() { + + continue + } + tsi, found := tsm.get(current, currentSum.Attributes()) if !found { // initialize everything. @@ -380,11 +387,20 @@ func adjustMetricSum(tsm *timeseriesMap, current pmetric.Metric) { } } -func adjustMetricSummary(tsm *timeseriesMap, current pmetric.Metric) { +func (a *initialPointAdjuster) adjustMetricSummary(tsm *timeseriesMap, current pmetric.Metric) { currentPoints := current.Summary().DataPoints() for i := 0; i < currentPoints.Len(); i++ { currentSummary := currentPoints.At(i) + + // start timestamp was set from _created + if a.useCreatedMetric && + !currentSummary.Flags().NoRecordedValue() && + currentSummary.StartTimestamp() < currentSummary.Timestamp() { + + continue + } + tsi, found := tsm.get(current, currentSummary.Attributes()) if !found { // initialize everything. diff --git a/component/otelcol/receiver/prometheus/internal/metrics_adjuster_test.go b/component/otelcol/receiver/prometheus/internal/metrics_adjuster_test.go index 2504bc721226..df38dea9e968 100644 --- a/component/otelcol/receiver/prometheus/internal/metrics_adjuster_test.go +++ b/component/otelcol/receiver/prometheus/internal/metrics_adjuster_test.go @@ -1,16 +1,5 @@ // Copyright The OpenTelemetry Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. +// SPDX-License-Identifier: Apache-2.0 package internal @@ -19,7 +8,7 @@ import ( "time" "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" + "go.opentelemetry.io/collector/pdata/pcommon" "go.opentelemetry.io/collector/pdata/pmetric" semconv "go.opentelemetry.io/collector/semconv/v1.8.0" "go.uber.org/zap" @@ -79,7 +68,7 @@ func TestGauge(t *testing.T) { adjusted: metrics(gaugeMetric(gauge1, doublePoint(k1v1k2v2, t3, t3, 55))), }, } - runScript(t, NewInitialPointAdjuster(zap.NewNop(), time.Minute), "job", "0", script) + runScript(t, NewInitialPointAdjuster(zap.NewNop(), time.Minute, true), "job", "0", script) } func TestSum(t *testing.T) { @@ -110,7 +99,7 @@ func TestSum(t *testing.T) { adjusted: metrics(sumMetric(sum1, doublePoint(k1v1k2v2, t3, t5, 72))), }, } - runScript(t, NewInitialPointAdjuster(zap.NewNop(), time.Minute), "job", "0", script) + runScript(t, NewInitialPointAdjuster(zap.NewNop(), time.Minute, true), "job", "0", script) } func TestSummaryNoCount(t *testing.T) { @@ -137,7 +126,7 @@ func TestSummaryNoCount(t *testing.T) { }, } - runScript(t, NewInitialPointAdjuster(zap.NewNop(), time.Minute), "job", "0", script) + runScript(t, NewInitialPointAdjuster(zap.NewNop(), time.Minute, true), "job", "0", script) } func TestSummaryFlagNoRecordedValue(t *testing.T) { @@ -154,7 +143,7 @@ func TestSummaryFlagNoRecordedValue(t *testing.T) { }, } - runScript(t, NewInitialPointAdjuster(zap.NewNop(), time.Minute), "job", "0", script) + runScript(t, NewInitialPointAdjuster(zap.NewNop(), time.Minute, true), "job", "0", script) } func TestSummary(t *testing.T) { @@ -197,7 +186,7 @@ func TestSummary(t *testing.T) { }, } - runScript(t, NewInitialPointAdjuster(zap.NewNop(), time.Minute), "job", "0", script) + runScript(t, NewInitialPointAdjuster(zap.NewNop(), time.Minute, true), "job", "0", script) } func TestHistogram(t *testing.T) { @@ -220,7 +209,7 @@ func TestHistogram(t *testing.T) { adjusted: metrics(histogramMetric(histogram1, histogramPoint(k1v1k2v2, t3, t4, bounds0, []uint64{7, 4, 2, 12}))), }, } - runScript(t, NewInitialPointAdjuster(zap.NewNop(), time.Minute), "job", "0", script) + runScript(t, NewInitialPointAdjuster(zap.NewNop(), time.Minute, true), "job", "0", script) } func TestHistogramFlagNoRecordedValue(t *testing.T) { @@ -237,7 +226,7 @@ func TestHistogramFlagNoRecordedValue(t *testing.T) { }, } - runScript(t, NewInitialPointAdjuster(zap.NewNop(), time.Minute), "job", "0", script) + runScript(t, NewInitialPointAdjuster(zap.NewNop(), time.Minute, true), "job", "0", script) } func TestHistogramFlagNoRecordedValueFirstObservation(t *testing.T) { @@ -254,7 +243,7 @@ func TestHistogramFlagNoRecordedValueFirstObservation(t *testing.T) { }, } - runScript(t, NewInitialPointAdjuster(zap.NewNop(), time.Minute), "job", "0", script) + runScript(t, NewInitialPointAdjuster(zap.NewNop(), time.Minute, true), "job", "0", script) } func TestSummaryFlagNoRecordedValueFirstObservation(t *testing.T) { @@ -271,7 +260,7 @@ func TestSummaryFlagNoRecordedValueFirstObservation(t *testing.T) { }, } - runScript(t, NewInitialPointAdjuster(zap.NewNop(), time.Minute), "job", "0", script) + runScript(t, NewInitialPointAdjuster(zap.NewNop(), time.Minute, true), "job", "0", script) } func TestGaugeFlagNoRecordedValueFirstObservation(t *testing.T) { @@ -288,7 +277,7 @@ func TestGaugeFlagNoRecordedValueFirstObservation(t *testing.T) { }, } - runScript(t, NewInitialPointAdjuster(zap.NewNop(), time.Minute), "job", "0", script) + runScript(t, NewInitialPointAdjuster(zap.NewNop(), time.Minute, true), "job", "0", script) } func TestSumFlagNoRecordedValueFirstObservation(t *testing.T) { @@ -305,7 +294,7 @@ func TestSumFlagNoRecordedValueFirstObservation(t *testing.T) { }, } - runScript(t, NewInitialPointAdjuster(zap.NewNop(), time.Minute), "job", "0", script) + runScript(t, NewInitialPointAdjuster(zap.NewNop(), time.Minute, true), "job", "0", script) } func TestMultiMetrics(t *testing.T) { @@ -369,7 +358,7 @@ func TestMultiMetrics(t *testing.T) { ), }, } - runScript(t, NewInitialPointAdjuster(zap.NewNop(), time.Minute), "job", "0", script) + runScript(t, NewInitialPointAdjuster(zap.NewNop(), time.Minute, true), "job", "0", script) } func TestNewDataPointsAdded(t *testing.T) { @@ -431,7 +420,7 @@ func TestNewDataPointsAdded(t *testing.T) { ), }, } - runScript(t, NewInitialPointAdjuster(zap.NewNop(), time.Minute), "job", "0", script) + runScript(t, NewInitialPointAdjuster(zap.NewNop(), time.Minute, true), "job", "0", script) } func TestMultiTimeseries(t *testing.T) { @@ -490,7 +479,7 @@ func TestMultiTimeseries(t *testing.T) { ), }, } - runScript(t, NewInitialPointAdjuster(zap.NewNop(), time.Minute), "job", "0", script) + runScript(t, NewInitialPointAdjuster(zap.NewNop(), time.Minute, true), "job", "0", script) } func TestEmptyLabels(t *testing.T) { @@ -516,7 +505,7 @@ func TestEmptyLabels(t *testing.T) { adjusted: metrics(sumMetric(sum1, doublePoint(k1vEmptyk2vEmptyk3vEmpty, t1, t3, 88))), }, } - runScript(t, NewInitialPointAdjuster(zap.NewNop(), time.Minute), "job", "0", script) + runScript(t, NewInitialPointAdjuster(zap.NewNop(), time.Minute, true), "job", "0", script) } func TestTsGC(t *testing.T) { @@ -570,7 +559,7 @@ func TestTsGC(t *testing.T) { }, } - ma := NewInitialPointAdjuster(zap.NewNop(), time.Minute) + ma := NewInitialPointAdjuster(zap.NewNop(), time.Minute, true) // run round 1 runScript(t, ma, "job", "0", script1) @@ -630,7 +619,7 @@ func TestJobGC(t *testing.T) { } gcInterval := 10 * time.Millisecond - ma := NewInitialPointAdjuster(zap.NewNop(), gcInterval) + ma := NewInitialPointAdjuster(zap.NewNop(), gcInterval, true) // run job 1, round 1 - all entries marked runScript(t, ma, "job1", "0", job1Script1) @@ -654,13 +643,6 @@ type metricsAdjusterTest struct { adjusted pmetric.Metrics } -func marshalMetric(t *testing.T, m pmetric.Metrics) string { - jm := &pmetric.JSONMarshaler{} - bytes, err := jm.MarshalMetrics(m) - assert.NoError(t, err) - return string(bytes) -} - func runScript(t *testing.T, ma MetricsAdjuster, job, instance string, tests []*metricsAdjusterTest) { for _, test := range tests { t.Run(test.description, func(t *testing.T) { @@ -674,7 +656,26 @@ func runScript(t *testing.T, ma MetricsAdjuster, job, instance string, tests []* // Add the instance/job to the expected metrics as well. test.adjusted.ResourceMetrics().At(0).Resource().Attributes().PutStr(semconv.AttributeServiceInstanceID, instance) test.adjusted.ResourceMetrics().At(0).Resource().Attributes().PutStr(semconv.AttributeServiceName, job) - require.JSONEq(t, marshalMetric(t, test.adjusted), marshalMetric(t, adjusted)) + assert.EqualValues(t, test.adjusted, adjusted) }) } } + +func BenchmarkGetAttributesSignature(b *testing.B) { + attrs := pcommon.NewMap() + attrs.PutStr("key1", "some-random-test-value-1") + attrs.PutStr("key2", "some-random-test-value-2") + attrs.PutStr("key6", "some-random-test-value-6") + attrs.PutStr("key3", "some-random-test-value-3") + attrs.PutStr("key4", "some-random-test-value-4") + attrs.PutStr("key5", "some-random-test-value-5") + attrs.PutStr("key7", "some-random-test-value-7") + attrs.PutStr("key8", "some-random-test-value-8") + + b.ResetTimer() + b.ReportAllocs() + + for i := 0; i < b.N; i++ { + getAttributesSignature(attrs) + } +} diff --git a/component/otelcol/receiver/prometheus/internal/metricsutil_test.go b/component/otelcol/receiver/prometheus/internal/metricsutil_test.go index ea29c0e61171..4ba25cfe846e 100644 --- a/component/otelcol/receiver/prometheus/internal/metricsutil_test.go +++ b/component/otelcol/receiver/prometheus/internal/metricsutil_test.go @@ -1,16 +1,5 @@ // Copyright The OpenTelemetry Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. +// SPDX-License-Identifier: Apache-2.0 package internal diff --git a/component/otelcol/receiver/prometheus/internal/prom_to_otlp.go b/component/otelcol/receiver/prometheus/internal/prom_to_otlp.go index 9b14cd9d053c..1b0d00a589ea 100644 --- a/component/otelcol/receiver/prometheus/internal/prom_to_otlp.go +++ b/component/otelcol/receiver/prometheus/internal/prom_to_otlp.go @@ -1,16 +1,5 @@ // Copyright The OpenTelemetry Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. +// SPDX-License-Identifier: Apache-2.0 package internal // import "github.com/open-telemetry/opentelemetry-collector-contrib/receiver/prometheusreceiver/internal" diff --git a/component/otelcol/receiver/prometheus/internal/prom_to_otlp_test.go b/component/otelcol/receiver/prometheus/internal/prom_to_otlp_test.go index 63f167ca5363..a532637e1e2d 100644 --- a/component/otelcol/receiver/prometheus/internal/prom_to_otlp_test.go +++ b/component/otelcol/receiver/prometheus/internal/prom_to_otlp_test.go @@ -1,16 +1,5 @@ // Copyright The OpenTelemetry Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. +// SPDX-License-Identifier: Apache-2.0 package internal @@ -284,7 +273,6 @@ func TestCreateNodeAndResourcePromToOTLP(t *testing.T) { t.Run(tt.name, func(t *testing.T) { got := CreateResource(tt.job, tt.instance, tt.sdLabels) require.Equal(t, tt.want.Attributes().AsRaw(), got.Attributes().AsRaw()) - require.Equal(t, tt.want.DroppedAttributesCount(), got.DroppedAttributesCount()) }) } } diff --git a/component/otelcol/receiver/prometheus/internal/staleness_end_to_end_test.go b/component/otelcol/receiver/prometheus/internal/staleness_end_to_end_test.go new file mode 100644 index 000000000000..224c3cd4e1f0 --- /dev/null +++ b/component/otelcol/receiver/prometheus/internal/staleness_end_to_end_test.go @@ -0,0 +1,241 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +package internal_test + +import ( + "context" + "fmt" + "io" + "net/http" + "net/http/httptest" + "net/url" + "os" + "strings" + "testing" + "time" + + "go.uber.org/atomic" + + "github.com/gogo/protobuf/proto" + "github.com/golang/snappy" + "github.com/prometheus/prometheus/model/value" + "github.com/prometheus/prometheus/prompb" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.opentelemetry.io/collector/component" + "go.opentelemetry.io/collector/confmap" + "go.opentelemetry.io/collector/confmap/provider/fileprovider" + "go.opentelemetry.io/collector/exporter" + "go.opentelemetry.io/collector/otelcol" + "go.opentelemetry.io/collector/processor" + "go.opentelemetry.io/collector/processor/batchprocessor" + "go.opentelemetry.io/collector/receiver" + "go.uber.org/zap" + "go.uber.org/zap/zapcore" + + "github.com/open-telemetry/opentelemetry-collector-contrib/exporter/prometheusremotewriteexporter" + "github.com/open-telemetry/opentelemetry-collector-contrib/receiver/prometheusreceiver" +) + +// Test that staleness markers are emitted for timeseries that intermittently disappear. +// This test runs the entire collector and end-to-end scrapes then checks with the +// Prometheus remotewrite exporter that staleness markers are emitted per timeseries. +// See https://github.com/open-telemetry/opentelemetry-collector/issues/3413 +func TestStalenessMarkersEndToEnd(t *testing.T) { + if testing.Short() { + t.Skip("This test can take a long time") + } + + ctx, cancel := context.WithCancel(context.Background()) + + // 1. Setup the server that sends series that intermittently appear and disappear. + n := &atomic.Uint64{} + scrapeServer := httptest.NewServer(http.HandlerFunc(func(rw http.ResponseWriter, req *http.Request) { + // Increment the scrape count atomically per scrape. + i := n.Add(1) + + select { + case <-ctx.Done(): + return + default: + } + + // Alternate metrics per scrape so that every one of + // them will be reported as stale. + if i%2 == 0 { + fmt.Fprintf(rw, ` +# HELP jvm_memory_bytes_used Used bytes of a given JVM memory area. +# TYPE jvm_memory_bytes_used gauge +jvm_memory_bytes_used{area="heap"} %.1f`, float64(i)) + } else { + fmt.Fprintf(rw, ` +# HELP jvm_memory_pool_bytes_used Used bytes of a given JVM memory pool. +# TYPE jvm_memory_pool_bytes_used gauge +jvm_memory_pool_bytes_used{pool="CodeHeap 'non-nmethods'"} %.1f`, float64(i)) + } + })) + defer scrapeServer.Close() + + serverURL, err := url.Parse(scrapeServer.URL) + require.NoError(t, err) + + // 2. Set up the Prometheus RemoteWrite endpoint. + prweUploads := make(chan *prompb.WriteRequest) + prweServer := httptest.NewServer(http.HandlerFunc(func(rw http.ResponseWriter, req *http.Request) { + // Snappy decode the uploads. + payload, rerr := io.ReadAll(req.Body) + require.NoError(t, rerr) + + recv := make([]byte, len(payload)) + decoded, derr := snappy.Decode(recv, payload) + require.NoError(t, derr) + + writeReq := new(prompb.WriteRequest) + require.NoError(t, proto.Unmarshal(decoded, writeReq)) + + select { + case <-ctx.Done(): + return + case prweUploads <- writeReq: + } + })) + defer prweServer.Close() + + // 3. Set the OpenTelemetry Prometheus receiver. + cfg := fmt.Sprintf(` +receivers: + prometheus: + config: + scrape_configs: + - job_name: 'test' + scrape_interval: 100ms + static_configs: + - targets: [%q] + +processors: + batch: +exporters: + prometheusremotewrite: + endpoint: %q + tls: + insecure: true + +service: + pipelines: + metrics: + receivers: [prometheus] + processors: [batch] + exporters: [prometheusremotewrite]`, serverURL.Host, prweServer.URL) + + confFile, err := os.CreateTemp(os.TempDir(), "conf-") + require.Nil(t, err) + defer os.Remove(confFile.Name()) + _, err = confFile.Write([]byte(cfg)) + require.Nil(t, err) + // 4. Run the OpenTelemetry Collector. + receivers, err := receiver.MakeFactoryMap(prometheusreceiver.NewFactory()) + require.Nil(t, err) + exporters, err := exporter.MakeFactoryMap(prometheusremotewriteexporter.NewFactory()) + require.Nil(t, err) + processors, err := processor.MakeFactoryMap(batchprocessor.NewFactory()) + require.Nil(t, err) + + factories := otelcol.Factories{ + Receivers: receivers, + Exporters: exporters, + Processors: processors, + } + + fmp := fileprovider.New() + configProvider, err := otelcol.NewConfigProvider( + otelcol.ConfigProviderSettings{ + ResolverSettings: confmap.ResolverSettings{ + URIs: []string{confFile.Name()}, + Providers: map[string]confmap.Provider{fmp.Scheme(): fmp}, + }, + }) + require.NoError(t, err) + + appSettings := otelcol.CollectorSettings{ + Factories: factories, + ConfigProvider: configProvider, + BuildInfo: component.BuildInfo{ + Command: "otelcol", + Description: "OpenTelemetry Collector", + Version: "tests", + }, + LoggingOptions: []zap.Option{ + // Turn off the verbose logging from the collector. + zap.WrapCore(func(zapcore.Core) zapcore.Core { + return zapcore.NewNopCore() + }), + }, + } + + app, err := otelcol.NewCollector(appSettings) + require.Nil(t, err) + + go func() { + assert.NoError(t, app.Run(context.Background())) + }() + defer app.Shutdown() + + // Wait until the collector has actually started. + for notYetStarted := true; notYetStarted; { + state := app.GetState() + switch state { + case otelcol.StateRunning, otelcol.StateClosed, otelcol.StateClosing: + notYetStarted = false + case otelcol.StateStarting: + } + time.Sleep(10 * time.Millisecond) + } + + // 5. Let's wait on 10 fetches. + var wReqL []*prompb.WriteRequest + for i := 0; i < 10; i++ { + wReqL = append(wReqL, <-prweUploads) + } + defer cancel() + + // 6. Assert that we encounter the stale markers aka special NaNs for the various time series. + staleMarkerCount := 0 + totalSamples := 0 + require.True(t, len(wReqL) > 0, "Expecting at least one WriteRequest") + for i, wReq := range wReqL { + name := fmt.Sprintf("WriteRequest#%d", i) + require.True(t, len(wReq.Timeseries) > 0, "Expecting at least 1 timeSeries for:: "+name) + for j, ts := range wReq.Timeseries { + fullName := fmt.Sprintf("%s/TimeSeries#%d", name, j) + assert.True(t, len(ts.Samples) > 0, "Expected at least 1 Sample in:: "+fullName) + + // We are strictly counting series directly included in the scrapes, and no + // internal timeseries like "up" nor "scrape_seconds" etc. + metricName := "" + for _, label := range ts.Labels { + if label.Name == "__name__" { + metricName = label.Value + } + } + if !strings.HasPrefix(metricName, "jvm") { + continue + } + + for _, sample := range ts.Samples { + totalSamples++ + if value.IsStaleNaN(sample.Value) { + staleMarkerCount++ + } + } + } + } + + require.True(t, totalSamples > 0, "Expected at least 1 sample") + // On every alternative scrape the prior scrape will be reported as sale. + // Expect at least: + // * The first scrape will NOT return stale markers + // * (N-1 / alternatives) = ((10-1) / 2) = ~40% chance of stale markers being emitted. + chance := float64(staleMarkerCount) / float64(totalSamples) + require.True(t, chance >= 0.4, fmt.Sprintf("Expected at least one stale marker: %.3f", chance)) +} diff --git a/component/otelcol/receiver/prometheus/internal/starttimemetricadjuster.go b/component/otelcol/receiver/prometheus/internal/starttimemetricadjuster.go index a3169049e132..9195136e7841 100644 --- a/component/otelcol/receiver/prometheus/internal/starttimemetricadjuster.go +++ b/component/otelcol/receiver/prometheus/internal/starttimemetricadjuster.go @@ -1,16 +1,5 @@ // Copyright The OpenTelemetry Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. +// SPDX-License-Identifier: Apache-2.0 package internal // import "github.com/open-telemetry/opentelemetry-collector-contrib/receiver/prometheusreceiver/internal" @@ -79,6 +68,9 @@ func (stma *startTimeMetricAdjuster) AdjustMetrics(metrics pmetric.Metrics) erro dp.SetStartTimestamp(startTimeTs) } + case pmetric.MetricTypeEmpty, pmetric.MetricTypeExponentialHistogram: + fallthrough + default: stma.logger.Warn("Unknown metric type", zap.String("type", metric.Type().String())) } @@ -110,6 +102,8 @@ func (stma *startTimeMetricAdjuster) getStartTime(metrics pmetric.Metrics) (floa } return metric.Sum().DataPoints().At(0).DoubleValue(), nil + case pmetric.MetricTypeEmpty, pmetric.MetricTypeHistogram, pmetric.MetricTypeExponentialHistogram, pmetric.MetricTypeSummary: + fallthrough default: return 0, errUnsupportedTypeStartTimeMetric } diff --git a/component/otelcol/receiver/prometheus/internal/starttimemetricadjuster_test.go b/component/otelcol/receiver/prometheus/internal/starttimemetricadjuster_test.go index 0d4e1c66a277..89e4b10f8e5f 100644 --- a/component/otelcol/receiver/prometheus/internal/starttimemetricadjuster_test.go +++ b/component/otelcol/receiver/prometheus/internal/starttimemetricadjuster_test.go @@ -1,16 +1,5 @@ // Copyright The OpenTelemetry Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. +// SPDX-License-Identifier: Apache-2.0 package internal @@ -150,6 +139,7 @@ func TestStartTimeMetricMatch(t *testing.T) { for l := 0; l < dps.Len(); l++ { assert.Equal(t, tt.expectedStartTime, dps.At(l).StartTimestamp()) } + case pmetric.MetricTypeEmpty, pmetric.MetricTypeGauge, pmetric.MetricTypeExponentialHistogram: } } } diff --git a/component/otelcol/receiver/prometheus/internal/transaction.go b/component/otelcol/receiver/prometheus/internal/transaction.go index d942e2947d57..6b59674c199d 100644 --- a/component/otelcol/receiver/prometheus/internal/transaction.go +++ b/component/otelcol/receiver/prometheus/internal/transaction.go @@ -1,16 +1,5 @@ // Copyright The OpenTelemetry Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. +// SPDX-License-Identifier: Apache-2.0 package internal // import "github.com/open-telemetry/opentelemetry-collector-contrib/receiver/prometheusreceiver/internal" @@ -28,6 +17,7 @@ import ( "github.com/prometheus/prometheus/model/value" "github.com/prometheus/prometheus/scrape" "github.com/prometheus/prometheus/storage" + "go.opentelemetry.io/collector/component" "go.opentelemetry.io/collector/consumer" "go.opentelemetry.io/collector/pdata/pcommon" "go.opentelemetry.io/collector/pdata/pmetric" @@ -37,20 +27,36 @@ import ( ) const ( - targetMetricName = "target_info" + targetMetricName = "target_info" + scopeMetricName = "otel_scope_info" + scopeNameLabel = "otel_scope_name" + scopeVersionLabel = "otel_scope_version" + receiverName = "otelcol/prometheusreceiver" ) type transaction struct { - isNew bool - ctx context.Context - families map[string]*metricFamily - mc scrape.MetricMetadataStore - sink consumer.Metrics - externalLabels labels.Labels - nodeResource pcommon.Resource - logger *zap.Logger - metricAdjuster MetricsAdjuster - obsrecv *receiverhelper.ObsReport + isNew bool + trimSuffixes bool + ctx context.Context + families map[scopeID]map[string]*metricFamily + mc scrape.MetricMetadataStore + sink consumer.Metrics + externalLabels labels.Labels + nodeResource pcommon.Resource + scopeAttributes map[scopeID]pcommon.Map + logger *zap.Logger + buildInfo component.BuildInfo + metricAdjuster MetricsAdjuster + obsrecv *receiverhelper.ObsReport + // Used as buffer to calculate series ref hash. + bufBytes []byte +} + +var emptyScopeID scopeID + +type scopeID struct { + name string + version string } func newTransaction( @@ -59,22 +65,27 @@ func newTransaction( sink consumer.Metrics, externalLabels labels.Labels, settings receiver.CreateSettings, - obsrecv *receiverhelper.ObsReport) *transaction { + obsrecv *receiverhelper.ObsReport, + trimSuffixes bool) *transaction { return &transaction{ - ctx: ctx, - families: make(map[string]*metricFamily), - isNew: true, - sink: sink, - metricAdjuster: metricAdjuster, - externalLabels: externalLabels, - logger: settings.Logger, - obsrecv: obsrecv, + ctx: ctx, + families: make(map[scopeID]map[string]*metricFamily), + isNew: true, + trimSuffixes: trimSuffixes, + sink: sink, + metricAdjuster: metricAdjuster, + externalLabels: externalLabels, + logger: settings.Logger, + buildInfo: settings.BuildInfo, + obsrecv: obsrecv, + bufBytes: make([]byte, 0, 1024), + scopeAttributes: make(map[scopeID]pcommon.Map), } } // Append always returns 0 to disable label caching. -func (t *transaction) Append(ref storage.SeriesRef, ls labels.Labels, atMs int64, val float64) (storage.SeriesRef, error) { +func (t *transaction) Append(_ storage.SeriesRef, ls labels.Labels, atMs int64, val float64) (storage.SeriesRef, error) { select { case <-t.ctx.Done(): return 0, errTransactionAborted @@ -123,27 +134,87 @@ func (t *transaction) Append(ref storage.SeriesRef, ls labels.Labels, atMs int64 // For the `target_info` metric we need to convert it to resource attributes. if metricName == targetMetricName { - return 0, t.AddTargetInfo(ls) + t.AddTargetInfo(ls) + return 0, nil + } + + // For the `otel_scope_info` metric we need to convert it to scope attributes. + if metricName == scopeMetricName { + t.addScopeInfo(ls) + return 0, nil + } + + curMF := t.getOrCreateMetricFamily(getScopeID(ls), metricName) + err := curMF.addSeries(t.getSeriesRef(ls, curMF.mtype), metricName, ls, atMs, val) + if err != nil { + t.logger.Warn("failed to add datapoint", zap.Error(err), zap.String("metric_name", metricName), zap.Any("labels", ls)) } - curMF, ok := t.families[metricName] + return 0, nil // never return errors, as that fails the whole scrape +} + +func (t *transaction) getOrCreateMetricFamily(scope scopeID, mn string) *metricFamily { + _, ok := t.families[scope] + if !ok { + t.families[scope] = make(map[string]*metricFamily) + } + curMf, ok := t.families[scope][mn] if !ok { - familyName := normalizeMetricName(metricName) - if mf, ok := t.families[familyName]; ok && mf.includesMetric(metricName) { - curMF = mf + fn := mn + if _, ok := t.mc.GetMetadata(mn); !ok { + fn = normalizeMetricName(mn) + } + if mf, ok := t.families[scope][fn]; ok && mf.includesMetric(mn) { + curMf = mf } else { - curMF = newMetricFamily(metricName, t.mc, t.logger) - t.families[curMF.name] = curMF + curMf = newMetricFamily(mn, t.mc, t.logger) + t.families[scope][curMf.name] = curMf + } + } + return curMf +} + +func (t *transaction) AppendExemplar(_ storage.SeriesRef, l labels.Labels, e exemplar.Exemplar) (storage.SeriesRef, error) { + select { + case <-t.ctx.Done(): + return 0, errTransactionAborted + default: + } + + if t.isNew { + if err := t.initTransaction(l); err != nil { + return 0, err } } - return 0, curMF.Add(metricName, ls, atMs, val) + l = l.WithoutEmpty() + + if dupLabel, hasDup := l.HasDuplicateLabelNames(); hasDup { + return 0, fmt.Errorf("invalid sample: non-unique label names: %q", dupLabel) + } + + mn := l.Get(model.MetricNameLabel) + if mn == "" { + return 0, errMetricNameNotFound + } + + mf := t.getOrCreateMetricFamily(getScopeID(l), mn) + mf.addExemplar(t.getSeriesRef(l, mf.mtype), e) + + return 0, nil } -func (t *transaction) AppendExemplar(ref storage.SeriesRef, l labels.Labels, e exemplar.Exemplar) (storage.SeriesRef, error) { +func (t *transaction) AppendHistogram(_ storage.SeriesRef, _ labels.Labels, _ int64, _ *histogram.Histogram, _ *histogram.FloatHistogram) (storage.SeriesRef, error) { + //TODO: implement this func return 0, nil } +func (t *transaction) getSeriesRef(ls labels.Labels, mtype pmetric.MetricType) uint64 { + var hash uint64 + hash, t.bufBytes = getSeriesRef(t.bufBytes, ls, mtype) + return hash +} + // getMetrics returns all metrics to the given slice. // The only error returned by this function is errNoDataToBuild. func (t *transaction) getMetrics(resource pcommon.Resource) (pmetric.Metrics, error) { @@ -154,15 +225,47 @@ func (t *transaction) getMetrics(resource pcommon.Resource) (pmetric.Metrics, er md := pmetric.NewMetrics() rms := md.ResourceMetrics().AppendEmpty() resource.CopyTo(rms.Resource()) - metrics := rms.ScopeMetrics().AppendEmpty().Metrics() - for _, mf := range t.families { - mf.appendMetric(metrics) + for scope, mfs := range t.families { + ils := rms.ScopeMetrics().AppendEmpty() + // If metrics don't include otel_scope_name or otel_scope_version + // labels, use the receiver name and version. + if scope == emptyScopeID { + ils.Scope().SetName(receiverName) + ils.Scope().SetVersion(t.buildInfo.Version) + } else { + // Otherwise, use the scope that was provided with the metrics. + ils.Scope().SetName(scope.name) + ils.Scope().SetVersion(scope.version) + // If we got an otel_scope_info metric for that scope, get scope + // attributes from it. + attributes, ok := t.scopeAttributes[scope] + if ok { + attributes.CopyTo(ils.Scope().Attributes()) + } + } + metrics := ils.Metrics() + for _, mf := range mfs { + mf.appendMetric(metrics, t.trimSuffixes) + } } return md, nil } +func getScopeID(ls labels.Labels) scopeID { + var scope scopeID + for _, lbl := range ls { + if lbl.Name == scopeNameLabel { + scope.name = lbl.Value + } + if lbl.Name == scopeVersionLabel { + scope.version = lbl.Value + } + } + return scope +} + func (t *transaction) initTransaction(labels labels.Labels) error { target, ok := scrape.TargetFromContext(t.ctx) if !ok { @@ -213,26 +316,41 @@ func (t *transaction) Rollback() error { return nil } -func (t *transaction) UpdateMetadata(ref storage.SeriesRef, l labels.Labels, m metadata.Metadata) (storage.SeriesRef, error) { +func (t *transaction) UpdateMetadata(_ storage.SeriesRef, _ labels.Labels, _ metadata.Metadata) (storage.SeriesRef, error) { //TODO: implement this func return 0, nil } -func (t *transaction) AppendHistogram(ref storage.SeriesRef, l labels.Labels, ts int64, h *histogram.Histogram, fh *histogram.FloatHistogram) (storage.SeriesRef, error) { - //TODO: implement this func - return 0, nil -} - -func (t *transaction) AddTargetInfo(labels labels.Labels) error { +func (t *transaction) AddTargetInfo(labels labels.Labels) { attrs := t.nodeResource.Attributes() - for _, lbl := range labels { if lbl.Name == model.JobLabel || lbl.Name == model.InstanceLabel || lbl.Name == model.MetricNameLabel { continue } + attrs.PutStr(lbl.Name, lbl.Value) + } +} +func (t *transaction) addScopeInfo(labels labels.Labels) { + attrs := pcommon.NewMap() + scope := scopeID{} + for _, lbl := range labels { + if lbl.Name == model.JobLabel || lbl.Name == model.InstanceLabel || lbl.Name == model.MetricNameLabel { + continue + } + if lbl.Name == scopeNameLabel { + scope.name = lbl.Value + continue + } + if lbl.Name == scopeVersionLabel { + scope.version = lbl.Value + continue + } attrs.PutStr(lbl.Name, lbl.Value) } + t.scopeAttributes[scope] = attrs +} - return nil +func getSeriesRef(bytes []byte, ls labels.Labels, mtype pmetric.MetricType) (uint64, []byte) { + return ls.HashWithoutLabels(bytes, getSortedNotUsefulLabels(mtype)...) } diff --git a/component/otelcol/receiver/prometheus/internal/transaction_test.go b/component/otelcol/receiver/prometheus/internal/transaction_test.go index 279093185a95..f8e7fb286cdb 100644 --- a/component/otelcol/receiver/prometheus/internal/transaction_test.go +++ b/component/otelcol/receiver/prometheus/internal/transaction_test.go @@ -1,16 +1,5 @@ // Copyright The OpenTelemetry Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. +// SPDX-License-Identifier: Apache-2.0 package internal @@ -21,6 +10,7 @@ import ( "time" "github.com/prometheus/common/model" + "github.com/prometheus/prometheus/model/exemplar" "github.com/prometheus/prometheus/model/labels" "github.com/prometheus/prometheus/model/metadata" "github.com/prometheus/prometheus/scrape" @@ -32,6 +22,8 @@ import ( "go.opentelemetry.io/collector/pdata/pmetric" "go.opentelemetry.io/collector/receiver/receiverhelper" "go.opentelemetry.io/collector/receiver/receivertest" + "go.uber.org/zap" + "go.uber.org/zap/zaptest/observer" ) const ( @@ -61,24 +53,24 @@ var ( ) func TestTransactionCommitWithoutAdding(t *testing.T) { - tr := newTransaction(scrapeCtx, &startTimeAdjuster{startTime: startTimestamp}, consumertest.NewNop(), nil, receivertest.NewNopCreateSettings(), nopObsRecv(t)) + tr := newTransaction(scrapeCtx, &startTimeAdjuster{startTime: startTimestamp}, consumertest.NewNop(), nil, receivertest.NewNopCreateSettings(), nopObsRecv(t), false) assert.NoError(t, tr.Commit()) } func TestTransactionRollbackDoesNothing(t *testing.T) { - tr := newTransaction(scrapeCtx, &startTimeAdjuster{startTime: startTimestamp}, consumertest.NewNop(), nil, receivertest.NewNopCreateSettings(), nopObsRecv(t)) + tr := newTransaction(scrapeCtx, &startTimeAdjuster{startTime: startTimestamp}, consumertest.NewNop(), nil, receivertest.NewNopCreateSettings(), nopObsRecv(t), false) assert.NoError(t, tr.Rollback()) } func TestTransactionUpdateMetadataDoesNothing(t *testing.T) { - tr := newTransaction(scrapeCtx, &startTimeAdjuster{startTime: startTimestamp}, consumertest.NewNop(), nil, receivertest.NewNopCreateSettings(), nopObsRecv(t)) + tr := newTransaction(scrapeCtx, &startTimeAdjuster{startTime: startTimestamp}, consumertest.NewNop(), nil, receivertest.NewNopCreateSettings(), nopObsRecv(t), false) _, err := tr.UpdateMetadata(0, labels.New(), metadata.Metadata{}) assert.NoError(t, err) } func TestTransactionAppendNoTarget(t *testing.T) { badLabels := labels.FromStrings(model.MetricNameLabel, "counter_test") - tr := newTransaction(scrapeCtx, &startTimeAdjuster{startTime: startTimestamp}, consumertest.NewNop(), nil, receivertest.NewNopCreateSettings(), nopObsRecv(t)) + tr := newTransaction(scrapeCtx, &startTimeAdjuster{startTime: startTimestamp}, consumertest.NewNop(), nil, receivertest.NewNopCreateSettings(), nopObsRecv(t), false) _, err := tr.Append(0, badLabels, time.Now().Unix()*1000, 1.0) assert.Error(t, err) } @@ -88,7 +80,7 @@ func TestTransactionAppendNoMetricName(t *testing.T) { model.InstanceLabel: "localhost:8080", model.JobLabel: "test2", }) - tr := newTransaction(scrapeCtx, &startTimeAdjuster{startTime: startTimestamp}, consumertest.NewNop(), nil, receivertest.NewNopCreateSettings(), nopObsRecv(t)) + tr := newTransaction(scrapeCtx, &startTimeAdjuster{startTime: startTimestamp}, consumertest.NewNop(), nil, receivertest.NewNopCreateSettings(), nopObsRecv(t), false) _, err := tr.Append(0, jobNotFoundLb, time.Now().Unix()*1000, 1.0) assert.ErrorIs(t, err, errMetricNameNotFound) @@ -96,7 +88,7 @@ func TestTransactionAppendNoMetricName(t *testing.T) { } func TestTransactionAppendEmptyMetricName(t *testing.T) { - tr := newTransaction(scrapeCtx, &startTimeAdjuster{startTime: startTimestamp}, consumertest.NewNop(), nil, receivertest.NewNopCreateSettings(), nopObsRecv(t)) + tr := newTransaction(scrapeCtx, &startTimeAdjuster{startTime: startTimestamp}, consumertest.NewNop(), nil, receivertest.NewNopCreateSettings(), nopObsRecv(t), false) _, err := tr.Append(0, labels.FromMap(map[string]string{ model.InstanceLabel: "localhost:8080", model.JobLabel: "test2", @@ -107,7 +99,7 @@ func TestTransactionAppendEmptyMetricName(t *testing.T) { func TestTransactionAppendResource(t *testing.T) { sink := new(consumertest.MetricsSink) - tr := newTransaction(scrapeCtx, &startTimeAdjuster{startTime: startTimestamp}, sink, nil, receivertest.NewNopCreateSettings(), nopObsRecv(t)) + tr := newTransaction(scrapeCtx, &startTimeAdjuster{startTime: startTimestamp}, sink, nil, receivertest.NewNopCreateSettings(), nopObsRecv(t), false) _, err := tr.Append(0, labels.FromMap(map[string]string{ model.InstanceLabel: "localhost:8080", model.JobLabel: "test", @@ -128,6 +120,28 @@ func TestTransactionAppendResource(t *testing.T) { require.Equal(t, expectedResource, gotResource) } +func TestReceiverVersionAndNameAreAttached(t *testing.T) { + sink := new(consumertest.MetricsSink) + tr := newTransaction(scrapeCtx, &startTimeAdjuster{startTime: startTimestamp}, sink, nil, receivertest.NewNopCreateSettings(), nopObsRecv(t), false) + _, err := tr.Append(0, labels.FromMap(map[string]string{ + model.InstanceLabel: "localhost:8080", + model.JobLabel: "test", + model.MetricNameLabel: "counter_test", + }), time.Now().Unix()*1000, 1.0) + assert.NoError(t, err) + assert.NoError(t, tr.Commit()) + + expectedResource := CreateResource("test", "localhost:8080", labels.FromStrings(model.SchemeLabel, "http")) + mds := sink.AllMetrics() + require.Len(t, mds, 1) + gotResource := mds[0].ResourceMetrics().At(0).Resource() + require.Equal(t, expectedResource, gotResource) + + gotScope := mds[0].ResourceMetrics().At(0).ScopeMetrics().At(0).Scope() + require.Equal(t, receiverName, gotScope.Name()) + require.Equal(t, component.NewDefaultBuildInfo().Version, gotScope.Version()) +} + func TestTransactionCommitErrorWhenAdjusterError(t *testing.T) { goodLabels := labels.FromMap(map[string]string{ model.InstanceLabel: "localhost:8080", @@ -136,7 +150,7 @@ func TestTransactionCommitErrorWhenAdjusterError(t *testing.T) { }) sink := new(consumertest.MetricsSink) adjusterErr := errors.New("adjuster error") - tr := newTransaction(scrapeCtx, &errorAdjuster{err: adjusterErr}, sink, nil, receivertest.NewNopCreateSettings(), nopObsRecv(t)) + tr := newTransaction(scrapeCtx, &errorAdjuster{err: adjusterErr}, sink, nil, receivertest.NewNopCreateSettings(), nopObsRecv(t), false) _, err := tr.Append(0, goodLabels, time.Now().Unix()*1000, 1.0) assert.NoError(t, err) assert.ErrorIs(t, tr.Commit(), adjusterErr) @@ -145,7 +159,7 @@ func TestTransactionCommitErrorWhenAdjusterError(t *testing.T) { // Ensure that we reject duplicate label keys. See https://github.com/open-telemetry/wg-prometheus/issues/44. func TestTransactionAppendDuplicateLabels(t *testing.T) { sink := new(consumertest.MetricsSink) - tr := newTransaction(scrapeCtx, &startTimeAdjuster{startTime: startTimestamp}, sink, nil, receivertest.NewNopCreateSettings(), nopObsRecv(t)) + tr := newTransaction(scrapeCtx, &startTimeAdjuster{startTime: startTimestamp}, sink, nil, receivertest.NewNopCreateSettings(), nopObsRecv(t), false) dupLabels := labels.FromStrings( model.InstanceLabel, "0.0.0.0:8855", @@ -163,7 +177,18 @@ func TestTransactionAppendDuplicateLabels(t *testing.T) { func TestTransactionAppendHistogramNoLe(t *testing.T) { sink := new(consumertest.MetricsSink) - tr := newTransaction(scrapeCtx, &startTimeAdjuster{startTime: startTimestamp}, sink, nil, receivertest.NewNopCreateSettings(), nopObsRecv(t)) + receiverSettings := receivertest.NewNopCreateSettings() + core, observedLogs := observer.New(zap.InfoLevel) + receiverSettings.Logger = zap.New(core) + tr := newTransaction( + scrapeCtx, + &startTimeAdjuster{startTime: startTimestamp}, + sink, + nil, + receiverSettings, + nopObsRecv(t), + false, + ) goodLabels := labels.FromStrings( model.InstanceLabel, "0.0.0.0:8855", @@ -172,12 +197,28 @@ func TestTransactionAppendHistogramNoLe(t *testing.T) { ) _, err := tr.Append(0, goodLabels, 1917, 1.0) - require.ErrorIs(t, err, errEmptyLeLabel) + require.NoError(t, err) + assert.Equal(t, 1, observedLogs.Len()) + assert.Equal(t, 1, observedLogs.FilterMessage("failed to add datapoint").Len()) + + assert.NoError(t, tr.Commit()) + assert.Len(t, sink.AllMetrics(), 0) } func TestTransactionAppendSummaryNoQuantile(t *testing.T) { sink := new(consumertest.MetricsSink) - tr := newTransaction(scrapeCtx, &startTimeAdjuster{startTime: startTimestamp}, sink, nil, receivertest.NewNopCreateSettings(), nopObsRecv(t)) + receiverSettings := receivertest.NewNopCreateSettings() + core, observedLogs := observer.New(zap.InfoLevel) + receiverSettings.Logger = zap.New(core) + tr := newTransaction( + scrapeCtx, + &startTimeAdjuster{startTime: startTimestamp}, + sink, + nil, + receiverSettings, + nopObsRecv(t), + false, + ) goodLabels := labels.FromStrings( model.InstanceLabel, "0.0.0.0:8855", @@ -186,18 +227,139 @@ func TestTransactionAppendSummaryNoQuantile(t *testing.T) { ) _, err := tr.Append(0, goodLabels, 1917, 1.0) - require.ErrorIs(t, err, errEmptyQuantileLabel) + require.NoError(t, err) + assert.Equal(t, 1, observedLogs.Len()) + assert.Equal(t, 1, observedLogs.FilterMessage("failed to add datapoint").Len()) + + assert.NoError(t, tr.Commit()) + assert.Len(t, sink.AllMetrics(), 0) +} + +func TestTransactionAppendValidAndInvalid(t *testing.T) { + sink := new(consumertest.MetricsSink) + receiverSettings := receivertest.NewNopCreateSettings() + core, observedLogs := observer.New(zap.InfoLevel) + receiverSettings.Logger = zap.New(core) + tr := newTransaction( + scrapeCtx, + &startTimeAdjuster{startTime: startTimestamp}, + sink, + nil, + receiverSettings, + nopObsRecv(t), + false, + ) + + // a valid counter + _, err := tr.Append(0, labels.FromMap(map[string]string{ + model.InstanceLabel: "localhost:8080", + model.JobLabel: "test", + model.MetricNameLabel: "counter_test", + }), time.Now().Unix()*1000, 1.0) + assert.NoError(t, err) + + // summary without quantiles, should be ignored + summarylabels := labels.FromStrings( + model.InstanceLabel, "0.0.0.0:8855", + model.JobLabel, "test", + model.MetricNameLabel, "summary_test", + ) + + _, err = tr.Append(0, summarylabels, 1917, 1.0) + require.NoError(t, err) + + assert.Equal(t, 1, observedLogs.Len()) + assert.Equal(t, 1, observedLogs.FilterMessage("failed to add datapoint").Len()) + + assert.NoError(t, tr.Commit()) + expectedResource := CreateResource("test", "localhost:8080", labels.FromStrings(model.SchemeLabel, "http")) + mds := sink.AllMetrics() + require.Len(t, mds, 1) + gotResource := mds[0].ResourceMetrics().At(0).Resource() + require.Equal(t, expectedResource, gotResource) + require.Equal(t, 1, mds[0].MetricCount()) +} + +func TestAppendExemplarWithNoMetricName(t *testing.T) { + sink := new(consumertest.MetricsSink) + tr := newTransaction(scrapeCtx, &startTimeAdjuster{startTime: startTimestamp}, sink, nil, receivertest.NewNopCreateSettings(), nopObsRecv(t), false) + + labels := labels.FromStrings( + model.InstanceLabel, "0.0.0.0:8855", + model.JobLabel, "test", + ) + + _, err := tr.AppendExemplar(0, labels, exemplar.Exemplar{Value: 0}) + assert.Equal(t, errMetricNameNotFound, err) +} + +func TestAppendExemplarWithEmptyMetricName(t *testing.T) { + sink := new(consumertest.MetricsSink) + tr := newTransaction(scrapeCtx, &startTimeAdjuster{startTime: startTimestamp}, sink, nil, receivertest.NewNopCreateSettings(), nopObsRecv(t), false) + + labels := labels.FromStrings( + model.InstanceLabel, "0.0.0.0:8855", + model.JobLabel, "test", + model.MetricNameLabel, "", + ) + _, err := tr.AppendExemplar(0, labels, exemplar.Exemplar{Value: 0}) + assert.Equal(t, errMetricNameNotFound, err) +} + +func TestAppendExemplarWithDuplicateLabels(t *testing.T) { + sink := new(consumertest.MetricsSink) + tr := newTransaction(scrapeCtx, &startTimeAdjuster{startTime: startTimestamp}, sink, nil, receivertest.NewNopCreateSettings(), nopObsRecv(t), false) + + labels := labels.FromStrings( + model.InstanceLabel, "0.0.0.0:8855", + model.JobLabel, "test", + model.MetricNameLabel, "", + "a", "b", + "a", "c", + ) + _, err := tr.AppendExemplar(0, labels, exemplar.Exemplar{Value: 0}) + require.Error(t, err) + assert.Contains(t, err.Error(), `invalid sample: non-unique label names: "a"`) +} + +func TestAppendExemplarWithoutAddingMetric(t *testing.T) { + sink := new(consumertest.MetricsSink) + tr := newTransaction(scrapeCtx, &startTimeAdjuster{startTime: startTimestamp}, sink, nil, receivertest.NewNopCreateSettings(), nopObsRecv(t), false) + + labels := labels.FromStrings( + model.InstanceLabel, "0.0.0.0:8855", + model.JobLabel, "test", + model.MetricNameLabel, "counter_test", + "a", "b", + ) + _, err := tr.AppendExemplar(0, labels, exemplar.Exemplar{Value: 0}) + assert.NoError(t, err) +} + +func TestAppendExemplarWithNoLabels(t *testing.T) { + sink := new(consumertest.MetricsSink) + tr := newTransaction(scrapeCtx, &startTimeAdjuster{startTime: startTimestamp}, sink, nil, receivertest.NewNopCreateSettings(), nopObsRecv(t), false) + + _, err := tr.AppendExemplar(0, nil, exemplar.Exemplar{Value: 0}) + assert.Equal(t, errNoJobInstance, err) +} + +func TestAppendExemplarWithEmptyLabelArray(t *testing.T) { + sink := new(consumertest.MetricsSink) + tr := newTransaction(scrapeCtx, &startTimeAdjuster{startTime: startTimestamp}, sink, nil, receivertest.NewNopCreateSettings(), nopObsRecv(t), false) + + _, err := tr.AppendExemplar(0, []labels.Label{}, exemplar.Exemplar{Value: 0}) + assert.Equal(t, errNoJobInstance, err) } func nopObsRecv(t *testing.T) *receiverhelper.ObsReport { - res, err := receiverhelper.NewObsReport(receiverhelper.ObsReportSettings{ + obsrecv, err := receiverhelper.NewObsReport(receiverhelper.ObsReportSettings{ ReceiverID: component.NewID("prometheus"), Transport: transport, ReceiverCreateSettings: receivertest.NewNopCreateSettings(), }) - - assert.NoError(t, err) - return res + require.NoError(t, err) + return obsrecv } func TestMetricBuilderCounters(t *testing.T) { @@ -207,7 +369,58 @@ func TestMetricBuilderCounters(t *testing.T) { inputs: []*testScrapedPage{ { pts: []*testDataPoint{ - createDataPoint("counter_test", 100, "foo", "bar"), + createDataPoint("counter_test", 100, nil, "foo", "bar"), + }, + }, + }, + wants: func() []pmetric.Metrics { + md0 := pmetric.NewMetrics() + mL0 := md0.ResourceMetrics().AppendEmpty().ScopeMetrics().AppendEmpty().Metrics() + m0 := mL0.AppendEmpty() + m0.SetName("counter_test") + sum := m0.SetEmptySum() + sum.SetAggregationTemporality(pmetric.AggregationTemporalityCumulative) + sum.SetIsMonotonic(true) + pt0 := sum.DataPoints().AppendEmpty() + pt0.SetDoubleValue(100.0) + pt0.SetStartTimestamp(startTimestamp) + pt0.SetTimestamp(tsNanos) + pt0.Attributes().PutStr("foo", "bar") + + return []pmetric.Metrics{md0} + }, + }, + { + name: "single-item-with-exemplars", + inputs: []*testScrapedPage{ + { + pts: []*testDataPoint{ + createDataPoint( + "counter_test", + 100, + []exemplar.Exemplar{ + { + Value: 1, + Ts: 1663113420863, + Labels: []labels.Label{{Name: model.MetricNameLabel, Value: "counter_test"}, {Name: model.JobLabel, Value: "job"}, {Name: model.InstanceLabel, Value: "instance"}, {Name: "foo", Value: "bar"}}, + }, + { + Value: 1, + Ts: 1663113420863, + Labels: []labels.Label{{Name: "foo", Value: "bar"}, {Name: "trace_id", Value: ""}, {Name: "span_id", Value: ""}}, + }, + { + Value: 1, + Ts: 1663113420863, + Labels: []labels.Label{{Name: "foo", Value: "bar"}, {Name: "trace_id", Value: "10a47365b8aa04e08291fab9deca84db6170"}, {Name: "span_id", Value: "719cee4a669fd7d109ff"}}, + }, + { + Value: 1, + Ts: 1663113420863, + Labels: []labels.Label{{Name: "foo", Value: "bar"}, {Name: "trace_id", Value: "174137cab66dc880"}, {Name: "span_id", Value: "dfa4597a9d"}}, + }, + }, + "foo", "bar"), }, }, }, @@ -225,6 +438,33 @@ func TestMetricBuilderCounters(t *testing.T) { pt0.SetTimestamp(tsNanos) pt0.Attributes().PutStr("foo", "bar") + e0 := pt0.Exemplars().AppendEmpty() + e0.SetTimestamp(timestampFromMs(1663113420863)) + e0.SetDoubleValue(1) + e0.FilteredAttributes().PutStr(model.MetricNameLabel, "counter_test") + e0.FilteredAttributes().PutStr(model.JobLabel, "job") + e0.FilteredAttributes().PutStr(model.InstanceLabel, "instance") + e0.FilteredAttributes().PutStr("foo", "bar") + + e1 := pt0.Exemplars().AppendEmpty() + e1.SetTimestamp(timestampFromMs(1663113420863)) + e1.SetDoubleValue(1) + e1.FilteredAttributes().PutStr("foo", "bar") + + e2 := pt0.Exemplars().AppendEmpty() + e2.SetTimestamp(timestampFromMs(1663113420863)) + e2.SetDoubleValue(1) + e2.FilteredAttributes().PutStr("foo", "bar") + e2.SetTraceID([16]byte{0x10, 0xa4, 0x73, 0x65, 0xb8, 0xaa, 0x04, 0xe0, 0x82, 0x91, 0xfa, 0xb9, 0xde, 0xca, 0x84, 0xdb}) + e2.SetSpanID([8]byte{0x71, 0x9c, 0xee, 0x4a, 0x66, 0x9f, 0xd7, 0xd1}) + + e3 := pt0.Exemplars().AppendEmpty() + e3.SetTimestamp(timestampFromMs(1663113420863)) + e3.SetDoubleValue(1) + e3.FilteredAttributes().PutStr("foo", "bar") + e3.SetTraceID([16]byte{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x17, 0x41, 0x37, 0xca, 0xb6, 0x6d, 0xc8, 0x80}) + e3.SetSpanID([8]byte{0x00, 0x00, 0x00, 0xdf, 0xa4, 0x59, 0x7a, 0x9d}) + return []pmetric.Metrics{md0} }, }, @@ -233,8 +473,8 @@ func TestMetricBuilderCounters(t *testing.T) { inputs: []*testScrapedPage{ { pts: []*testDataPoint{ - createDataPoint("counter_test", 150, "foo", "bar"), - createDataPoint("counter_test", 25, "foo", "other"), + createDataPoint("counter_test", 150, nil, "foo", "bar"), + createDataPoint("counter_test", 25, nil, "foo", "other"), }, }, }, @@ -266,9 +506,9 @@ func TestMetricBuilderCounters(t *testing.T) { inputs: []*testScrapedPage{ { pts: []*testDataPoint{ - createDataPoint("counter_test", 150, "foo", "bar"), - createDataPoint("counter_test", 25, "foo", "other"), - createDataPoint("counter_test2", 100, "foo", "bar"), + createDataPoint("counter_test", 150, nil, "foo", "bar"), + createDataPoint("counter_test", 25, nil, "foo", "other"), + createDataPoint("counter_test2", 100, nil, "foo", "bar"), }, }, }, @@ -311,7 +551,7 @@ func TestMetricBuilderCounters(t *testing.T) { inputs: []*testScrapedPage{ { pts: []*testDataPoint{ - createDataPoint("poor_name_count", 100, "foo", "bar"), + createDataPoint("poor_name_count", 100, nil, "foo", "bar"), }, }, }, @@ -348,12 +588,77 @@ func TestMetricBuilderGauges(t *testing.T) { inputs: []*testScrapedPage{ { pts: []*testDataPoint{ - createDataPoint("gauge_test", 100, "foo", "bar"), + createDataPoint("gauge_test", 100, nil, "foo", "bar"), + }, + }, + { + pts: []*testDataPoint{ + createDataPoint("gauge_test", 90, nil, "foo", "bar"), + }, + }, + }, + wants: func() []pmetric.Metrics { + md0 := pmetric.NewMetrics() + mL0 := md0.ResourceMetrics().AppendEmpty().ScopeMetrics().AppendEmpty().Metrics() + m0 := mL0.AppendEmpty() + m0.SetName("gauge_test") + gauge0 := m0.SetEmptyGauge() + pt0 := gauge0.DataPoints().AppendEmpty() + pt0.SetDoubleValue(100.0) + pt0.SetStartTimestamp(0) + pt0.SetTimestamp(tsNanos) + pt0.Attributes().PutStr("foo", "bar") + + md1 := pmetric.NewMetrics() + mL1 := md1.ResourceMetrics().AppendEmpty().ScopeMetrics().AppendEmpty().Metrics() + m1 := mL1.AppendEmpty() + m1.SetName("gauge_test") + gauge1 := m1.SetEmptyGauge() + pt1 := gauge1.DataPoints().AppendEmpty() + pt1.SetDoubleValue(90.0) + pt1.SetStartTimestamp(0) + pt1.SetTimestamp(tsPlusIntervalNanos) + pt1.Attributes().PutStr("foo", "bar") + + return []pmetric.Metrics{md0, md1} + }, + }, + { + name: "one-gauge-with-exemplars", + inputs: []*testScrapedPage{ + { + pts: []*testDataPoint{ + createDataPoint( + "gauge_test", + 100, + []exemplar.Exemplar{ + { + Value: 2, + Ts: 1663350815890, + Labels: []labels.Label{{Name: model.MetricNameLabel, Value: "counter_test"}, {Name: model.JobLabel, Value: "job"}, {Name: model.InstanceLabel, Value: "instance"}, {Name: "foo", Value: "bar"}}, + }, + { + Value: 2, + Ts: 1663350815890, + Labels: []labels.Label{{Name: "foo", Value: "bar"}, {Name: "trace_id", Value: ""}, {Name: "span_id", Value: ""}}, + }, + { + Value: 2, + Ts: 1663350815890, + Labels: []labels.Label{{Name: "foo", Value: "bar"}, {Name: "trace_id", Value: "10a47365b8aa04e08291fab9deca84db6170"}, {Name: "span_id", Value: "719cee4a669fd7d109ff"}}, + }, + { + Value: 2, + Ts: 1663350815890, + Labels: []labels.Label{{Name: "foo", Value: "bar"}, {Name: "trace_id", Value: "174137cab66dc880"}, {Name: "span_id", Value: "dfa4597a9d"}}, + }, + }, + "foo", "bar"), }, }, { pts: []*testDataPoint{ - createDataPoint("gauge_test", 90, "foo", "bar"), + createDataPoint("gauge_test", 90, nil, "foo", "bar"), }, }, }, @@ -369,6 +674,33 @@ func TestMetricBuilderGauges(t *testing.T) { pt0.SetTimestamp(tsNanos) pt0.Attributes().PutStr("foo", "bar") + e0 := pt0.Exemplars().AppendEmpty() + e0.SetTimestamp(timestampFromMs(1663350815890)) + e0.SetDoubleValue(2) + e0.FilteredAttributes().PutStr(model.MetricNameLabel, "counter_test") + e0.FilteredAttributes().PutStr(model.JobLabel, "job") + e0.FilteredAttributes().PutStr(model.InstanceLabel, "instance") + e0.FilteredAttributes().PutStr("foo", "bar") + + e1 := pt0.Exemplars().AppendEmpty() + e1.SetTimestamp(timestampFromMs(1663350815890)) + e1.SetDoubleValue(2) + e1.FilteredAttributes().PutStr("foo", "bar") + + e2 := pt0.Exemplars().AppendEmpty() + e2.SetTimestamp(timestampFromMs(1663350815890)) + e2.SetDoubleValue(2) + e2.FilteredAttributes().PutStr("foo", "bar") + e2.SetTraceID([16]byte{0x10, 0xa4, 0x73, 0x65, 0xb8, 0xaa, 0x04, 0xe0, 0x82, 0x91, 0xfa, 0xb9, 0xde, 0xca, 0x84, 0xdb}) + e2.SetSpanID([8]byte{0x71, 0x9c, 0xee, 0x4a, 0x66, 0x9f, 0xd7, 0xd1}) + + e3 := pt0.Exemplars().AppendEmpty() + e3.SetTimestamp(timestampFromMs(1663350815890)) + e3.SetDoubleValue(2) + e3.FilteredAttributes().PutStr("foo", "bar") + e3.SetTraceID([16]byte{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x17, 0x41, 0x37, 0xca, 0xb6, 0x6d, 0xc8, 0x80}) + e3.SetSpanID([8]byte{0x00, 0x00, 0x00, 0xdf, 0xa4, 0x59, 0x7a, 0x9d}) + md1 := pmetric.NewMetrics() mL1 := md1.ResourceMetrics().AppendEmpty().ScopeMetrics().AppendEmpty().Metrics() m1 := mL1.AppendEmpty() @@ -388,8 +720,8 @@ func TestMetricBuilderGauges(t *testing.T) { inputs: []*testScrapedPage{ { pts: []*testDataPoint{ - createDataPoint("gauge_test", 100, "foo", "bar"), - createDataPoint("gauge_test", 200, "bar", "foo"), + createDataPoint("gauge_test", 100, nil, "foo", "bar"), + createDataPoint("gauge_test", 200, nil, "bar", "foo"), }, }, }, @@ -421,13 +753,13 @@ func TestMetricBuilderGauges(t *testing.T) { inputs: []*testScrapedPage{ { pts: []*testDataPoint{ - createDataPoint("gauge_test", 100, "foo", "bar"), - createDataPoint("gauge_test", 200, "bar", "foo"), + createDataPoint("gauge_test", 100, nil, "foo", "bar"), + createDataPoint("gauge_test", 200, nil, "bar", "foo"), }, }, { pts: []*testDataPoint{ - createDataPoint("gauge_test", 20, "foo", "bar"), + createDataPoint("gauge_test", 20, nil, "foo", "bar"), }, }, }, @@ -479,7 +811,7 @@ func TestMetricBuilderUntyped(t *testing.T) { inputs: []*testScrapedPage{ { pts: []*testDataPoint{ - createDataPoint("unknown_test", 100, "foo", "bar"), + createDataPoint("unknown_test", 100, nil, "foo", "bar"), }, }, }, @@ -503,9 +835,9 @@ func TestMetricBuilderUntyped(t *testing.T) { inputs: []*testScrapedPage{ { pts: []*testDataPoint{ - createDataPoint("something_not_exists", 100, "foo", "bar"), - createDataPoint("theother_not_exists", 200, "foo", "bar"), - createDataPoint("theother_not_exists", 300, "bar", "foo"), + createDataPoint("something_not_exists", 100, nil, "foo", "bar"), + createDataPoint("theother_not_exists", 200, nil, "foo", "bar"), + createDataPoint("theother_not_exists", 300, nil, "bar", "foo"), }, }, }, @@ -541,7 +873,7 @@ func TestMetricBuilderUntyped(t *testing.T) { inputs: []*testScrapedPage{ { pts: []*testDataPoint{ - createDataPoint("some_count", 100, "foo", "bar"), + createDataPoint("some_count", 100, nil, "foo", "bar"), }, }, }, @@ -575,11 +907,73 @@ func TestMetricBuilderHistogram(t *testing.T) { inputs: []*testScrapedPage{ { pts: []*testDataPoint{ - createDataPoint("hist_test_bucket", 1, "foo", "bar", "le", "10"), - createDataPoint("hist_test_bucket", 2, "foo", "bar", "le", "20"), - createDataPoint("hist_test_bucket", 10, "foo", "bar", "le", "+inf"), - createDataPoint("hist_test_sum", 99, "foo", "bar"), - createDataPoint("hist_test_count", 10, "foo", "bar"), + createDataPoint("hist_test_bucket", 1, nil, "foo", "bar", "le", "10"), + createDataPoint("hist_test_bucket", 2, nil, "foo", "bar", "le", "20"), + createDataPoint("hist_test_bucket", 10, nil, "foo", "bar", "le", "+inf"), + createDataPoint("hist_test_sum", 99, nil, "foo", "bar"), + createDataPoint("hist_test_count", 10, nil, "foo", "bar"), + }, + }, + }, + wants: func() []pmetric.Metrics { + md0 := pmetric.NewMetrics() + mL0 := md0.ResourceMetrics().AppendEmpty().ScopeMetrics().AppendEmpty().Metrics() + m0 := mL0.AppendEmpty() + m0.SetName("hist_test") + hist0 := m0.SetEmptyHistogram() + hist0.SetAggregationTemporality(pmetric.AggregationTemporalityCumulative) + pt0 := hist0.DataPoints().AppendEmpty() + pt0.SetCount(10) + pt0.SetSum(99) + pt0.ExplicitBounds().FromRaw([]float64{10, 20}) + pt0.BucketCounts().FromRaw([]uint64{1, 1, 8}) + pt0.SetTimestamp(tsNanos) + pt0.SetStartTimestamp(startTimestamp) + pt0.Attributes().PutStr("foo", "bar") + + return []pmetric.Metrics{md0} + }, + }, + { + name: "single item with exemplars", + inputs: []*testScrapedPage{ + { + pts: []*testDataPoint{ + createDataPoint( + "hist_test_bucket", + 1, + []exemplar.Exemplar{ + { + Value: 1, + Ts: 1663113420863, + Labels: []labels.Label{{Name: model.MetricNameLabel, Value: "counter_test"}, {Name: model.JobLabel, Value: "job"}, {Name: model.InstanceLabel, Value: "instance"}, {Name: "foo", Value: "bar"}}, + }, + { + Value: 1, + Ts: 1663113420863, + Labels: []labels.Label{{Name: "foo", Value: "bar"}, {Name: "trace_id", Value: ""}, {Name: "span_id", Value: ""}, {Name: "le", Value: "20"}}, + }, + { + Value: 1, + Ts: 1663113420863, + Labels: []labels.Label{{Name: "foo", Value: "bar"}, {Name: "trace_id", Value: "10a47365b8aa04e08291fab9deca84db6170"}, {Name: "traceid", Value: "e3688e1aa2961786"}, {Name: "span_id", Value: "719cee4a669fd7d109ff"}}, + }, + { + Value: 1, + Ts: 1663113420863, + Labels: []labels.Label{{Name: "foo", Value: "bar"}, {Name: "trace_id", Value: "174137cab66dc880"}, {Name: "span_id", Value: "dfa4597a9d"}}, + }, + { + Value: 1, + Ts: 1663113420863, + Labels: []labels.Label{{Name: "foo", Value: "bar"}, {Name: "trace_id", Value: "174137cab66dc88"}, {Name: "span_id", Value: "dfa4597a9"}}, + }, + }, + "foo", "bar", "le", "10"), + createDataPoint("hist_test_bucket", 2, nil, "foo", "bar", "le", "20"), + createDataPoint("hist_test_bucket", 10, nil, "foo", "bar", "le", "+inf"), + createDataPoint("hist_test_sum", 99, nil, "foo", "bar"), + createDataPoint("hist_test_count", 10, nil, "foo", "bar"), }, }, }, @@ -599,6 +993,42 @@ func TestMetricBuilderHistogram(t *testing.T) { pt0.SetStartTimestamp(startTimestamp) pt0.Attributes().PutStr("foo", "bar") + e0 := pt0.Exemplars().AppendEmpty() + e0.SetTimestamp(timestampFromMs(1663113420863)) + e0.SetDoubleValue(1) + e0.FilteredAttributes().PutStr(model.MetricNameLabel, "counter_test") + e0.FilteredAttributes().PutStr(model.JobLabel, "job") + e0.FilteredAttributes().PutStr(model.InstanceLabel, "instance") + e0.FilteredAttributes().PutStr("foo", "bar") + + e1 := pt0.Exemplars().AppendEmpty() + e1.SetTimestamp(timestampFromMs(1663113420863)) + e1.SetDoubleValue(1) + e1.FilteredAttributes().PutStr("foo", "bar") + e1.FilteredAttributes().PutStr("le", "20") + + e2 := pt0.Exemplars().AppendEmpty() + e2.SetTimestamp(timestampFromMs(1663113420863)) + e2.SetDoubleValue(1) + e2.FilteredAttributes().PutStr("foo", "bar") + e2.FilteredAttributes().PutStr("traceid", "e3688e1aa2961786") + e2.SetTraceID([16]byte{0x10, 0xa4, 0x73, 0x65, 0xb8, 0xaa, 0x04, 0xe0, 0x82, 0x91, 0xfa, 0xb9, 0xde, 0xca, 0x84, 0xdb}) + e2.SetSpanID([8]byte{0x71, 0x9c, 0xee, 0x4a, 0x66, 0x9f, 0xd7, 0xd1}) + + e3 := pt0.Exemplars().AppendEmpty() + e3.SetTimestamp(timestampFromMs(1663113420863)) + e3.SetDoubleValue(1) + e3.FilteredAttributes().PutStr("foo", "bar") + e3.SetTraceID([16]byte{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x17, 0x41, 0x37, 0xca, 0xb6, 0x6d, 0xc8, 0x80}) + e3.SetSpanID([8]byte{0x00, 0x00, 0x00, 0xdf, 0xa4, 0x59, 0x7a, 0x9d}) + + e4 := pt0.Exemplars().AppendEmpty() + e4.SetTimestamp(timestampFromMs(1663113420863)) + e4.SetDoubleValue(1) + e4.FilteredAttributes().PutStr("foo", "bar") + e4.FilteredAttributes().PutStr("trace_id", "174137cab66dc88") + e4.FilteredAttributes().PutStr("span_id", "dfa4597a9") + return []pmetric.Metrics{md0} }, }, @@ -607,16 +1037,16 @@ func TestMetricBuilderHistogram(t *testing.T) { inputs: []*testScrapedPage{ { pts: []*testDataPoint{ - createDataPoint("hist_test_bucket", 1, "foo", "bar", "le", "10"), - createDataPoint("hist_test_bucket", 2, "foo", "bar", "le", "20"), - createDataPoint("hist_test_bucket", 10, "foo", "bar", "le", "+inf"), - createDataPoint("hist_test_sum", 99, "foo", "bar"), - createDataPoint("hist_test_count", 10, "foo", "bar"), - createDataPoint("hist_test_bucket", 1, "key2", "v2", "le", "10"), - createDataPoint("hist_test_bucket", 2, "key2", "v2", "le", "20"), - createDataPoint("hist_test_bucket", 3, "key2", "v2", "le", "+inf"), - createDataPoint("hist_test_sum", 50, "key2", "v2"), - createDataPoint("hist_test_count", 3, "key2", "v2"), + createDataPoint("hist_test_bucket", 1, nil, "foo", "bar", "le", "10"), + createDataPoint("hist_test_bucket", 2, nil, "foo", "bar", "le", "20"), + createDataPoint("hist_test_bucket", 10, nil, "foo", "bar", "le", "+inf"), + createDataPoint("hist_test_sum", 99, nil, "foo", "bar"), + createDataPoint("hist_test_count", 10, nil, "foo", "bar"), + createDataPoint("hist_test_bucket", 1, nil, "key2", "v2", "le", "10"), + createDataPoint("hist_test_bucket", 2, nil, "key2", "v2", "le", "20"), + createDataPoint("hist_test_bucket", 3, nil, "key2", "v2", "le", "+inf"), + createDataPoint("hist_test_sum", 50, nil, "key2", "v2"), + createDataPoint("hist_test_count", 3, nil, "key2", "v2"), }, }, }, @@ -653,21 +1083,21 @@ func TestMetricBuilderHistogram(t *testing.T) { inputs: []*testScrapedPage{ { pts: []*testDataPoint{ - createDataPoint("hist_test_bucket", 1, "foo", "bar", "le", "10"), - createDataPoint("hist_test_bucket", 2, "foo", "bar", "le", "20"), - createDataPoint("hist_test_bucket", 10, "foo", "bar", "le", "+inf"), - createDataPoint("hist_test_sum", 99, "foo", "bar"), - createDataPoint("hist_test_count", 10, "foo", "bar"), - createDataPoint("hist_test_bucket", 1, "key2", "v2", "le", "10"), - createDataPoint("hist_test_bucket", 2, "key2", "v2", "le", "20"), - createDataPoint("hist_test_bucket", 3, "key2", "v2", "le", "+inf"), - createDataPoint("hist_test_sum", 50, "key2", "v2"), - createDataPoint("hist_test_count", 3, "key2", "v2"), - createDataPoint("hist_test2_bucket", 1, "foo", "bar", "le", "10"), - createDataPoint("hist_test2_bucket", 2, "foo", "bar", "le", "20"), - createDataPoint("hist_test2_bucket", 3, "foo", "bar", "le", "+inf"), - createDataPoint("hist_test2_sum", 50, "foo", "bar"), - createDataPoint("hist_test2_count", 3, "foo", "bar"), + createDataPoint("hist_test_bucket", 1, nil, "foo", "bar", "le", "10"), + createDataPoint("hist_test_bucket", 2, nil, "foo", "bar", "le", "20"), + createDataPoint("hist_test_bucket", 10, nil, "foo", "bar", "le", "+inf"), + createDataPoint("hist_test_sum", 99, nil, "foo", "bar"), + createDataPoint("hist_test_count", 10, nil, "foo", "bar"), + createDataPoint("hist_test_bucket", 1, nil, "key2", "v2", "le", "10"), + createDataPoint("hist_test_bucket", 2, nil, "key2", "v2", "le", "20"), + createDataPoint("hist_test_bucket", 3, nil, "key2", "v2", "le", "+inf"), + createDataPoint("hist_test_sum", 50, nil, "key2", "v2"), + createDataPoint("hist_test_count", 3, nil, "key2", "v2"), + createDataPoint("hist_test2_bucket", 1, nil, "foo", "bar", "le", "10"), + createDataPoint("hist_test2_bucket", 2, nil, "foo", "bar", "le", "20"), + createDataPoint("hist_test2_bucket", 3, nil, "foo", "bar", "le", "+inf"), + createDataPoint("hist_test2_sum", 50, nil, "foo", "bar"), + createDataPoint("hist_test2_count", 3, nil, "foo", "bar"), }, }, }, @@ -717,11 +1147,11 @@ func TestMetricBuilderHistogram(t *testing.T) { inputs: []*testScrapedPage{ { pts: []*testDataPoint{ - createDataPoint("hist_test_bucket", 10, "foo", "bar", "le", "+inf"), - createDataPoint("hist_test_bucket", 1, "foo", "bar", "le", "10"), - createDataPoint("hist_test_bucket", 2, "foo", "bar", "le", "20"), - createDataPoint("hist_test_sum", 99, "foo", "bar"), - createDataPoint("hist_test_count", 10, "foo", "bar"), + createDataPoint("hist_test_bucket", 10, nil, "foo", "bar", "le", "+inf"), + createDataPoint("hist_test_bucket", 1, nil, "foo", "bar", "le", "10"), + createDataPoint("hist_test_bucket", 2, nil, "foo", "bar", "le", "20"), + createDataPoint("hist_test_sum", 99, nil, "foo", "bar"), + createDataPoint("hist_test_count", 10, nil, "foo", "bar"), }, }, }, @@ -745,14 +1175,14 @@ func TestMetricBuilderHistogram(t *testing.T) { }, }, { - // this won't likely happen in real env, as prometheus won't generate histogram with less than 3 buckets + // this won't likely happen in real env, as prometheus wont generate histogram with less than 3 buckets name: "only-one-bucket", inputs: []*testScrapedPage{ { pts: []*testDataPoint{ - createDataPoint("hist_test_bucket", 3, "foo", "bar", "le", "+inf"), - createDataPoint("hist_test_count", 3, "foo", "bar"), - createDataPoint("hist_test_sum", 100, "foo", "bar"), + createDataPoint("hist_test_bucket", 3, nil, "foo", "bar", "le", "+inf"), + createDataPoint("hist_test_count", 3, nil, "foo", "bar"), + createDataPoint("hist_test_sum", 100, nil, "foo", "bar"), }, }, }, @@ -775,14 +1205,14 @@ func TestMetricBuilderHistogram(t *testing.T) { }, }, { - // this won't likely happen in real env, as prometheus won't generate histogram with less than 3 buckets + // this won't likely happen in real env, as prometheus wont generate histogram with less than 3 buckets name: "only-one-bucket-noninf", inputs: []*testScrapedPage{ { pts: []*testDataPoint{ - createDataPoint("hist_test_bucket", 3, "foo", "bar", "le", "20"), - createDataPoint("hist_test_count", 3, "foo", "bar"), - createDataPoint("hist_test_sum", 100, "foo", "bar"), + createDataPoint("hist_test_bucket", 3, nil, "foo", "bar", "le", "20"), + createDataPoint("hist_test_count", 3, nil, "foo", "bar"), + createDataPoint("hist_test_sum", 100, nil, "foo", "bar"), }, }, }, @@ -796,7 +1226,8 @@ func TestMetricBuilderHistogram(t *testing.T) { pt0 := hist0.DataPoints().AppendEmpty() pt0.SetCount(3) pt0.SetSum(100) - pt0.BucketCounts().FromRaw([]uint64{3}) + pt0.BucketCounts().FromRaw([]uint64{3, 0}) + pt0.ExplicitBounds().FromRaw([]float64{20}) pt0.SetTimestamp(tsNanos) pt0.SetStartTimestamp(startTimestamp) pt0.Attributes().PutStr("foo", "bar") @@ -809,10 +1240,10 @@ func TestMetricBuilderHistogram(t *testing.T) { inputs: []*testScrapedPage{ { pts: []*testDataPoint{ - createDataPoint("hist_test_bucket", 1, "foo", "bar", "le", "10"), - createDataPoint("hist_test_bucket", 2, "foo", "bar", "le", "20"), - createDataPoint("hist_test_bucket", 3, "foo", "bar", "le", "+inf"), - createDataPoint("hist_test_count", 3, "foo", "bar"), + createDataPoint("hist_test_bucket", 1, nil, "foo", "bar", "le", "10"), + createDataPoint("hist_test_bucket", 2, nil, "foo", "bar", "le", "20"), + createDataPoint("hist_test_bucket", 3, nil, "foo", "bar", "le", "+inf"), + createDataPoint("hist_test_count", 3, nil, "foo", "bar"), }, }, }, @@ -839,13 +1270,27 @@ func TestMetricBuilderHistogram(t *testing.T) { inputs: []*testScrapedPage{ { pts: []*testDataPoint{ - createDataPoint("hist_test_sum", 99), - createDataPoint("hist_test_count", 10), + createDataPoint("hist_test_sum", 99, nil, "foo", "bar"), + createDataPoint("hist_test_count", 10, nil, "foo", "bar"), }, }, }, wants: func() []pmetric.Metrics { - return []pmetric.Metrics{pmetric.NewMetrics()} + md0 := pmetric.NewMetrics() + mL0 := md0.ResourceMetrics().AppendEmpty().ScopeMetrics().AppendEmpty().Metrics() + m0 := mL0.AppendEmpty() + m0.SetName("hist_test") + hist0 := m0.SetEmptyHistogram() + hist0.SetAggregationTemporality(pmetric.AggregationTemporalityCumulative) + pt0 := hist0.DataPoints().AppendEmpty() + pt0.SetCount(10) + pt0.SetSum(99) + pt0.BucketCounts().FromRaw([]uint64{10}) + pt0.SetTimestamp(tsNanos) + pt0.SetStartTimestamp(startTimestamp) + pt0.Attributes().PutStr("foo", "bar") + + return []pmetric.Metrics{md0} }, }, { @@ -853,10 +1298,10 @@ func TestMetricBuilderHistogram(t *testing.T) { inputs: []*testScrapedPage{ { pts: []*testDataPoint{ - createDataPoint("hist_test_bucket", 1, "foo", "bar", "le", "10"), - createDataPoint("hist_test_bucket", 2, "foo", "bar", "le", "20"), - createDataPoint("hist_test_bucket", 3, "foo", "bar", "le", "+inf"), - createDataPoint("hist_test_sum", 99, "foo", "bar"), + createDataPoint("hist_test_bucket", 1, nil, "foo", "bar", "le", "10"), + createDataPoint("hist_test_bucket", 2, nil, "foo", "bar", "le", "20"), + createDataPoint("hist_test_bucket", 3, nil, "foo", "bar", "le", "+inf"), + createDataPoint("hist_test_sum", 99, nil, "foo", "bar"), }, }, }, @@ -880,7 +1325,7 @@ func TestMetricBuilderSummary(t *testing.T) { inputs: []*testScrapedPage{ { pts: []*testDataPoint{ - createDataPoint("summary_test", 5, "foo", "bar", "quantile", "1"), + createDataPoint("summary_test", 5, nil, "foo", "bar", "quantile", "1"), }, }, }, @@ -893,10 +1338,10 @@ func TestMetricBuilderSummary(t *testing.T) { inputs: []*testScrapedPage{ { pts: []*testDataPoint{ - createDataPoint("summary_test", 1, "foo", "bar", "quantile", "0.5"), - createDataPoint("summary_test", 2, "foo", "bar", "quantile", "0.75"), - createDataPoint("summary_test", 5, "foo", "bar", "quantile", "1"), - createDataPoint("summary_test_sum", 500, "foo", "bar"), + createDataPoint("summary_test", 1, nil, "foo", "bar", "quantile", "0.5"), + createDataPoint("summary_test", 2, nil, "foo", "bar", "quantile", "0.75"), + createDataPoint("summary_test", 5, nil, "foo", "bar", "quantile", "1"), + createDataPoint("summary_test_sum", 500, nil, "foo", "bar"), }, }, }, @@ -909,10 +1354,10 @@ func TestMetricBuilderSummary(t *testing.T) { inputs: []*testScrapedPage{ { pts: []*testDataPoint{ - createDataPoint("summary_test", 1, "foo", "bar", "quantile", "0.5"), - createDataPoint("summary_test", 2, "foo", "bar", "quantile", "0.75"), - createDataPoint("summary_test", 5, "foo", "bar", "quantile", "1"), - createDataPoint("summary_test_count", 500, "foo", "bar"), + createDataPoint("summary_test", 1, nil, "foo", "bar", "quantile", "0.5"), + createDataPoint("summary_test", 2, nil, "foo", "bar", "quantile", "0.75"), + createDataPoint("summary_test", 5, nil, "foo", "bar", "quantile", "1"), + createDataPoint("summary_test_count", 500, nil, "foo", "bar"), }, }, }, @@ -938,7 +1383,6 @@ func TestMetricBuilderSummary(t *testing.T) { q100 := qvL.AppendEmpty() q100.SetQuantile(1) q100.SetValue(5.0) - return []pmetric.Metrics{md0} }, }, @@ -947,8 +1391,8 @@ func TestMetricBuilderSummary(t *testing.T) { inputs: []*testScrapedPage{ { pts: []*testDataPoint{ - createDataPoint("summary_test_sum", 100, "foo", "bar"), - createDataPoint("summary_test_count", 500, "foo", "bar"), + createDataPoint("summary_test_sum", 100, nil, "foo", "bar"), + createDataPoint("summary_test_count", 500, nil, "foo", "bar"), }, }, }, @@ -973,11 +1417,11 @@ func TestMetricBuilderSummary(t *testing.T) { inputs: []*testScrapedPage{ { pts: []*testDataPoint{ - createDataPoint("summary_test", 1, "foo", "bar", "quantile", "0.5"), - createDataPoint("summary_test", 2, "foo", "bar", "quantile", "0.75"), - createDataPoint("summary_test", 5, "foo", "bar", "quantile", "1"), - createDataPoint("summary_test_sum", 100, "foo", "bar"), - createDataPoint("summary_test_count", 500, "foo", "bar"), + createDataPoint("summary_test", 1, nil, "foo", "bar", "quantile", "0.5"), + createDataPoint("summary_test", 2, nil, "foo", "bar", "quantile", "0.75"), + createDataPoint("summary_test", 5, nil, "foo", "bar", "quantile", "1"), + createDataPoint("summary_test_sum", 100, nil, "foo", "bar"), + createDataPoint("summary_test_count", 500, nil, "foo", "bar"), }, }, }, @@ -1028,12 +1472,17 @@ func (tt buildTestData) run(t *testing.T) { st := ts for i, page := range tt.inputs { sink := new(consumertest.MetricsSink) - tr := newTransaction(scrapeCtx, &startTimeAdjuster{startTime: startTimestamp}, sink, nil, receivertest.NewNopCreateSettings(), nopObsRecv(t)) + tr := newTransaction(scrapeCtx, &startTimeAdjuster{startTime: startTimestamp}, sink, nil, receivertest.NewNopCreateSettings(), nopObsRecv(t), false) for _, pt := range page.pts { // set ts for testing pt.t = st _, err := tr.Append(0, pt.lb, pt.t, pt.v) assert.NoError(t, err) + + for _, e := range pt.exemplars { + _, err := tr.AppendExemplar(0, pt.lb, e) + assert.NoError(t, err) + } } assert.NoError(t, tr.Commit()) mds := sink.AllMetrics() @@ -1084,6 +1533,7 @@ func (s *startTimeAdjuster) AdjustMetrics(metrics pmetric.Metrics) error { for l := 0; l < dps.Len(); l++ { dps.At(l).SetStartTimestamp(s.startTime) } + case pmetric.MetricTypeEmpty, pmetric.MetricTypeGauge, pmetric.MetricTypeExponentialHistogram: } } } @@ -1092,16 +1542,17 @@ func (s *startTimeAdjuster) AdjustMetrics(metrics pmetric.Metrics) error { } type testDataPoint struct { - lb labels.Labels - t int64 - v float64 + lb labels.Labels + t int64 + v float64 + exemplars []exemplar.Exemplar } type testScrapedPage struct { pts []*testDataPoint } -func createDataPoint(mname string, value float64, tagPairs ...string) *testDataPoint { +func createDataPoint(mname string, value float64, es []exemplar.Exemplar, tagPairs ...string) *testDataPoint { var lbls []string lbls = append(lbls, tagPairs...) lbls = append(lbls, model.MetricNameLabel, mname) @@ -1109,9 +1560,10 @@ func createDataPoint(mname string, value float64, tagPairs ...string) *testDataP lbls = append(lbls, model.InstanceLabel, "instance") return &testDataPoint{ - lb: labels.FromStrings(lbls...), - t: ts, - v: value, + lb: labels.FromStrings(lbls...), + t: ts, + v: value, + exemplars: es, } } diff --git a/component/otelcol/receiver/prometheus/internal/util.go b/component/otelcol/receiver/prometheus/internal/util.go index 3f8633a7b31f..405a181f47a5 100644 --- a/component/otelcol/receiver/prometheus/internal/util.go +++ b/component/otelcol/receiver/prometheus/internal/util.go @@ -1,16 +1,5 @@ // Copyright The OpenTelemetry Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. +// SPDX-License-Identifier: Apache-2.0 package internal // import "github.com/open-telemetry/opentelemetry-collector-contrib/receiver/prometheusreceiver/internal" @@ -33,6 +22,7 @@ const ( metricsSuffixSum = "_sum" metricSuffixTotal = "_total" metricSuffixInfo = "_info" + metricSuffixCreated = "_created" startTimeMetricName = "process_start_time_seconds" scrapeUpMetricName = "up" @@ -41,11 +31,11 @@ const ( ) var ( - trimmableSuffixes = []string{metricsSuffixBucket, metricsSuffixCount, metricsSuffixSum, metricSuffixTotal, metricSuffixInfo} + trimmableSuffixes = []string{metricsSuffixBucket, metricsSuffixCount, metricsSuffixSum, metricSuffixTotal, metricSuffixInfo, metricSuffixCreated} errNoDataToBuild = errors.New("there's no data to build") errNoBoundaryLabel = errors.New("given metricType has no 'le' or 'quantile' label") - errEmptyQuantileLabel = errors.New("'quantile' label on summary metric missing is empty") - errEmptyLeLabel = errors.New("'le' label on histogram metric id missing or empty") + errEmptyQuantileLabel = errors.New("'quantile' label on summary metric is missing or empty") + errEmptyLeLabel = errors.New("'le' label on histogram metric is missing or empty") errMetricNameNotFound = errors.New("metricName not found from labels") errTransactionAborted = errors.New("transaction aborted") errNoJobInstance = errors.New("job or instance cannot be found from labels") @@ -66,6 +56,8 @@ func getSortedNotUsefulLabels(mType pmetric.MetricType) []string { return notUsefulLabelsHistogram case pmetric.MetricTypeSummary: return notUsefulLabelsSummary + case pmetric.MetricTypeEmpty, pmetric.MetricTypeGauge, pmetric.MetricTypeSum, pmetric.MetricTypeExponentialHistogram: + fallthrough default: return notUsefulLabelsOther } @@ -82,7 +74,7 @@ func timestampFromMs(timeAtMs int64) pcommon.Timestamp { } func getBoundary(metricType pmetric.MetricType, labels labels.Labels) (float64, error) { - val := "" + var val string switch metricType { case pmetric.MetricTypeHistogram: val = labels.Get(model.BucketLabel) @@ -94,6 +86,8 @@ func getBoundary(metricType pmetric.MetricType, labels labels.Labels) (float64, if val == "" { return 0, errEmptyQuantileLabel } + case pmetric.MetricTypeEmpty, pmetric.MetricTypeGauge, pmetric.MetricTypeSum, pmetric.MetricTypeExponentialHistogram: + fallthrough default: return 0, errNoBoundaryLabel } @@ -120,6 +114,8 @@ func convToMetricType(metricType textparse.MetricType) (pmetric.MetricType, bool return pmetric.MetricTypeSummary, true case textparse.MetricTypeInfo, textparse.MetricTypeStateset: return pmetric.MetricTypeSum, false + case textparse.MetricTypeGaugeHistogram: + fallthrough default: // including: textparse.MetricTypeGaugeHistogram return pmetric.MetricTypeEmpty, false diff --git a/component/otelcol/receiver/prometheus/internal/util_test.go b/component/otelcol/receiver/prometheus/internal/util_test.go index a9d23c3dbb09..3bea1ac42471 100644 --- a/component/otelcol/receiver/prometheus/internal/util_test.go +++ b/component/otelcol/receiver/prometheus/internal/util_test.go @@ -1,16 +1,5 @@ // Copyright The OpenTelemetry Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. +// SPDX-License-Identifier: Apache-2.0 package internal // import "github.com/open-telemetry/opentelemetry-collector-contrib/receiver/prometheusreceiver/internal" @@ -111,7 +100,7 @@ func TestConvToMetricType(t *testing.T) { wantMonotonic: false, }, { - name: "textparse.metric_gauge_histogram", + name: "textparse.metric_gauge_hostogram", mtype: textparse.MetricTypeGaugeHistogram, want: pmetric.MetricTypeEmpty, wantMonotonic: false, diff --git a/component/otelcol/receiver/prometheus/prometheus.go b/component/otelcol/receiver/prometheus/prometheus.go index b0493f4d6982..f1b4d8a7d422 100644 --- a/component/otelcol/receiver/prometheus/prometheus.go +++ b/component/otelcol/receiver/prometheus/prometheus.go @@ -100,9 +100,20 @@ func (c *Component) Update(newConfig component.Arguments) error { useStartTimeMetric = false startTimeMetricRegex *regexp.Regexp + // Start time for Summary, Histogram and Sum metrics can be retrieved from `_created` metrics. + useCreatedMetric = false + + // Trimming the metric suffixes is used to remove the metric type and the unit and the end of the metric name. + // To trim the unit, the opentelemetry code uses the MetricMetadataStore which is currently not supported by the agent. + // When supported, this could be added as an arg. + trimMetricSuffixes = false + gcInterval = 5 * time.Minute ) settings := otelreceiver.CreateSettings{ + + ID: otelcomponent.NewID(otelcomponent.Type(c.opts.ID)), + TelemetrySettings: otelcomponent.TelemetrySettings{ Logger: zapadapter.New(c.opts.Logger), @@ -129,8 +140,9 @@ func (c *Component) Update(newConfig component.Arguments) error { gcInterval, useStartTimeMetric, startTimeMetricRegex, - otelcomponent.NewID(otelcomponent.Type(c.opts.ID)), + useCreatedMetric, labels.Labels{}, + trimMetricSuffixes, ) if err != nil { return err diff --git a/component/otelcol/receiver/prometheus/prometheus_test.go b/component/otelcol/receiver/prometheus/prometheus_test.go index 3002916eb64a..3877bad7c990 100644 --- a/component/otelcol/receiver/prometheus/prometheus_test.go +++ b/component/otelcol/receiver/prometheus/prometheus_test.go @@ -13,6 +13,7 @@ import ( "github.com/grafana/agent/pkg/util" "github.com/grafana/river" "github.com/prometheus/common/model" + "github.com/prometheus/prometheus/model/exemplar" "github.com/prometheus/prometheus/model/labels" "github.com/prometheus/prometheus/scrape" "github.com/stretchr/testify/require" @@ -59,16 +60,33 @@ func Test(t *testing.T) { {Name: model.JobLabel, Value: "testJob"}, {Name: model.InstanceLabel, Value: "otelcol.receiver.prometheus"}, {Name: "foo", Value: "bar"}, + {Name: model.MetricNameLabel, Value: "otel_scope_info"}, + {Name: "otel_scope_name", Value: "go.opentelemetry.io.contrib.instrumentation.net.http.otelhttp"}, + {Name: "otel_scope_version", Value: "v0.24.0"}, } ts := time.Now().Unix() v := 100. + exemplarLabels := labels.Labels{ + {Name: model.MetricNameLabel, Value: "testMetric"}, + {Name: "trace_id", Value: "123456789abcdef0123456789abcdef0"}, + {Name: "span_id", Value: "123456789abcdef0"}, + } + exemplar := exemplar.Exemplar{ + Value: 2, + Ts: ts, + HasTs: true, + Labels: exemplarLabels, + } + ctx := context.Background() ctx = scrape.ContextWithMetricMetadataStore(ctx, flowprometheus.NoopMetadataStore{}) ctx = scrape.ContextWithTarget(ctx, &scrape.Target{}) app := exports.Receiver.Appender(ctx) _, err := app.Append(0, l, ts, v) require.NoError(t, err) + _, err = app.AppendExemplar(0, l, exemplar) + require.NoError(t, err) require.NoError(t, app.Commit()) }() @@ -79,6 +97,14 @@ func Test(t *testing.T) { case m := <-metricCh: require.Equal(t, 1, m.MetricCount()) require.Equal(t, "testMetric", m.ResourceMetrics().At(0).ScopeMetrics().At(0).Metrics().At(0).Name()) + require.Equal(t, "go.opentelemetry.io.contrib.instrumentation.net.http.otelhttp", m.ResourceMetrics().At(0).ScopeMetrics().At(0).Scope().Name()) + require.Equal(t, "v0.24.0", m.ResourceMetrics().At(0).ScopeMetrics().At(0).Scope().Version()) + require.Equal(t, "Gauge", m.ResourceMetrics().At(0).ScopeMetrics().At(0).Metrics().At(0).Type().String()) + require.Equal(t, 1, m.ResourceMetrics().At(0).ScopeMetrics().At(0).Metrics().At(0).Gauge().DataPoints().Len()) + require.Equal(t, 1, m.ResourceMetrics().At(0).ScopeMetrics().At(0).Metrics().At(0).Gauge().DataPoints().At(0).Exemplars().Len()) + require.Equal(t, "123456789abcdef0123456789abcdef0", m.ResourceMetrics().At(0).ScopeMetrics().At(0).Metrics().At(0).Gauge().DataPoints().At(0).Exemplars().At(0).TraceID().String()) + require.Equal(t, "123456789abcdef0", m.ResourceMetrics().At(0).ScopeMetrics().At(0).Metrics().At(0).Gauge().DataPoints().At(0).Exemplars().At(0).SpanID().String()) + require.Equal(t, 2.0, m.ResourceMetrics().At(0).ScopeMetrics().At(0).Metrics().At(0).Gauge().DataPoints().At(0).Exemplars().At(0).DoubleValue()) } } diff --git a/docs/sources/flow/release-notes.md b/docs/sources/flow/release-notes.md index 62dd29b8c6b9..f8053bf3c0b3 100644 --- a/docs/sources/flow/release-notes.md +++ b/docs/sources/flow/release-notes.md @@ -29,9 +29,15 @@ Other release notes for the different {{< param "PRODUCT_ROOT_NAME" >}} variants [release-notes-operator]: {{< relref "../operator/release-notes.md" >}} {{% /admonition %}} - ## v0.39 +### Breaking change: `otelcol.receiver.prometheus` will drop all `otel_scope_info` metrics when converting them to OTLP + +* If the `otel_scope_info` metric has the `otel_scope_name` and `otel_scope_version` labels, + their values are used to set the OTLP Instrumentation Scope name and version, respectively. +* Labels for `otel_scope_info` metrics other than `otel_scope_name` and `otel_scope_version` + are added as scope attributes with the matching name and version. + ### Breaking change: label for `target` block in `prometheus.exporter.blackbox` is removed Previously in `prometheus.exporter.blackbox`, the `target` block requires a label which is used in job's name. diff --git a/go.mod b/go.mod index 6c82f970e4b7..9e65fea46485 100644 --- a/go.mod +++ b/go.mod @@ -611,7 +611,9 @@ require github.com/ianlancetaylor/demangle v0.0.0-20230524184225-eabc099b10ab require ( github.com/githubexporter/github-exporter v0.0.0-20231025122338-656e7dc33fe7 github.com/natefinch/atomic v1.0.1 + github.com/open-telemetry/opentelemetry-collector-contrib/exporter/prometheusremotewriteexporter v0.87.0 github.com/open-telemetry/opentelemetry-collector-contrib/processor/filterprocessor v0.87.0 + github.com/open-telemetry/opentelemetry-collector-contrib/receiver/prometheusreceiver v0.87.0 github.com/open-telemetry/opentelemetry-collector-contrib/receiver/vcenterreceiver v0.87.0 go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v0.42.0 ) @@ -631,13 +633,20 @@ require ( github.com/metalmatze/signal v0.0.0-20210307161603-1c9aa721a97a // indirect github.com/minio/asm2plan9s v0.0.0-20200509001527-cdd76441f9d8 // indirect github.com/minio/c2goasm v0.0.0-20190812172519-36a3d3bbc4f3 // indirect + github.com/mitchellh/hashstructure/v2 v2.0.2 // indirect github.com/open-telemetry/opentelemetry-collector-contrib/internal/k8sconfig v0.87.0 // indirect github.com/open-telemetry/opentelemetry-collector-contrib/internal/kafka v0.87.0 // indirect + github.com/open-telemetry/opentelemetry-collector-contrib/pkg/translator/prometheusremotewrite v0.87.0 // indirect github.com/openshift/api v3.9.0+incompatible // indirect github.com/openshift/client-go v0.0.0-20210521082421-73d9475a9142 // indirect github.com/prometheus-community/prom-label-proxy v0.6.0 // indirect github.com/sercand/kuberesolver/v4 v4.0.0 // indirect github.com/sony/gobreaker v0.5.0 // indirect + github.com/tidwall/gjson v1.10.2 // indirect + github.com/tidwall/match v1.1.1 // indirect + github.com/tidwall/pretty v1.2.0 // indirect + github.com/tidwall/tinylru v1.1.0 // indirect + github.com/tidwall/wal v1.1.7 // indirect go.opentelemetry.io/otel/exporters/otlp/otlpmetric v0.42.0 // indirect go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v0.42.0 // indirect go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.19.0 // indirect diff --git a/go.sum b/go.sum index 648c487e8ed2..e1ebb63ddbe4 100644 --- a/go.sum +++ b/go.sum @@ -1601,7 +1601,6 @@ github.com/mitchellh/go-testing-interface v1.14.1/go.mod h1:gfgS7OtZj6MA4U1UrDRp github.com/mitchellh/go-wordwrap v1.0.0/go.mod h1:ZXFpozHsX6DPmq2I0TCekCxypsnAUbP2oI0UX1GXzOo= github.com/mitchellh/gox v0.4.0/go.mod h1:Sd9lOJ0+aimLBi73mGofS1ycjY8lL3uZM3JPS42BGNg= github.com/mitchellh/hashstructure v0.0.0-20170609045927-2bca23e0e452/go.mod h1:QjSHrPWS+BGUVBYkbTZWEnOh3G1DutKwClXU/ABz6AQ= -github.com/mitchellh/hashstructure v1.1.0 h1:P6P1hdjqAAknpY/M1CGipelZgp+4y9ja9kmUZPXP+H0= github.com/mitchellh/hashstructure/v2 v2.0.2 h1:vGKWl0YJqUNxE8d+h8f6NJLcCJrgbhC4NcD46KavDd4= github.com/mitchellh/hashstructure/v2 v2.0.2/go.mod h1:MG3aRVU/N29oo/V/IhBX8GR/zz4kQkprJgF2EVszyDE= github.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0QubkSMEySY= @@ -1731,6 +1730,8 @@ github.com/open-telemetry/opentelemetry-collector-contrib/exporter/loadbalancing github.com/open-telemetry/opentelemetry-collector-contrib/exporter/loadbalancingexporter v0.87.0/go.mod h1:JXVmcuySy3xyo3JjoU+CrNWy/C12Fw6JB1HWXf26HwQ= github.com/open-telemetry/opentelemetry-collector-contrib/exporter/prometheusexporter v0.87.0 h1:5LmBAlLycadwA3AHI2rqPuDjx1HFb/PSn3946Eyp3Jw= github.com/open-telemetry/opentelemetry-collector-contrib/exporter/prometheusexporter v0.87.0/go.mod h1:lgOFfu/GLf6LbvZwlumkUv3iBLqRdtBentZKcrrqb3Y= +github.com/open-telemetry/opentelemetry-collector-contrib/exporter/prometheusremotewriteexporter v0.87.0 h1:52+RVfmzj+JePVJuD07gfppdzF9fsKASIRGzTC05QIg= +github.com/open-telemetry/opentelemetry-collector-contrib/exporter/prometheusremotewriteexporter v0.87.0/go.mod h1:VQ7QIry+qNpzGr2/1HrS/IzV9JXoWnqLIsgSi3qPvhM= github.com/open-telemetry/opentelemetry-collector-contrib/extension/basicauthextension v0.87.0 h1:p4pPpRv9zOT/kOQT8GJPhl2drySkTDIpLEhLjXjo5yc= github.com/open-telemetry/opentelemetry-collector-contrib/extension/basicauthextension v0.87.0/go.mod h1:vbU0PUtyhWa3iwIJn7blygKYVnt2GzEAA66zlPbLz90= github.com/open-telemetry/opentelemetry-collector-contrib/extension/bearertokenauthextension v0.87.0 h1:UVFqhd0y7IGSabrHUiDX4efC7qW71tq/FyDFPcBFaJE= @@ -1775,6 +1776,8 @@ github.com/open-telemetry/opentelemetry-collector-contrib/pkg/translator/opencen github.com/open-telemetry/opentelemetry-collector-contrib/pkg/translator/opencensus v0.87.0/go.mod h1:hKArXrn+iYk888KKQThhdPEgPf2GMay2CBe7NnTnmTs= github.com/open-telemetry/opentelemetry-collector-contrib/pkg/translator/prometheus v0.87.0 h1:1eceF0bEseOnk7K6U5OdrEcFKvxEdjnqHTzwNAw2pxA= github.com/open-telemetry/opentelemetry-collector-contrib/pkg/translator/prometheus v0.87.0/go.mod h1:shG9MpBWsBTzns2MYKRFiRymJXhdNb3snGyjgTW5mDg= +github.com/open-telemetry/opentelemetry-collector-contrib/pkg/translator/prometheusremotewrite v0.87.0 h1:uQDcjWlVodE6nYzsRI5LPxZ0X0Ki3fYLJJ3SFK8+MgM= +github.com/open-telemetry/opentelemetry-collector-contrib/pkg/translator/prometheusremotewrite v0.87.0/go.mod h1:xeUhbksYHZ6PkkKidaK95zztJOQcemwxdS+SHracC3Y= github.com/open-telemetry/opentelemetry-collector-contrib/pkg/translator/zipkin v0.87.0 h1:RljU9Xodt7Ptc0enTRuTwUotGi2BuiWBqCUVQwT1otY= github.com/open-telemetry/opentelemetry-collector-contrib/pkg/translator/zipkin v0.87.0/go.mod h1:ybZnD0ldx1tEm6xgJ5wP5tK2x6AY8PNpTonCpOBVI6Y= github.com/open-telemetry/opentelemetry-collector-contrib/processor/attributesprocessor v0.87.0 h1:9MVdMcdtc+Gl0DAaeZ+PdJzskIg1K8FKuYql4h6pQC0= @@ -2185,8 +2188,18 @@ github.com/testcontainers/testcontainers-go/modules/k3s v0.0.0-20230615142642-c1 github.com/tg123/go-htpasswd v1.2.1 h1:i4wfsX1KvvkyoMiHZzjS0VzbAPWfxzI8INcZAKtutoU= github.com/tg123/go-htpasswd v1.2.1/go.mod h1:erHp1B86KXdwQf1X5ZrLb7erXZnWueEQezb2dql4q58= github.com/tidwall/gjson v1.6.0/go.mod h1:P256ACg0Mn+j1RXIDXoss50DeIABTYK1PULOJHhxOls= +github.com/tidwall/gjson v1.10.2 h1:APbLGOM0rrEkd8WBw9C24nllro4ajFuJu0Sc9hRz8Bo= +github.com/tidwall/gjson v1.10.2/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= github.com/tidwall/match v1.0.1/go.mod h1:LujAq0jyVjBy028G1WhWfIzbpQfMO8bBZ6Tyb0+pL9E= +github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA= +github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM= github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= +github.com/tidwall/pretty v1.2.0 h1:RWIZEg2iJ8/g6fDDYzMpobmaoGh5OLl4AXtGUGPcqCs= +github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= +github.com/tidwall/tinylru v1.1.0 h1:XY6IUfzVTU9rpwdhKUF6nQdChgCdGjkMfLzbWyiau6I= +github.com/tidwall/tinylru v1.1.0/go.mod h1:3+bX+TJ2baOLMWTnlyNWHh4QMnFyARg2TLTQ6OFbzw8= +github.com/tidwall/wal v1.1.7 h1:emc1TRjIVsdKKSnpwGBAcsAGg0767SvUk8+ygx7Bb+4= +github.com/tidwall/wal v1.1.7/go.mod h1:r6lR1j27W9EPalgHiB7zLJDYu3mzW5BQP5KrzBpYY/E= github.com/tilinna/clock v1.1.0 h1:6IQQQCo6KoBxVudv6gwtY8o4eDfhHo8ojA5dP0MfhSs= github.com/tilinna/clock v1.1.0/go.mod h1:ZsP7BcY7sEEz7ktc0IVy8Us6boDrK8VradlKRUGfOao= github.com/tinylib/msgp v1.1.5/go.mod h1:eQsjooMTnV42mHu917E26IogZ2930nFyBQdofk10Udg=