diff --git a/CHANGELOG.md b/CHANGELOG.md index 9bc8c33ff95c..424ac4c787f6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -63,6 +63,8 @@ Main (unreleased) - Added additional http client proxy configurations to components for `no_proxy`, `proxy_from_environment`, and `proxy_connect_header`. (@erikbaranowski) +- Batch staleness tracking to reduce mutex contention and increase performance. (@mattdurham) + ### Bugfixes - Fix an issue in `remote.s3` where the exported content of an object would be an empty string if `remote.s3` failed to fully retrieve @@ -97,6 +99,9 @@ Main (unreleased) - Updated docs for MSSQL Integration to show additional authentication capabilities. (@StefanKurek) +- `grafana-agent` and `grafana-agent-flow` fallback to default X.509 trusted root certificates + when the `GODEBUG=x509usefallbackroots=1` environment variable is set. (@hainenber) + v0.39.2 (2024-1-31) -------------------- diff --git a/cmd/grafana-agent-flow/main.go b/cmd/grafana-agent-flow/main.go index e64f007a9e38..60ad1707f0dd 100644 --- a/cmd/grafana-agent-flow/main.go +++ b/cmd/grafana-agent-flow/main.go @@ -17,6 +17,10 @@ import ( // Register integrations _ "github.com/grafana/agent/pkg/integrations/install" + + // Embed a set of fallback X.509 trusted roots + // Allows the app to work correctly even when the OS does not provide a verifier or systems roots pool + _ "golang.org/x/crypto/x509roots/fallback" ) func init() { diff --git a/cmd/grafana-agent/main.go b/cmd/grafana-agent/main.go index f2aa40fc860e..976d5654812c 100644 --- a/cmd/grafana-agent/main.go +++ b/cmd/grafana-agent/main.go @@ -21,6 +21,10 @@ import ( // Register integrations _ "github.com/grafana/agent/pkg/integrations/install" + + // Embed a set of fallback X.509 trusted roots + // Allows the app to work correctly even when the OS does not provide a verifier or systems roots pool + _ "golang.org/x/crypto/x509roots/fallback" ) func init() { diff --git a/component/prometheus/fanout.go b/component/prometheus/fanout.go index 32055c01cee7..d5533d820fa5 100644 --- a/component/prometheus/fanout.go +++ b/component/prometheus/fanout.go @@ -12,7 +12,6 @@ import ( "github.com/prometheus/prometheus/model/histogram" "github.com/prometheus/prometheus/model/labels" "github.com/prometheus/prometheus/model/metadata" - "github.com/prometheus/prometheus/model/value" "github.com/prometheus/prometheus/scrape" "github.com/prometheus/prometheus/storage" ) @@ -75,11 +74,12 @@ func (f *Fanout) Appender(ctx context.Context) storage.Appender { ctx = scrape.ContextWithMetricMetadataStore(ctx, NoopMetadataStore{}) app := &appender{ - children: make([]storage.Appender, 0), - componentID: f.componentID, - writeLatency: f.writeLatency, - samplesCounter: f.samplesCounter, - ls: f.ls, + children: make([]storage.Appender, 0), + componentID: f.componentID, + writeLatency: f.writeLatency, + samplesCounter: f.samplesCounter, + ls: f.ls, + stalenessTrackers: make([]labelstore.StalenessTracker, 0), } for _, x := range f.children { @@ -92,12 +92,13 @@ func (f *Fanout) Appender(ctx context.Context) storage.Appender { } type appender struct { - children []storage.Appender - componentID string - writeLatency prometheus.Histogram - samplesCounter prometheus.Counter - start time.Time - ls labelstore.LabelStore + children []storage.Appender + componentID string + writeLatency prometheus.Histogram + samplesCounter prometheus.Counter + start time.Time + ls labelstore.LabelStore + stalenessTrackers []labelstore.StalenessTracker } var _ storage.Appender = (*appender)(nil) @@ -110,12 +111,11 @@ func (a *appender) Append(ref storage.SeriesRef, l labels.Labels, t int64, v flo if ref == 0 { ref = storage.SeriesRef(a.ls.GetOrAddGlobalRefID(l)) } - if value.IsStaleNaN(v) { - a.ls.AddStaleMarker(uint64(ref), l) - } else { - // Tested this to ensure it had no cpu impact, since it is called so often. - a.ls.RemoveStaleMarker(uint64(ref)) - } + a.stalenessTrackers = append(a.stalenessTrackers, labelstore.StalenessTracker{ + GlobalRefID: uint64(ref), + Labels: l, + Value: v, + }) var multiErr error updated := false for _, x := range a.children { @@ -136,6 +136,7 @@ func (a *appender) Append(ref storage.SeriesRef, l labels.Labels, t int64, v flo func (a *appender) Commit() error { defer a.recordLatency() var multiErr error + a.ls.TrackStaleness(a.stalenessTrackers) for _, x := range a.children { err := x.Commit() if err != nil { @@ -148,6 +149,7 @@ func (a *appender) Commit() error { // Rollback satisfies the Appender interface. func (a *appender) Rollback() error { defer a.recordLatency() + a.ls.TrackStaleness(a.stalenessTrackers) var multiErr error for _, x := range a.children { err := x.Rollback() diff --git a/component/prometheus/interceptor.go b/component/prometheus/interceptor.go index c9b88fc6785a..ff33ac8026c6 100644 --- a/component/prometheus/interceptor.go +++ b/component/prometheus/interceptor.go @@ -8,7 +8,6 @@ import ( "github.com/prometheus/prometheus/model/histogram" "github.com/prometheus/prometheus/model/labels" "github.com/prometheus/prometheus/model/metadata" - "github.com/prometheus/prometheus/model/value" "github.com/prometheus/prometheus/storage" ) @@ -80,8 +79,9 @@ func WithHistogramHook(f func(ref storage.SeriesRef, l labels.Labels, t int64, h // Appender satisfies the Appendable interface. func (f *Interceptor) Appender(ctx context.Context) storage.Appender { app := &interceptappender{ - interceptor: f, - ls: f.ls, + interceptor: f, + ls: f.ls, + stalenessTrackers: make([]labelstore.StalenessTracker, 0), } if f.next != nil { app.child = f.next.Appender(ctx) @@ -90,9 +90,10 @@ func (f *Interceptor) Appender(ctx context.Context) storage.Appender { } type interceptappender struct { - interceptor *Interceptor - child storage.Appender - ls labelstore.LabelStore + interceptor *Interceptor + child storage.Appender + ls labelstore.LabelStore + stalenessTrackers []labelstore.StalenessTracker } var _ storage.Appender = (*interceptappender)(nil) @@ -102,13 +103,11 @@ func (a *interceptappender) Append(ref storage.SeriesRef, l labels.Labels, t int if ref == 0 { ref = storage.SeriesRef(a.ls.GetOrAddGlobalRefID(l)) } - - if value.IsStaleNaN(v) { - a.ls.AddStaleMarker(uint64(ref), l) - } else { - // Tested this to ensure it had no cpu impact, since it is called so often. - a.ls.RemoveStaleMarker(uint64(ref)) - } + a.stalenessTrackers = append(a.stalenessTrackers, labelstore.StalenessTracker{ + GlobalRefID: uint64(ref), + Labels: l, + Value: v, + }) if a.interceptor.onAppend != nil { return a.interceptor.onAppend(ref, l, t, v, a.child) @@ -121,6 +120,7 @@ func (a *interceptappender) Append(ref storage.SeriesRef, l labels.Labels, t int // Commit satisfies the Appender interface. func (a *interceptappender) Commit() error { + a.ls.TrackStaleness(a.stalenessTrackers) if a.child == nil { return nil } @@ -129,6 +129,7 @@ func (a *interceptappender) Commit() error { // Rollback satisfies the Appender interface. func (a *interceptappender) Rollback() error { + a.ls.TrackStaleness(a.stalenessTrackers) if a.child == nil { return nil } diff --git a/docs/sources/flow/reference/components/loki.process.md b/docs/sources/flow/reference/components/loki.process.md index 91d25d7fdeee..80978ec4c035 100644 --- a/docs/sources/flow/reference/components/loki.process.md +++ b/docs/sources/flow/reference/components/loki.process.md @@ -961,7 +961,7 @@ The following arguments are supported: | Name | Type | Description | Default | Required | | ------------------ | -------------- | ------------------------------------------------------------------------------- | ------- | -------- | | `labels` | `list(string)` | The values from the extracted data and labels to pack with the log entry. | | yes | -| `ingest_timestamp` | `bool` | Whether to replace the log entry timestamp with the time the `pack` stage runs. | `true | no | +| `ingest_timestamp` | `bool` | Whether to replace the log entry timestamp with the time the `pack` stage runs. | `true` | no | This stage lets you embed extracted values and labels together with the log line, by packing them into a JSON object. The original message is stored under @@ -1459,6 +1459,11 @@ The following arguments are supported: | `location` | `string` | IANA Timezone Database location to use when parsing. | `""` | no | | `action_on_failure` | `string` | What to do when the timestamp can't be extracted or parsed. | `"fudge"` | no | +{{< admonition type="note" >}} +Be careful with further stages which may also override the timestamp. +For example, a `stage.pack` with `ingest_timestamp` set to `true` could replace the timestamp which `stage.timestamp` had set earlier in the pipeline. +{{< /admonition >}} + The `source` field defines which value from the shared map of extracted values the stage should attempt to parse as a timestamp. diff --git a/docs/sources/flow/reference/components/otelcol.auth.headers.md b/docs/sources/flow/reference/components/otelcol.auth.headers.md index bd93a9045e31..6b70a021de35 100644 --- a/docs/sources/flow/reference/components/otelcol.auth.headers.md +++ b/docs/sources/flow/reference/components/otelcol.auth.headers.md @@ -70,9 +70,12 @@ Exactly one of `value` or `from_context` must be provided for each `header` block. The `value` attribute sets the value of the header directly. +Alternatively, `from_context` can be used to dynamically retrieve the header value from request metadata. -Alternatively, `from_context` can be used to dynamically retrieve the header -value from request metadata. +For `from_context` to work, other components in the pipeline also need to be configured appropriately: +* If an `otelcol.processor.batch` is present in the pipeline, it must be configured to preserve client metadata. + Do this by adding the value that `from_context` needs to the `metadata_keys` of the batch processor. +* `otelcol` receivers must be configured with `include_metadata` set to `true` so that metadata keys are available to the pipeline. ## Exported fields @@ -96,10 +99,29 @@ configuration. This example configures [otelcol.exporter.otlp][] to use custom headers: ```river -otelcol.exporter.otlp "example" { - client { - endpoint = "my-otlp-grpc-server:4317" - auth = otelcol.auth.headers.creds.handler +otelcol.receiver.otlp "default" { + http { + include_metadata = true + } + grpc { + include_metadata = true + } + + output { + metrics = [otelcol.processor.batch.default.input] + logs = [otelcol.processor.batch.default.input] + traces = [otelcol.processor.batch.default.input] + } +} + +otelcol.processor.batch "default" { + // Preserve the tenant_id metadata. + metadata_keys = ["tenant_id"] + + output { + metrics = [otelcol.exporter.otlp.production.input] + logs = [otelcol.exporter.otlp.production.input] + traces = [otelcol.exporter.otlp.production.input] } } @@ -114,6 +136,13 @@ otelcol.auth.headers "creds" { value = "user_id" } } + +otelcol.exporter.otlp "production" { + client { + endpoint = env("OTLP_SERVER_ENDPOINT") + auth = otelcol.auth.headers.creds.handler + } +} ``` [otelcol.exporter.otlp]: {{< relref "./otelcol.exporter.otlp.md" >}} diff --git a/go.mod b/go.mod index eee36bc67aa4..207afe279450 100644 --- a/go.mod +++ b/go.mod @@ -616,6 +616,7 @@ require ( github.com/open-telemetry/opentelemetry-collector-contrib/receiver/prometheusreceiver v0.87.0 github.com/open-telemetry/opentelemetry-collector-contrib/receiver/vcenterreceiver v0.87.0 go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v0.42.0 + golang.org/x/crypto/x509roots/fallback v0.0.0-20240208163226-62c9f1799c91 k8s.io/apimachinery v0.28.3 ) diff --git a/go.sum b/go.sum index 73da63596501..3678d04048f4 100644 --- a/go.sum +++ b/go.sum @@ -2513,6 +2513,8 @@ golang.org/x/crypto v0.7.0/go.mod h1:pYwdfH91IfpZVANVyUOhSIPZaFoJGxTFbZhFTx+dXZU golang.org/x/crypto v0.10.0/go.mod h1:o4eNf7Ede1fv+hwOwZsTHl9EsPFO6q6ZvYR8vYfY45I= golang.org/x/crypto v0.18.0 h1:PGVlW0xEltQnzFZ55hkuX5+KLyrMYhHld1YHO4AKcdc= golang.org/x/crypto v0.18.0/go.mod h1:R0j02AL6hcrfOiy9T4ZYp/rcWeMxM3L6QYxlOuEG1mg= +golang.org/x/crypto/x509roots/fallback v0.0.0-20240208163226-62c9f1799c91 h1:Lyizcy9jX02jYR0ceBkL6S+jRys8Uepf7wt1vrz6Ras= +golang.org/x/crypto/x509roots/fallback v0.0.0-20240208163226-62c9f1799c91/go.mod h1:kNa9WdvYnzFwC79zRpLRMJbdEFlhyM5RPFBBZp/wWH8= golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20180807140117-3d87b88a115f/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= diff --git a/operations/helm/charts/grafana-agent/CHANGELOG.md b/operations/helm/charts/grafana-agent/CHANGELOG.md index f6b9caf9aedb..5af50bd3ac0b 100644 --- a/operations/helm/charts/grafana-agent/CHANGELOG.md +++ b/operations/helm/charts/grafana-agent/CHANGELOG.md @@ -10,6 +10,15 @@ internal API changes are not present. Unreleased ---------- +0.32.0 (2024-02-15) +------------------- + +### Enhancements + +- Allow setting scheme for readiness checks when using tls. (@captncraig) + +- Update Grafana Agent version to v0.39.2. (@captncraig) + 0.31.1 (2024-01-19) ------------------- diff --git a/operations/helm/charts/grafana-agent/Chart.yaml b/operations/helm/charts/grafana-agent/Chart.yaml index 76675dcf65a7..bc15d163221a 100644 --- a/operations/helm/charts/grafana-agent/Chart.yaml +++ b/operations/helm/charts/grafana-agent/Chart.yaml @@ -2,8 +2,8 @@ apiVersion: v2 name: grafana-agent description: 'Grafana Agent' type: application -version: 0.31.1 -appVersion: 'v0.39.1' +version: 0.32.0 +appVersion: 'v0.39.2' dependencies: - name: crds diff --git a/operations/helm/charts/grafana-agent/README.md b/operations/helm/charts/grafana-agent/README.md index 4904736239cc..343463584ba5 100644 --- a/operations/helm/charts/grafana-agent/README.md +++ b/operations/helm/charts/grafana-agent/README.md @@ -1,6 +1,6 @@ # Grafana Agent Helm chart -![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![Version: 0.31.1](https://img.shields.io/badge/Version-0.31.1-informational?style=flat-square) ![AppVersion: v0.39.1](https://img.shields.io/badge/AppVersion-v0.39.1-informational?style=flat-square) +![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![Version: 0.32.0](https://img.shields.io/badge/Version-0.32.0-informational?style=flat-square) ![AppVersion: v0.39.2](https://img.shields.io/badge/AppVersion-v0.39.2-informational?style=flat-square) Helm chart for deploying [Grafana Agent][] to Kubernetes. @@ -53,6 +53,7 @@ use the older mode (called "static mode"), set the `agent.mode` value to | agent.extraPorts | list | `[]` | Extra ports to expose on the Agent | | agent.listenAddr | string | `"0.0.0.0"` | Address to listen for traffic on. 0.0.0.0 exposes the UI to other containers. | | agent.listenPort | int | `80` | Port to listen for traffic on. | +| agent.listenScheme | string | `"HTTP"` | Scheme is needed for readiness probes. If enabling tls in your configs, set to "HTTPS" | | agent.mode | string | `"flow"` | Mode to run Grafana Agent in. Can be "flow" or "static". | | agent.mounts.dockercontainers | bool | `false` | Mount /var/lib/docker/containers from the host into the container for log collection. | | agent.mounts.extra | list | `[]` | Extra volume mounts to add into the Grafana Agent container. Does not affect the watch container. | diff --git a/operations/helm/charts/grafana-agent/templates/containers/_agent.yaml b/operations/helm/charts/grafana-agent/templates/containers/_agent.yaml index 32d081b58dc4..0066f198ca0c 100644 --- a/operations/helm/charts/grafana-agent/templates/containers/_agent.yaml +++ b/operations/helm/charts/grafana-agent/templates/containers/_agent.yaml @@ -55,6 +55,7 @@ httpGet: path: /-/ready port: {{ .Values.agent.listenPort }} + scheme: {{ .Values.agent.listenScheme }} initialDelaySeconds: 10 timeoutSeconds: 1 {{- with .Values.agent.resources }} diff --git a/operations/helm/charts/grafana-agent/values.yaml b/operations/helm/charts/grafana-agent/values.yaml index 9050976a43b9..c053280c209a 100644 --- a/operations/helm/charts/grafana-agent/values.yaml +++ b/operations/helm/charts/grafana-agent/values.yaml @@ -53,6 +53,9 @@ agent: # -- Port to listen for traffic on. listenPort: 80 + # -- Scheme is needed for readiness probes. If enabling tls in your configs, set to "HTTPS" + listenScheme: HTTP + # -- Base path where the UI is exposed. uiPathPrefix: / diff --git a/operations/helm/tests/additional-serviceaccount-label/grafana-agent/templates/controllers/daemonset.yaml b/operations/helm/tests/additional-serviceaccount-label/grafana-agent/templates/controllers/daemonset.yaml index ac5ccd4389b9..41d527d5dbea 100644 --- a/operations/helm/tests/additional-serviceaccount-label/grafana-agent/templates/controllers/daemonset.yaml +++ b/operations/helm/tests/additional-serviceaccount-label/grafana-agent/templates/controllers/daemonset.yaml @@ -27,7 +27,7 @@ spec: serviceAccountName: grafana-agent containers: - name: grafana-agent - image: docker.io/grafana/agent:v0.39.1 + image: docker.io/grafana/agent:v0.39.2 imagePullPolicy: IfNotPresent args: - run @@ -51,6 +51,7 @@ spec: httpGet: path: /-/ready port: 80 + scheme: HTTP initialDelaySeconds: 10 timeoutSeconds: 1 volumeMounts: diff --git a/operations/helm/tests/clustering/grafana-agent/templates/controllers/statefulset.yaml b/operations/helm/tests/clustering/grafana-agent/templates/controllers/statefulset.yaml index fb64c8abfc9b..5088ba03f04e 100644 --- a/operations/helm/tests/clustering/grafana-agent/templates/controllers/statefulset.yaml +++ b/operations/helm/tests/clustering/grafana-agent/templates/controllers/statefulset.yaml @@ -30,7 +30,7 @@ spec: serviceAccountName: grafana-agent containers: - name: grafana-agent - image: docker.io/grafana/agent:v0.39.1 + image: docker.io/grafana/agent:v0.39.2 imagePullPolicy: IfNotPresent args: - run @@ -56,6 +56,7 @@ spec: httpGet: path: /-/ready port: 80 + scheme: HTTP initialDelaySeconds: 10 timeoutSeconds: 1 volumeMounts: diff --git a/operations/helm/tests/controller-volumes-extra/grafana-agent/templates/controllers/daemonset.yaml b/operations/helm/tests/controller-volumes-extra/grafana-agent/templates/controllers/daemonset.yaml index 2b36fc32980a..26696e473513 100644 --- a/operations/helm/tests/controller-volumes-extra/grafana-agent/templates/controllers/daemonset.yaml +++ b/operations/helm/tests/controller-volumes-extra/grafana-agent/templates/controllers/daemonset.yaml @@ -27,7 +27,7 @@ spec: serviceAccountName: grafana-agent containers: - name: grafana-agent - image: docker.io/grafana/agent:v0.39.1 + image: docker.io/grafana/agent:v0.39.2 imagePullPolicy: IfNotPresent args: - run @@ -51,6 +51,7 @@ spec: httpGet: path: /-/ready port: 80 + scheme: HTTP initialDelaySeconds: 10 timeoutSeconds: 1 volumeMounts: diff --git a/operations/helm/tests/create-daemonset-hostnetwork/grafana-agent/templates/controllers/daemonset.yaml b/operations/helm/tests/create-daemonset-hostnetwork/grafana-agent/templates/controllers/daemonset.yaml index 252d5e276878..749461246e5d 100644 --- a/operations/helm/tests/create-daemonset-hostnetwork/grafana-agent/templates/controllers/daemonset.yaml +++ b/operations/helm/tests/create-daemonset-hostnetwork/grafana-agent/templates/controllers/daemonset.yaml @@ -27,7 +27,7 @@ spec: serviceAccountName: grafana-agent containers: - name: grafana-agent - image: docker.io/grafana/agent:v0.39.1 + image: docker.io/grafana/agent:v0.39.2 imagePullPolicy: IfNotPresent args: - run @@ -51,6 +51,7 @@ spec: httpGet: path: /-/ready port: 80 + scheme: HTTP initialDelaySeconds: 10 timeoutSeconds: 1 volumeMounts: diff --git a/operations/helm/tests/create-daemonset/grafana-agent/templates/controllers/daemonset.yaml b/operations/helm/tests/create-daemonset/grafana-agent/templates/controllers/daemonset.yaml index ac5ccd4389b9..41d527d5dbea 100644 --- a/operations/helm/tests/create-daemonset/grafana-agent/templates/controllers/daemonset.yaml +++ b/operations/helm/tests/create-daemonset/grafana-agent/templates/controllers/daemonset.yaml @@ -27,7 +27,7 @@ spec: serviceAccountName: grafana-agent containers: - name: grafana-agent - image: docker.io/grafana/agent:v0.39.1 + image: docker.io/grafana/agent:v0.39.2 imagePullPolicy: IfNotPresent args: - run @@ -51,6 +51,7 @@ spec: httpGet: path: /-/ready port: 80 + scheme: HTTP initialDelaySeconds: 10 timeoutSeconds: 1 volumeMounts: diff --git a/operations/helm/tests/create-deployment-autoscaling/grafana-agent/templates/controllers/deployment.yaml b/operations/helm/tests/create-deployment-autoscaling/grafana-agent/templates/controllers/deployment.yaml index 98397b6c00f4..b889386e95f6 100644 --- a/operations/helm/tests/create-deployment-autoscaling/grafana-agent/templates/controllers/deployment.yaml +++ b/operations/helm/tests/create-deployment-autoscaling/grafana-agent/templates/controllers/deployment.yaml @@ -27,7 +27,7 @@ spec: serviceAccountName: grafana-agent containers: - name: grafana-agent - image: docker.io/grafana/agent:v0.39.1 + image: docker.io/grafana/agent:v0.39.2 imagePullPolicy: IfNotPresent args: - run @@ -51,6 +51,7 @@ spec: httpGet: path: /-/ready port: 80 + scheme: HTTP initialDelaySeconds: 10 timeoutSeconds: 1 resources: diff --git a/operations/helm/tests/create-deployment/grafana-agent/templates/controllers/deployment.yaml b/operations/helm/tests/create-deployment/grafana-agent/templates/controllers/deployment.yaml index 5ccee146ffef..a721958449e4 100644 --- a/operations/helm/tests/create-deployment/grafana-agent/templates/controllers/deployment.yaml +++ b/operations/helm/tests/create-deployment/grafana-agent/templates/controllers/deployment.yaml @@ -28,7 +28,7 @@ spec: serviceAccountName: grafana-agent containers: - name: grafana-agent - image: docker.io/grafana/agent:v0.39.1 + image: docker.io/grafana/agent:v0.39.2 imagePullPolicy: IfNotPresent args: - run @@ -52,6 +52,7 @@ spec: httpGet: path: /-/ready port: 80 + scheme: HTTP initialDelaySeconds: 10 timeoutSeconds: 1 volumeMounts: diff --git a/operations/helm/tests/create-statefulset-autoscaling/grafana-agent/templates/controllers/statefulset.yaml b/operations/helm/tests/create-statefulset-autoscaling/grafana-agent/templates/controllers/statefulset.yaml index 132210c7c2e3..1fad47bde2e9 100644 --- a/operations/helm/tests/create-statefulset-autoscaling/grafana-agent/templates/controllers/statefulset.yaml +++ b/operations/helm/tests/create-statefulset-autoscaling/grafana-agent/templates/controllers/statefulset.yaml @@ -29,7 +29,7 @@ spec: serviceAccountName: grafana-agent containers: - name: grafana-agent - image: docker.io/grafana/agent:v0.39.1 + image: docker.io/grafana/agent:v0.39.2 imagePullPolicy: IfNotPresent args: - run @@ -53,6 +53,7 @@ spec: httpGet: path: /-/ready port: 80 + scheme: HTTP initialDelaySeconds: 10 timeoutSeconds: 1 resources: diff --git a/operations/helm/tests/create-statefulset/grafana-agent/templates/controllers/statefulset.yaml b/operations/helm/tests/create-statefulset/grafana-agent/templates/controllers/statefulset.yaml index 6f7d11f76fa9..97eea71b733e 100644 --- a/operations/helm/tests/create-statefulset/grafana-agent/templates/controllers/statefulset.yaml +++ b/operations/helm/tests/create-statefulset/grafana-agent/templates/controllers/statefulset.yaml @@ -30,7 +30,7 @@ spec: serviceAccountName: grafana-agent containers: - name: grafana-agent - image: docker.io/grafana/agent:v0.39.1 + image: docker.io/grafana/agent:v0.39.2 imagePullPolicy: IfNotPresent args: - run @@ -54,6 +54,7 @@ spec: httpGet: path: /-/ready port: 80 + scheme: HTTP initialDelaySeconds: 10 timeoutSeconds: 1 volumeMounts: diff --git a/operations/helm/tests/custom-config/grafana-agent/templates/controllers/daemonset.yaml b/operations/helm/tests/custom-config/grafana-agent/templates/controllers/daemonset.yaml index ac5ccd4389b9..41d527d5dbea 100644 --- a/operations/helm/tests/custom-config/grafana-agent/templates/controllers/daemonset.yaml +++ b/operations/helm/tests/custom-config/grafana-agent/templates/controllers/daemonset.yaml @@ -27,7 +27,7 @@ spec: serviceAccountName: grafana-agent containers: - name: grafana-agent - image: docker.io/grafana/agent:v0.39.1 + image: docker.io/grafana/agent:v0.39.2 imagePullPolicy: IfNotPresent args: - run @@ -51,6 +51,7 @@ spec: httpGet: path: /-/ready port: 80 + scheme: HTTP initialDelaySeconds: 10 timeoutSeconds: 1 volumeMounts: diff --git a/operations/helm/tests/default-values/grafana-agent/templates/controllers/daemonset.yaml b/operations/helm/tests/default-values/grafana-agent/templates/controllers/daemonset.yaml index ac5ccd4389b9..41d527d5dbea 100644 --- a/operations/helm/tests/default-values/grafana-agent/templates/controllers/daemonset.yaml +++ b/operations/helm/tests/default-values/grafana-agent/templates/controllers/daemonset.yaml @@ -27,7 +27,7 @@ spec: serviceAccountName: grafana-agent containers: - name: grafana-agent - image: docker.io/grafana/agent:v0.39.1 + image: docker.io/grafana/agent:v0.39.2 imagePullPolicy: IfNotPresent args: - run @@ -51,6 +51,7 @@ spec: httpGet: path: /-/ready port: 80 + scheme: HTTP initialDelaySeconds: 10 timeoutSeconds: 1 volumeMounts: diff --git a/operations/helm/tests/enable-servicemonitor/grafana-agent/templates/controllers/daemonset.yaml b/operations/helm/tests/enable-servicemonitor/grafana-agent/templates/controllers/daemonset.yaml index ac5ccd4389b9..41d527d5dbea 100644 --- a/operations/helm/tests/enable-servicemonitor/grafana-agent/templates/controllers/daemonset.yaml +++ b/operations/helm/tests/enable-servicemonitor/grafana-agent/templates/controllers/daemonset.yaml @@ -27,7 +27,7 @@ spec: serviceAccountName: grafana-agent containers: - name: grafana-agent - image: docker.io/grafana/agent:v0.39.1 + image: docker.io/grafana/agent:v0.39.2 imagePullPolicy: IfNotPresent args: - run @@ -51,6 +51,7 @@ spec: httpGet: path: /-/ready port: 80 + scheme: HTTP initialDelaySeconds: 10 timeoutSeconds: 1 volumeMounts: diff --git a/operations/helm/tests/envFrom/grafana-agent/templates/controllers/daemonset.yaml b/operations/helm/tests/envFrom/grafana-agent/templates/controllers/daemonset.yaml index 85fb01959587..448a41609886 100644 --- a/operations/helm/tests/envFrom/grafana-agent/templates/controllers/daemonset.yaml +++ b/operations/helm/tests/envFrom/grafana-agent/templates/controllers/daemonset.yaml @@ -27,7 +27,7 @@ spec: serviceAccountName: grafana-agent containers: - name: grafana-agent - image: docker.io/grafana/agent:v0.39.1 + image: docker.io/grafana/agent:v0.39.2 imagePullPolicy: IfNotPresent args: - run @@ -54,6 +54,7 @@ spec: httpGet: path: /-/ready port: 80 + scheme: HTTP initialDelaySeconds: 10 timeoutSeconds: 1 volumeMounts: diff --git a/operations/helm/tests/existing-config/grafana-agent/templates/controllers/daemonset.yaml b/operations/helm/tests/existing-config/grafana-agent/templates/controllers/daemonset.yaml index b6a93df6f9f8..aecad9744502 100644 --- a/operations/helm/tests/existing-config/grafana-agent/templates/controllers/daemonset.yaml +++ b/operations/helm/tests/existing-config/grafana-agent/templates/controllers/daemonset.yaml @@ -27,7 +27,7 @@ spec: serviceAccountName: grafana-agent containers: - name: grafana-agent - image: docker.io/grafana/agent:v0.39.1 + image: docker.io/grafana/agent:v0.39.2 imagePullPolicy: IfNotPresent args: - run @@ -51,6 +51,7 @@ spec: httpGet: path: /-/ready port: 80 + scheme: HTTP initialDelaySeconds: 10 timeoutSeconds: 1 volumeMounts: diff --git a/operations/helm/tests/extra-env/grafana-agent/templates/controllers/daemonset.yaml b/operations/helm/tests/extra-env/grafana-agent/templates/controllers/daemonset.yaml index 4f45d0fc40ed..6b458bf8acbe 100644 --- a/operations/helm/tests/extra-env/grafana-agent/templates/controllers/daemonset.yaml +++ b/operations/helm/tests/extra-env/grafana-agent/templates/controllers/daemonset.yaml @@ -27,7 +27,7 @@ spec: serviceAccountName: grafana-agent containers: - name: grafana-agent - image: docker.io/grafana/agent:v0.39.1 + image: docker.io/grafana/agent:v0.39.2 imagePullPolicy: IfNotPresent args: - run @@ -60,6 +60,7 @@ spec: httpGet: path: /-/ready port: 80 + scheme: HTTP initialDelaySeconds: 10 timeoutSeconds: 1 volumeMounts: diff --git a/operations/helm/tests/extra-ports/grafana-agent/templates/controllers/daemonset.yaml b/operations/helm/tests/extra-ports/grafana-agent/templates/controllers/daemonset.yaml index c07733057e80..a0d812daf634 100644 --- a/operations/helm/tests/extra-ports/grafana-agent/templates/controllers/daemonset.yaml +++ b/operations/helm/tests/extra-ports/grafana-agent/templates/controllers/daemonset.yaml @@ -27,7 +27,7 @@ spec: serviceAccountName: grafana-agent containers: - name: grafana-agent - image: docker.io/grafana/agent:v0.39.1 + image: docker.io/grafana/agent:v0.39.2 imagePullPolicy: IfNotPresent args: - run @@ -54,6 +54,7 @@ spec: httpGet: path: /-/ready port: 80 + scheme: HTTP initialDelaySeconds: 10 timeoutSeconds: 1 volumeMounts: diff --git a/operations/helm/tests/faro-ingress/grafana-agent/templates/controllers/daemonset.yaml b/operations/helm/tests/faro-ingress/grafana-agent/templates/controllers/daemonset.yaml index c955003a200e..9b1d7375d803 100644 --- a/operations/helm/tests/faro-ingress/grafana-agent/templates/controllers/daemonset.yaml +++ b/operations/helm/tests/faro-ingress/grafana-agent/templates/controllers/daemonset.yaml @@ -27,7 +27,7 @@ spec: serviceAccountName: grafana-agent containers: - name: grafana-agent - image: docker.io/grafana/agent:v0.39.1 + image: docker.io/grafana/agent:v0.39.2 imagePullPolicy: IfNotPresent args: - run @@ -54,6 +54,7 @@ spec: httpGet: path: /-/ready port: 80 + scheme: HTTP initialDelaySeconds: 10 timeoutSeconds: 1 volumeMounts: diff --git a/operations/helm/tests/global-image-pullsecrets/grafana-agent/templates/controllers/daemonset.yaml b/operations/helm/tests/global-image-pullsecrets/grafana-agent/templates/controllers/daemonset.yaml index 1f05afd30c17..35ed8c3eee6c 100644 --- a/operations/helm/tests/global-image-pullsecrets/grafana-agent/templates/controllers/daemonset.yaml +++ b/operations/helm/tests/global-image-pullsecrets/grafana-agent/templates/controllers/daemonset.yaml @@ -32,7 +32,7 @@ spec: - name: global-cred containers: - name: grafana-agent - image: docker.io/grafana/agent:v0.39.1 + image: docker.io/grafana/agent:v0.39.2 imagePullPolicy: IfNotPresent args: - run @@ -56,6 +56,7 @@ spec: httpGet: path: /-/ready port: 80 + scheme: HTTP initialDelaySeconds: 10 timeoutSeconds: 1 volumeMounts: diff --git a/operations/helm/tests/global-image-registry/grafana-agent/templates/controllers/daemonset.yaml b/operations/helm/tests/global-image-registry/grafana-agent/templates/controllers/daemonset.yaml index 0048873d187e..c18af15deea3 100644 --- a/operations/helm/tests/global-image-registry/grafana-agent/templates/controllers/daemonset.yaml +++ b/operations/helm/tests/global-image-registry/grafana-agent/templates/controllers/daemonset.yaml @@ -27,7 +27,7 @@ spec: serviceAccountName: grafana-agent containers: - name: grafana-agent - image: quay.io/grafana/agent:v0.39.1 + image: quay.io/grafana/agent:v0.39.2 imagePullPolicy: IfNotPresent args: - run @@ -51,6 +51,7 @@ spec: httpGet: path: /-/ready port: 80 + scheme: HTTP initialDelaySeconds: 10 timeoutSeconds: 1 volumeMounts: diff --git a/operations/helm/tests/initcontainers/grafana-agent/templates/controllers/daemonset.yaml b/operations/helm/tests/initcontainers/grafana-agent/templates/controllers/daemonset.yaml index 0d58403356bb..fb649aa53fb6 100644 --- a/operations/helm/tests/initcontainers/grafana-agent/templates/controllers/daemonset.yaml +++ b/operations/helm/tests/initcontainers/grafana-agent/templates/controllers/daemonset.yaml @@ -45,7 +45,7 @@ spec: name: geoip containers: - name: grafana-agent - image: docker.io/grafana/agent:v0.39.1 + image: docker.io/grafana/agent:v0.39.2 imagePullPolicy: IfNotPresent args: - run @@ -69,6 +69,7 @@ spec: httpGet: path: /-/ready port: 80 + scheme: HTTP initialDelaySeconds: 10 timeoutSeconds: 1 volumeMounts: diff --git a/operations/helm/tests/local-image-pullsecrets/grafana-agent/templates/controllers/daemonset.yaml b/operations/helm/tests/local-image-pullsecrets/grafana-agent/templates/controllers/daemonset.yaml index 2ce6b7ad7a21..dd90c71c2af9 100644 --- a/operations/helm/tests/local-image-pullsecrets/grafana-agent/templates/controllers/daemonset.yaml +++ b/operations/helm/tests/local-image-pullsecrets/grafana-agent/templates/controllers/daemonset.yaml @@ -29,7 +29,7 @@ spec: - name: local-cred containers: - name: grafana-agent - image: docker.io/grafana/agent:v0.39.1 + image: docker.io/grafana/agent:v0.39.2 imagePullPolicy: IfNotPresent args: - run @@ -53,6 +53,7 @@ spec: httpGet: path: /-/ready port: 80 + scheme: HTTP initialDelaySeconds: 10 timeoutSeconds: 1 volumeMounts: diff --git a/operations/helm/tests/local-image-registry/grafana-agent/templates/controllers/daemonset.yaml b/operations/helm/tests/local-image-registry/grafana-agent/templates/controllers/daemonset.yaml index 0048873d187e..c18af15deea3 100644 --- a/operations/helm/tests/local-image-registry/grafana-agent/templates/controllers/daemonset.yaml +++ b/operations/helm/tests/local-image-registry/grafana-agent/templates/controllers/daemonset.yaml @@ -27,7 +27,7 @@ spec: serviceAccountName: grafana-agent containers: - name: grafana-agent - image: quay.io/grafana/agent:v0.39.1 + image: quay.io/grafana/agent:v0.39.2 imagePullPolicy: IfNotPresent args: - run @@ -51,6 +51,7 @@ spec: httpGet: path: /-/ready port: 80 + scheme: HTTP initialDelaySeconds: 10 timeoutSeconds: 1 volumeMounts: diff --git a/operations/helm/tests/nodeselectors-and-tolerations/grafana-agent/templates/controllers/daemonset.yaml b/operations/helm/tests/nodeselectors-and-tolerations/grafana-agent/templates/controllers/daemonset.yaml index b4c896139945..9d70a1b0c49e 100644 --- a/operations/helm/tests/nodeselectors-and-tolerations/grafana-agent/templates/controllers/daemonset.yaml +++ b/operations/helm/tests/nodeselectors-and-tolerations/grafana-agent/templates/controllers/daemonset.yaml @@ -27,7 +27,7 @@ spec: serviceAccountName: grafana-agent containers: - name: grafana-agent - image: docker.io/grafana/agent:v0.39.1 + image: docker.io/grafana/agent:v0.39.2 imagePullPolicy: IfNotPresent args: - run @@ -51,6 +51,7 @@ spec: httpGet: path: /-/ready port: 80 + scheme: HTTP initialDelaySeconds: 10 timeoutSeconds: 1 volumeMounts: diff --git a/operations/helm/tests/pod_annotations/grafana-agent/templates/controllers/daemonset.yaml b/operations/helm/tests/pod_annotations/grafana-agent/templates/controllers/daemonset.yaml index 8fb72ca6f523..fc507885ae2c 100644 --- a/operations/helm/tests/pod_annotations/grafana-agent/templates/controllers/daemonset.yaml +++ b/operations/helm/tests/pod_annotations/grafana-agent/templates/controllers/daemonset.yaml @@ -28,7 +28,7 @@ spec: serviceAccountName: grafana-agent containers: - name: grafana-agent - image: docker.io/grafana/agent:v0.39.1 + image: docker.io/grafana/agent:v0.39.2 imagePullPolicy: IfNotPresent args: - run @@ -52,6 +52,7 @@ spec: httpGet: path: /-/ready port: 80 + scheme: HTTP initialDelaySeconds: 10 timeoutSeconds: 1 volumeMounts: diff --git a/operations/helm/tests/sidecars/grafana-agent/templates/controllers/daemonset.yaml b/operations/helm/tests/sidecars/grafana-agent/templates/controllers/daemonset.yaml index bb9b6a1ba64d..998f9770e590 100644 --- a/operations/helm/tests/sidecars/grafana-agent/templates/controllers/daemonset.yaml +++ b/operations/helm/tests/sidecars/grafana-agent/templates/controllers/daemonset.yaml @@ -27,7 +27,7 @@ spec: serviceAccountName: grafana-agent containers: - name: grafana-agent - image: docker.io/grafana/agent:v0.39.1 + image: docker.io/grafana/agent:v0.39.2 imagePullPolicy: IfNotPresent args: - run @@ -51,6 +51,7 @@ spec: httpGet: path: /-/ready port: 80 + scheme: HTTP initialDelaySeconds: 10 timeoutSeconds: 1 volumeMounts: diff --git a/operations/helm/tests/static-mode/grafana-agent/templates/controllers/daemonset.yaml b/operations/helm/tests/static-mode/grafana-agent/templates/controllers/daemonset.yaml index f676d9f3e046..47367e4d5ae4 100644 --- a/operations/helm/tests/static-mode/grafana-agent/templates/controllers/daemonset.yaml +++ b/operations/helm/tests/static-mode/grafana-agent/templates/controllers/daemonset.yaml @@ -27,7 +27,7 @@ spec: serviceAccountName: grafana-agent containers: - name: grafana-agent - image: docker.io/grafana/agent:v0.39.1 + image: docker.io/grafana/agent:v0.39.2 imagePullPolicy: IfNotPresent args: - -config.file=/etc/agent/config.yaml @@ -48,6 +48,7 @@ spec: httpGet: path: /-/ready port: 80 + scheme: HTTP initialDelaySeconds: 10 timeoutSeconds: 1 volumeMounts: diff --git a/operations/helm/tests/topologyspreadconstraints/grafana-agent/templates/controllers/deployment.yaml b/operations/helm/tests/topologyspreadconstraints/grafana-agent/templates/controllers/deployment.yaml index b28114e09e4d..79db950e4208 100644 --- a/operations/helm/tests/topologyspreadconstraints/grafana-agent/templates/controllers/deployment.yaml +++ b/operations/helm/tests/topologyspreadconstraints/grafana-agent/templates/controllers/deployment.yaml @@ -28,7 +28,7 @@ spec: serviceAccountName: grafana-agent containers: - name: grafana-agent - image: docker.io/grafana/agent:v0.39.1 + image: docker.io/grafana/agent:v0.39.2 imagePullPolicy: IfNotPresent args: - run @@ -52,6 +52,7 @@ spec: httpGet: path: /-/ready port: 80 + scheme: HTTP initialDelaySeconds: 10 timeoutSeconds: 1 volumeMounts: diff --git a/operations/helm/tests/with-digests/grafana-agent/templates/controllers/daemonset.yaml b/operations/helm/tests/with-digests/grafana-agent/templates/controllers/daemonset.yaml index fbe3f266523b..d52c502d3c31 100644 --- a/operations/helm/tests/with-digests/grafana-agent/templates/controllers/daemonset.yaml +++ b/operations/helm/tests/with-digests/grafana-agent/templates/controllers/daemonset.yaml @@ -51,6 +51,7 @@ spec: httpGet: path: /-/ready port: 80 + scheme: HTTP initialDelaySeconds: 10 timeoutSeconds: 1 volumeMounts: diff --git a/service/labelstore/data.go b/service/labelstore/data.go index 628bc8fd89e7..21739d8f140f 100644 --- a/service/labelstore/data.go +++ b/service/labelstore/data.go @@ -3,7 +3,6 @@ package labelstore import "github.com/prometheus/prometheus/model/labels" type LabelStore interface { - // GetOrAddLink returns the global id for the values, if none found one will be created based on the lbls. GetOrAddLink(componentID string, localRefID uint64, lbls labels.Labels) uint64 @@ -16,12 +15,16 @@ type LabelStore interface { // GetLocalRefID gets the mapping from global to local id specific to a component. Returns 0 if nothing found. GetLocalRefID(componentID string, globalRefID uint64) uint64 - // AddStaleMarker adds a stale marker to a reference, that reference will then get removed on the next check. - AddStaleMarker(globalRefID uint64, l labels.Labels) - - // RemoveStaleMarker removes the stale marker for a reference, keeping it around. - RemoveStaleMarker(globalRefID uint64) + // TrackStaleness adds a stale marker if NaN, then that reference will be removed on the next check. If not a NaN + // then if tracked will remove it. + TrackStaleness(ids []StalenessTracker) // CheckAndRemoveStaleMarkers identifies any series with a stale marker and removes those entries from the LabelStore. CheckAndRemoveStaleMarkers() } + +type StalenessTracker struct { + GlobalRefID uint64 + Value float64 + Labels labels.Labels +} diff --git a/service/labelstore/service.go b/service/labelstore/service.go index f8ca82ba5ba4..79de6f772928 100644 --- a/service/labelstore/service.go +++ b/service/labelstore/service.go @@ -11,6 +11,7 @@ import ( flow_service "github.com/grafana/agent/service" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/prometheus/model/labels" + "github.com/prometheus/prometheus/model/value" ) const ServiceName = "labelstore" @@ -73,6 +74,7 @@ func (s *service) Describe(m chan<- *prometheus.Desc) { m <- s.totalIDs m <- s.idsInRemoteWrapping } + func (s *service) Collect(m chan<- prometheus.Metric) { s.mut.Lock() defer s.mut.Unlock() @@ -196,24 +198,34 @@ func (s *service) GetLocalRefID(componentID string, globalRefID uint64) uint64 { return local } -// AddStaleMarker adds a stale marker -func (s *service) AddStaleMarker(globalRefID uint64, l labels.Labels) { - s.mut.Lock() - defer s.mut.Unlock() - - s.staleGlobals[globalRefID] = &staleMarker{ - lastMarkedStale: time.Now(), - labelHash: l.Hash(), - globalID: globalRefID, +func (s *service) TrackStaleness(ids []StalenessTracker) { + var ( + toAdd = make([]*staleMarker, 0) + toRemove = make([]uint64, 0) + now = time.Now() + ) + + for _, id := range ids { + if value.IsStaleNaN(id.Value) { + toAdd = append(toAdd, &staleMarker{ + globalID: id.GlobalRefID, + lastMarkedStale: now, + labelHash: id.Labels.Hash(), + }) + } else { + toRemove = append(toRemove, id.GlobalRefID) + } } -} -// RemoveStaleMarker removes a stale marker -func (s *service) RemoveStaleMarker(globalRefID uint64) { s.mut.Lock() defer s.mut.Unlock() - delete(s.staleGlobals, globalRefID) + for _, marker := range toAdd { + s.staleGlobals[marker.globalID] = marker + } + for _, id := range toRemove { + delete(s.staleGlobals, id) + } } // staleDuration determines how long we should wait after a stale value is received to GC that value diff --git a/service/labelstore/service_test.go b/service/labelstore/service_test.go index 5a16ce9d8d95..7ea0bf748fe1 100644 --- a/service/labelstore/service_test.go +++ b/service/labelstore/service_test.go @@ -1,12 +1,16 @@ package labelstore import ( + "math" + "strconv" + "sync" "testing" "time" "github.com/go-kit/log" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/prometheus/model/labels" + "github.com/prometheus/prometheus/model/value" "github.com/stretchr/testify/require" ) @@ -122,7 +126,13 @@ func TestStaleness(t *testing.T) { global1 := mapping.GetOrAddLink("1", 1, l) _ = mapping.GetOrAddLink("2", 1, l2) - mapping.AddStaleMarker(global1, l) + mapping.TrackStaleness([]StalenessTracker{ + { + GlobalRefID: global1, + Value: math.Float64frombits(value.StaleNaN), + Labels: l, + }, + }) require.Len(t, mapping.staleGlobals, 1) require.Len(t, mapping.labelsHashToGlobal, 2) staleDuration = 1 * time.Millisecond @@ -141,8 +151,54 @@ func TestRemovingStaleness(t *testing.T) { }) global1 := mapping.GetOrAddLink("1", 1, l) - mapping.AddStaleMarker(global1, l) + mapping.TrackStaleness([]StalenessTracker{ + { + GlobalRefID: global1, + Value: math.Float64frombits(value.StaleNaN), + Labels: l, + }, + }) + require.Len(t, mapping.staleGlobals, 1) - mapping.RemoveStaleMarker(global1) + // This should remove it from staleness tracking. + mapping.TrackStaleness([]StalenessTracker{ + { + GlobalRefID: global1, + Value: 1, + Labels: l, + }, + }) require.Len(t, mapping.staleGlobals, 0) } + +func BenchmarkStaleness(b *testing.B) { + b.StopTimer() + ls := New(log.NewNopLogger(), prometheus.DefaultRegisterer) + + tracking := make([]StalenessTracker, 100_000) + for i := 0; i < 100_000; i++ { + l := labels.FromStrings("id", strconv.Itoa(i)) + gid := ls.GetOrAddGlobalRefID(l) + var val float64 + if i%2 == 0 { + val = float64(i) + } else { + val = math.Float64frombits(value.StaleNaN) + } + tracking[i] = StalenessTracker{ + GlobalRefID: gid, + Value: val, + Labels: l, + } + } + b.StartTimer() + var wg sync.WaitGroup + for i := 0; i < b.N; i++ { + wg.Add(1) + go func() { + ls.TrackStaleness(tracking) + wg.Done() + }() + } + wg.Wait() +}