Add track_timestamps_staleness

CHANGELOG.md

@@ -25,6 +25,8 @@ Main (unreleased)
 - A new `otelcol.processor.resourcedetection` component which inserts resource attributes 
   to OTLP telemetry based on the host on which Grafana Agent is running. (@ptodev)
 
+- Expose track_timestamps_staleness on Prometheus scraping, to fix the issue where container metrics live for 5 minutes after the container disappears. (@ptodev)
+
 ### Enhancements
 
 - Include line numbers in profiles produced by `pyrsocope.java` component. (@korniltsev)

component/prometheus/scrape/scrape.go

@@ -51,6 +51,8 @@ type Arguments struct {
 	HonorLabels bool `river:"honor_labels,attr,optional"`
 	// Indicator whether the scraped timestamps should be respected.
 	HonorTimestamps bool `river:"honor_timestamps,attr,optional"`
+	// Indicator whether to track the staleness of the scraped timestamps.
+	TrackTimestampsStaleness bool `river:"track_timestamps_staleness,attr,optional"`
 	// A set of query parameters with which the target is scraped.
 	Params url.Values `river:"params,attr,optional"`
 	// Whether to scrape a classic histogram that is also exposed as a native histogram.
@@ -94,13 +96,14 @@ type Arguments struct {
 // SetToDefault implements river.Defaulter.
 func (arg *Arguments) SetToDefault() {
 	*arg = Arguments{
-		MetricsPath:      "/metrics",
-		Scheme:           "http",
-		HonorLabels:      false,
-		HonorTimestamps:  true,
-		HTTPClientConfig: component_config.DefaultHTTPClientConfig,
-		ScrapeInterval:   1 * time.Minute,  // From config.DefaultGlobalConfig
-		ScrapeTimeout:    10 * time.Second, // From config.DefaultGlobalConfig
+		MetricsPath:              "/metrics",
+		Scheme:                   "http",
+		HonorLabels:              false,
+		HonorTimestamps:          true,
+		TrackTimestampsStaleness: false,
+		HTTPClientConfig:         component_config.DefaultHTTPClientConfig,
+		ScrapeInterval:           1 * time.Minute,  // From config.DefaultGlobalConfig
+		ScrapeTimeout:            10 * time.Second, // From config.DefaultGlobalConfig
 	}
 }
 
@@ -287,6 +290,7 @@ func getPromScrapeConfigs(jobName string, c Arguments) *config.ScrapeConfig {
 	}
 	dec.HonorLabels = c.HonorLabels
 	dec.HonorTimestamps = c.HonorTimestamps
+	dec.TrackTimestampsStaleness = c.TrackTimestampsStaleness
 	dec.Params = c.Params
 	dec.ScrapeClassicHistograms = c.ScrapeClassicHistograms
 	dec.ScrapeInterval = model.Duration(c.ScrapeInterval)

component/prometheus/scrape/scrape_test.go

@@ -29,6 +29,7 @@ func TestRiverConfig(t *testing.T) {
 	forward_to      = []
 	scrape_interval = "10s"
 	job_name        = "local"
+	track_timestamps_staleness = true
 
 	bearer_token = "token"
 	proxy_url = "http://0.0.0.0:11111"

converter/internal/prometheusconvert/component/scrape.go

@@ -52,6 +52,7 @@ func toScrapeArguments(scrapeConfig *prom_config.ScrapeConfig, forwardTo []stora
 		JobName:                   scrapeConfig.JobName,
 		HonorLabels:               scrapeConfig.HonorLabels,
 		HonorTimestamps:           scrapeConfig.HonorTimestamps,
+		TrackTimestampsStaleness:  scrapeConfig.TrackTimestampsStaleness,
 		Params:                    scrapeConfig.Params,
 		ScrapeClassicHistograms:   scrapeConfig.ScrapeClassicHistograms,
 		ScrapeInterval:            time.Duration(scrapeConfig.ScrapeInterval),

converter/internal/prometheusconvert/testdata/scrape.river

@@ -9,11 +9,12 @@ prometheus.scrape "prometheus_1" {
 			app         = "foo",
 		}],
 	)
-	forward_to       = [prometheus.remote_write.default.receiver]
-	job_name         = "prometheus-1"
-	honor_timestamps = false
-	scrape_interval  = "10s"
-	scrape_timeout   = "5s"
+	forward_to                 = [prometheus.remote_write.default.receiver]
+	job_name                   = "prometheus-1"
+	honor_timestamps           = false
+	track_timestamps_staleness = true
+	scrape_interval            = "10s"
+	scrape_timeout             = "5s"
 
 	basic_auth {
 		username = "user"

docs/sources/flow/reference/components/prometheus.scrape.md

@@ -51,6 +51,7 @@ Name | Type | Description | Default | Required
 `enable_protobuf_negotiation` | `bool`     | Whether to enable protobuf negotiation with the client. | `false` | no
 `honor_labels`                | `bool`     | Indicator whether the scraped metrics should remain unmodified. | `false` | no
 `honor_timestamps`            | `bool`     | Indicator whether the scraped timestamps should be respected. | `true` | no
+`track_timestamps_staleness`  | `bool`     | Indicator whether to track the staleness of the scraped timestamps. | `false` | no
 `params`                      | `map(list(string))` | A set of query parameters with which the target is scraped. | | no
 `scrape_classic_histograms`   | `bool`     | Whether to scrape a classic histogram that is also exposed as a native histogram. | `false` | no
 `scrape_interval`             | `duration` | How frequently to scrape the targets of this scrape configuration. | `"60s"` | no
@@ -76,6 +77,20 @@ Name | Type | Description | Default | Required
  - [`authorization` block][authorization].
  - [`oauth2` block][oauth2].
 
+`track_timestamps_staleness` controls whether Prometheus tracks [staleness][prom-staleness] of metrics which with an explicit timestamp present in scraped data.
+* An "explicit timestamp" is an optional timestamp in the [Prometheus metrics exposition format][prom-text-exposition-format]. For example, this sample has a timestamp of `1395066363000`:
+  ```
+  http_requests_total{method="post",code="200"} 1027 1395066363000
+  ```
+* If `track_timestamps_staleness` is set to `true`, a staleness marker will be inserted when a metric is no longer present or the target is down.
+* A "staleness marker" is just a {{< term "sample" >}}sample{{< /term >}} with a specific NaN value which is reserved for internal use by Prometheus.
+* It is recommended to set `track_timestamps_staleness` to `true` if the database where metrics are written to has enabled [out of order ingestion][mimir-ooo].
+* If `track_timestamps_staleness` is set to `false`, samples with explicit timestamps will only be labeled as stale after a certain time period, which in Prometheus is 5 minutes by default.
+
+[prom-text-exposition-format]: https://prometheus.io/docs/instrumenting/exposition_formats/#text-based-format
+[prom-staleness]: https://prometheus.io/docs/prometheus/latest/querying/basics/#staleness
+[mimir-ooo]: https://grafana.com/docs/mimir/latest/configure/configure-out-of-order-samples-ingestion/
+
 ## Blocks
 
 The following blocks are supported inside the definition of `prometheus.scrape`: