chore: Make metric for dequeued tasks in bloom-gateway a Histogram (#…

…14413) This change allows to observe the distribution of how many tasks are dequeued at once over time. Signed-off-by: Christian Haudum <[email protected]>
grafana · Oct 8, 2024 · 91c7d34 · 91c7d34
1 parent 1f30892
commit 91c7d34
Show file tree

Hide file tree

Showing 2 changed files with 7 additions and 6 deletions.
diff --git a/pkg/bloomgateway/metrics.go b/pkg/bloomgateway/metrics.go
@@ -119,7 +119,7 @@ type workerMetrics struct {
 	dequeueDuration    *prometheus.HistogramVec
 	queueDuration      *prometheus.HistogramVec
 	processDuration    *prometheus.HistogramVec
-	tasksDequeued      *prometheus.CounterVec
+	tasksDequeued      *prometheus.HistogramVec
 	tasksProcessed     *prometheus.CounterVec
 	blocksNotAvailable *prometheus.CounterVec
 	blockQueryLatency  *prometheus.HistogramVec
@@ -147,11 +147,12 @@ func newWorkerMetrics(registerer prometheus.Registerer, namespace, subsystem str
 			Name:      "process_duration_seconds",
 			Help:      "Time spent processing tasks in seconds",
 		}, append(labels, "status")),
-		tasksDequeued: r.NewCounterVec(prometheus.CounterOpts{
+		tasksDequeued: r.NewHistogramVec(prometheus.HistogramOpts{
 			Namespace: namespace,
 			Subsystem: subsystem,
-			Name:      "tasks_dequeued_total",
-			Help:      "Total amount of tasks that the worker dequeued from the queue",
+			Name:      "tasks_dequeued",
+			Help:      "Total amount of tasks that the worker dequeued from the queue at once",
+			Buckets:   prometheus.ExponentialBuckets(1, 2, 8), // [1, 2, ..., 128]
 		}, append(labels, "status")),
 		tasksProcessed: r.NewCounterVec(prometheus.CounterOpts{
 			Namespace: namespace,

diff --git a/pkg/bloomgateway/worker.go b/pkg/bloomgateway/worker.go
@@ -76,7 +76,7 @@ func (w *worker) running(_ context.Context) error {
 			if err == queue.ErrStopped && len(items) == 0 {
 				return err
 			}
-			w.metrics.tasksDequeued.WithLabelValues(w.id, labelFailure).Inc()
+			w.metrics.tasksDequeued.WithLabelValues(w.id, labelFailure).Observe(1)
 			level.Error(w.logger).Log("msg", "failed to dequeue tasks", "err", err, "items", len(items))
 		}
 		idx = newIdx
@@ -86,7 +86,7 @@ func (w *worker) running(_ context.Context) error {
 			continue
 		}
 
-		w.metrics.tasksDequeued.WithLabelValues(w.id, labelSuccess).Add(float64(len(items)))
+		w.metrics.tasksDequeued.WithLabelValues(w.id, labelSuccess).Observe(float64(len(items)))
 
 		tasks := make([]Task, 0, len(items))
 		for _, item := range items {