From 83996138ff2ff2032b984fc9afa9abe38aeea6cc Mon Sep 17 00:00:00 2001 From: gabemontero Date: Wed, 7 Aug 2024 14:14:19 -0400 Subject: [PATCH] check namespace config for node throttle metric Now that @vdemeester 's fix so that the background metric polling goroutines for pipelinerunmetrics and taskrunmetrics can access config.observability, we can properly address issue around verifiying non-default config around adding the namespace label for the node and quota throttle metrics that @divyansh42 @khrm and myself discovered last week. Signed-off-by: gabemontero rh-pre-commit.version: 2.3.1 rh-pre-commit.check-secrets: ENABLED --- pkg/taskrunmetrics/metrics.go | 6 +++++- pkg/taskrunmetrics/metrics_test.go | 12 ++++++++++++ 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/pkg/taskrunmetrics/metrics.go b/pkg/taskrunmetrics/metrics.go index 2a4fa887a24..1c5bd91ab1b 100644 --- a/pkg/taskrunmetrics/metrics.go +++ b/pkg/taskrunmetrics/metrics.go @@ -528,7 +528,11 @@ func (r *Recorder) RunningTaskRuns(ctx context.Context, lister listers.TaskRunLi metrics.Record(ctx, runningTRsThrottledByQuota.M(float64(cnt))) } for ns, cnt := range trsThrottledByNode { - ctx, err := tag.New(ctx, []tag.Mutator{tag.Insert(namespaceTag, ns)}...) + var mutators []tag.Mutator + if addNamespaceLabelToQuotaThrottleMetric { + mutators = []tag.Mutator{tag.Insert(namespaceTag, ns)} + } + ctx, err := tag.New(ctx, mutators...) if err != nil { return err } diff --git a/pkg/taskrunmetrics/metrics_test.go b/pkg/taskrunmetrics/metrics_test.go index d2a02ef35a3..3c4502cfa89 100644 --- a/pkg/taskrunmetrics/metrics_test.go +++ b/pkg/taskrunmetrics/metrics_test.go @@ -583,6 +583,18 @@ func TestRecordRunningTaskRunsThrottledCounts(t *testing.T) { reason: pod.ReasonExceededNodeResources, nodeCount: 3, }, + { + status: corev1.ConditionUnknown, + reason: pod.ReasonExceededResourceQuota, + quotaCount: 3, + addNS: true, + }, + { + status: corev1.ConditionUnknown, + reason: pod.ReasonExceededNodeResources, + nodeCount: 3, + addNS: true, + }, { status: corev1.ConditionUnknown, reason: v1.TaskRunReasonResolvingTaskRef,