diff --git a/pkg/controllers/cronfederatedhpa/cronfederatedhpa_controller.go b/pkg/controllers/cronfederatedhpa/cronfederatedhpa_controller.go index 3b159cf7f3ba..981e23e87381 100755 --- a/pkg/controllers/cronfederatedhpa/cronfederatedhpa_controller.go +++ b/pkg/controllers/cronfederatedhpa/cronfederatedhpa_controller.go @@ -15,6 +15,7 @@ package cronfederatedhpa import ( "context" + "time" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/equality" @@ -28,6 +29,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/controller" autoscalingv1alpha1 "github.com/karmada-io/karmada/pkg/apis/autoscaling/v1alpha1" + "github.com/karmada-io/karmada/pkg/metrics" "github.com/karmada-io/karmada/pkg/sharedcli/ratelimiterflag" "github.com/karmada-io/karmada/pkg/util/helper" ) @@ -70,6 +72,9 @@ func (c *CronFHPAController) Reconcile(ctx context.Context, req controllerruntim return controllerruntime.Result{}, nil } + var err error + defer metrics.ObserveProcessCronFederatedHPALatency(err, time.Now()) + origRuleSets := sets.New[string]() for _, history := range cronFHPA.Status.ExecutionHistories { origRuleSets.Insert(history.RuleName) @@ -84,7 +89,7 @@ func (c *CronFHPAController) Reconcile(ctx context.Context, req controllerruntim newRuleSets := sets.New[string]() for _, rule := range cronFHPA.Spec.Rules { - if err := c.processCronRule(cronFHPA, rule); err != nil { + if err = c.processCronRule(cronFHPA, rule); err != nil { return controllerruntime.Result{Requeue: true}, err } newRuleSets.Insert(rule.Name) @@ -96,7 +101,7 @@ func (c *CronFHPAController) Reconcile(ctx context.Context, req controllerruntim continue } c.CronHandler.StopRuleExecutor(req.NamespacedName.String(), name) - if err := c.removeCronFHPAHistory(cronFHPA, name); err != nil { + if err = c.removeCronFHPAHistory(cronFHPA, name); err != nil { return controllerruntime.Result{Requeue: true}, err } } diff --git a/pkg/controllers/cronfederatedhpa/cronfederatedhpa_job.go b/pkg/controllers/cronfederatedhpa/cronfederatedhpa_job.go index ff302ce26c5d..4132459b9295 100755 --- a/pkg/controllers/cronfederatedhpa/cronfederatedhpa_job.go +++ b/pkg/controllers/cronfederatedhpa/cronfederatedhpa_job.go @@ -34,6 +34,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" autoscalingv1alpha1 "github.com/karmada-io/karmada/pkg/apis/autoscaling/v1alpha1" + "github.com/karmada-io/karmada/pkg/metrics" "github.com/karmada-io/karmada/pkg/util" "github.com/karmada-io/karmada/pkg/util/helper" ) @@ -86,7 +87,9 @@ func RunCronFederatedHPARule(c *CronFederatedHPAJob) { } var scaleErr error + start := time.Now() defer func() { + metrics.ObserveProcessCronFederatedHPARuleLatency(scaleErr, start) if scaleErr != nil { c.eventRecorder.Event(cronFHPA, corev1.EventTypeWarning, "ScaleFailed", scaleErr.Error()) err = c.addFailedExecutionHistory(cronFHPA, scaleErr.Error()) diff --git a/pkg/metrics/resource.go b/pkg/metrics/resource.go index c649bfa3d976..2ff643d98bad 100644 --- a/pkg/metrics/resource.go +++ b/pkg/metrics/resource.go @@ -9,12 +9,14 @@ import ( ) const ( - resourceMatchPolicyDurationMetricsName = "resource_match_policy_duration_seconds" - resourceApplyPolicyDurationMetricsName = "resource_apply_policy_duration_seconds" - policyApplyAttemptsMetricsName = "policy_apply_attempts_total" - syncWorkDurationMetricsName = "binding_sync_work_duration_seconds" - syncWorkloadDurationMetricsName = "work_sync_workload_duration_seconds" - policyPreemptionMetricsName = "policy_preemption_total" + resourceMatchPolicyDurationMetricsName = "resource_match_policy_duration_seconds" + resourceApplyPolicyDurationMetricsName = "resource_apply_policy_duration_seconds" + policyApplyAttemptsMetricsName = "policy_apply_attempts_total" + syncWorkDurationMetricsName = "binding_sync_work_duration_seconds" + syncWorkloadDurationMetricsName = "work_sync_workload_duration_seconds" + policyPreemptionMetricsName = "policy_preemption_total" + cronFederatedHPADurationMetricsName = "cron_federated_hpa_process_duration_seconds" + cronFederatedHPARuleDurationMetricsName = "cron_federated_hpa_rule_process_duration_seconds" ) var ( @@ -51,6 +53,18 @@ var ( Name: policyPreemptionMetricsName, Help: "Number of preemption for the resource template. By the result, 'error' means a resource template failed to be preempted by other propagation policies. Otherwise 'success'.", }, []string{"result"}) + + cronFederatedHPADurationHistogram = prometheus.NewHistogramVec(prometheus.HistogramOpts{ + Name: cronFederatedHPADurationMetricsName, + Help: "Duration in seconds to process a cron federated HPA. By the result, 'error' means a cron federated HPA failed to be processed. Otherwise 'success'.", + Buckets: prometheus.ExponentialBuckets(0.001, 2, 12), + }, []string{"result"}) + + cronFederatedHPARuleDurationHistogram = prometheus.NewHistogramVec(prometheus.HistogramOpts{ + Name: cronFederatedHPARuleDurationMetricsName, + Help: "Duration in seconds to process a cron federated HPA rule. By the result, 'error' means a cron federated HPA rule failed to be processed. Otherwise 'success'.", + Buckets: prometheus.ExponentialBuckets(0.001, 2, 12), + }, []string{"result"}) ) // ObserveFindMatchedPolicyLatency records the duration for the resource finding a matched policy. @@ -79,6 +93,16 @@ func CountPolicyPreemption(err error) { policyPreemptionCounter.WithLabelValues(utilmetrics.GetResultByError(err)).Inc() } +// ObserveProcessCronFederatedHPALatency records the duration to process a cron federated HPA. +func ObserveProcessCronFederatedHPALatency(err error, start time.Time) { + cronFederatedHPADurationHistogram.WithLabelValues(utilmetrics.GetResultByError(err)).Observe(utilmetrics.DurationInSeconds(start)) +} + +// ObserveProcessCronFederatedHPARuleLatency records the duration to process a cron federated HPA rule. +func ObserveProcessCronFederatedHPARuleLatency(err error, start time.Time) { + cronFederatedHPARuleDurationHistogram.WithLabelValues(utilmetrics.GetResultByError(err)).Observe(utilmetrics.DurationInSeconds(start)) +} + // ResourceCollectors returns the collectors about resources. func ResourceCollectors() []prometheus.Collector { return []prometheus.Collector{