Skip to content

Commit

Permalink
add cron federated hpa metrics
Browse files Browse the repository at this point in the history
Signed-off-by: whitewindmills <[email protected]>
  • Loading branch information
whitewindmills committed Aug 17, 2023
1 parent 999c7ae commit e41c4bc
Show file tree
Hide file tree
Showing 3 changed files with 40 additions and 8 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@ package cronfederatedhpa

import (
"context"
"time"

corev1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/equality"
Expand All @@ -28,6 +29,7 @@ import (
"sigs.k8s.io/controller-runtime/pkg/controller"

autoscalingv1alpha1 "github.com/karmada-io/karmada/pkg/apis/autoscaling/v1alpha1"
"github.com/karmada-io/karmada/pkg/metrics"
"github.com/karmada-io/karmada/pkg/sharedcli/ratelimiterflag"
"github.com/karmada-io/karmada/pkg/util/helper"
)
Expand Down Expand Up @@ -70,6 +72,9 @@ func (c *CronFHPAController) Reconcile(ctx context.Context, req controllerruntim
return controllerruntime.Result{}, nil
}

var err error
defer metrics.ObserveProcessCronFederatedHPALatency(err, time.Now())

origRuleSets := sets.New[string]()
for _, history := range cronFHPA.Status.ExecutionHistories {
origRuleSets.Insert(history.RuleName)
Expand All @@ -84,7 +89,7 @@ func (c *CronFHPAController) Reconcile(ctx context.Context, req controllerruntim

newRuleSets := sets.New[string]()
for _, rule := range cronFHPA.Spec.Rules {
if err := c.processCronRule(cronFHPA, rule); err != nil {
if err = c.processCronRule(cronFHPA, rule); err != nil {
return controllerruntime.Result{Requeue: true}, err
}
newRuleSets.Insert(rule.Name)
Expand All @@ -96,7 +101,7 @@ func (c *CronFHPAController) Reconcile(ctx context.Context, req controllerruntim
continue
}
c.CronHandler.StopRuleExecutor(req.NamespacedName.String(), name)
if err := c.removeCronFHPAHistory(cronFHPA, name); err != nil {
if err = c.removeCronFHPAHistory(cronFHPA, name); err != nil {
return controllerruntime.Result{Requeue: true}, err
}
}
Expand Down
3 changes: 3 additions & 0 deletions pkg/controllers/cronfederatedhpa/cronfederatedhpa_job.go
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,7 @@ import (
"sigs.k8s.io/controller-runtime/pkg/client"

autoscalingv1alpha1 "github.com/karmada-io/karmada/pkg/apis/autoscaling/v1alpha1"
"github.com/karmada-io/karmada/pkg/metrics"
"github.com/karmada-io/karmada/pkg/util"
"github.com/karmada-io/karmada/pkg/util/helper"
)
Expand Down Expand Up @@ -86,7 +87,9 @@ func RunCronFederatedHPARule(c *CronFederatedHPAJob) {
}

var scaleErr error
start := time.Now()
defer func() {
metrics.ObserveProcessCronFederatedHPARuleLatency(scaleErr, start)
if scaleErr != nil {
c.eventRecorder.Event(cronFHPA, corev1.EventTypeWarning, "ScaleFailed", scaleErr.Error())
err = c.addFailedExecutionHistory(cronFHPA, scaleErr.Error())
Expand Down
36 changes: 30 additions & 6 deletions pkg/metrics/resource.go
Original file line number Diff line number Diff line change
Expand Up @@ -9,12 +9,14 @@ import (
)

const (
resourceMatchPolicyDurationMetricsName = "resource_match_policy_duration_seconds"
resourceApplyPolicyDurationMetricsName = "resource_apply_policy_duration_seconds"
policyApplyAttemptsMetricsName = "policy_apply_attempts_total"
syncWorkDurationMetricsName = "binding_sync_work_duration_seconds"
syncWorkloadDurationMetricsName = "work_sync_workload_duration_seconds"
policyPreemptionMetricsName = "policy_preemption_total"
resourceMatchPolicyDurationMetricsName = "resource_match_policy_duration_seconds"
resourceApplyPolicyDurationMetricsName = "resource_apply_policy_duration_seconds"
policyApplyAttemptsMetricsName = "policy_apply_attempts_total"
syncWorkDurationMetricsName = "binding_sync_work_duration_seconds"
syncWorkloadDurationMetricsName = "work_sync_workload_duration_seconds"
policyPreemptionMetricsName = "policy_preemption_total"
cronFederatedHPADurationMetricsName = "cron_federated_hpa_process_duration_seconds"
cronFederatedHPARuleDurationMetricsName = "cron_federated_hpa_rule_process_duration_seconds"
)

var (
Expand Down Expand Up @@ -51,6 +53,18 @@ var (
Name: policyPreemptionMetricsName,
Help: "Number of preemption for the resource template. By the result, 'error' means a resource template failed to be preempted by other propagation policies. Otherwise 'success'.",
}, []string{"result"})

cronFederatedHPADurationHistogram = prometheus.NewHistogramVec(prometheus.HistogramOpts{
Name: cronFederatedHPADurationMetricsName,
Help: "Duration in seconds to process a cron federated HPA. By the result, 'error' means a cron federated HPA failed to be processed. Otherwise 'success'.",
Buckets: prometheus.ExponentialBuckets(0.001, 2, 12),
}, []string{"result"})

cronFederatedHPARuleDurationHistogram = prometheus.NewHistogramVec(prometheus.HistogramOpts{
Name: cronFederatedHPARuleDurationMetricsName,
Help: "Duration in seconds to process a cron federated HPA rule. By the result, 'error' means a cron federated HPA rule failed to be processed. Otherwise 'success'.",
Buckets: prometheus.ExponentialBuckets(0.001, 2, 12),
}, []string{"result"})
)

// ObserveFindMatchedPolicyLatency records the duration for the resource finding a matched policy.
Expand Down Expand Up @@ -79,6 +93,16 @@ func CountPolicyPreemption(err error) {
policyPreemptionCounter.WithLabelValues(utilmetrics.GetResultByError(err)).Inc()
}

// ObserveProcessCronFederatedHPALatency records the duration to process a cron federated HPA.
func ObserveProcessCronFederatedHPALatency(err error, start time.Time) {
cronFederatedHPADurationHistogram.WithLabelValues(utilmetrics.GetResultByError(err)).Observe(utilmetrics.DurationInSeconds(start))
}

// ObserveProcessCronFederatedHPARuleLatency records the duration to process a cron federated HPA rule.
func ObserveProcessCronFederatedHPARuleLatency(err error, start time.Time) {
cronFederatedHPARuleDurationHistogram.WithLabelValues(utilmetrics.GetResultByError(err)).Observe(utilmetrics.DurationInSeconds(start))
}

// ResourceCollectors returns the collectors about resources.
func ResourceCollectors() []prometheus.Collector {
return []prometheus.Collector{
Expand Down

0 comments on commit e41c4bc

Please sign in to comment.