feat: Add metrics for the number of triggers by source

cmd/api/main.go

@@ -99,6 +99,10 @@ func main() {
 	}
 	defer searchIndex.Stop() //nolint
 
+	stats := newTriggerStats(logger, database, telemetry.Metrics)
+	stats.Start()
+	defer stats.Stop() //nolint
+
 	if !searchIndex.IsReady() {
 		logger.Fatal().Msg("Search index is not ready, exit")
 	}

cmd/api/trigger_stats.go

@@ -0,0 +1,66 @@
+package main
+
+import (
+	"fmt"
+	"time"
+
+	"github.com/moira-alert/moira"
+	"github.com/moira-alert/moira/metrics"
+	"gopkg.in/tomb.v2"
+)
+
+type triggerStats struct {
+	tomb     tomb.Tomb
+	metrics  *metrics.TriggersMetrics
+	database moira.Database
+	logger   moira.Logger
+}
+
+func newTriggerStats(
+	logger moira.Logger,
+	database moira.Database,
+	metricsRegistry metrics.Registry,
+) *triggerStats {
+	return &triggerStats{
+		logger:   logger,
+		database: database,
+		metrics:  metrics.ConfigureTriggersMetrics(metricsRegistry),
+	}
+}
+
+func (stats *triggerStats) Start() {
+	stats.tomb.Go(stats.startCheckingTriggerCount)
+}
+
+func (stats *triggerStats) startCheckingTriggerCount() error {
+	checkTicker := time.NewTicker(time.Second * 60)
+	for {
+		select {
+		case <-stats.tomb.Dying():
+			return nil
+
+		case <-checkTicker.C:
+			stats.checkTriggerCount()
+		}
+	}
+}
+
+func (stats *triggerStats) Stop() error {
+	stats.tomb.Kill(nil)
+	return stats.tomb.Wait()
+}
+
+func (stats *triggerStats) checkTriggerCount() {
+	triggersCount, err := stats.database.GetTriggerCount()
+	if err != nil {
+		stats.logger.Warning().
+			Error(err).
+			Msg("Failed to fetch triggers count")
+		return
+	}
+
+	for source, count := range triggersCount {
+		stats.metrics.Update(source, count)
+		stats.logger.Debug().Msg(fmt.Sprintf("source: %s, count: %d", string(source), count))
+	}
+}

database/redis/trigger.go

@@ -52,6 +52,38 @@ func (connector *DbConnector) GetPrometheusTriggerIDs() ([]string, error) {
 	return triggerIds, nil
 }
 
+func (connector *DbConnector) GetTriggerCount() (map[moira.TriggerSource]int64, error) {
+	pipe := (*connector.client).TxPipeline()
+
+	total := pipe.SCard(connector.context, triggersListKey)
+	remote := pipe.SCard(connector.context, remoteTriggersListKey)
+	prometheus := pipe.SCard(connector.context, prometheusTriggersListKey)
+
+	_, err := pipe.Exec(connector.context)
+	if err != nil {
+		return nil, err
+	}
+
+	totalCount, err := total.Result()
+	if err != nil {
+		return nil, err
+	}
+	remoteCount, err := remote.Result()
+	if err != nil {
+		return nil, err
+	}
+	prometheusCount, err := prometheus.Result()
+	if err != nil {
+		return nil, err
+	}
+
+	return map[moira.TriggerSource]int64{
+		moira.GraphiteLocal:    totalCount - remoteCount - prometheusCount,
+		moira.GraphiteRemote:   remoteCount,
+		moira.PrometheusRemote: prometheusCount,
+	}, nil
+}
+
 // GetTrigger gets trigger and trigger tags by given ID and return it in merged object
 func (connector *DbConnector) GetTrigger(triggerID string) (moira.Trigger, error) {
 	pipe := (*connector.client).TxPipeline()

interfaces.go

@@ -38,6 +38,8 @@ type Database interface {
 	GetRemoteTriggerIDs() ([]string, error)
 	GetPrometheusTriggerIDs() ([]string, error)
 
+	GetTriggerCount() (map[TriggerSource]int64, error)
+
 	GetTrigger(triggerID string) (Trigger, error)
 	GetTriggers(triggerIDs []string) ([]*Trigger, error)
 	GetTriggerChecks(triggerIDs []string) ([]*TriggerCheck, error)

metrics/triggers.go

@@ -0,0 +1,23 @@
+package metrics
+
+import "github.com/moira-alert/moira"
+
+const triggersMetricsPrefix = "triggersMetrics"
+
+type TriggersMetrics struct {
+	countByTriggerSource map[moira.TriggerSource]Histogram
+}
+
+func ConfigureTriggersMetrics(registry Registry) *TriggersMetrics {
+	return &TriggersMetrics{
+		countByTriggerSource: map[moira.TriggerSource]Histogram{
+			moira.GraphiteLocal:    registry.NewHistogram(triggersMetricsPrefix, string(moira.GraphiteLocal)),
+			moira.GraphiteRemote:   registry.NewHistogram(triggersMetricsPrefix, string(moira.GraphiteRemote)),
+			moira.PrometheusRemote: registry.NewHistogram(triggersMetricsPrefix, string(moira.PrometheusRemote)),
+		},
+	}
+}
+
+func (metrics *TriggersMetrics) Update(source moira.TriggerSource, count int64) {
+	metrics.countByTriggerSource[source].Update(count)
+}