feat: Add metrics for the number of triggers by source (#904)

cmd/api/main.go

@@ -99,6 +99,10 @@ func main() {
 	}
 	defer searchIndex.Stop() //nolint
 
+	stats := newTriggerStats(logger, database, telemetry.Metrics)
+	stats.Start()
+	defer stats.Stop() //nolint
+
 	if !searchIndex.IsReady() {
 		logger.Fatal().Msg("Search index is not ready, exit")
 	}

cmd/api/trigger_stats.go

@@ -0,0 +1,66 @@
+package main
+
+import (
+	"fmt"
+	"time"
+
+	"github.com/moira-alert/moira"
+	"github.com/moira-alert/moira/metrics"
+	"gopkg.in/tomb.v2"
+)
+
+type triggerStats struct {
+	tomb     tomb.Tomb
+	metrics  *metrics.TriggersMetrics
+	database moira.Database
+	logger   moira.Logger
+}
+
+func newTriggerStats(
+	logger moira.Logger,
+	database moira.Database,
+	metricsRegistry metrics.Registry,
+) *triggerStats {
+	return &triggerStats{
+		logger:   logger,
+		database: database,
+		metrics:  metrics.NewTriggersMetrics(metricsRegistry),
+	}
+}
+
+func (stats *triggerStats) Start() {
+	stats.tomb.Go(stats.startCheckingTriggerCount)
+}
+
+func (stats *triggerStats) startCheckingTriggerCount() error {
+	checkTicker := time.NewTicker(time.Second * 60)
+	for {
+		select {
+		case <-stats.tomb.Dying():
+			return nil
+
+		case <-checkTicker.C:
+			stats.checkTriggerCount()
+		}
+	}
+}
+
+func (stats *triggerStats) Stop() error {
+	stats.tomb.Kill(nil)
+	return stats.tomb.Wait()
+}
+
+func (stats *triggerStats) checkTriggerCount() {
+	triggersCount, err := stats.database.GetTriggerCount()
+	if err != nil {
+		stats.logger.Warning().
+			Error(err).
+			Msg("Failed to fetch triggers count")
+		return
+	}
+
+	for source, count := range triggersCount {
+		stats.metrics.Mark(source, count)
+		stats.logger.Debug().Msg(fmt.Sprintf("source: %s, count: %d", string(source), count))
+	}
+}

cmd/api/trigger_stats_test.go

@@ -0,0 +1,49 @@
+package main
+
+import (
+	"testing"
+
+	"github.com/golang/mock/gomock"
+	"github.com/moira-alert/moira"
+	"github.com/moira-alert/moira/logging/zerolog_adapter"
+	mock_moira_alert "github.com/moira-alert/moira/mock/moira-alert"
+	mock_metrics "github.com/moira-alert/moira/mock/moira-alert/metrics"
+	. "github.com/smartystreets/goconvey/convey"
+)
+
+func TestTriggerStatsCheckTriggerCount(t *testing.T) {
+	Convey("Given db returns correct results", t, func() {
+		mockCtrl := gomock.NewController(t)
+		defer mockCtrl.Finish()
+
+		registry := mock_metrics.NewMockRegistry(mockCtrl)
+
+		graphiteLocalCount := int64(12)
+		graphiteRemoteCount := int64(24)
+		promethteusRemoteCount := int64(42)
+
+		graphiteLocalMeter := mock_metrics.NewMockMeter(mockCtrl)
+		graphiteRemoteMeter := mock_metrics.NewMockMeter(mockCtrl)
+		promethteusRemoteMeter := mock_metrics.NewMockMeter(mockCtrl)
+
+		registry.EXPECT().NewMeter("triggers", string(moira.GraphiteLocal), "count").Return(graphiteLocalMeter)
+		registry.EXPECT().NewMeter("triggers", string(moira.GraphiteRemote), "count").Return(graphiteRemoteMeter)
+		registry.EXPECT().NewMeter("triggers", string(moira.PrometheusRemote), "count").Return(promethteusRemoteMeter)
+
+		dataBase := mock_moira_alert.NewMockDatabase(mockCtrl)
+		dataBase.EXPECT().GetTriggerCount().Return(map[moira.TriggerSource]int64{
+			moira.GraphiteLocal:    graphiteLocalCount,
+			moira.GraphiteRemote:   graphiteRemoteCount,
+			moira.PrometheusRemote: promethteusRemoteCount,
+		}, nil)
+
+		graphiteLocalMeter.EXPECT().Mark(graphiteLocalCount)
+		graphiteRemoteMeter.EXPECT().Mark(graphiteRemoteCount)
+		promethteusRemoteMeter.EXPECT().Mark(promethteusRemoteCount)
+
+		logger, _ := zerolog_adapter.GetLogger("Test")
+		triggerStats := newTriggerStats(logger, dataBase, registry)
+
+		triggerStats.checkTriggerCount()
+	})
+}

database/redis/trigger.go

@@ -52,6 +52,38 @@ func (connector *DbConnector) GetPrometheusTriggerIDs() ([]string, error) {
 	return triggerIds, nil
 }
 
+func (connector *DbConnector) GetTriggerCount() (map[moira.TriggerSource]int64, error) {
+	pipe := (*connector.client).TxPipeline()
+
+	total := pipe.SCard(connector.context, triggersListKey)
+	remote := pipe.SCard(connector.context, remoteTriggersListKey)
+	prometheus := pipe.SCard(connector.context, prometheusTriggersListKey)
+
+	_, err := pipe.Exec(connector.context)
+	if err != nil {
+		return nil, err
+	}
+
+	totalCount, err := total.Result()
+	if err != nil {
+		return nil, err
+	}
+	remoteCount, err := remote.Result()
+	if err != nil {
+		return nil, err
+	}
+	prometheusCount, err := prometheus.Result()
+	if err != nil {
+		return nil, err
+	}
+
+	return map[moira.TriggerSource]int64{
+		moira.GraphiteLocal:    totalCount - remoteCount - prometheusCount,
+		moira.GraphiteRemote:   remoteCount,
+		moira.PrometheusRemote: prometheusCount,
+	}, nil
+}
+
 // GetTrigger gets trigger and trigger tags by given ID and return it in merged object
 func (connector *DbConnector) GetTrigger(triggerID string) (moira.Trigger, error) {
 	pipe := (*connector.client).TxPipeline()

filter/metrics_parser_test.go

@@ -169,7 +169,7 @@ func TestParseMetric(t *testing.T) {
 		// ...
 		// [n=19] One.two.three 123 1234567890.6790847778320312500
 
-		for i := 1; i < 20; i++ {
+		for i := 1; i < 7; i++ {
 			rawTimestamp := strconv.FormatFloat(float64(testTimestamp)+rand.Float64(), 'f', i, 64)
 			rawMetric := "One.two.three 123 " + rawTimestamp
 			validMetric := ValidMetricCase{rawMetric, "One.two.three", "One.two.three", map[string]string{}, 123, testTimestamp}

generate_mocks.sh

@@ -22,4 +22,7 @@ mockgen -destination=mock/heartbeat/heartbeat.go -package=mock_heartbeat github.
 mockgen -destination=mock/clock/clock.go -package=mock_clock github.com/moira-alert/moira Clock
 mockgen -destination=mock/notifier/mattermost/client.go -package=mock_mattermost github.com/moira-alert/moira/senders/mattermost Client
 
+mockgen -destination=mock/moira-alert/metrics/registry.go -package=mock_moira_alert github.com/moira-alert/moira/metrics Registry
+mockgen -destination=mock/moira-alert/metrics/meter.go -package=mock_moira_alert github.com/moira-alert/moira/metrics Meter
+
 git add mock/*

interfaces.go

@@ -38,6 +38,8 @@ type Database interface {
 	GetRemoteTriggerIDs() ([]string, error)
 	GetPrometheusTriggerIDs() ([]string, error)
 
+	GetTriggerCount() (map[TriggerSource]int64, error)
+
 	GetTrigger(triggerID string) (Trigger, error)
 	GetTriggers(triggerIDs []string) ([]*Trigger, error)
 	GetTriggerChecks(triggerIDs []string) ([]*TriggerCheck, error)

metrics/prometheus.go

@@ -78,8 +78,9 @@ type prometheusMeter struct {
 	summary prometheus.Summary
 }
 
-func (source *prometheusMeter) Mark(int64) {
+func (source *prometheusMeter) Mark(value int64) {
 	atomic.AddInt64(&source.count, 1)
+	source.summary.Observe(float64(value))
 }
 
 func (source *prometheusMeter) Count() int64 {

metrics/triggers.go

@@ -0,0 +1,24 @@
+package metrics
+
+import "github.com/moira-alert/moira"
+
+// Collection of metrics for trigger count metrics
+type TriggersMetrics struct {
+	countByTriggerSource map[moira.TriggerSource]Meter
+}
+
+// Creates and configurates the instance of TriggersMetrics
+func NewTriggersMetrics(registry Registry) *TriggersMetrics {
+	return &TriggersMetrics{
+		countByTriggerSource: map[moira.TriggerSource]Meter{
+			moira.GraphiteLocal:    registry.NewMeter("triggers", string(moira.GraphiteLocal), "count"),
+			moira.GraphiteRemote:   registry.NewMeter("triggers", string(moira.GraphiteRemote), "count"),
+			moira.PrometheusRemote: registry.NewMeter("triggers", string(moira.PrometheusRemote), "count"),
+		},
+	}
+}
+
+// Marks the number of trigger for given trigger source
+func (metrics *TriggersMetrics) Mark(source moira.TriggerSource, count int64) {
+	metrics.countByTriggerSource[source].Mark(count)
+}