fix bug with reusing one heartbeater base in all heartbeaters

moira-alert · Nov 8, 2024 · 7d1678f · 7d1678f
1 parent 3d43d51
commit 7d1678f
Show file tree

Hide file tree

Showing 4 changed files with 26 additions and 13 deletions.
diff --git a/local/api.yml b/local/api.yml
@@ -59,11 +59,17 @@ web:
       label: MS Teams
     - type: mattermost
       label: Mattermost
+    - type: webhook
+      label: Webhook
   feature_flags:
     is_plotting_available: true
     is_plotting_default_on: true
     is_subscription_to_all_tags_available: true
     is_readonly_enabled: false
+  emergency_contact_types:
+    - heartbeat_local_checker
+    - heartbeat_notifier
+    - heartbeat_database
 notification_history:
   ttl: 48h
   query_limit: 10000

diff --git a/local/notifier.yml b/local/notifier.yml
@@ -51,7 +51,7 @@ notifier:
             need_turn_off_notifier: true
             alert:
               name: Notifier Problems
-        notice_interval: 30s
+        notice_interval: 20s
         check_interval: 5s
         contacts:
             - type: webhook
@@ -64,21 +64,25 @@ notifier:
             need_turn_off_notifier: true
             alert:
               name: Database Problems
-            redis_disconnect_delay: 60s
+            redis_disconnect_delay: 10s
           notifier:
             enabled: true
             need_turn_off_notifier: true
             alert:
               name: Notifier Problems
-        notice_interval: 10s
+        notice_interval: 20s
         check_interval: 5s
     controller:
       enabled: true
       heartbeaters:
           database:
             enabled: true
             need_turn_off_notifier: true
-            redis_disconnect_delay: 60s
+            redis_disconnect_delay: 10s
+          local_checker:
+            enabled: true
+            need_turn_off_notifier: true
+            last_check_delay: 10s
           notifier:
             enabled: true
             need_turn_off_notifier: true

diff --git a/notifier/selfstate/controller/controller.go b/notifier/selfstate/controller/controller.go
@@ -72,11 +72,10 @@ func createHeartbeaters( //nolint:gocyclo
 	database moira.Database,
 	clock moira.Clock,
 ) []heartbeat.Heartbeater {
-	hearbeaterBase := heartbeat.NewHeartbeaterBase(logger, database, clock)
-
 	heartbeaters := make([]heartbeat.Heartbeater, 0)
 
 	if heartbeatersCfg.DatabaseCfg.Enabled && heartbeatersCfg.DatabaseCfg.NeedTurnOffNotifier {
+		hearbeaterBase := heartbeat.NewHeartbeaterBase(logger, database, clock)
 		databaseHeartbeater, err := heartbeat.NewDatabaseHeartbeater(heartbeatersCfg.DatabaseCfg, hearbeaterBase)
 		if err != nil {
 			logger.Error().
@@ -89,6 +88,7 @@ func createHeartbeaters( //nolint:gocyclo
 	}
 
 	if heartbeatersCfg.FilterCfg.Enabled && heartbeatersCfg.FilterCfg.NeedTurnOffNotifier {
+		hearbeaterBase := heartbeat.NewHeartbeaterBase(logger, database, clock)
 		filterHeartbeater, err := heartbeat.NewFilterHeartbeater(heartbeatersCfg.FilterCfg, hearbeaterBase)
 		if err != nil {
 			logger.Error().
@@ -101,6 +101,7 @@ func createHeartbeaters( //nolint:gocyclo
 	}
 
 	if heartbeatersCfg.LocalCheckerCfg.Enabled && heartbeatersCfg.LocalCheckerCfg.NeedTurnOffNotifier {
+		hearbeaterBase := heartbeat.NewHeartbeaterBase(logger, database, clock)
 		localCheckerHeartbeater, err := heartbeat.NewLocalCheckerHeartbeater(heartbeatersCfg.LocalCheckerCfg, hearbeaterBase)
 		if err != nil {
 			logger.Error().
@@ -113,6 +114,7 @@ func createHeartbeaters( //nolint:gocyclo
 	}
 
 	if heartbeatersCfg.RemoteCheckerCfg.Enabled && heartbeatersCfg.RemoteCheckerCfg.NeedTurnOffNotifier {
+		hearbeaterBase := heartbeat.NewHeartbeaterBase(logger, database, clock)
 		remoteCheckerHeartbeater, err := heartbeat.NewRemoteCheckerHeartbeater(heartbeatersCfg.RemoteCheckerCfg, hearbeaterBase)
 		if err != nil {
 			logger.Error().
@@ -125,6 +127,7 @@ func createHeartbeaters( //nolint:gocyclo
 	}
 
 	if heartbeatersCfg.NotifierCfg.Enabled && heartbeatersCfg.NotifierCfg.NeedTurnOffNotifier {
+		hearbeaterBase := heartbeat.NewHeartbeaterBase(logger, database, clock)
 		notifierHeartbeater, err := heartbeat.NewNotifierHeartbeater(heartbeatersCfg.NotifierCfg, hearbeaterBase)
 		if err != nil {
 			logger.Error().

diff --git a/notifier/selfstate/controller/controller_test.go b/notifier/selfstate/controller/controller_test.go
@@ -105,7 +105,7 @@ func TestCreateHeartbeaters(t *testing.T) {
 	Convey("Test createHeartbeaters", t, func() {
 		Convey("Without any heartbeater", func() {
 			hbCfg := heartbeat.HeartbeatersConfig{}
-			mockClock.EXPECT().NowUTC().Return(testTime)
+			mockClock.EXPECT().NowUTC().Return(testTime).AnyTimes()
 			heartbeaters := createHeartbeaters(hbCfg, mockLogger, mockDatabase, mockClock)
 			So(heartbeaters, ShouldBeEmpty)
 		})
@@ -119,7 +119,7 @@ func TestCreateHeartbeaters(t *testing.T) {
 					RedisDisconnectDelay: time.Minute,
 				},
 			}
-			mockClock.EXPECT().NowUTC().Return(testTime)
+			mockClock.EXPECT().NowUTC().Return(testTime).AnyTimes()
 			heartbeaters := createHeartbeaters(hbCfg, mockLogger, mockDatabase, mockClock)
 			So(heartbeaters, ShouldBeEmpty)
 		})
@@ -134,7 +134,7 @@ func TestCreateHeartbeaters(t *testing.T) {
 					RedisDisconnectDelay: time.Minute,
 				},
 			}
-			mockClock.EXPECT().NowUTC().Return(testTime)
+			mockClock.EXPECT().NowUTC().Return(testTime).AnyTimes()
 			heartbeaters := createHeartbeaters(hbCfg, mockLogger, mockDatabase, mockClock)
 			So(heartbeaters, ShouldHaveLength, 1)
 		})
@@ -156,7 +156,7 @@ func TestCreateHeartbeaters(t *testing.T) {
 					MetricReceivedDelay: time.Minute,
 				},
 			}
-			mockClock.EXPECT().NowUTC().Return(testTime)
+			mockClock.EXPECT().NowUTC().Return(testTime).AnyTimes()
 			heartbeaters := createHeartbeaters(hbCfg, mockLogger, mockDatabase, mockClock)
 			So(heartbeaters, ShouldHaveLength, 2)
 		})
@@ -185,7 +185,7 @@ func TestCreateHeartbeaters(t *testing.T) {
 					LocalCheckDelay: time.Minute,
 				},
 			}
-			mockClock.EXPECT().NowUTC().Return(testTime)
+			mockClock.EXPECT().NowUTC().Return(testTime).AnyTimes()
 			heartbeaters := createHeartbeaters(hbCfg, mockLogger, mockDatabase, mockClock)
 			So(heartbeaters, ShouldHaveLength, 3)
 		})
@@ -221,7 +221,7 @@ func TestCreateHeartbeaters(t *testing.T) {
 					RemoteCheckDelay: time.Minute,
 				},
 			}
-			mockClock.EXPECT().NowUTC().Return(testTime)
+			mockClock.EXPECT().NowUTC().Return(testTime).AnyTimes()
 			heartbeaters := createHeartbeaters(hbCfg, mockLogger, mockDatabase, mockClock)
 			So(heartbeaters, ShouldHaveLength, 4)
 		})
@@ -263,7 +263,7 @@ func TestCreateHeartbeaters(t *testing.T) {
 					},
 				},
 			}
-			mockClock.EXPECT().NowUTC().Return(testTime)
+			mockClock.EXPECT().NowUTC().Return(testTime).AnyTimes()
 			heartbeaters := createHeartbeaters(hbCfg, mockLogger, mockDatabase, mockClock)
 			So(heartbeaters, ShouldHaveLength, 5)
 		})