Merge pull request #49 from blockpane/release/v2.2.0

v2.2.0 adding slack support and fixes secp256k1 valcons keys

example-config.yml

@@ -41,6 +41,13 @@ telegram:
   # The group ID for the chat where messages will be sent. Google how to find this, will include better info later.
   channel: "-666666666"
 
+# Slack settings
+slack:
+  # Send alerts to Slack?
+  enabled: no
+  # The webhook can be added in the Slack app directory.
+  webhook: https://hooks.slack.com/services/AAAAAAAAAAAAAAAAAAAAAAA/bbbbbbbbbbbbbbbbbbbbbbbb
+
 # The various chains to be monitored. Create a new entry for each chain. The name itself can be arbitrary, but a
 # user-friendly name is recommended.
 chains:
@@ -103,6 +110,11 @@ chains:
         api_key: "" # uses default if blank
         channel: "" # uses default if blank
 
+      # Slack settings
+      slack:
+          enabled: yes
+          webhook: "" # uses default if blank
+
     # This section covers our RPC providers. No LCD (aka REST) endpoints are used, only TM's RPC endpoints
     # Multiple hosts are encouraged, and will be tried sequentially until a working endpoint is discovered.
     nodes:

td2/alert.go

@@ -18,6 +18,7 @@ type alertMsg struct {
 	pd   bool
 	disc bool
 	tg   bool
+	slk  bool
 
 	severity string
 	resolved bool
@@ -32,6 +33,9 @@ type alertMsg struct {
 
 	discHook     string
 	discMentions string
+
+	slkHook      string
+	slkMentions  string
 }
 
 type notifyDest uint8
@@ -40,12 +44,14 @@ const (
 	pd notifyDest = iota
 	tg
 	di
+	slk
 )
 
 type alarmCache struct {
 	SentPdAlarms   map[string]time.Time            `json:"sent_pd_alarms"`
 	SentTgAlarms   map[string]time.Time            `json:"sent_tg_alarms"`
 	SentDiAlarms   map[string]time.Time            `json:"sent_di_alarms"`
+	SentSlkAlarms  map[string]time.Time            `json:"sent_slk_alarms"`
 	AllAlarms      map[string]map[string]time.Time `json:"sent_all_alarms"`
 	flappingAlarms map[string]map[string]time.Time
 	notifyMux      sync.RWMutex
@@ -87,6 +93,7 @@ var alarms = &alarmCache{
 	SentPdAlarms:   make(map[string]time.Time),
 	SentTgAlarms:   make(map[string]time.Time),
 	SentDiAlarms:   make(map[string]time.Time),
+	SentSlkAlarms:  make(map[string]time.Time),
 	AllAlarms:      make(map[string]map[string]time.Time),
 	flappingAlarms: make(map[string]map[string]time.Time),
 	notifyMux:      sync.RWMutex{},
@@ -110,6 +117,9 @@ func shouldNotify(msg *alertMsg, dest notifyDest) bool {
 	case di:
 		whichMap = alarms.SentDiAlarms
 		service = "Discord"
+	case slk:
+		whichMap = alarms.SentSlkAlarms
+		service = "Slack"
 	}
 
 	switch {
@@ -145,6 +155,65 @@ func shouldNotify(msg *alertMsg, dest notifyDest) bool {
 	return true
 }
 
+func notifySlack(msg *alertMsg) (err error) {
+	if !msg.slk {
+		return
+	}
+	data, err := json.Marshal(buildSlackMessage(msg))
+	if err != nil {
+		return
+	}
+
+	req, err := http.NewRequest("POST", msg.slkHook, bytes.NewBuffer(data))
+	if err != nil {
+		return
+	}
+
+	client := &http.Client{}
+	resp, err := client.Do(req)
+	if err != nil {
+		return
+	}
+	_ = resp.Body.Close()
+
+	if resp.StatusCode != 200 {
+		return fmt.Errorf("could not notify slack for %s got %d response", msg.chain, resp.StatusCode)
+	}
+
+	return
+}
+
+type SlackMessage struct {
+	Text        string       `json:"text"`
+	Attachments []Attachment `json:"attachments"`
+}
+
+type Attachment struct {
+	Text      string `json:"text"`
+	Color     string `json:"color"`
+	Title     string `json:"title"`
+	TitleLink string `json:"title_link"`
+}
+
+func buildSlackMessage(msg *alertMsg) *SlackMessage {
+	prefix := "🚨 ALERT: "
+	color := "danger"
+	if msg.resolved {
+		msg.message = "OK: " + msg.message
+		prefix = "💜 Resolved: "
+		color = "good"
+	}
+	return &SlackMessage{
+		Text: msg.message,
+		Attachments: []Attachment{
+			Attachment{
+				Title:      fmt.Sprintf("TenderDuty %s %s %s", prefix, msg.chain, msg.slkMentions),
+				Color:     color,
+			},
+		},
+	}
+}
+
 func notifyDiscord(msg *alertMsg) (err error) {
 	if !msg.disc {
 		return nil
@@ -292,6 +361,7 @@ func (c *Config) alert(chainName, message, severity string, resolved bool, id *s
 		pd:           c.Pagerduty.Enabled && c.Chains[chainName].Alerts.Pagerduty.Enabled,
 		disc:         c.Discord.Enabled && c.Chains[chainName].Alerts.Discord.Enabled,
 		tg:           c.Telegram.Enabled && c.Chains[chainName].Alerts.Telegram.Enabled,
+		slk:          c.Slack.Enabled && c.Chains[chainName].Alerts.Slack.Enabled,
 		severity:     severity,
 		resolved:     resolved,
 		chain:        chainName,
@@ -303,6 +373,7 @@ func (c *Config) alert(chainName, message, severity string, resolved bool, id *s
 		tgMentions:   strings.Join(c.Chains[chainName].Alerts.Telegram.Mentions, " "),
 		discHook:     c.Chains[chainName].Alerts.Discord.Webhook,
 		discMentions: strings.Join(c.Chains[chainName].Alerts.Discord.Mentions, " "),
+		slkHook:      c.Chains[chainName].Alerts.Slack.Webhook,
 	}
 	c.alertChan <- a
 	c.chainsMux.RUnlock()

td2/run.go

@@ -49,6 +49,10 @@ func Run(configFile, stateFile, chainConfigDirectory string, password *string) e
 					if e != nil {
 						l(msg.chain, "error sending alert to telegram", e.Error())
 					}
+					e = notifySlack(msg)
+					if e != nil {
+						l(msg.chain, "error sending alert to slack", e.Error())
+					}
 				}(alert)
 			case <-td.ctx.Done():
 				return

td2/types.go

@@ -60,6 +60,8 @@ type Config struct {
 	Discord DiscordConfig `yaml:"discord"`
 	// Telegram api information
 	Telegram TeleConfig `yaml:"telegram"`
+	// Slack webhook information
+	Slack SlackConfig `yaml:"slack"`
 
 	chainsMux sync.RWMutex // prevents concurrent map access for Chains
 	// Chains has settings for each validator to monitor. The map's name does not need to match the chain-id.
@@ -173,6 +175,8 @@ type AlertConfig struct {
 	Discord DiscordConfig `yaml:"discord"`
 	// Telegram webhook information
 	Telegram TeleConfig `yaml:"telegram"`
+	// Slack webhook information
+	Slack SlackConfig `yaml:"slack"`
 }
 
 // NodeConfig holds the basic information for a node to connect to.
@@ -199,7 +203,6 @@ type DiscordConfig struct {
 	Enabled  bool     `yaml:"enabled"`
 	Webhook  string   `yaml:"webhook"`
 	Mentions []string `yaml:"mentions"`
-	// TODO
 }
 
 // TeleConfig holds the information needed to publish to a Telegram webhook for sending alerts
@@ -208,7 +211,13 @@ type TeleConfig struct {
 	ApiKey   string   `yaml:"api_key"`
 	Channel  string   `yaml:"channel"`
 	Mentions []string `yaml:"mentions"`
-	// TODO
+}
+
+// SlackConfig holds the information needed to publish to a Slack webhook for sending alerts
+type SlackConfig struct {
+	Enabled  bool     `yaml:"enabled"`
+	Webhook  string   `yaml:"webhook"`
+	Mentions []string `yaml:"mentions"`
 }
 
 // validateConfig is a non-exhaustive check for common problems with the configuration. Needs love.
@@ -236,14 +245,6 @@ func validateConfig(c *Config) (fatal bool, problems []string) {
 		problems = append(problems, "warning: setting 'node_down_alert_minutes' to less than three minutes might result in false alarms")
 	}
 
-	// if c.Discord.Enabled {
-	// 	// TODO
-	// }
-
-	// if c.Telegram.Enabled {
-	// 	// TODO
-	// }
-
 	var wantsPublic bool
 	for k, v := range c.Chains {
 		if v.blocksResults == nil {
@@ -278,6 +279,10 @@ func validateConfig(c *Config) (fatal bool, problems []string) {
 			v.Alerts.Discord.Webhook = c.Discord.Webhook
 			v.Alerts.Discord.Mentions = c.Discord.Mentions
 		}
+		if v.Alerts.Slack.Webhook == "" {
+			v.Alerts.Slack.Webhook = c.Slack.Webhook
+			v.Alerts.Slack.Mentions = c.Slack.Mentions
+		}
 		if v.Alerts.Telegram.ApiKey == "" {
 			v.Alerts.Telegram.ApiKey = c.Telegram.ApiKey
 			v.Alerts.Telegram.Mentions = c.Telegram.Mentions
@@ -291,6 +296,9 @@ func validateConfig(c *Config) (fatal bool, problems []string) {
 		}
 
 		switch {
+		case v.Alerts.Slack.Enabled && !c.Slack.Enabled:
+			problems = append(problems, fmt.Sprintf("warn: %20s is configured for slack alerts, but it is not enabled", k))
+			fallthrough
 		case v.Alerts.Discord.Enabled && !c.Discord.Enabled:
 			problems = append(problems, fmt.Sprintf("warn: %20s is configured for discord alerts, but it is not enabled", k))
 			fallthrough
@@ -302,7 +310,7 @@ func validateConfig(c *Config) (fatal bool, problems []string) {
 		case !v.Alerts.ConsecutiveAlerts && !v.Alerts.PercentageAlerts && !v.Alerts.AlertIfInactive && !v.Alerts.AlertIfNoServers:
 			problems = append(problems, fmt.Sprintf("warn: %20s has no alert types configured", k))
 			fallthrough
-		case !v.Alerts.Pagerduty.Enabled && !v.Alerts.Discord.Enabled && !v.Alerts.Telegram.Enabled:
+		case !v.Alerts.Pagerduty.Enabled && !v.Alerts.Discord.Enabled && !v.Alerts.Telegram.Enabled && !v.Alerts.Slack.Enabled:
 			problems = append(problems, fmt.Sprintf("warn: %20s has no notifications configured", k))
 		}
 		if td.EnableDash {
@@ -467,11 +475,12 @@ func loadConfig(yamlFile, stateFile, chainConfigDirectory string, password *stri
 
 	// handle cached data. FIXME: incomplete.
 	c.alarms = &alarmCache{
-		SentPdAlarms: make(map[string]time.Time),
-		SentTgAlarms: make(map[string]time.Time),
-		SentDiAlarms: make(map[string]time.Time),
-		AllAlarms:    make(map[string]map[string]time.Time),
-		notifyMux:    sync.RWMutex{},
+		SentPdAlarms:  make(map[string]time.Time),
+		SentTgAlarms:  make(map[string]time.Time),
+		SentDiAlarms:  make(map[string]time.Time),
+		SentSlkAlarms: make(map[string]time.Time),
+		AllAlarms:     make(map[string]map[string]time.Time),
+		notifyMux:     sync.RWMutex{},
 	}
 
 	//#nosec -- variable specified on command line
@@ -509,6 +518,10 @@ func loadConfig(yamlFile, stateFile, chainConfigDirectory string, password *stri
 			alarms.SentDiAlarms = saved.Alarms.SentDiAlarms
 			clearStale(alarms.SentDiAlarms, "Discord", c.Pagerduty.Enabled, staleHours)
 		}
+		if saved.Alarms.SentSlkAlarms != nil {
+			alarms.SentSlkAlarms = saved.Alarms.SentSlkAlarms
+			clearStale(alarms.SentSlkAlarms, "Slack", c.Pagerduty.Enabled, staleHours)
+		}
 		if saved.Alarms.AllAlarms != nil {
 			alarms.AllAlarms = saved.Alarms.AllAlarms
 			for _, alrm := range saved.Alarms.AllAlarms {

td2/validator.go

@@ -5,6 +5,7 @@ import (
 	"errors"
 	"fmt"
 	"github.com/cosmos/cosmos-sdk/crypto/keys/ed25519"
+	"github.com/cosmos/cosmos-sdk/crypto/keys/secp256k1"
 	"github.com/cosmos/cosmos-sdk/types/bech32"
 	slashing "github.com/cosmos/cosmos-sdk/x/slashing/types"
 	staking "github.com/cosmos/cosmos-sdk/x/staking/types"
@@ -151,10 +152,30 @@ func getVal(ctx context.Context, client *rpchttp.HTTP, valoper string) (pub []by
 	if err != nil {
 		return
 	}
-	pk := ed25519.PubKey{}
-	err = pk.Unmarshal(val.Validator.ConsensusPubkey.Value)
-	if err != nil {
-		return
+	if val.Validator.ConsensusPubkey == nil {
+		return nil, "", false, false, errors.New("got invalid consensus pubkey for " + valoper)
+	}
+
+	pubBytes := make([]byte, 0)
+	switch val.Validator.ConsensusPubkey.TypeUrl {
+	case "/cosmos.crypto.ed25519.PubKey":
+		pk := ed25519.PubKey{}
+		err = pk.Unmarshal(val.Validator.ConsensusPubkey.Value)
+		if err != nil {
+			return
+		}
+		pubBytes = pk.Address().Bytes()
+	case "/cosmos.crypto.secp256k1.PubKey":
+		pk := secp256k1.PubKey{}
+		err = pk.Unmarshal(val.Validator.ConsensusPubkey.Value)
+		if err != nil {
+			return
+		}
+		pubBytes = pk.Address().Bytes()
+	}
+	if len(pubBytes) == 0 {
+		return nil, "", false, false, errors.New("could not get pubkey for" + valoper)
 	}
-	return pk.Address().Bytes(), val.Validator.GetMoniker(), val.Validator.Jailed, val.Validator.Status == 3, nil
+
+	return pubBytes, val.Validator.GetMoniker(), val.Validator.Jailed, val.Validator.Status == 3, nil
 }