add metrics for UP/DOWN nodes (#619)

projecteru2 · Oct 17, 2023 · ce8d1bc · ce8d1bc
1 parent 9265573
commit ce8d1bc
Show file tree

Hide file tree

Showing 2 changed files with 48 additions and 2 deletions.
diff --git a/metrics/handler.go b/metrics/handler.go
@@ -20,14 +20,27 @@ func (m *Metrics) ResourceMiddleware(cluster cluster.Cluster) func(http.Handler)
 			if err != nil {
 				logger.Error(ctx, err, "Get all nodes err")
 			}
+			podUpDownNodes := map[string][]int{}
 			for node := range nodes {
+				if podUpDownNodes[node.Podname] == nil {
+					podUpDownNodes[node.Podname] = []int{0, 0}
+				}
+				if node.IsDown() {
+					podUpDownNodes[node.Podname][1]++
+				} else {
+					podUpDownNodes[node.Podname][0]++
+				}
 				metrics, err := m.rmgr.GetNodeMetrics(ctx, node)
 				if err != nil {
 					logger.Error(ctx, err, "Get metrics failed")
 					continue
 				}
 				m.SendMetrics(ctx, metrics...)
 			}
+			for podname, ud := range podUpDownNodes {
+				m.SendPodUpDownNodes(ctx, podname, ud[0], ud[1])
+			}
+
 			h.ServeHTTP(w, r)
 		})
 	}

diff --git a/metrics/metrics.go b/metrics/metrics.go
@@ -23,8 +23,13 @@ import (
 const (
 	deployCountKey  = "core.%s.deploy.count"
 	deployCountName = "core_deploy"
-	gaugeType       = "gauge"
-	counterType     = "counter"
+	upNodesKey      = "core.up.nodes"
+	upNodesName     = "core_up_nodes"
+	downNodesKey    = "core.down.nodes"
+	downNodesName   = "core_down_nodes"
+
+	gaugeType   = "gauge"
+	counterType = "counter"
 )
 
 // Metrics define metrics
@@ -53,6 +58,24 @@ func (m *Metrics) SendDeployCount(ctx context.Context, n int) {
 	m.SendMetrics(ctx, metrics)
 }
 
+func (m *Metrics) SendPodUpDownNodes(ctx context.Context, podname string, up, down int) {
+	log.WithFunc("metrics.SendUpDownNodes").Info(ctx, "Update deploy counter")
+	m1 := &plugintypes.Metrics{
+		Name:   upNodesName,
+		Labels: []string{m.Hostname, podname},
+		Key:    upNodesKey,
+		Value:  strconv.Itoa(up),
+	}
+
+	m2 := &plugintypes.Metrics{
+		Name:   downNodesName,
+		Labels: []string{m.Hostname, podname},
+		Key:    downNodesKey,
+		Value:  strconv.Itoa(down),
+	}
+	m.SendMetrics(ctx, m1, m2)
+}
+
 // SendMetrics update metrics
 func (m *Metrics) SendMetrics(ctx context.Context, metrics ...*plugintypes.Metrics) {
 	logger := log.WithFunc("metrics.SendMetrics")
@@ -208,6 +231,16 @@ func InitMetrics(ctx context.Context, config types.Config, metricsDescriptions [
 		Help: "core deploy counter",
 	}, []string{"hostname"})
 
+	Client.Collectors[upNodesName] = prometheus.NewGaugeVec(prometheus.GaugeOpts{
+		Name: upNodesName,
+		Help: "number of up nodes",
+	}, []string{"hostname", "podname"})
+
+	Client.Collectors[downNodesName] = prometheus.NewGaugeVec(prometheus.GaugeOpts{
+		Name: downNodesName,
+		Help: "number of down nodes",
+	}, []string{"hostname", "podname"})
+
 	once.Do(func() {
 		prometheus.MustRegister(maps.Values(Client.Collectors)...)
 	})