[horus] Try to add downtime recovery (#444)

apache · Oct 10, 2024 · 1be672d · 1be672d
1 parent 156e5fa
commit 1be672d
Show file tree

Hide file tree

Showing 4 changed files with 24 additions and 16 deletions.
diff --git a/app/horus/base/config/config.go b/app/horus/base/config/config.go
@@ -73,6 +73,7 @@ type DowntimeConfiguration struct {
 	KubeMultiple         map[string]string      `yaml:"kubeMultiple"`
 	AbnormalityQL        []string               `yaml:"abnormalityQL"`
 	AbnormalInfoSystemQL string                 `yaml:"abnormalInfoSystemQL"`
+	AbnormalRecoveryQL   []string               `yaml:"abnormalRecoveryQL"`
 	AllSystemUser        string                 `yaml:"allSystemUser"`
 	AllSystemPassword    string                 `yaml:"allSystemPassword"`
 	DingTalk             *DingTalkConfiguration `yaml:"dingTalk"`

diff --git a/app/horus/base/db/db.go b/app/horus/base/db/db.go
@@ -25,21 +25,23 @@ import (
 )
 
 type NodeDataInfo struct {
-	Id              int64     `json:"id"`
-	NodeName        string    `json:"node_name" xorm:"node_name"`
-	NodeIP          string    `json:"node_ip" xorm:"node_ip"`
-	Sn              string    `json:"sn"`
-	ClusterName     string    `json:"cluster_name" xorm:"cluster_name"`
-	ModuleName      string    `json:"module_name" xorm:"module_name"`
-	Reason          string    `json:"reason"`
-	Restart         int       `json:"restart"`
-	Repair          int       `json:"repair"`
-	RepairTicketUrl string    `json:"repair_ticket_url" xorm:"repair_ticket_url"`
-	FirstDate       string    `json:"first_date" xorm:"first_date"`
-	CreateTime      time.Time `json:"create_time" xorm:"create_time created"`
-	UpdateTime      time.Time `json:"update_time" xorm:"update_time updated"`
-	RecoveryMark    int64     `json:"recovery_mark" xorm:"recovery_mark"`
-	RecoveryQL      string    `json:"recovery_ql" xorm:"recovery_ql"`
+	Id                   int64     `json:"id"`
+	NodeName             string    `json:"node_name" xorm:"node_name"`
+	NodeIP               string    `json:"node_ip" xorm:"node_ip"`
+	Sn                   string    `json:"sn"`
+	ClusterName          string    `json:"cluster_name" xorm:"cluster_name"`
+	ModuleName           string    `json:"module_name" xorm:"module_name"`
+	Reason               string    `json:"reason"`
+	Restart              int       `json:"restart"`
+	Repair               int       `json:"repair"`
+	RepairTicketUrl      string    `json:"repair_ticket_url" xorm:"repair_ticket_url"`
+	FirstDate            string    `json:"first_date" xorm:"first_date"`
+	CreateTime           time.Time `json:"create_time" xorm:"create_time created"`
+	UpdateTime           time.Time `json:"update_time" xorm:"update_time updated"`
+	RecoveryMark         int64     `json:"recovery_mark" xorm:"recovery_mark"`
+	RecoveryQL           string    `json:"recovery_ql" xorm:"recovery_ql"`
+	DownTimeRecoveryQL   []string  `json:"downtime_recovery_ql xorm:downtime_recovery_ql"`
+	DownTimeRecoveryMark int64     `json:"downtime_recovery_mark xorm:downtime_recovery_mark"`
 }
 
 type PodDataInfo struct {

diff --git a/app/horus/core/horuser/node_downtime.go b/app/horus/core/horuser/node_downtime.go
@@ -147,6 +147,7 @@ func (h *Horuser) DownTimeNodes(clusterName, addr string) {
 			NodeIP:      nodeIP,
 			ClusterName: clusterName,
 			ModuleName:  NODE_DOWN,
+			RecoveryQL: ,
 		}
 		exist, _ := write.Check()
 		if exist {

diff --git a/manifests/horus/horus.yaml b/manifests/horus/horus.yaml
@@ -75,6 +75,10 @@ nodeDownTime:
     node_os_info{node="%s"}
   allSystemUser: "zxj"
   allSystemPassword: "1"
+  abnormalRecoveryQL:
+    - 100 - (avg by (node) (rate(node_cpu_seconds_total{mode="idle",node="%s"}[5m])) * 100) < 20
+    - (avg by (node) (node_memory_MemFree_bytes{node="%s"} / node_memory_MemTotal_bytes{node="%s"} )) * 100 < 25
+    # - node_filesystem_avail_bytes{mountpoint="/"} / node_filesystem_size_bytes{mountpoint="/"} * 100 > 15
   kubeMultiple:
     cluster: config.1
   dingTalk:
@@ -87,7 +91,7 @@ nodeDownTime:
     title: "自定义通知"
 
 podStagnationCleaner:
-  enabled: false
+  enabled: true
   intervalSecond: 15
   doubleSecond: 60
   fieldSelector: "status.phase!=Running"