diff --git a/backend/docs/docs.go b/backend/docs/docs.go index 5d188b7..2b03838 100644 --- a/backend/docs/docs.go +++ b/backend/docs/docs.go @@ -4420,7 +4420,10 @@ const docTemplate = `{ } }, "serviceName": { - "type": "string" + "type": "array", + "items": { + "type": "string" + } } } }, @@ -4779,7 +4782,7 @@ const docTemplate = `{ "request.GetServiceRouteRequest": { "type": "object", "properties": { - "serviceNames": { + "serviceName": { "type": "array", "items": { "type": "string" @@ -5213,6 +5216,12 @@ const docTemplate = `{ "type": "string" } }, + "serviceName": { + "type": "array", + "items": { + "type": "string" + } + }, "tableName": { "type": "string" } @@ -6188,6 +6197,9 @@ const docTemplate = `{ "response.LogParseResponse": { "type": "object", "properties": { + "parseInfo": { + "type": "string" + }, "parseName": { "type": "string" }, @@ -6201,7 +6213,10 @@ const docTemplate = `{ } }, "serviceName": { - "type": "string" + "type": "array", + "items": { + "type": "string" + } } } }, diff --git a/backend/docs/swagger.json b/backend/docs/swagger.json index ff4f616..afb4f22 100644 --- a/backend/docs/swagger.json +++ b/backend/docs/swagger.json @@ -4412,7 +4412,10 @@ } }, "serviceName": { - "type": "string" + "type": "array", + "items": { + "type": "string" + } } } }, @@ -4771,7 +4774,7 @@ "request.GetServiceRouteRequest": { "type": "object", "properties": { - "serviceNames": { + "serviceName": { "type": "array", "items": { "type": "string" @@ -5205,6 +5208,12 @@ "type": "string" } }, + "serviceName": { + "type": "array", + "items": { + "type": "string" + } + }, "tableName": { "type": "string" } @@ -6180,6 +6189,9 @@ "response.LogParseResponse": { "type": "object", "properties": { + "parseInfo": { + "type": "string" + }, "parseName": { "type": "string" }, @@ -6193,7 +6205,10 @@ } }, "serviceName": { - "type": "string" + "type": "array", + "items": { + "type": "string" + } } } }, diff --git a/backend/docs/swagger.yaml b/backend/docs/swagger.yaml index ab3e279..eefeb2a 100644 --- a/backend/docs/swagger.yaml +++ b/backend/docs/swagger.yaml @@ -615,7 +615,9 @@ definitions: type: string type: object serviceName: - type: string + items: + type: string + type: array type: object request.AddOtherTableRequest: properties: @@ -857,7 +859,7 @@ definitions: type: object request.GetServiceRouteRequest: properties: - serviceNames: + serviceName: items: type: string type: array @@ -1159,6 +1161,10 @@ definitions: additionalProperties: type: string type: object + serviceName: + items: + type: string + type: array tableName: type: string type: object @@ -1811,6 +1817,8 @@ definitions: type: object response.LogParseResponse: properties: + parseInfo: + type: string parseName: type: string parseRule: @@ -1820,7 +1828,9 @@ definitions: type: string type: object serviceName: - type: string + items: + type: string + type: array type: object response.LogQueryResponse: properties: diff --git a/backend/pkg/api/log/handler.go b/backend/pkg/api/log/handler.go index 2d6484e..793c8ff 100644 --- a/backend/pkg/api/log/handler.go +++ b/backend/pkg/api/log/handler.go @@ -22,11 +22,6 @@ type Handler interface { // @Router /api/log/fault/content [post] GetFaultLogContent() core.HandlerFunc - // UpdateLogTable 更新日志表 - // @Tags API.log - // @Router /api/log/update [post] - UpdateLogTable() core.HandlerFunc - // QueryLog 查询全量日志 // @Tags API.log // @Router /api/log/query [post] diff --git a/backend/pkg/model/request/log_parse.go b/backend/pkg/model/request/log_parse.go index eaa36de..1e1b6ff 100644 --- a/backend/pkg/model/request/log_parse.go +++ b/backend/pkg/model/request/log_parse.go @@ -10,13 +10,14 @@ type UpdateLogParseRequest struct { TableName string `json:"tableName"` ParseInfo string `json:"parseInfo"` ParseName string `json:"parseName"` + Service []string `json:"serviceName"` RouteRule map[string]string `json:"routeRule"` ParseRule string `json:"parseRule"` } type AddLogParseRequest struct { ParseName string `json:"parseName"` - Service string `json:"serviceName"` + Service []string `json:"serviceName"` ParseInfo string `json:"parseInfo"` RouteRule map[string]string `json:"routeRule"` ParseRule string `json:"parseRule"` @@ -24,7 +25,7 @@ type AddLogParseRequest struct { } type GetServiceRouteRequest struct { - Service []string `form:"serviceNames"` + Service []string `form:"serviceName"` } type LogTable struct { diff --git a/backend/pkg/model/response/log_parse.go b/backend/pkg/model/response/log_parse.go index f9f02a2..a0a587a 100644 --- a/backend/pkg/model/response/log_parse.go +++ b/backend/pkg/model/response/log_parse.go @@ -1,7 +1,8 @@ package response type LogParseResponse struct { - Service string `json:"serviceName"` + ParseInfo string `json:"parseInfo"` + Service []string `json:"serviceName"` ParseName string `json:"parseName"` RouteRule map[string]string `json:"routeRule"` ParseRule string `json:"parseRule"` diff --git a/backend/pkg/router/router_api.go b/backend/pkg/router/router_api.go index 3b15d69..7a285e1 100644 --- a/backend/pkg/router/router_api.go +++ b/backend/pkg/router/router_api.go @@ -66,7 +66,7 @@ func setApiRouter(r *resource) { logApi.POST("/fault/pagelist", logHandler.GetFaultLogPageList()) logApi.POST("/fault/content", logHandler.GetFaultLogContent()) - logApi.POST("/update", logHandler.UpdateLogTable()) + //logApi.POST("/update", logHandler.UpdateLogTable()) logApi.POST("/query", logHandler.QueryLog()) logApi.POST("/chart", logHandler.GetLogChart()) diff --git a/backend/pkg/services/log/service_add_log_parse_rule.go b/backend/pkg/services/log/service_add_log_parse_rule.go index bf5f0d8..dcd1d93 100644 --- a/backend/pkg/services/log/service_add_log_parse_rule.go +++ b/backend/pkg/services/log/service_add_log_parse_rule.go @@ -3,6 +3,7 @@ package log import ( "encoding/json" "fmt" + "regexp" "strings" "github.com/CloudDetail/apo/backend/pkg/model/request" @@ -26,13 +27,28 @@ func getRouteRule(routeMap map[string]string) string { return strings.Join(res, " || ") } +var fieldsRegexp = regexp.MustCompile(`\?P<(?P\w+)>`) + func (s *service) AddLogParseRule(req *request.AddLogParseRequest) (*response.LogParseResponse, error) { // 先去建表 logReq := &request.LogTableRequest{ TableName: req.ParseName, } + matchesFields := fieldsRegexp.FindAllStringSubmatch(req.ParseRule, -1) + + fields := make([]request.Field, 0) + for _, match := range matchesFields { + if match[1] == "msg" { + continue + } + fields = append(fields, request.Field{ + Name: match[1], + Type: "String", + }) + } + logReq.TTL = req.LogTable.TTL - logReq.Fields = req.LogTable.Fields + logReq.Fields = fields logReq.Buffer = req.LogTable.Buffer logReq.FillerValue() @@ -84,7 +100,7 @@ func (s *service) AddLogParseRule(req *request.AddLogParseRequest) (*response.Lo DataBase: logReq.DataBase, Cluster: logReq.Cluster, Fields: string(fieldsJSON), - Service: req.Service, + Service: strings.Join(req.Service, ","), } err = s.dbRepo.OperateLogTableInfo(&log, database.INSERT) if err != nil { diff --git a/backend/pkg/services/log/service_get_log_parse_rule.go b/backend/pkg/services/log/service_get_log_parse_rule.go index 6f2d00d..9c8760d 100644 --- a/backend/pkg/services/log/service_get_log_parse_rule.go +++ b/backend/pkg/services/log/service_get_log_parse_rule.go @@ -47,9 +47,10 @@ func (s *service) GetLogParseRule(req *request.QueryLogParseRequest) (*response. } return &response.LogParseResponse{ - Service: model.Service, + Service: strings.Split(model.Service, ","), ParseName: model.ParseName, ParseRule: model.ParseRule, + ParseInfo: model.ParseInfo, RouteRule: getRouteRuleMap(model.RouteRule), }, nil } diff --git a/backend/pkg/services/log/service_log_table.go b/backend/pkg/services/log/service_log_table.go index 8344ea2..a2315b5 100644 --- a/backend/pkg/services/log/service_log_table.go +++ b/backend/pkg/services/log/service_log_table.go @@ -88,14 +88,12 @@ func (s *service) UpdateLogTable(req *request.LogTableRequest) (*response.LogTab } err := s.dbRepo.OperateLogTableInfo(logtable, database.QUERY) if err != nil { - res.Err = err.Error() - return res, nil + return nil, err } var fields []request.Field err = json.Unmarshal([]byte(logtable.Fields), &fields) if err != nil { - res.Err = err.Error() - return res, nil + return nil, err } fieldsname := map[string]struct{}{} newFields := []request.Field{} @@ -110,19 +108,16 @@ func (s *service) UpdateLogTable(req *request.LogTableRequest) (*response.LogTab sqls, err := s.chRepo.UpdateLogTable(req, newFields, fields) res.Sqls = sqls if err != nil { - res.Err = err.Error() - return res, nil + return nil, err } fieldsJSON, err := json.Marshal(req.Fields) if err != nil { - res.Err = err.Error() - return res, nil + return nil, err } logtable.Fields = string(fieldsJSON) err = s.dbRepo.OperateLogTableInfo(logtable, database.UPDATE) if err != nil { - res.Err = err.Error() - return res, nil + return nil, err } return res, nil } diff --git a/backend/pkg/services/log/service_update_log_parse_rule.go b/backend/pkg/services/log/service_update_log_parse_rule.go index 60ea30b..ed45772 100644 --- a/backend/pkg/services/log/service_update_log_parse_rule.go +++ b/backend/pkg/services/log/service_update_log_parse_rule.go @@ -9,6 +9,30 @@ import ( ) func (s *service) UpdateLogParseRule(req *request.UpdateLogParseRequest) (*response.LogParseResponse, error) { + //更新日志表 + matchesFields := fieldsRegexp.FindAllStringSubmatch(req.ParseRule, -1) + + fields := make([]request.Field, 0) + for _, match := range matchesFields { + if match[1] == "msg" { + continue + } + fields = append(fields, request.Field{ + Name: match[1], + Type: "String", + }) + } + logReq := &request.LogTableRequest{ + DataBase: req.DataBase, + TableName: req.TableName, + Fields: fields, + } + logReq.FillerValue() + _, err := s.UpdateLogTable(logReq) + if err != nil { + return nil, err + } + // 更新k8s configmap res := &response.LogParseResponse{ ParseName: req.ParseName, @@ -37,7 +61,9 @@ func (s *service) UpdateLogParseRule(req *request.UpdateLogParseRequest) (*respo if err != nil { return nil, err } - // 更新sqlite表信息 + + // 调整整个表结构 + log := database.LogTableInfo{ ParseRule: req.ParseRule, ParseInfo: req.ParseInfo,