Skip to content

Commit

Permalink
mv upate log table
Browse files Browse the repository at this point in the history
  • Loading branch information
@fuwx295
fuwx295 committed Oct 16, 2024
1 parent e9f23a9 commit f4a8547
Show file tree
Hide file tree
Showing 11 changed files with 107 additions and 32 deletions.
21 changes: 18 additions & 3 deletions backend/docs/docs.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4420,7 +4420,10 @@ const docTemplate = `{
}
},
"serviceName": {
"type": "string"
"type": "array",
"items": {
"type": "string"
}
}
}
},
Expand Down Expand Up @@ -4779,7 +4782,7 @@ const docTemplate = `{
"request.GetServiceRouteRequest": {
"type": "object",
"properties": {
"serviceNames": {
"serviceName": {
"type": "array",
"items": {
"type": "string"
Expand Down Expand Up @@ -5213,6 +5216,12 @@ const docTemplate = `{
"type": "string"
}
},
"serviceName": {
"type": "array",
"items": {
"type": "string"
}
},
"tableName": {
"type": "string"
}
Expand Down Expand Up @@ -6188,6 +6197,9 @@ const docTemplate = `{
"response.LogParseResponse": {
"type": "object",
"properties": {
"parseInfo": {
"type": "string"
},
"parseName": {
"type": "string"
},
Expand All @@ -6201,7 +6213,10 @@ const docTemplate = `{
}
},
"serviceName": {
"type": "string"
"type": "array",
"items": {
"type": "string"
}
}
}
},
Expand Down
21 changes: 18 additions & 3 deletions backend/docs/swagger.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4412,7 +4412,10 @@
}
},
"serviceName": {
"type": "string"
"type": "array",
"items": {
"type": "string"
}
}
}
},
Expand Down Expand Up @@ -4771,7 +4774,7 @@
"request.GetServiceRouteRequest": {
"type": "object",
"properties": {
"serviceNames": {
"serviceName": {
"type": "array",
"items": {
"type": "string"
Expand Down Expand Up @@ -5205,6 +5208,12 @@
"type": "string"
}
},
"serviceName": {
"type": "array",
"items": {
"type": "string"
}
},
"tableName": {
"type": "string"
}
Expand Down Expand Up @@ -6180,6 +6189,9 @@
"response.LogParseResponse": {
"type": "object",
"properties": {
"parseInfo": {
"type": "string"
},
"parseName": {
"type": "string"
},
Expand All @@ -6193,7 +6205,10 @@
}
},
"serviceName": {
"type": "string"
"type": "array",
"items": {
"type": "string"
}
}
}
},
Expand Down
16 changes: 13 additions & 3 deletions backend/docs/swagger.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -615,7 +615,9 @@ definitions:
type: string
type: object
serviceName:
type: string
items:
type: string
type: array
type: object
request.AddOtherTableRequest:
properties:
Expand Down Expand Up @@ -857,7 +859,7 @@ definitions:
type: object
request.GetServiceRouteRequest:
properties:
serviceNames:
serviceName:
items:
type: string
type: array
Expand Down Expand Up @@ -1159,6 +1161,10 @@ definitions:
additionalProperties:
type: string
type: object
serviceName:
items:
type: string
type: array
tableName:
type: string
type: object
Expand Down Expand Up @@ -1811,6 +1817,8 @@ definitions:
type: object
response.LogParseResponse:
properties:
parseInfo:
type: string
parseName:
type: string
parseRule:
Expand All @@ -1820,7 +1828,9 @@ definitions:
type: string
type: object
serviceName:
type: string
items:
type: string
type: array
type: object
response.LogQueryResponse:
properties:
Expand Down
5 changes: 0 additions & 5 deletions backend/pkg/api/log/handler.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,11 +22,6 @@ type Handler interface {
// @Router /api/log/fault/content [post]
GetFaultLogContent() core.HandlerFunc

// UpdateLogTable 更新日志表
// @Tags API.log
// @Router /api/log/update [post]
UpdateLogTable() core.HandlerFunc

// QueryLog 查询全量日志
// @Tags API.log
// @Router /api/log/query [post]
Expand Down
5 changes: 3 additions & 2 deletions backend/pkg/model/request/log_parse.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,21 +10,22 @@ type UpdateLogParseRequest struct {
TableName string `json:"tableName"`
ParseInfo string `json:"parseInfo"`
ParseName string `json:"parseName"`
Service []string `json:"serviceName"`
RouteRule map[string]string `json:"routeRule"`
ParseRule string `json:"parseRule"`
}

type AddLogParseRequest struct {
ParseName string `json:"parseName"`
Service string `json:"serviceName"`
Service []string `json:"serviceName"`
ParseInfo string `json:"parseInfo"`
RouteRule map[string]string `json:"routeRule"`
ParseRule string `json:"parseRule"`
LogTable LogTable `json:"logTable"`
}

type GetServiceRouteRequest struct {
Service []string `form:"serviceNames"`
Service []string `form:"serviceName"`
}

type LogTable struct {
Expand Down
3 changes: 2 additions & 1 deletion backend/pkg/model/response/log_parse.go
View file
Original file line number Diff line number Diff line change
@@ -1,7 +1,8 @@
package response

type LogParseResponse struct {
Service string `json:"serviceName"`
ParseInfo string `json:"parseInfo"`
Service []string `json:"serviceName"`
ParseName string `json:"parseName"`
RouteRule map[string]string `json:"routeRule"`
ParseRule string `json:"parseRule"`
Expand Down
2 changes: 1 addition & 1 deletion backend/pkg/router/router_api.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -66,7 +66,7 @@ func setApiRouter(r *resource) {
logApi.POST("/fault/pagelist", logHandler.GetFaultLogPageList())
logApi.POST("/fault/content", logHandler.GetFaultLogContent())

logApi.POST("/update", logHandler.UpdateLogTable())
//logApi.POST("/update", logHandler.UpdateLogTable())

logApi.POST("/query", logHandler.QueryLog())
logApi.POST("/chart", logHandler.GetLogChart())
Expand Down
20 changes: 18 additions & 2 deletions backend/pkg/services/log/service_add_log_parse_rule.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ package log
import (
"encoding/json"
"fmt"
"regexp"
"strings"

"github.com/CloudDetail/apo/backend/pkg/model/request"
Expand All @@ -26,13 +27,28 @@ func getRouteRule(routeMap map[string]string) string {
return strings.Join(res, " || ")
}

var fieldsRegexp = regexp.MustCompile(`\?P<(?P<name>\w+)>`)

func (s *service) AddLogParseRule(req *request.AddLogParseRequest) (*response.LogParseResponse, error) {
// 先去建表
logReq := &request.LogTableRequest{
TableName: req.ParseName,
}
matchesFields := fieldsRegexp.FindAllStringSubmatch(req.ParseRule, -1)

fields := make([]request.Field, 0)
for _, match := range matchesFields {
if match[1] == "msg" {
continue
}
fields = append(fields, request.Field{
Name: match[1],
Type: "String",
})
}

logReq.TTL = req.LogTable.TTL
logReq.Fields = req.LogTable.Fields
logReq.Fields = fields
logReq.Buffer = req.LogTable.Buffer
logReq.FillerValue()

Expand Down Expand Up @@ -84,7 +100,7 @@ func (s *service) AddLogParseRule(req *request.AddLogParseRequest) (*response.Lo
DataBase: logReq.DataBase,
Cluster: logReq.Cluster,
Fields: string(fieldsJSON),
Service: req.Service,
Service: strings.Join(req.Service, ","),
}
err = s.dbRepo.OperateLogTableInfo(&log, database.INSERT)
if err != nil {
Expand Down
3 changes: 2 additions & 1 deletion backend/pkg/services/log/service_get_log_parse_rule.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -47,9 +47,10 @@ func (s *service) GetLogParseRule(req *request.QueryLogParseRequest) (*response.
}

return &response.LogParseResponse{
Service: model.Service,
Service: strings.Split(model.Service, ","),
ParseName: model.ParseName,
ParseRule: model.ParseRule,
ParseInfo: model.ParseInfo,
RouteRule: getRouteRuleMap(model.RouteRule),
}, nil
}
15 changes: 5 additions & 10 deletions backend/pkg/services/log/service_log_table.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -88,14 +88,12 @@ func (s *service) UpdateLogTable(req *request.LogTableRequest) (*response.LogTab
}
err := s.dbRepo.OperateLogTableInfo(logtable, database.QUERY)
if err != nil {
res.Err = err.Error()
return res, nil
return nil, err
}
var fields []request.Field
err = json.Unmarshal([]byte(logtable.Fields), &fields)
if err != nil {
res.Err = err.Error()
return res, nil
return nil, err
}
fieldsname := map[string]struct{}{}
newFields := []request.Field{}
Expand All @@ -110,19 +108,16 @@ func (s *service) UpdateLogTable(req *request.LogTableRequest) (*response.LogTab
sqls, err := s.chRepo.UpdateLogTable(req, newFields, fields)
res.Sqls = sqls
if err != nil {
res.Err = err.Error()
return res, nil
return nil, err
}
fieldsJSON, err := json.Marshal(req.Fields)
if err != nil {
res.Err = err.Error()
return res, nil
return nil, err
}
logtable.Fields = string(fieldsJSON)
err = s.dbRepo.OperateLogTableInfo(logtable, database.UPDATE)
if err != nil {
res.Err = err.Error()
return res, nil
return nil, err
}
return res, nil
}
28 changes: 27 additions & 1 deletion backend/pkg/services/log/service_update_log_parse_rule.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,30 @@ import (
)

func (s *service) UpdateLogParseRule(req *request.UpdateLogParseRequest) (*response.LogParseResponse, error) {
//更新日志表
matchesFields := fieldsRegexp.FindAllStringSubmatch(req.ParseRule, -1)

fields := make([]request.Field, 0)
for _, match := range matchesFields {
if match[1] == "msg" {
continue
}
fields = append(fields, request.Field{
Name: match[1],
Type: "String",
})
}
logReq := &request.LogTableRequest{
DataBase: req.DataBase,
TableName: req.TableName,
Fields: fields,
}
logReq.FillerValue()
_, err := s.UpdateLogTable(logReq)
if err != nil {
return nil, err
}

// 更新k8s configmap
res := &response.LogParseResponse{
ParseName: req.ParseName,
Expand Down Expand Up @@ -37,7 +61,9 @@ func (s *service) UpdateLogParseRule(req *request.UpdateLogParseRequest) (*respo
if err != nil {
return nil, err
}
// 更新sqlite表信息

// 调整整个表结构

log := database.LogTableInfo{
ParseRule: req.ParseRule,
ParseInfo: req.ParseInfo,
Expand Down

0 comments on commit f4a8547

Please sign in to comment.