From be82b44901bb65b8c3129b36519a92d86db11bd2 Mon Sep 17 00:00:00 2001 From: fuwx Date: Thu, 17 Oct 2024 10:33:24 +0800 Subject: [PATCH] fix log parsename --- backend/docs/docs.go | 83 ------------------- backend/docs/swagger.json | 83 ------------------- backend/docs/swagger.yaml | 54 ------------ backend/pkg/api/log/func_logtable.go | 43 ---------- backend/pkg/code/code.go | 4 - backend/pkg/code/en.go | 4 - backend/pkg/code/zh-cn.go | 3 - backend/pkg/model/response/log_index.go | 1 - .../log/service_add_log_parse_rule.go | 6 +- .../log/service_get_log_parse_rule.go | 3 + backend/pkg/services/log/service_log_index.go | 3 +- 11 files changed, 7 insertions(+), 280 deletions(-) delete mode 100644 backend/pkg/api/log/func_logtable.go diff --git a/backend/docs/docs.go b/backend/docs/docs.go index 2b03838..0e86506 100644 --- a/backend/docs/docs.go +++ b/backend/docs/docs.go @@ -1252,46 +1252,6 @@ const docTemplate = `{ } } }, - "/api/log/update": { - "post": { - "description": "更新日志表", - "consumes": [ - "application/json" - ], - "produces": [ - "application/json" - ], - "tags": [ - "API.log" - ], - "summary": "更新日志表", - "parameters": [ - { - "description": "请求信息", - "name": "Request", - "in": "body", - "required": true, - "schema": { - "$ref": "#/definitions/request.LogTableRequest" - } - } - ], - "responses": { - "200": { - "description": "OK", - "schema": { - "$ref": "#/definitions/response.LogTableResponse" - } - }, - "400": { - "description": "Bad Request", - "schema": { - "$ref": "#/definitions/code.Failure" - } - } - } - } - }, "/api/mock": { "get": { "description": "xx列表", @@ -5000,35 +4960,6 @@ const docTemplate = `{ "request.LogTableInfoRequest": { "type": "object" }, - "request.LogTableRequest": { - "type": "object", - "properties": { - "buffer": { - "$ref": "#/definitions/request.BufferEngineConfig" - }, - "cluster": { - "type": "string" - }, - "dataBase": { - "type": "string" - }, - "fields": { - "type": "array", - "items": { - "$ref": "#/definitions/request.Field" - } - }, - "replica": { - "type": "boolean" - }, - "tableName": { - "type": "string" - }, - "ttl": { - "type": "integer" - } - } - }, "request.Operation": { "type": "string", "enum": [ @@ -6289,20 +6220,6 @@ const docTemplate = `{ } } }, - "response.LogTableResponse": { - "type": "object", - "properties": { - "error": { - "type": "string" - }, - "sqls": { - "type": "array", - "items": { - "type": "string" - } - } - } - }, "response.MonitorStatus": { "type": "object", "properties": { diff --git a/backend/docs/swagger.json b/backend/docs/swagger.json index afb4f22..70c40c3 100644 --- a/backend/docs/swagger.json +++ b/backend/docs/swagger.json @@ -1244,46 +1244,6 @@ } } }, - "/api/log/update": { - "post": { - "description": "更新日志表", - "consumes": [ - "application/json" - ], - "produces": [ - "application/json" - ], - "tags": [ - "API.log" - ], - "summary": "更新日志表", - "parameters": [ - { - "description": "请求信息", - "name": "Request", - "in": "body", - "required": true, - "schema": { - "$ref": "#/definitions/request.LogTableRequest" - } - } - ], - "responses": { - "200": { - "description": "OK", - "schema": { - "$ref": "#/definitions/response.LogTableResponse" - } - }, - "400": { - "description": "Bad Request", - "schema": { - "$ref": "#/definitions/code.Failure" - } - } - } - } - }, "/api/mock": { "get": { "description": "xx列表", @@ -4992,35 +4952,6 @@ "request.LogTableInfoRequest": { "type": "object" }, - "request.LogTableRequest": { - "type": "object", - "properties": { - "buffer": { - "$ref": "#/definitions/request.BufferEngineConfig" - }, - "cluster": { - "type": "string" - }, - "dataBase": { - "type": "string" - }, - "fields": { - "type": "array", - "items": { - "$ref": "#/definitions/request.Field" - } - }, - "replica": { - "type": "boolean" - }, - "tableName": { - "type": "string" - }, - "ttl": { - "type": "integer" - } - } - }, "request.Operation": { "type": "string", "enum": [ @@ -6281,20 +6212,6 @@ } } }, - "response.LogTableResponse": { - "type": "object", - "properties": { - "error": { - "type": "string" - }, - "sqls": { - "type": "array", - "items": { - "type": "string" - } - } - } - }, "response.MonitorStatus": { "type": "object", "properties": { diff --git a/backend/docs/swagger.yaml b/backend/docs/swagger.yaml index eefeb2a..a4c2022 100644 --- a/backend/docs/swagger.yaml +++ b/backend/docs/swagger.yaml @@ -1008,25 +1008,6 @@ definitions: type: object request.LogTableInfoRequest: type: object - request.LogTableRequest: - properties: - buffer: - $ref: '#/definitions/request.BufferEngineConfig' - cluster: - type: string - dataBase: - type: string - fields: - items: - $ref: '#/definitions/request.Field' - type: array - replica: - type: boolean - tableName: - type: string - ttl: - type: integer - type: object request.Operation: enum: - EQUAL @@ -1877,15 +1858,6 @@ definitions: $ref: '#/definitions/response.Parse' type: array type: object - response.LogTableResponse: - properties: - error: - type: string - sqls: - items: - type: string - type: array - type: object response.MonitorStatus: properties: isAlive: @@ -2908,32 +2880,6 @@ paths: summary: 获取日志表信息 tags: - API.log - /api/log/update: - post: - consumes: - - application/json - description: 更新日志表 - parameters: - - description: 请求信息 - in: body - name: Request - required: true - schema: - $ref: '#/definitions/request.LogTableRequest' - produces: - - application/json - responses: - "200": - description: OK - schema: - $ref: '#/definitions/response.LogTableResponse' - "400": - description: Bad Request - schema: - $ref: '#/definitions/code.Failure' - summary: 更新日志表 - tags: - - API.log /api/mock: get: consumes: diff --git a/backend/pkg/api/log/func_logtable.go b/backend/pkg/api/log/func_logtable.go deleted file mode 100644 index ce7102a..0000000 --- a/backend/pkg/api/log/func_logtable.go +++ /dev/null @@ -1,43 +0,0 @@ -package log - -import ( - "net/http" - - "github.com/CloudDetail/apo/backend/pkg/code" - "github.com/CloudDetail/apo/backend/pkg/core" - "github.com/CloudDetail/apo/backend/pkg/model/request" -) - -// UpdateLogTable 更新日志表 -// @Summary 更新日志表 -// @Description 更新日志表 -// @Tags API.log -// @Accept json -// @Produce json -// @Param Request body request.LogTableRequest true "请求信息" -// @Success 200 {object} response.LogTableResponse -// @Failure 400 {object} code.Failure -// @Router /api/log/update [post] -func (h *handler) UpdateLogTable() core.HandlerFunc { - return func(c core.Context) { - req := new(request.LogTableRequest) - if err := c.ShouldBindJSON(req); err != nil { - c.AbortWithError(core.Error( - http.StatusBadRequest, - code.ParamBindError, - code.Text(code.ParamBindError)).WithError(err), - ) - return - } - resp, err := h.logService.UpdateLogTable(req) - if err != nil { - c.AbortWithError(core.Error( - http.StatusBadRequest, - code.UpateLogTableError, - code.Text(code.UpateLogTableError)).WithError(err), - ) - return - } - c.Payload(resp) - } -} diff --git a/backend/pkg/code/code.go b/backend/pkg/code/code.go index cdb4f2b..6f146e4 100644 --- a/backend/pkg/code/code.go +++ b/backend/pkg/code/code.go @@ -40,10 +40,6 @@ const ( GetFaultLogPageListError = "B0401" GetFaultLogContentError = "B0402" - CreateLogTableError = "B0403" - DropLogTableError = "B0404" - UpateLogTableError = "B0405" - QueryLogError = "B0406" GetLogChartError = "B0407" GetLogIndexError = "B0408" diff --git a/backend/pkg/code/en.go b/backend/pkg/code/en.go index 490ca05..8d82863 100644 --- a/backend/pkg/code/en.go +++ b/backend/pkg/code/en.go @@ -30,10 +30,6 @@ var enText = map[string]string{ GetFaultLogContentError: "Failed to get fault log content", GetMonitorStatusError: "Failed to get monitor status", - CreateLogTableError: "Failed to create log table", - DropLogTableError: "Failed to drop log table", - UpateLogTableError: "Failed to update log table", - QueryLogError: "Failed to query all logs", GetLogChartError: "Failed to get log chart", GetLogIndexError: "Failed to get log index", diff --git a/backend/pkg/code/zh-cn.go b/backend/pkg/code/zh-cn.go index b35501e..5e1c3f5 100644 --- a/backend/pkg/code/zh-cn.go +++ b/backend/pkg/code/zh-cn.go @@ -29,9 +29,6 @@ var zhCnText = map[string]string{ GetFaultLogPageListError: "获取故障现场日志分页列表失败", GetFaultLogContentError: "获取故障现场日志内容失败", - CreateLogTableError: "创建全量日志表出错", - DropLogTableError: "删除全量日志表出错", - UpateLogTableError: "更新全量日志表出错", QueryLogError: "查询全量日志失败", GetLogChartError: "获取全量日志图表数据失败", diff --git a/backend/pkg/model/response/log_index.go b/backend/pkg/model/response/log_index.go index 86f867d..d83013d 100644 --- a/backend/pkg/model/response/log_index.go +++ b/backend/pkg/model/response/log_index.go @@ -2,7 +2,6 @@ package response type LogIndexResponse struct { Indexs []IndexItem `json:"indexs"` - Err string `json:"error"` } type IndexItem struct { diff --git a/backend/pkg/services/log/service_add_log_parse_rule.go b/backend/pkg/services/log/service_add_log_parse_rule.go index dcd1d93..a545091 100644 --- a/backend/pkg/services/log/service_add_log_parse_rule.go +++ b/backend/pkg/services/log/service_add_log_parse_rule.go @@ -32,7 +32,7 @@ var fieldsRegexp = regexp.MustCompile(`\?P<(?P\w+)>`) func (s *service) AddLogParseRule(req *request.AddLogParseRequest) (*response.LogParseResponse, error) { // 先去建表 logReq := &request.LogTableRequest{ - TableName: req.ParseName, + TableName: "logs_" + req.ParseName, } matchesFields := fieldsRegexp.FindAllStringSubmatch(req.ParseRule, -1) @@ -69,7 +69,7 @@ func (s *service) AddLogParseRule(req *request.AddLogParseRequest) (*response.Lo } p := vector.ParseInfo{ ParseName: req.ParseName, - TableName: req.ParseName, + TableName: "logs_" + req.ParseName, ParseRule: req.ParseRule, RouteRule: getRouteRule(req.RouteRule), } @@ -96,7 +96,7 @@ func (s *service) AddLogParseRule(req *request.AddLogParseRequest) (*response.Lo ParseName: req.ParseName, ParseRule: req.ParseRule, RouteRule: getRouteRule(req.RouteRule), - Table: req.ParseName, + Table: "logs_" + req.ParseName, DataBase: logReq.DataBase, Cluster: logReq.Cluster, Fields: string(fieldsJSON), diff --git a/backend/pkg/services/log/service_get_log_parse_rule.go b/backend/pkg/services/log/service_get_log_parse_rule.go index 9c8760d..a79949e 100644 --- a/backend/pkg/services/log/service_get_log_parse_rule.go +++ b/backend/pkg/services/log/service_get_log_parse_rule.go @@ -22,6 +22,9 @@ func getRouteRuleMap(routeRule string) map[string]string { if len(matches) == 2 { key := matches[0][1] value := matches[1][1] + if key == "k8s.pod.name" { + continue + } res[key] = append(res[key], value) } } diff --git a/backend/pkg/services/log/service_log_index.go b/backend/pkg/services/log/service_log_index.go index 939abb2..1bcec4f 100644 --- a/backend/pkg/services/log/service_log_index.go +++ b/backend/pkg/services/log/service_log_index.go @@ -11,8 +11,7 @@ func (s *service) GetLogIndex(req *request.LogIndexRequest) (*response.LogIndexR res := &response.LogIndexResponse{} list, sum, err := s.chRepo.GetLogIndex(req) if err != nil { - res.Err = err.Error() - return res, nil + return nil, err } indexs := make([]response.IndexItem, 0) var count uint64