From f89ac346346efaa6f77ee6ede1df21017b3405fc Mon Sep 17 00:00:00 2001 From: fuwx Date: Fri, 18 Oct 2024 13:36:11 +0800 Subject: [PATCH] add query context --- backend/docs/docs.go | 77 +++++++++++++++++++ backend/docs/swagger.json | 77 +++++++++++++++++++ backend/docs/swagger.yaml | 50 ++++++++++++ backend/pkg/api/log/func_querylogcontext.go | 43 +++++++++++ backend/pkg/api/log/handler.go | 5 ++ backend/pkg/code/code.go | 7 +- backend/pkg/code/en.go | 7 +- backend/pkg/code/zh-cn.go | 7 +- backend/pkg/model/request/log_query.go | 7 ++ backend/pkg/model/response/log_query.go | 5 ++ backend/pkg/repository/clickhouse/dao.go | 1 + .../clickhouse/dao_query_log_context.go | 68 ++++++++++++++++ .../pkg/repository/clickhouse/factory/log.go | 2 +- .../pkg/repository/clickhouse/factory/view.go | 2 +- backend/pkg/router/router_api.go | 2 +- backend/pkg/services/log/service.go | 2 + backend/pkg/services/log/service_log_table.go | 4 +- .../services/log/service_query_log_context.go | 62 +++++++++++++++ 18 files changed, 414 insertions(+), 14 deletions(-) create mode 100644 backend/pkg/api/log/func_querylogcontext.go create mode 100644 backend/pkg/repository/clickhouse/dao_query_log_context.go create mode 100644 backend/pkg/services/log/service_query_log_context.go diff --git a/backend/docs/docs.go b/backend/docs/docs.go index f105fd1..6a7324b 100644 --- a/backend/docs/docs.go +++ b/backend/docs/docs.go @@ -695,6 +695,46 @@ const docTemplate = `{ } } }, + "/api/log/context": { + "post": { + "description": "获取日志上下文", + "consumes": [ + "application/json" + ], + "produces": [ + "application/json" + ], + "tags": [ + "API.log" + ], + "summary": "获取日志上下文", + "parameters": [ + { + "description": "请求信息", + "name": "Request", + "in": "body", + "required": true, + "schema": { + "$ref": "#/definitions/request.LogQueryContextRequest" + } + } + ], + "responses": { + "200": { + "description": "OK", + "schema": { + "$ref": "#/definitions/response.LogQueryContextResponse" + } + }, + "400": { + "description": "Bad Request", + "schema": { + "$ref": "#/definitions/code.Failure" + } + } + } + } + }, "/api/log/fault/content": { "post": { "description": "获取故障现场日志内容", @@ -4904,6 +4944,26 @@ const docTemplate = `{ } } }, + "request.LogQueryContextRequest": { + "type": "object", + "properties": { + "dataBase": { + "type": "string" + }, + "tableName": { + "type": "string" + }, + "tags": { + "type": "object", + "additionalProperties": { + "type": "string" + } + }, + "timestamp": { + "type": "integer" + } + } + }, "request.LogQueryRequest": { "type": "object", "required": [ @@ -6151,6 +6211,23 @@ const docTemplate = `{ } } }, + "response.LogQueryContextResponse": { + "type": "object", + "properties": { + "back": { + "type": "array", + "items": { + "$ref": "#/definitions/response.LogItem" + } + }, + "front": { + "type": "array", + "items": { + "$ref": "#/definitions/response.LogItem" + } + } + } + }, "response.LogQueryResponse": { "type": "object", "properties": { diff --git a/backend/docs/swagger.json b/backend/docs/swagger.json index 0ce43ae..a513379 100644 --- a/backend/docs/swagger.json +++ b/backend/docs/swagger.json @@ -687,6 +687,46 @@ } } }, + "/api/log/context": { + "post": { + "description": "获取日志上下文", + "consumes": [ + "application/json" + ], + "produces": [ + "application/json" + ], + "tags": [ + "API.log" + ], + "summary": "获取日志上下文", + "parameters": [ + { + "description": "请求信息", + "name": "Request", + "in": "body", + "required": true, + "schema": { + "$ref": "#/definitions/request.LogQueryContextRequest" + } + } + ], + "responses": { + "200": { + "description": "OK", + "schema": { + "$ref": "#/definitions/response.LogQueryContextResponse" + } + }, + "400": { + "description": "Bad Request", + "schema": { + "$ref": "#/definitions/code.Failure" + } + } + } + } + }, "/api/log/fault/content": { "post": { "description": "获取故障现场日志内容", @@ -4896,6 +4936,26 @@ } } }, + "request.LogQueryContextRequest": { + "type": "object", + "properties": { + "dataBase": { + "type": "string" + }, + "tableName": { + "type": "string" + }, + "tags": { + "type": "object", + "additionalProperties": { + "type": "string" + } + }, + "timestamp": { + "type": "integer" + } + } + }, "request.LogQueryRequest": { "type": "object", "required": [ @@ -6143,6 +6203,23 @@ } } }, + "response.LogQueryContextResponse": { + "type": "object", + "properties": { + "back": { + "type": "array", + "items": { + "$ref": "#/definitions/response.LogItem" + } + }, + "front": { + "type": "array", + "items": { + "$ref": "#/definitions/response.LogItem" + } + } + } + }, "response.LogQueryResponse": { "type": "object", "properties": { diff --git a/backend/docs/swagger.yaml b/backend/docs/swagger.yaml index 9c34811..975f16f 100644 --- a/backend/docs/swagger.yaml +++ b/backend/docs/swagger.yaml @@ -971,6 +971,19 @@ definitions: required: - endTime type: object + request.LogQueryContextRequest: + properties: + dataBase: + type: string + tableName: + type: string + tags: + additionalProperties: + type: string + type: object + timestamp: + type: integer + type: object request.LogQueryRequest: properties: dataBase: @@ -1813,6 +1826,17 @@ definitions: type: string type: array type: object + response.LogQueryContextResponse: + properties: + back: + items: + $ref: '#/definitions/response.LogItem' + type: array + front: + items: + $ref: '#/definitions/response.LogItem' + type: array + type: object response.LogQueryResponse: properties: cost: @@ -2518,6 +2542,32 @@ paths: summary: 获取日志趋势图 tags: - API.log + /api/log/context: + post: + consumes: + - application/json + description: 获取日志上下文 + parameters: + - description: 请求信息 + in: body + name: Request + required: true + schema: + $ref: '#/definitions/request.LogQueryContextRequest' + produces: + - application/json + responses: + "200": + description: OK + schema: + $ref: '#/definitions/response.LogQueryContextResponse' + "400": + description: Bad Request + schema: + $ref: '#/definitions/code.Failure' + summary: 获取日志上下文 + tags: + - API.log /api/log/fault/content: post: description: 获取故障现场日志内容 diff --git a/backend/pkg/api/log/func_querylogcontext.go b/backend/pkg/api/log/func_querylogcontext.go new file mode 100644 index 0000000..bca0a50 --- /dev/null +++ b/backend/pkg/api/log/func_querylogcontext.go @@ -0,0 +1,43 @@ +package log + +import ( + "net/http" + + "github.com/CloudDetail/apo/backend/pkg/code" + "github.com/CloudDetail/apo/backend/pkg/core" + "github.com/CloudDetail/apo/backend/pkg/model/request" +) + +// QueryLogContext 获取日志上下文 +// @Summary 获取日志上下文 +// @Description 获取日志上下文 +// @Tags API.log +// @Accept json +// @Produce json +// @Param Request body request.LogQueryContextRequest true "请求信息" +// @Success 200 {object} response.LogQueryContextResponse +// @Failure 400 {object} code.Failure +// @Router /api/log/context [post] +func (h *handler) QueryLogContext() core.HandlerFunc { + return func(c core.Context) { + req := new(request.LogQueryContextRequest) + if err := c.ShouldBindJSON(req); err != nil { + c.AbortWithError(core.Error( + http.StatusBadRequest, + code.ParamBindError, + code.Text(code.ParamBindError)).WithError(err), + ) + return + } + resp, err := h.logService.QueryLogContext(req) + if err != nil { + c.AbortWithError(core.Error( + http.StatusBadRequest, + code.QueryLogContextError, + code.Text(code.QueryLogContextError)).WithError(err), + ) + return + } + c.Payload(resp) + } +} diff --git a/backend/pkg/api/log/handler.go b/backend/pkg/api/log/handler.go index 793c8ff..4d575a4 100644 --- a/backend/pkg/api/log/handler.go +++ b/backend/pkg/api/log/handler.go @@ -86,6 +86,11 @@ type Handler interface { // @Tags API.log // @Router /api/log/other/delete [delete] DeleteOtherTable() core.HandlerFunc + + // QueryLogContext 获取日志上下文 + // @Tags API.log + // @Router /api/log/context [post] + QueryLogContext() core.HandlerFunc } type handler struct { diff --git a/backend/pkg/code/code.go b/backend/pkg/code/code.go index 6f146e4..781c61d 100644 --- a/backend/pkg/code/code.go +++ b/backend/pkg/code/code.go @@ -40,9 +40,10 @@ const ( GetFaultLogPageListError = "B0401" GetFaultLogContentError = "B0402" - QueryLogError = "B0406" - GetLogChartError = "B0407" - GetLogIndexError = "B0408" + QueryLogContextError = "B0405" + QueryLogError = "B0406" + GetLogChartError = "B0407" + GetLogIndexError = "B0408" GetLogTableInfoError = "B0409" diff --git a/backend/pkg/code/en.go b/backend/pkg/code/en.go index 8d82863..316fc93 100644 --- a/backend/pkg/code/en.go +++ b/backend/pkg/code/en.go @@ -30,9 +30,10 @@ var enText = map[string]string{ GetFaultLogContentError: "Failed to get fault log content", GetMonitorStatusError: "Failed to get monitor status", - QueryLogError: "Failed to query all logs", - GetLogChartError: "Failed to get log chart", - GetLogIndexError: "Failed to get log index", + QueryLogContextError: "Failed to query log context", + QueryLogError: "Failed to query all logs", + GetLogChartError: "Failed to get log chart", + GetLogIndexError: "Failed to get log index", GetLogTableInfoError: "Failed to get log table info", GetLogParseRuleError: "Failed to get log parse rule", diff --git a/backend/pkg/code/zh-cn.go b/backend/pkg/code/zh-cn.go index 5e1c3f5..e889d09 100644 --- a/backend/pkg/code/zh-cn.go +++ b/backend/pkg/code/zh-cn.go @@ -30,9 +30,10 @@ var zhCnText = map[string]string{ GetFaultLogPageListError: "获取故障现场日志分页列表失败", GetFaultLogContentError: "获取故障现场日志内容失败", - QueryLogError: "查询全量日志失败", - GetLogChartError: "获取全量日志图表数据失败", - GetLogIndexError: "获取全量日志索引失败", + QueryLogContextError: "查询日志上下文失败", + QueryLogError: "查询全量日志失败", + GetLogChartError: "获取全量日志图表数据失败", + GetLogIndexError: "获取全量日志索引失败", GetLogTableInfoError: "获取日志表信息失败", GetLogParseRuleError: "获取日志表解析规则失败", diff --git a/backend/pkg/model/request/log_query.go b/backend/pkg/model/request/log_query.go index a2caf44..3d72e45 100644 --- a/backend/pkg/model/request/log_query.go +++ b/backend/pkg/model/request/log_query.go @@ -22,3 +22,10 @@ type LogIndexRequest struct { LogField string `json:"logField"` Query string `json:"query"` } + +type LogQueryContextRequest struct { + TableName string `json:"tableName"` + DataBase string `json:"dataBase"` + Tags map[string]string `json:"tags"` + Time int64 `json:"timestamp"` +} diff --git a/backend/pkg/model/response/log_query.go b/backend/pkg/model/response/log_query.go index 188d588..0eab38e 100644 --- a/backend/pkg/model/response/log_query.go +++ b/backend/pkg/model/response/log_query.go @@ -15,3 +15,8 @@ type LogItem struct { Tags map[string]interface{} `json:"tags"` Time int64 `json:"timestamp"` } + +type LogQueryContextResponse struct { + Front []LogItem `json:"front"` + Back []LogItem `json:"back"` +} diff --git a/backend/pkg/repository/clickhouse/dao.go b/backend/pkg/repository/clickhouse/dao.go index 629cd87..2292269 100644 --- a/backend/pkg/repository/clickhouse/dao.go +++ b/backend/pkg/repository/clickhouse/dao.go @@ -63,6 +63,7 @@ type Repo interface { queryRowsData(sql string) ([]map[string]any, error) QueryAllLogs(req *request.LogQueryRequest) ([]map[string]any, string, error) + QueryLogContext(req *request.LogQueryContextRequest) ([]map[string]any, []map[string]any, error) GetLogChart(req *request.LogQueryRequest) ([]map[string]any, int64, error) GetLogIndex(req *request.LogIndexRequest) (map[string]uint64, uint64, error) diff --git a/backend/pkg/repository/clickhouse/dao_query_log_context.go b/backend/pkg/repository/clickhouse/dao_query_log_context.go new file mode 100644 index 0000000..d334b75 --- /dev/null +++ b/backend/pkg/repository/clickhouse/dao_query_log_context.go @@ -0,0 +1,68 @@ +package clickhouse + +import ( + "fmt" + + "github.com/CloudDetail/apo/backend/pkg/model/request" +) + +var keys = []string{"source", "container_id", "pid", "container_name", "host_ip", "host_name", "k8s_namespace_name", "k8s_pod_name"} + +func isKey(key string) bool { + for _, k := range keys { + if k == key { + return true + } + } + return false +} + +func tagsCondition(tags map[string]string) string { + var res string + for k, v := range tags { + if isKey(k) { + res += fmt.Sprintf(`AND %s='%s'`, k, v) + } + } + if res == "" { + res = "AND (1='1')" + } + + return res +} + +func reverseSlice(s []map[string]any) { + for i, j := 0, len(s)-1; i < j; i, j = i+1, j-1 { + s[i], s[j] = s[j], s[i] + } +} +func (ch *chRepo) QueryLogContext(req *request.LogQueryContextRequest) ([]map[string]any, []map[string]any, error) { + //condition := NewQueryCondition(req.StartTime, req.EndTime, req.TimeField, req.Query) + logtime := req.Time / 1000000 + timefront := fmt.Sprintf("toUnixTimestamp(timestamp) < %d AND toUnixTimestamp(timestamp) > %d ", logtime, logtime-60) + tags := tagsCondition(req.Tags) + //查前面50条，反转 + bySqlfront := NewByLimitBuilder(). + OrderBy("timestamp", false). + Limit(50). + String() + frontSql := fmt.Sprintf(querySQl, req.DataBase, req.TableName, timefront+tags, bySqlfront) + front, err := ch.queryRowsData(frontSql) + if err != nil { + front = []map[string]any{} + } + reverseSlice(front) + timeend := fmt.Sprintf("toUnixTimestamp(timestamp) >= %d AND toUnixTimestamp(timestamp) < %d ", logtime, logtime+60) + bySqlend := NewByLimitBuilder(). + OrderBy("timestamp", true). + Limit(50). + String() + endSql := fmt.Sprintf(querySQl, req.DataBase, req.TableName, timeend+tags, bySqlend) + end, err := ch.queryRowsData(endSql) + if err != nil { + end = []map[string]any{} + } + + return front, end, nil + +} diff --git a/backend/pkg/repository/clickhouse/factory/log.go b/backend/pkg/repository/clickhouse/factory/log.go index 9a1be37..0ad779b 100644 --- a/backend/pkg/repository/clickhouse/factory/log.go +++ b/backend/pkg/repository/clickhouse/factory/log.go @@ -42,7 +42,7 @@ INDEX idx_content content TYPE tokenbf_v1(32768, 3, 0) GRANULARITY 1 ) %s PARTITION BY toDate(timestamp) -ORDER BY (host_ip, toUnixTimestamp(timestamp)) +ORDER BY (host_ip, timestamp) %s SETTINGS index_granularity = 8192, ttl_only_drop_parts = 1; ` diff --git a/backend/pkg/repository/clickhouse/factory/view.go b/backend/pkg/repository/clickhouse/factory/view.go index cb4c88b..b5083ca 100644 --- a/backend/pkg/repository/clickhouse/factory/view.go +++ b/backend/pkg/repository/clickhouse/factory/view.go @@ -22,7 +22,7 @@ k8s_namespace_name LowCardinality(String) CODEC(ZSTD(1)), k8s_pod_name LowCardinality(String) CODEC(ZSTD(1))%s ) AS SELECT -toDateTime64(parseDateTimeBestEffort(timestamp), 9) AS timestamp, +parseDateTime64BestEffort(timestamp, 9) AS timestamp, content AS content, _source_ AS source, _container_id_ AS container_id, diff --git a/backend/pkg/router/router_api.go b/backend/pkg/router/router_api.go index 88a20f9..89fe05f 100644 --- a/backend/pkg/router/router_api.go +++ b/backend/pkg/router/router_api.go @@ -66,7 +66,7 @@ func setApiRouter(r *resource) { logApi.POST("/fault/pagelist", logHandler.GetFaultLogPageList()) logApi.POST("/fault/content", logHandler.GetFaultLogContent()) - //logApi.POST("/update", logHandler.UpdateLogTable()) + logApi.POST("/context", logHandler.QueryLogContext()) logApi.POST("/query", logHandler.QueryLog()) logApi.POST("/chart", logHandler.GetLogChart()) diff --git a/backend/pkg/services/log/service.go b/backend/pkg/services/log/service.go index 54493e9..8cff214 100644 --- a/backend/pkg/services/log/service.go +++ b/backend/pkg/services/log/service.go @@ -27,6 +27,8 @@ type Service interface { // 查询全量日志 QueryLog(req *request.LogQueryRequest) (*response.LogQueryResponse, error) + + QueryLogContext(req *request.LogQueryContextRequest) (*response.LogQueryContextResponse, error) // 日志趋势图 GetLogChart(req *request.LogQueryRequest) (*response.LogChartResponse, error) // 字段分析 diff --git a/backend/pkg/services/log/service_log_table.go b/backend/pkg/services/log/service_log_table.go index f795c98..5d2043b 100644 --- a/backend/pkg/services/log/service_log_table.go +++ b/backend/pkg/services/log/service_log_table.go @@ -9,9 +9,9 @@ import ( ) const ( - defaultParseInfo = "默认Java日志解析, 从日志字段中解析出level、thread、method信息" + defaultParseInfo = "默认应用Java格式的日志解析规则, 从满足条件的日志中解析出level、thread、method信息" defaultParseName = "all_logs" - defaultRouteRule = `!starts_with(string!(."k8s.pod.name"), "apo")` + defaultRouteRule = `starts_with(string!(."k8s.pod.name"), "apo")` defaultParseRule = `.msg, err = parse_regex(.content, r' \[(?P.*?)\] \[(?P.*?)\] \[(?P.*?)\(.*?\)\] - (?P.*)') if err == null { .content = encode_json(.msg) diff --git a/backend/pkg/services/log/service_query_log_context.go b/backend/pkg/services/log/service_query_log_context.go new file mode 100644 index 0000000..b126742 --- /dev/null +++ b/backend/pkg/services/log/service_query_log_context.go @@ -0,0 +1,62 @@ +package log + +import ( + "errors" + "time" + + "github.com/CloudDetail/apo/backend/pkg/model/request" + "github.com/CloudDetail/apo/backend/pkg/model/response" +) + +func log2item(logs []map[string]any) ([]response.LogItem, error) { + var timestamp int64 + logitems := make([]response.LogItem, len(logs)) + for i, log := range logs { + content := log["content"] + delete(log, "content") + + for k, v := range log { + if k == "timestamp" { + ts, ok := v.(time.Time) + if ok { + timestamp = ts.UnixMicro() + } else { + return nil, errors.New("timestamp type error") + } + delete(log, k) + } + vMap, ok := v.(map[string]string) + if ok { + for k2, v2 := range vMap { + log[k+"."+k2] = v2 + } + delete(log, k) + } + } + + logitems[i] = response.LogItem{ + Content: content, + Tags: log, + Time: timestamp, + } + } + return logitems, nil +} + +func (s *service) QueryLogContext(req *request.LogQueryContextRequest) (*response.LogQueryContextResponse, error) { + res := &response.LogQueryContextResponse{} + front, end, _ := s.chRepo.QueryLogContext(req) + + frontItem, err := log2item(front) + if err != nil { + return nil, err + } + endItem, err := log2item(end) + if err != nil { + return nil, err + } + + res.Front = frontItem + res.Back = endItem + return res, nil +}