From c2c3d272e9095e04dac7f08639e342de7d16cb45 Mon Sep 17 00:00:00 2001 From: zhixiongdu027 Date: Mon, 15 Aug 2022 14:20:12 +0800 Subject: [PATCH] compatible with multiple versions of es --- .../tarsframework/templates/tars-tarslog.yaml | 4 +- src/LogServer/ESIndex.h | 189 +++++++++++++++--- src/LogServer/ESReader.cpp | 78 ++++---- 3 files changed, 200 insertions(+), 71 deletions(-) diff --git a/helm/tarsframework/templates/tars-tarslog.yaml b/helm/tarsframework/templates/tars-tarslog.yaml index 603dc226..e481aac3 100644 --- a/helm/tarsframework/templates/tars-tarslog.yaml +++ b/helm/tarsframework/templates/tars-tarslog.yaml @@ -26,13 +26,13 @@ metadata: tars.io/ServerApp: tars tars.io/ServerName: tarslog tars.io/SubType: tars - tars.io/Template: tars.es + tars.io/Template: tars.cpp spec: app: tars server: tarslog subType: tars tars: - template: tars.es + template: tars.cpp profile: | diff --git a/src/LogServer/ESIndex.h b/src/LogServer/ESIndex.h index c56b8677..ec12ddf6 100644 --- a/src/LogServer/ESIndex.h +++ b/src/LogServer/ESIndex.h @@ -13,34 +13,70 @@ constexpr char LogIndexTemplate[] = R"( "mappings": { "properties": { "trace": { - "type": "keyword" + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } }, "span": { - "type": "keyword" + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } }, "parent": { - "type": "keyword" - }, - "type": { - "type": "keyword" + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } }, "master": { - "type": "keyword" + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } }, "slave": { - "type": "keyword" + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } }, "function": { - "type": "keyword" + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } }, "time": { "type": "long" }, + "type": { + "type": "text" + }, "ret": { - "type": "keyword" + "type": "text" }, "data": { - "type": "keyword" + "type": "text" } } } @@ -53,13 +89,31 @@ constexpr char TraceIndexTemplate[] = R"( "mappings": { "properties": { "trace": { - "type": "keyword" + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } }, "tSpan": { - "type": "keyword" + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } }, "tMaster": { - "type": "keyword" + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } }, "tsTime": { "type": "long" @@ -68,25 +122,61 @@ constexpr char TraceIndexTemplate[] = R"( "type": "long" }, "sHash": { - "type": "keyword" + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } }, "fHash": { - "type": "keyword" + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } }, "spans": { "type": "nested", "properties": { "span": { - "type": "keyword" + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } }, "parent": { - "type": "keyword" + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } }, "master": { - "type": "keyword" + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } }, "slave": { - "type": "keyword" + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } }, "csTime": { "type": "long" @@ -101,22 +191,37 @@ constexpr char TraceIndexTemplate[] = R"( "type": "long" }, "csData": { - "type": "keyword" + "type": "text" }, "srData": { - "type": "keyword" + "type": "text" }, "ssData": { - "type": "keyword" + "type": "text" }, "crData": { - "type": "keyword" + "type": "text" }, "ret": { - "type": "keyword" + "type": "text" }, "children": { - "type": "keyword" + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "function": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } } } } @@ -131,13 +236,25 @@ constexpr char GraphIndexTemplate[] = R"( "mappings": { "properties": { "type": { - "type": "keyword" + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } }, "vertexes": { "type": "nested", "properties": { "vertex": { - "type": "keyword" + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } }, "callCount": { "type": "long" @@ -151,10 +268,22 @@ constexpr char GraphIndexTemplate[] = R"( "type": "nested", "properties": { "fromVertex": { - "type": "keyword" + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } }, "toVertex": { - "type": "keyword" + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } }, "callCount": { "type": "long" diff --git a/src/LogServer/ESReader.cpp b/src/LogServer/ESReader.cpp index 58e3c37d..1090020a 100644 --- a/src/LogServer/ESReader.cpp +++ b/src/LogServer/ESReader.cpp @@ -1,4 +1,4 @@ -#include "ESReader.h" +#include "ESReader.h" #include "TraceData.h" #include "ESIndex.h" #include @@ -6,11 +6,11 @@ #include "ESClient.h" #include "servant/RemoteLogger.h" -static int readESResponse(const std::string& response, const std::function&)>& f) +static int readESResponse(const std::string& response, const std::function&)>& f) { auto jsonPtr = TC_Json::getValue(response); auto&& jsonValuePtr = JsonValueObjPtr::dynamicCast(jsonPtr); - const auto& jsonValue = jsonValuePtr->value; +// const auto& jsonValue = jsonValuePtr->value; auto&& firstHitsPtr = jsonValuePtr->get("hits"); if (firstHitsPtr.get() == nullptr) { @@ -42,7 +42,7 @@ static int readESResponse(const std::string& response, const std::function& ts) +int ESReader::listTrace(const string& date, int64_t beginTime, int64_t endTime, const string& serverName, vector & ts) { constexpr char ListTraceTemplate[] = R"( { @@ -73,10 +73,10 @@ int ESReader::listTrace(const string& date, int64_t beginTime, int64_t endTime, } )"; string body = ListTraceTemplate; - map replaceMap = { - { "BEGIN_TIME", to_string(beginTime) }, - { "END_TIME", to_string(endTime) }, - { "SERVER", serverName } + map replaceMap = { + { "BEGIN_TIME", to_string(beginTime) }, + { "END_TIME", to_string(endTime) }, + { "SERVER", serverName } }; body = TC_Common::replace(body, replaceMap); auto url = "/" + buildTraceIndexByDate(date) + "/_search"; @@ -87,7 +87,7 @@ int ESReader::listTrace(const string& date, int64_t beginTime, int64_t endTime, TLOGERROR("do elk request error\n, \tRequest: " << body.substr(0, 2048) << "\n, \t" << response << endl); return -1; } - if (readESResponse(response, [&ts, &serverName](const TC_AutoPtr& ptr)mutable -> int + if (readESResponse(response, [&ts](const TC_AutoPtr & ptr)mutable -> int { auto v = JsonValueStringPtr::dynamicCast(ptr->get("trace"))->value; ts.emplace_back(v); @@ -100,7 +100,7 @@ int ESReader::listTrace(const string& date, int64_t beginTime, int64_t endTime, return 0; } -int ESReader::listFunction(const string& date, const string& serverName, set& fs) +int ESReader::listFunction(const string& date, const string& serverName, set & fs) { constexpr char ListFunctionTemplate[] = R"( { @@ -122,7 +122,7 @@ int ESReader::listFunction(const string& date, const string& serverName, set replaceMap = { - { "SERVER", serverName } + map replaceMap = { + { "SERVER", serverName } }; body = TC_Common::replace(body, replaceMap); auto url = "/" + buildGraphIndexByDate(date) + "/_search"; @@ -144,10 +144,10 @@ int ESReader::listFunction(const string& date, const string& serverName, set& ptr)mutable -> int + if (readESResponse(response, [&fs, &serverName](const TC_AutoPtr & ptr)mutable -> int { auto jsonObj = JsonValueObjPtr::dynamicCast(ptr); auto vertexes = jsonObj->get("vertexes"); @@ -175,7 +175,7 @@ int ESReader::listFunction(const string& date, const string& serverName, set& ss) +int ESReader::listTraceSummary(const string& date, int64_t beginTime, int64_t endTime, const string& serverName, vector & ss) { //fixme trace may had no tsTime. constexpr char ListTraceSummaryTemplate[] = R"( @@ -194,10 +194,10 @@ int ESReader::listTraceSummary(const string& date, int64_t beginTime, int64_t en } )"; string body = ListTraceSummaryTemplate; - map replaceMap = { - { "BEGIN_TIME", to_string(beginTime) }, - { "END_TIME", to_string(endTime) }, - { "SERVER", serverName } + map replaceMap = { + { "BEGIN_TIME", to_string(beginTime) }, + { "END_TIME", to_string(endTime) }, + { "SERVER", serverName } }; body = TC_Common::replace(body, replaceMap); auto url = "/" + buildTraceIndexByDate(date) + "/_search"; @@ -205,10 +205,10 @@ int ESReader::listTraceSummary(const string& date, int64_t beginTime, int64_t en int res = ESClient::instance().doRequest(ESClientRequestMethod::Get, url, body, response); if (res != 200) { - TLOGERROR("do elk request error\n, \tRequest: " << body.substr(0, 2048) << "\n, \t" << response << endl); + TLOGERROR("do es request error\n, \tRequest: " << body.substr(0, 2048) << "\n, \t" << response << endl); return -1; } - if (readESResponse(response, [&ss, &serverName](const TC_AutoPtr& ptr)mutable -> int + if (readESResponse(response, [&ss](const TC_AutoPtr & ptr)mutable -> int { auto traceName = JsonValueStringPtr::dynamicCast(ptr->get("trace"))->value; auto tsTimeNum = JsonValueNumPtr::dynamicCast(ptr->get("tsTime"))->value; @@ -235,7 +235,7 @@ int ESReader::listTraceSummary(const string& date, int64_t beginTime, int64_t en return 0; } -int ESReader::getServerGraph(const string& date, const string& serverName, vector& graphs) +int ESReader::getServerGraph(const string& date, const string& serverName, vector & graphs) { constexpr char GetServerGraphTemplate[] = R"( { @@ -244,15 +244,15 @@ int ESReader::getServerGraph(const string& date, const string& serverName, vecto "bool":{ "filter":[ {"term":{"type":"server"}}, - {"nested":{"path":"vertexes","query":{"bool":{"must":[{"term":{"vertexes.vertex":"SERVER"}}]}}}} + {"nested":{"path":"vertexes","query":{"bool":{"must":[{"term":{"vertexes.vertex.keyword":"SERVER"}}]}}}} ] } } } )"; string body = GetServerGraphTemplate; - map replaceMap = { - { "SERVER", serverName } + map replaceMap = { + { "SERVER", serverName } }; body = TC_Common::replace(body, replaceMap); auto url = "/" + buildGraphIndexByDate(date) + "/_search"; @@ -260,10 +260,10 @@ int ESReader::getServerGraph(const string& date, const string& serverName, vecto int res = ESClient::instance().doRequest(ESClientRequestMethod::Get, url, body, response); if (res != 200) { - TLOGERROR("do elk request error\n, \tRequest: " << body.substr(0, 2048) << "\n, \t" << response << endl); + TLOGERROR("do es request error\n, \tRequest: " << body.substr(0, 2048) << "\n, \t" << response << endl); return -1; } - if (readESResponse(response, [&graphs](const TC_AutoPtr& ptr)mutable -> int + if (readESResponse(response, [&graphs](const TC_AutoPtr & ptr)mutable -> int { graphs.emplace_back(); graphs.rbegin()->readFromJson(ptr); @@ -276,7 +276,7 @@ int ESReader::getServerGraph(const string& date, const string& serverName, vecto return 0; } -int ESReader::getFunctionGraph(const string& date, const string& functionName, vector& graphs) +int ESReader::getFunctionGraph(const string& date, const string& functionName, vector & graphs) { constexpr char GetFunctionGraphTemplate[] = R"( { @@ -285,15 +285,15 @@ int ESReader::getFunctionGraph(const string& date, const string& functionName, v "bool":{ "filter":[ {"term":{"type":"function"}}, - {"nested":{"path":"vertexes","query":{"bool":{"must":[{"term":{"vertexes.vertex":"FUNCTION"}}]}}}} + {"nested":{"path":"vertexes","query":{"bool":{"must":[{"term":{"vertexes.vertex.keyword":"FUNCTION"}}]}}}} ] } } } )"; string body = GetFunctionGraphTemplate; - map replaceMap = { - { "FUNCTION", functionName } + map replaceMap = { + { "FUNCTION", functionName } }; body = TC_Common::replace(body, replaceMap); auto url = "/" + buildGraphIndexByDate(date) + "/_search"; @@ -301,10 +301,10 @@ int ESReader::getFunctionGraph(const string& date, const string& functionName, v int res = ESClient::instance().doRequest(ESClientRequestMethod::Get, url, body, response); if (res != 200) { - TLOGERROR("do elk request error\n, \tRequest: " << body.substr(0, 2048) << "\n, \t" << response << endl); + TLOGERROR("do es request error\n, \tRequest: " << body.substr(0, 2048) << "\n, \t" << response << endl); return -1; } - if (readESResponse(response, [&graphs](const TC_AutoPtr& ptr)mutable -> int + if (readESResponse(response, [&graphs](const TC_AutoPtr & ptr)mutable -> int { graphs.emplace_back(); graphs.rbegin()->readFromJson(ptr); @@ -324,15 +324,15 @@ int ESReader::getTrace(const string& date, const string& traceName, ITrace& trac "query":{ "bool":{ "filter":[ - {"term":{"trace":"TRACE"}} + {"term":{"trace.keyword":"TRACE"}} ] } } } )"; string body = GetTraceGraphTemplate; - map replaceMap = { - { "TRACE", traceName } + map replaceMap = { + { "TRACE", traceName } }; body = TC_Common::replace(body, replaceMap); auto url = "/" + buildTraceIndexByDate(date) + "/_search"; @@ -340,10 +340,10 @@ int ESReader::getTrace(const string& date, const string& traceName, ITrace& trac int res = ESClient::instance().doRequest(ESClientRequestMethod::Get, url, body, response); if (res != 200) { - TLOGERROR("do elk request error\n, \tRequest: " << body.substr(0, 2048) << "\n, \t" << response << endl); + TLOGERROR("do es request error\n, \tRequest: " << body.substr(0, 2048) << "\n, \t" << response << endl); return -1; } - if (readESResponse(response, [&trace](const TC_AutoPtr& ptr)mutable -> int + if (readESResponse(response, [&trace](const TC_AutoPtr & ptr)mutable -> int { trace.readFromJson(ptr); return 0;