diff --git a/core/CMakeLists.txt b/core/CMakeLists.txt index e680c21a8d..dbdcad6e93 100644 --- a/core/CMakeLists.txt +++ b/core/CMakeLists.txt @@ -113,10 +113,9 @@ include(${CMAKE_CURRENT_SOURCE_DIR}/plugin/flusher/links.cmake) # Subdirectories (modules). except for common, input, processor, flusher, observer, helper, spl, and provider. set(SUB_DIRECTORIES_LIST - application app_config checkpoint container_manager logger go_pipeline monitor profile_sender models + application app_config checkpoint container_manager logger go_pipeline monitor monitor/metric_constants profile_sender models config config/watcher pipeline pipeline/batch pipeline/limiter pipeline/plugin pipeline/plugin/creator pipeline/plugin/instance pipeline/plugin/interface pipeline/queue pipeline/route pipeline/serializer - monitor/metric_constants runner runner/sink/http protobuf/sls file_server file_server/event file_server/event_handler file_server/event_listener file_server/reader file_server/polling diff --git a/core/common/compression/Compressor.cpp b/core/common/compression/Compressor.cpp index 297c57f7bf..c5800016d9 100644 --- a/core/common/compression/Compressor.cpp +++ b/core/common/compression/Compressor.cpp @@ -16,7 +16,7 @@ #include -#include "monitor/MetricConstants.h" +#include "monitor/metric_constants/MetricConstants.h" using namespace std; @@ -26,10 +26,10 @@ void Compressor::SetMetricRecordRef(MetricLabels&& labels, DynamicMetricLabels&& WriteMetrics::GetInstance()->PrepareMetricsRecordRef( mMetricsRecordRef, std::move(labels), std::move(dynamicLabels)); mInItemsTotal = mMetricsRecordRef.CreateCounter(METRIC_COMPONENT_IN_ITEMS_TOTAL); - mInItemSizeBytes = mMetricsRecordRef.CreateCounter(METRIC_COMPONENT_IN_ITEM_SIZE_BYTES); + mInItemSizeBytes = mMetricsRecordRef.CreateCounter(METRIC_COMPONENT_IN_SIZE_BYTES); mOutItemsTotal = mMetricsRecordRef.CreateCounter(METRIC_COMPONENT_OUT_ITEMS_TOTAL); - mOutItemSizeBytes = mMetricsRecordRef.CreateCounter(METRIC_COMPONENT_OUT_ITEM_SIZE_BYTES); - mTotalProcessMs = mMetricsRecordRef.CreateCounter(METRIC_COMPONENT_TOTAL_PROCESS_MS); + mOutItemSizeBytes = mMetricsRecordRef.CreateCounter(METRIC_COMPONENT_OUT_SIZE_BYTES); + mTotalProcessMs = mMetricsRecordRef.CreateCounter(METRIC_COMPONENT_TOTAL_PROCESS_TIME_MS); mDiscardedItemsTotal = mMetricsRecordRef.CreateCounter(METRIC_COMPONENT_DISCARDED_ITEMS_TOTAL); mDiscardedItemSizeBytes = mMetricsRecordRef.CreateCounter(METRIC_COMPONENT_DISCARDED_ITEMS_SIZE_BYTES); } diff --git a/core/common/compression/CompressorFactory.cpp b/core/common/compression/CompressorFactory.cpp index 24c67061a2..0507b7716b 100644 --- a/core/common/compression/CompressorFactory.cpp +++ b/core/common/compression/CompressorFactory.cpp @@ -15,7 +15,7 @@ #include "common/compression/CompressorFactory.h" #include "common/ParamExtractor.h" -#include "monitor/MetricConstants.h" +#include "monitor/metric_constants/MetricConstants.h" #include "common/compression/LZ4Compressor.h" #include "common/compression/ZstdCompressor.h" @@ -61,8 +61,8 @@ unique_ptr CompressorFactory::Create(const Json::Value& config, } else { compressor = Create(defaultType); } - compressor->SetMetricRecordRef({{METRIC_LABEL_PROJECT, ctx.GetProjectName()}, - {METRIC_LABEL_PIPELINE_NAME, ctx.GetConfigName()}, + compressor->SetMetricRecordRef({{METRIC_LABEL_KEY_PROJECT, ctx.GetProjectName()}, + {METRIC_LABEL_KEY_PIPELINE_NAME, ctx.GetConfigName()}, {METRIC_LABEL_KEY_COMPONENT_NAME, METRIC_LABEL_VALUE_COMPONENT_NAME_COMPRESSOR}, {METRIC_LABEL_KEY_FLUSHER_NODE_ID, flusherId}}); return compressor; diff --git a/core/file_server/polling/PollingDirFile.cpp b/core/file_server/polling/PollingDirFile.cpp index 0a578f5ce8..4897624817 100644 --- a/core/file_server/polling/PollingDirFile.cpp +++ b/core/file_server/polling/PollingDirFile.cpp @@ -34,7 +34,7 @@ #include "file_server/polling/PollingModify.h" #include "logger/Logger.h" #include "monitor/LogtailAlarm.h" -#include "monitor/MetricConstants.h" +#include "monitor/metric_constants/MetricConstants.h" // Control the check frequency to call ClearUnavailableFileAndDir. DEFINE_FLAG_INT32(check_not_exist_file_dir_round, "clear not exist file dir cache, round", 20); diff --git a/core/file_server/polling/PollingModify.cpp b/core/file_server/polling/PollingModify.cpp index 73ff76b3cd..0a940cd4a3 100644 --- a/core/file_server/polling/PollingModify.cpp +++ b/core/file_server/polling/PollingModify.cpp @@ -28,7 +28,7 @@ #include "file_server/event/Event.h" #include "logger/Logger.h" #include "monitor/LogtailAlarm.h" -#include "monitor/MetricConstants.h" +#include "monitor/metric_constants/MetricConstants.h" using namespace std; diff --git a/core/file_server/reader/LogFileReader.cpp b/core/file_server/reader/LogFileReader.cpp index d9d7c6fc6d..2f849b5fcb 100644 --- a/core/file_server/reader/LogFileReader.cpp +++ b/core/file_server/reader/LogFileReader.cpp @@ -48,7 +48,7 @@ #include "logger/Logger.h" #include "monitor/LogFileProfiler.h" #include "monitor/LogtailAlarm.h" -#include "monitor/MetricConstants.h" +#include "monitor/metric_constants/MetricConstants.h" #include "pipeline/queue/ExactlyOnceQueueManager.h" #include "pipeline/queue/ProcessQueueManager.h" #include "pipeline/queue/QueueKeyManager.h" @@ -203,9 +203,9 @@ LogFileReader::LogFileReader(const std::string& hostLogPathDir, void LogFileReader::SetMetrics() { mMetricInited = false; - mMetricLabels = {{METRIC_LABEL_FILE_NAME, GetConvertedPath()}, - {METRIC_LABEL_FILE_DEV, std::to_string(GetDevInode().dev)}, - {METRIC_LABEL_FILE_INODE, std::to_string(GetDevInode().inode)}}; + mMetricLabels = {{METRIC_LABEL_KEY_FILE_NAME, GetConvertedPath()}, + {METRIC_LABEL_KEY_FILE_DEV, std::to_string(GetDevInode().dev)}, + {METRIC_LABEL_KEY_FILE_INODE, std::to_string(GetDevInode().inode)}}; mMetricsRecordRef = FileServer::GetInstance()->GetOrCreateReentrantMetricsRecordRef(GetConfigName(), mMetricLabels); if (mMetricsRecordRef == nullptr) { LOG_ERROR(sLogger, diff --git a/core/monitor/LogtailMetric.cpp b/core/monitor/LogtailMetric.cpp index 6b26300d8c..8e87dd08f2 100644 --- a/core/monitor/LogtailMetric.cpp +++ b/core/monitor/LogtailMetric.cpp @@ -25,6 +25,9 @@ using namespace sls_logs; namespace logtail { +const std::string LABEL_PREFIX = "label."; +const std::string VALUE_PREFIX = "value."; + MetricsRecord::MetricsRecord(MetricLabelsPtr labels, DynamicMetricLabelsPtr dynamicLabels) : mLabels(labels), mDynamicLabels(dynamicLabels), mDeleted(false) { } @@ -325,7 +328,9 @@ ReadMetrics::~ReadMetrics() { Clear(); } -void ReadMetrics::ReadAsLogGroup(std::map& logGroupMap) const { +void ReadMetrics::ReadAsLogGroup(std::string regionFieldName, + std::string defaultRegion, + std::map& logGroupMap) const { ReadLock lock(mReadWriteLock); MetricsRecord* tmp = mHead; while (tmp) { @@ -333,7 +338,7 @@ void ReadMetrics::ReadAsLogGroup(std::map& log for (auto item = tmp->GetLabels()->begin(); item != tmp->GetLabels()->end(); ++item) { std::pair pair = *item; - if (METRIC_FIELD_REGION == pair.first) { + if (regionFieldName == pair.first) { std::map::iterator iter; std::string region = pair.second; iter = logGroupMap.find(region); @@ -349,14 +354,14 @@ void ReadMetrics::ReadAsLogGroup(std::map& log } if (!logPtr) { std::map::iterator iter; - iter = logGroupMap.find(METRIC_REGION_DEFAULT); + iter = logGroupMap.find(defaultRegion); if (iter != logGroupMap.end()) { sls_logs::LogGroup* logGroup = iter->second; logPtr = logGroup->add_logs(); } else { sls_logs::LogGroup* logGroup = new sls_logs::LogGroup(); logPtr = logGroup->add_logs(); - logGroupMap.insert(std::pair(METRIC_REGION_DEFAULT, logGroup)); + logGroupMap.insert(std::pair(defaultRegion, logGroup)); } } auto now = GetCurrentLogtailTime(); diff --git a/core/monitor/LogtailMetric.h b/core/monitor/LogtailMetric.h index 6f4ef9e8ef..4901bdd35e 100644 --- a/core/monitor/LogtailMetric.h +++ b/core/monitor/LogtailMetric.h @@ -160,7 +160,9 @@ class ReadMetrics { static ReadMetrics* ptr = new ReadMetrics(); return ptr; } - void ReadAsLogGroup(std::map& logGroupMap) const; + void ReadAsLogGroup(std::string regionFieldName, + std::string defaultRegion, + std::map& logGroupMap) const; void ReadAsFileBuffer(std::string& metricsContent) const; void UpdateMetrics(); diff --git a/core/monitor/MetricExportor.cpp b/core/monitor/MetricExportor.cpp index f82a2bbb70..7ba9fb7b1f 100644 --- a/core/monitor/MetricExportor.cpp +++ b/core/monitor/MetricExportor.cpp @@ -24,8 +24,8 @@ #include "common/RuntimeUtil.h" #include "common/TimeUtil.h" #include "go_pipeline/LogtailPlugin.h" -#include "protobuf/sls/sls_logs.pb.h" #include "pipeline/PipelineManager.h" +#include "protobuf/sls/sls_logs.pb.h" using namespace sls_logs; using namespace std; @@ -34,12 +34,17 @@ DECLARE_FLAG_STRING(metrics_report_method); namespace logtail { +const string METRIC_REGION_FIELD_NAME = "region"; +const string METRIC_REGION_DEFAULT = "default"; +const string METRIC_SLS_LOGSTORE_NAME = "shennong_log_profile"; +const string METRIC_TOPIC_TYPE = "loong_collector_metric"; + const std::string METRIC_EXPORT_TYPE_GO = "direct"; const std::string METRIC_EXPORT_TYPE_CPP = "cpp_provided"; MetricExportor::MetricExportor() : mSendInterval(60), mLastSendTime(time(NULL) - (rand() % (mSendInterval / 10)) * 10) { mAgentMemGo = LoongCollectorMonitor::GetInstance()->GetIntGauge(METRIC_AGENT_MEMORY_GO); - mAgentGoRoutines = LoongCollectorMonitor::GetInstance()->GetIntGauge(METRIC_AGENT_GO_ROUTINES_TOTAL); + mAgentGoRoutines = LoongCollectorMonitor::GetInstance()->GetIntGauge(METRIC_AGENT_GO_ROUTINES_TOTAL); } void MetricExportor::PushMetrics(bool forceSend) { @@ -62,7 +67,7 @@ void MetricExportor::PushCppMetrics() { if ("sls" == STRING_FLAG(metrics_report_method)) { std::map logGroupMap; - ReadMetrics::GetInstance()->ReadAsLogGroup(logGroupMap); + ReadMetrics::GetInstance()->ReadAsLogGroup(METRIC_REGION_FIELD_NAME, METRIC_REGION_DEFAULT, logGroupMap); SendToSLS(logGroupMap); } else if ("file" == STRING_FLAG(metrics_report_method)) { std::string metricsContent; @@ -89,8 +94,7 @@ void MetricExportor::SendToSLS(std::map& logGr logGroup->set_source(LogFileProfiler::mIpAddr); logGroup->set_topic(METRIC_TOPIC_TYPE); if (METRIC_REGION_DEFAULT == iter->first) { - GetProfileSender()->SendToProfileProject(GetProfileSender()->GetDefaultProfileRegion(), - *logGroup); + GetProfileSender()->SendToProfileProject(GetProfileSender()->GetDefaultProfileRegion(), *logGroup); } else { GetProfileSender()->SendToProfileProject(iter->first, *logGroup); } diff --git a/core/monitor/Monitor.cpp b/core/monitor/Monitor.cpp index 801dfbc614..b5d0424805 100644 --- a/core/monitor/Monitor.cpp +++ b/core/monitor/Monitor.cpp @@ -704,18 +704,18 @@ LoongCollectorMonitor* LoongCollectorMonitor::GetInstance() { void LoongCollectorMonitor::Init() { // create metric record MetricLabels labels; - labels.emplace_back(METRIC_LABEL_INSTANCE_ID, Application::GetInstance()->GetInstanceId()); - labels.emplace_back(METRIC_LABEL_IP, LogFileProfiler::mIpAddr); - labels.emplace_back(METRIC_LABEL_OS, OS_NAME); - labels.emplace_back(METRIC_LABEL_OS_DETAIL, LogFileProfiler::mOsDetail); - labels.emplace_back(METRIC_LABEL_UUID, Application::GetInstance()->GetUUID()); - labels.emplace_back(METRIC_LABEL_VERSION, ILOGTAIL_VERSION); + labels.emplace_back(METRIC_LABEL_KEY_INSTANCE_ID, Application::GetInstance()->GetInstanceId()); + labels.emplace_back(METRIC_LABEL_KEY_IP, LogFileProfiler::mIpAddr); + labels.emplace_back(METRIC_LABEL_KEY_OS, OS_NAME); + labels.emplace_back(METRIC_LABEL_KEY_OS_DETAIL, LogFileProfiler::mOsDetail); + labels.emplace_back(METRIC_LABEL_KEY_UUID, Application::GetInstance()->GetUUID()); + labels.emplace_back(METRIC_LABEL_KEY_VERSION, ILOGTAIL_VERSION); DynamicMetricLabels dynamicLabels; - dynamicLabels.emplace_back(METRIC_LABEL_PROJECT, []() -> std::string { return FlusherSLS::GetAllProjects(); }); + dynamicLabels.emplace_back(METRIC_LABEL_KEY_PROJECT, []() -> std::string { return FlusherSLS::GetAllProjects(); }); #ifdef __ENTERPRISE__ - dynamicLabels.emplace_back(METRIC_LABEL_ALIUIDS, + dynamicLabels.emplace_back(METRIC_LABEL_KEY_ALIUIDS, []() -> std::string { return EnterpriseConfigProvider::GetInstance()->GetAliuidSet(); }); - dynamicLabels.emplace_back(METRIC_LABEL_USER_DEFINED_ID, []() -> std::string { + dynamicLabels.emplace_back(METRIC_LABEL_KEY_USER_DEFINED_ID, []() -> std::string { return EnterpriseConfigProvider::GetInstance()->GetUserDefinedIdSet(); }); #endif diff --git a/core/monitor/metric_constants/AgentMetrics.cpp b/core/monitor/metric_constants/AgentMetrics.cpp index d759489d89..658a1ccc49 100644 --- a/core/monitor/metric_constants/AgentMetrics.cpp +++ b/core/monitor/metric_constants/AgentMetrics.cpp @@ -12,28 +12,31 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include "../MetricConstants.h" +#include "MetricConstants.h" + +using namespace std; namespace logtail { // label keys -const std::string METRIC_LABEL_ALIUIDS = "aliuids"; -const std::string METRIC_LABEL_INSTANCE_ID = "instance_id"; -const std::string METRIC_LABEL_IP = "ip"; -const std::string METRIC_LABEL_OS = "os"; -const std::string METRIC_LABEL_OS_DETAIL = "os_detail"; -const std::string METRIC_LABEL_USER_DEFINED_ID = "user_defined_id"; -const std::string METRIC_LABEL_UUID = "uuid"; -const std::string METRIC_LABEL_VERSION = "version"; +const string METRIC_LABEL_KEY_ALIUIDS = "aliuids"; +const string METRIC_LABEL_KEY_INSTANCE_ID = "instance_id"; +const string METRIC_LABEL_KEY_IP = "ip"; +const string METRIC_LABEL_KEY_OS = "os"; +const string METRIC_LABEL_KEY_OS_DETAIL = "os_detail"; +const string METRIC_LABEL_KEY_PROJECT = "project"; +const string METRIC_LABEL_KEY_USER_DEFINED_ID = "user_defined_id"; +const string METRIC_LABEL_KEY_UUID = "uuid"; +const string METRIC_LABEL_KEY_VERSION = "version"; // metric keys -const std::string METRIC_AGENT_CPU = "agent_cpu_percent"; -const std::string METRIC_AGENT_GO_ROUTINES_TOTAL = "agent_go_routines_total"; -const std::string METRIC_AGENT_INSTANCE_CONFIG_TOTAL = "agent_instance_config_total"; // Not Implemented -const std::string METRIC_AGENT_MEMORY = "agent_memory_used_mb"; -const std::string METRIC_AGENT_MEMORY_GO = "agent_go_memory_used_mb"; -const std::string METRIC_AGENT_OPEN_FD_TOTAL = "agent_open_fd_total"; -const std::string METRIC_AGENT_PIPELINE_CONFIG_TOTAL = "agent_pipeline_config_total"; -const std::string METRIC_AGENT_PLUGIN_TOTAL = "agent_plugin_total"; // Not Implemented +const string METRIC_AGENT_CPU = "agent_cpu_percent"; +const string METRIC_AGENT_GO_ROUTINES_TOTAL = "agent_go_routines_total"; +const string METRIC_AGENT_INSTANCE_CONFIG_TOTAL = "agent_instance_config_total"; // Not Implemented +const string METRIC_AGENT_MEMORY = "agent_memory_used_mb"; +const string METRIC_AGENT_MEMORY_GO = "agent_go_memory_used_mb"; +const string METRIC_AGENT_OPEN_FD_TOTAL = "agent_open_fd_total"; +const string METRIC_AGENT_PIPELINE_CONFIG_TOTAL = "agent_pipeline_config_total"; +const string METRIC_AGENT_PLUGIN_TOTAL = "agent_plugin_total"; // Not Implemented -} \ No newline at end of file +} // namespace logtail diff --git a/core/monitor/metric_constants/CommonMetrics.cpp b/core/monitor/metric_constants/CommonMetrics.cpp deleted file mode 100644 index 5fbe0f600e..0000000000 --- a/core/monitor/metric_constants/CommonMetrics.cpp +++ /dev/null @@ -1,31 +0,0 @@ -// Copyright 2024 iLogtail Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "../MetricConstants.h" - -namespace logtail { - -const std::string METRIC_FIELD_REGION = "region"; -const std::string METRIC_REGION_DEFAULT = "default"; -const std::string METRIC_SLS_LOGSTORE_NAME = "shennong_log_profile"; -const std::string METRIC_TOPIC_TYPE = "loong_collector_metric"; -const std::string METRIC_TOPIC_FIELD_NAME = "__topic__"; - -const std::string LABEL_PREFIX = "label."; -const std::string VALUE_PREFIX = "value."; - -// common label keys -const std::string METRIC_LABEL_PROJECT = "project"; - -} \ No newline at end of file diff --git a/core/monitor/metric_constants/ComponentMetrics.cpp b/core/monitor/metric_constants/ComponentMetrics.cpp index 70168d6881..38012437ed 100644 --- a/core/monitor/metric_constants/ComponentMetrics.cpp +++ b/core/monitor/metric_constants/ComponentMetrics.cpp @@ -12,53 +12,54 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include "../MetricConstants.h" +#include "MetricConstants.h" + +using namespace std; namespace logtail { // label keys -const std::string METRIC_LABEL_KEY_COMPONENT_NAME = "component_name"; -const std::string METRIC_LABEL_KEY_FLUSHER_NODE_ID = "flusher_node_id"; -const std::string METRIC_LABEL_KEY_EXACTLY_ONCE_FLAG = "is_exactly_once"; -const std::string METRIC_LABEL_KEY_QUEUE_TYPE = "queue_type"; +const string METRIC_LABEL_KEY_COMPONENT_NAME = "component_name"; +const string METRIC_LABEL_KEY_FLUSHER_NODE_ID = "flusher_node_id"; +const string METRIC_LABEL_KEY_EXACTLY_ONCE_FLAG = "is_exactly_once"; +const string METRIC_LABEL_KEY_QUEUE_TYPE = "queue_type"; // label values -const std::string METRIC_LABEL_VALUE_COMPONENT_NAME_BATCHER = "batcher"; -const std::string METRIC_LABEL_VALUE_COMPONENT_NAME_COMPRESSOR = "compressor"; -const std::string METRIC_LABEL_VALUE_COMPONENT_NAME_PROCESS_QUEUE = "process_queue"; -const std::string METRIC_LABEL_VALUE_COMPONENT_NAME_ROUTER = "router"; -const std::string METRIC_LABEL_VALUE_COMPONENT_NAME_SENDER_QUEUE = "sender_queue"; -const std::string METRIC_LABEL_VALUE_COMPONENT_NAME_SERIALIZER = "serializer"; +const string METRIC_LABEL_VALUE_COMPONENT_NAME_BATCHER = "batcher"; +const string METRIC_LABEL_VALUE_COMPONENT_NAME_COMPRESSOR = "compressor"; +const string METRIC_LABEL_VALUE_COMPONENT_NAME_PROCESS_QUEUE = "process_queue"; +const string METRIC_LABEL_VALUE_COMPONENT_NAME_ROUTER = "router"; +const string METRIC_LABEL_VALUE_COMPONENT_NAME_SENDER_QUEUE = "sender_queue"; +const string METRIC_LABEL_VALUE_COMPONENT_NAME_SERIALIZER = "serializer"; // metric keys -const std::string METRIC_COMPONENT_IN_EVENTS_TOTAL = "component_in_events_total"; -const std::string METRIC_COMPONENT_IN_EVENT_GROUP_SIZE_BYTES = "component_in_event_group_size_bytes"; -const std::string METRIC_COMPONENT_IN_ITEMS_TOTAL = "component_in_items_total"; -const std::string METRIC_COMPONENT_IN_ITEM_SIZE_BYTES = "component_in_item_size_bytes"; -const std::string METRIC_COMPONENT_OUT_EVENTS_TOTAL = "component_out_events_total"; -const std::string METRIC_COMPONENT_OUT_ITEMS_TOTAL = "component_out_items_total"; -const std::string METRIC_COMPONENT_OUT_ITEM_SIZE_BYTES = "component_out_item_size_bytes"; -const std::string METRIC_COMPONENT_TOTAL_DELAY_MS = "component_total_delay_ms"; -const std::string METRIC_COMPONENT_TOTAL_PROCESS_MS = "component_total_process_ms"; -const std::string METRIC_COMPONENT_DISCARDED_ITEMS_TOTAL = "component_discarded_items_total"; -const std::string METRIC_COMPONENT_DISCARDED_ITEMS_SIZE_BYTES = "component_discarded_item_size_bytes"; +const string METRIC_COMPONENT_IN_EVENTS_TOTAL = "component_in_events_total"; +const string METRIC_COMPONENT_IN_SIZE_BYTES = "component_in_size_bytes"; +const string METRIC_COMPONENT_IN_ITEMS_TOTAL = "component_in_items_total"; +const string METRIC_COMPONENT_OUT_EVENTS_TOTAL = "component_out_events_total"; +const string METRIC_COMPONENT_OUT_ITEMS_TOTAL = "component_out_items_total"; +const string METRIC_COMPONENT_OUT_SIZE_BYTES = "component_out_size_bytes"; +const string METRIC_COMPONENT_TOTAL_DELAY_MS = "component_total_delay_ms"; +const string METRIC_COMPONENT_TOTAL_PROCESS_TIME_MS = "component_total_process_time_ms"; +const string METRIC_COMPONENT_DISCARDED_ITEMS_TOTAL = "component_discarded_items_total"; +const string METRIC_COMPONENT_DISCARDED_ITEMS_SIZE_BYTES = "component_discarded_item_size_bytes"; /********************************************************** -* batcher -**********************************************************/ -const std::string METRIC_COMPONENT_BATCHER_EVENT_BATCHES_TOTAL = "component_event_batches_total"; -const std::string METRIC_COMPONENT_BATCHER_BUFFERED_GROUPS_TOTAL = "component_buffered_groups_total"; -const std::string METRIC_COMPONENT_BATCHER_BUFFERED_EVENTS_TOTAL = "component_buffered_events_total"; -const std::string METRIC_COMPONENT_BATCHER_BUFFERED_SIZE_BYTES = "component_buffered_size_bytes"; + * batcher + **********************************************************/ +const string METRIC_COMPONENT_BATCHER_EVENT_BATCHES_TOTAL = "component_event_batches_total"; +const string METRIC_COMPONENT_BATCHER_BUFFERED_GROUPS_TOTAL = "component_buffered_groups_total"; +const string METRIC_COMPONENT_BATCHER_BUFFERED_EVENTS_TOTAL = "component_buffered_events_total"; +const string METRIC_COMPONENT_BATCHER_BUFFERED_SIZE_BYTES = "component_buffered_size_bytes"; /********************************************************** -* queue -**********************************************************/ -const std::string METRIC_COMPONENT_QUEUE_SIZE_TOTAL = "component_queue_size_total"; -const std::string METRIC_COMPONENT_QUEUE_SIZE_BYTES = "component_queue_size_bytes"; -const std::string METRIC_COMPONENT_QUEUE_VALID_TO_PUSH_FLAG = "component_valid_to_push"; -const std::string METRIC_COMPONENT_QUEUE_EXTRA_BUFFER_SIZE_TOTAL = "component_extra_buffer_size_total"; -const std::string METRIC_COMPONENT_QUEUE_EXTRA_BUFFER_SIZE_BYTES = "component_extra_buffer_size_bytes"; -const std::string METRIC_COMPONENT_QUEUE_DISCARDED_EVENTS_TOTAL = "component_discarded_events_total"; + * queue + **********************************************************/ +const string METRIC_COMPONENT_QUEUE_SIZE_TOTAL = "component_queue_size_total"; +const string METRIC_COMPONENT_QUEUE_SIZE_BYTES = "component_queue_size_bytes"; +const string METRIC_COMPONENT_QUEUE_VALID_TO_PUSH_FLAG = "component_valid_to_push"; +const string METRIC_COMPONENT_QUEUE_EXTRA_BUFFER_SIZE_TOTAL = "component_extra_buffer_size_total"; +const string METRIC_COMPONENT_QUEUE_EXTRA_BUFFER_SIZE_BYTES = "component_extra_buffer_size_bytes"; +const string METRIC_COMPONENT_QUEUE_DISCARDED_EVENTS_TOTAL = "component_discarded_events_total"; -} \ No newline at end of file +} // namespace logtail \ No newline at end of file diff --git a/core/monitor/MetricConstants.h b/core/monitor/metric_constants/MetricConstants.h similarity index 66% rename from core/monitor/MetricConstants.h rename to core/monitor/metric_constants/MetricConstants.h index 06ca0abd0b..3c74f446f4 100644 --- a/core/monitor/MetricConstants.h +++ b/core/monitor/metric_constants/MetricConstants.h @@ -19,35 +19,20 @@ namespace logtail { -////////////////////////////////////////////////////////////////////////// -// common -////////////////////////////////////////////////////////////////////////// - -extern const std::string METRIC_FIELD_REGION; -extern const std::string METRIC_REGION_DEFAULT; -extern const std::string METRIC_SLS_LOGSTORE_NAME; -extern const std::string METRIC_TOPIC_TYPE; -extern const std::string METRIC_TOPIC_FIELD_NAME; - -extern const std::string LABEL_PREFIX; -extern const std::string VALUE_PREFIX; - -// common label keys -extern const std::string METRIC_LABEL_PROJECT; - ////////////////////////////////////////////////////////////////////////// // agent ////////////////////////////////////////////////////////////////////////// // label keys -extern const std::string METRIC_LABEL_ALIUIDS; -extern const std::string METRIC_LABEL_INSTANCE_ID; -extern const std::string METRIC_LABEL_IP; -extern const std::string METRIC_LABEL_OS; -extern const std::string METRIC_LABEL_OS_DETAIL; -extern const std::string METRIC_LABEL_USER_DEFINED_ID; -extern const std::string METRIC_LABEL_UUID; -extern const std::string METRIC_LABEL_VERSION; +extern const std::string METRIC_LABEL_KEY_ALIUIDS; +extern const std::string METRIC_LABEL_KEY_INSTANCE_ID; +extern const std::string METRIC_LABEL_KEY_IP; +extern const std::string METRIC_LABEL_KEY_OS; +extern const std::string METRIC_LABEL_KEY_OS_DETAIL; +extern const std::string METRIC_LABEL_KEY_PROJECT; +extern const std::string METRIC_LABEL_KEY_USER_DEFINED_ID; +extern const std::string METRIC_LABEL_KEY_UUID; +extern const std::string METRIC_LABEL_KEY_VERSION; // metric keys extern const std::string METRIC_AGENT_CPU; @@ -64,15 +49,15 @@ extern const std::string METRIC_AGENT_PLUGIN_TOTAL; ////////////////////////////////////////////////////////////////////////// // label keys -extern const std::string METRIC_LABEL_LOGSTORE; -extern const std::string METRIC_LABEL_PIPELINE_NAME; -extern const std::string METRIC_LABEL_REGION; +extern const std::string METRIC_LABEL_KEY_LOGSTORE; +extern const std::string METRIC_LABEL_KEY_PIPELINE_NAME; +extern const std::string METRIC_LABEL_KEY_REGION; // metric keys extern const std::string METRIC_PIPELINE_PROCESSORS_IN_EVENTS_TOTAL; extern const std::string METRIC_PIPELINE_PROCESSORS_IN_EVENT_GROUPS_TOTAL; -extern const std::string METRIC_PIPELINE_PROCESSORS_IN_EVENT_GROUP_SIZE_BYTES; -extern const std::string METRIC_PIPELINE_PROCESSORS_TOTAL_DELAY_MS; +extern const std::string METRIC_PIPELINE_PROCESSORS_IN_SIZE_BYTES; +extern const std::string METRIC_PIPELINE_PROCESSORS_TOTAL_PROCESS_TIME_MS; extern const std::string METRIC_PIPELINE_START_TIME; ////////////////////////////////////////////////////////////////////////// @@ -80,10 +65,10 @@ extern const std::string METRIC_PIPELINE_START_TIME; ////////////////////////////////////////////////////////////////////////// // label keys -extern const std::string METRIC_LABEL_CHILD_NODE_ID; -extern const std::string METRIC_LABEL_NODE_ID; -extern const std::string METRIC_LABEL_PLUGIN_ID; -extern const std::string METRIC_LABEL_PLUGIN_TYPE; +extern const std::string METRIC_LABEL_KEY_CHILD_NODE_ID; +extern const std::string METRIC_LABEL_KEY_NODE_ID; +extern const std::string METRIC_LABEL_KEY_PLUGIN_ID; +extern const std::string METRIC_LABEL_KEY_PLUGIN_TYPE; // metric keys extern const std::string METRIC_PLUGIN_IN_EVENTS_TOTAL; @@ -92,51 +77,51 @@ extern const std::string METRIC_PLUGIN_IN_SIZE_BYTES; extern const std::string METRIC_PLUGIN_OUT_EVENTS_TOTAL; extern const std::string METRIC_PLUGIN_OUT_EVENT_GROUPS_TOTAL; extern const std::string METRIC_PLUGIN_OUT_SIZE_BYTES; -extern const std::string METRIC_PLUGIN_TOTAL_DELAY_TIME_MS; +extern const std::string METRIC_PLUGIN_TOTAL_DELAY_MS; extern const std::string METRIC_PLUGIN_TOTAL_PROCESS_TIME_MS; /********************************************************** -* input_file -* input_container_stdio -**********************************************************/ -extern const std::string METRIC_LABEL_FILE_DEV; -extern const std::string METRIC_LABEL_FILE_INODE; -extern const std::string METRIC_LABEL_FILE_NAME; + * input_file + * input_container_stdio + **********************************************************/ +extern const std::string METRIC_LABEL_KEY_FILE_DEV; +extern const std::string METRIC_LABEL_KEY_FILE_INODE; +extern const std::string METRIC_LABEL_KEY_FILE_NAME; extern const std::string METRIC_PLUGIN_MONITOR_FILE_TOTAL; extern const std::string METRIC_PLUGIN_SOURCE_READ_OFFSET_BYTES; extern const std::string METRIC_PLUGIN_SOURCE_SIZE_BYTES; /********************************************************** -* all processor （所有解析类的处理插件通用指标。Todo：目前统计还不全、不准确） -**********************************************************/ + * all processor （所有解析类的处理插件通用指标。Todo：目前统计还不全、不准确） + **********************************************************/ extern const std::string METRIC_PLUGIN_DISCARDED_EVENTS_TOTAL; extern const std::string METRIC_PLUGIN_OUT_FAILED_EVENTS_TOTAL; extern const std::string METRIC_PLUGIN_OUT_KEY_NOT_FOUND_EVENTS_TOTAL; extern const std::string METRIC_PLUGIN_OUT_SUCCESSFUL_EVENTS_TOTAL; /********************************************************** -* processor_parse_apsara_native -* processor_parse_timestamp_native -**********************************************************/ + * processor_parse_apsara_native + * processor_parse_timestamp_native + **********************************************************/ extern const std::string METRIC_PLUGIN_HISTORY_FAILURE_TOTAL; /********************************************************** -* processor_split_multiline_log_string_native -**********************************************************/ -extern const std::string METRIC_PLUGIN_SPLIT_MULTILINE_LOG_MATCHED_RECORDS_TOTAL; -extern const std::string METRIC_PLUGIN_SPLIT_MULTILINE_LOG_MATCHED_LINES_TOTAL; -extern const std::string METRIC_PLUGIN_SPLIT_MULTILINE_LOG_UNMATCHED_LINES_TOTAL; + * processor_split_multiline_log_string_native + **********************************************************/ +extern const std::string METRIC_PLUGIN_MATCHED_EVENTS_TOTAL; +extern const std::string METRIC_PLUGIN_MATCHED_LINES_TOTAL; +extern const std::string METRIC_PLUGIN_UNMATCHED_LINES_TOTAL; /********************************************************** -* processor_merge_multiline_log_native -**********************************************************/ -extern const std::string METRIC_PLUGIN_MERGE_MULTILINE_LOG_MERGED_RECORDS_TOTAL; -extern const std::string METRIC_PLUGIN_MERGE_MULTILINE_LOG_UNMATCHED_RECORDS_TOTAL; + * processor_merge_multiline_log_native + **********************************************************/ +extern const std::string METRIC_PLUGIN_MERGED_EVENTS_TOTAL; +extern const std::string METRIC_PLUGIN_UNMATCHED_EVENTS_TOTAL; /********************************************************** -* processor_parse_container_log_native -**********************************************************/ + * processor_parse_container_log_native + **********************************************************/ extern const std::string METRIC_PLUGIN_PARSE_STDERR_TOTAL; extern const std::string METRIC_PLUGIN_PARSE_STDOUT_TOTAL; @@ -160,28 +145,27 @@ extern const std::string METRIC_LABEL_VALUE_COMPONENT_NAME_SERIALIZER; // metric keys extern const std::string METRIC_COMPONENT_IN_EVENTS_TOTAL; -extern const std::string METRIC_COMPONENT_IN_EVENT_GROUP_SIZE_BYTES; +extern const std::string METRIC_COMPONENT_IN_SIZE_BYTES; extern const std::string METRIC_COMPONENT_IN_ITEMS_TOTAL; -extern const std::string METRIC_COMPONENT_IN_ITEM_SIZE_BYTES; extern const std::string METRIC_COMPONENT_OUT_EVENTS_TOTAL; extern const std::string METRIC_COMPONENT_OUT_ITEMS_TOTAL; -extern const std::string METRIC_COMPONENT_OUT_ITEM_SIZE_BYTES; +extern const std::string METRIC_COMPONENT_OUT_SIZE_BYTES; extern const std::string METRIC_COMPONENT_TOTAL_DELAY_MS; -extern const std::string METRIC_COMPONENT_TOTAL_PROCESS_MS; +extern const std::string METRIC_COMPONENT_TOTAL_PROCESS_TIME_MS; extern const std::string METRIC_COMPONENT_DISCARDED_ITEMS_TOTAL; extern const std::string METRIC_COMPONENT_DISCARDED_ITEMS_SIZE_BYTES; /********************************************************** -* batcher -**********************************************************/ + * batcher + **********************************************************/ extern const std::string METRIC_COMPONENT_BATCHER_EVENT_BATCHES_TOTAL; extern const std::string METRIC_COMPONENT_BATCHER_BUFFERED_GROUPS_TOTAL; extern const std::string METRIC_COMPONENT_BATCHER_BUFFERED_EVENTS_TOTAL; extern const std::string METRIC_COMPONENT_BATCHER_BUFFERED_SIZE_BYTES; /********************************************************** -* queue -**********************************************************/ + * queue + **********************************************************/ extern const std::string METRIC_COMPONENT_QUEUE_SIZE_TOTAL; extern const std::string METRIC_COMPONENT_QUEUE_SIZE_BYTES; extern const std::string METRIC_COMPONENT_QUEUE_VALID_TO_PUSH_FLAG; @@ -205,30 +189,29 @@ extern const std::string METRIC_LABEL_VALUE_RUNNER_NAME_PROCESSOR; // metric keys extern const std::string METRIC_RUNNER_IN_EVENTS_TOTAL; extern const std::string METRIC_RUNNER_IN_EVENT_GROUPS_TOTAL; -extern const std::string METRIC_RUNNER_IN_EVENT_GROUP_SIZE_BYTES; +extern const std::string METRIC_RUNNER_IN_SIZE_BYTES; extern const std::string METRIC_RUNNER_IN_ITEMS_TOTAL; -extern const std::string METRIC_RUNNER_IN_ITEM_SIZE_BYTES; extern const std::string METRIC_RUNNER_LAST_RUN_TIME; extern const std::string METRIC_RUNNER_OUT_ITEMS_TOTAL; extern const std::string METRIC_RUNNER_TOTAL_DELAY_MS; /********************************************************** -* http sink -**********************************************************/ + * http sink + **********************************************************/ extern const std::string METRIC_RUNNER_HTTP_SINK_OUT_SUCCESSFUL_ITEMS_TOTAL; extern const std::string METRIC_RUNNER_HTTP_SINK_OUT_FAILED_ITEMS_TOTAL; extern const std::string METRIC_RUNNER_HTTP_SINK_SENDING_ITEMS_TOTAL; extern const std::string METRIC_RUNNER_HTTP_SINK_SEND_CONCURRENCY; /********************************************************** -* flusher runner -**********************************************************/ -extern const std::string METRIC_RUNNER_FLUSHER_IN_ITEM_RAW_SIZE_BYTES; + * flusher runner + **********************************************************/ +extern const std::string METRIC_RUNNER_FLUSHER_IN_SIZE_BYTES; extern const std::string METRIC_RUNNER_FLUSHER_WAITING_ITEMS_TOTAL; /********************************************************** -* file server -**********************************************************/ + * file server + **********************************************************/ extern const std::string METRIC_RUNNER_FILE_WATCHED_DIRS_TOTAL; extern const std::string METRIC_RUNNER_FILE_ACTIVE_READERS_TOTAL; extern const std::string METRIC_RUNNER_FILE_ENABLE_FILE_INCLUDED_BY_MULTI_CONFIGS_FLAG; diff --git a/core/monitor/metric_constants/PipelineMetrics.cpp b/core/monitor/metric_constants/PipelineMetrics.cpp index 0b0c1f308a..44017501d6 100644 --- a/core/monitor/metric_constants/PipelineMetrics.cpp +++ b/core/monitor/metric_constants/PipelineMetrics.cpp @@ -12,21 +12,22 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include "../MetricConstants.h" +#include "MetricConstants.h" + +using namespace std; namespace logtail { // label keys -const std::string METRIC_LABEL_LOGSTORE = "logstore"; -const std::string METRIC_LABEL_PIPELINE_NAME = "pipeline_name"; -const std::string METRIC_LABEL_REGION = "region"; +const string METRIC_LABEL_KEY_LOGSTORE = "logstore"; +const string METRIC_LABEL_KEY_PIPELINE_NAME = "pipeline_name"; +const string METRIC_LABEL_KEY_REGION = "region"; // metric keys -const std::string METRIC_PIPELINE_PROCESSORS_IN_EVENTS_TOTAL = "pipeline_processors_in_events_total"; -const std::string METRIC_PIPELINE_PROCESSORS_IN_EVENT_GROUPS_TOTAL = "pipeline_processors_in_event_groups_total"; -const std::string METRIC_PIPELINE_PROCESSORS_IN_EVENT_GROUP_SIZE_BYTES - = "pipeline_processors_in_event_group_size_bytes"; -const std::string METRIC_PIPELINE_PROCESSORS_TOTAL_DELAY_MS = "pipeline_processors_total_delay_ms"; -const std::string METRIC_PIPELINE_START_TIME = "pipeline_start_time"; +const string METRIC_PIPELINE_PROCESSORS_IN_EVENTS_TOTAL = "pipeline_processors_in_events_total"; +const string METRIC_PIPELINE_PROCESSORS_IN_EVENT_GROUPS_TOTAL = "pipeline_processors_in_event_groups_total"; +const string METRIC_PIPELINE_PROCESSORS_IN_SIZE_BYTES = "pipeline_processors_in_size_bytes"; +const string METRIC_PIPELINE_PROCESSORS_TOTAL_PROCESS_TIME_MS = "pipeline_processors_total_process_time_ms"; +const string METRIC_PIPELINE_START_TIME = "pipeline_start_time"; -} \ No newline at end of file +} // namespace logtail diff --git a/core/monitor/metric_constants/PluginMetrics.cpp b/core/monitor/metric_constants/PluginMetrics.cpp index 0fbfe61eae..a992b8b7b3 100644 --- a/core/monitor/metric_constants/PluginMetrics.cpp +++ b/core/monitor/metric_constants/PluginMetrics.cpp @@ -12,74 +12,71 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include "../MetricConstants.h" +#include "MetricConstants.h" + +using namespace std; namespace logtail { // label keys -const std::string METRIC_LABEL_CHILD_NODE_ID = "child_node_id"; -const std::string METRIC_LABEL_NODE_ID = "node_id"; -const std::string METRIC_LABEL_PLUGIN_ID = "plugin_id"; -const std::string METRIC_LABEL_PLUGIN_TYPE = "plugin_type"; +const string METRIC_LABEL_KEY_CHILD_NODE_ID = "child_node_id"; +const string METRIC_LABEL_KEY_NODE_ID = "node_id"; +const string METRIC_LABEL_KEY_PLUGIN_ID = "plugin_id"; +const string METRIC_LABEL_KEY_PLUGIN_TYPE = "plugin_type"; // metric keys -const std::string METRIC_PLUGIN_IN_EVENTS_TOTAL = "plugin_in_events_total"; -const std::string METRIC_PLUGIN_IN_EVENT_GROUPS_TOTAL = "plugin_in_event_groups_total"; -const std::string METRIC_PLUGIN_IN_SIZE_BYTES = "plugin_in_size_bytes"; -const std::string METRIC_PLUGIN_OUT_EVENTS_TOTAL = "plugin_out_events_total"; -const std::string METRIC_PLUGIN_OUT_EVENT_GROUPS_TOTAL = "plugin_out_event_groups_total"; -const std::string METRIC_PLUGIN_OUT_SIZE_BYTES = "plugin_out_size_bytes"; -const std::string METRIC_PLUGIN_TOTAL_DELAY_TIME_MS = "plugin_total_delay_time_ms"; -const std::string METRIC_PLUGIN_TOTAL_PROCESS_TIME_MS = "plugin_total_process_time_ms"; +const string METRIC_PLUGIN_IN_EVENTS_TOTAL = "plugin_in_events_total"; +const string METRIC_PLUGIN_IN_EVENT_GROUPS_TOTAL = "plugin_in_event_groups_total"; +const string METRIC_PLUGIN_IN_SIZE_BYTES = "plugin_in_size_bytes"; +const string METRIC_PLUGIN_OUT_EVENTS_TOTAL = "plugin_out_events_total"; +const string METRIC_PLUGIN_OUT_EVENT_GROUPS_TOTAL = "plugin_out_event_groups_total"; +const string METRIC_PLUGIN_OUT_SIZE_BYTES = "plugin_out_size_bytes"; +const string METRIC_PLUGIN_TOTAL_DELAY_MS = "plugin_total_delay_ms"; +const string METRIC_PLUGIN_TOTAL_PROCESS_TIME_MS = "plugin_total_process_time_ms"; /********************************************************** -* input_file -* input_container_stdio -**********************************************************/ -const std::string METRIC_LABEL_FILE_DEV = "file_dev"; -const std::string METRIC_LABEL_FILE_INODE = "file_inode"; -const std::string METRIC_LABEL_FILE_NAME = "file_name"; + * input_file + * input_container_stdio + **********************************************************/ +const string METRIC_LABEL_KEY_FILE_DEV = "file_dev"; +const string METRIC_LABEL_KEY_FILE_INODE = "file_inode"; +const string METRIC_LABEL_KEY_FILE_NAME = "file_name"; -const std::string METRIC_PLUGIN_MONITOR_FILE_TOTAL = "plugin_monitor_file_total"; -const std::string METRIC_PLUGIN_SOURCE_READ_OFFSET_BYTES = "plugin_source_read_offset_bytes"; -const std::string METRIC_PLUGIN_SOURCE_SIZE_BYTES = "plugin_source_size_bytes"; +const string METRIC_PLUGIN_MONITOR_FILE_TOTAL = "plugin_monitor_file_total"; +const string METRIC_PLUGIN_SOURCE_READ_OFFSET_BYTES = "plugin_source_read_offset_bytes"; +const string METRIC_PLUGIN_SOURCE_SIZE_BYTES = "plugin_source_size_bytes"; /********************************************************** -* all processor （所有解析类的处理插件通用指标。Todo：目前统计还不全、不准确） -**********************************************************/ -const std::string METRIC_PLUGIN_DISCARDED_EVENTS_TOTAL = "plugin_discarded_events_total"; -const std::string METRIC_PLUGIN_OUT_FAILED_EVENTS_TOTAL = "plugin_out_failed_events_total"; -const std::string METRIC_PLUGIN_OUT_KEY_NOT_FOUND_EVENTS_TOTAL = "plugin_out_key_not_found_events_total"; -const std::string METRIC_PLUGIN_OUT_SUCCESSFUL_EVENTS_TOTAL = "plugin_out_successful_events_total"; + * all processor （所有解析类的处理插件通用指标。Todo：目前统计还不全、不准确） + **********************************************************/ +const string METRIC_PLUGIN_DISCARDED_EVENTS_TOTAL = "plugin_discarded_events_total"; +const string METRIC_PLUGIN_OUT_FAILED_EVENTS_TOTAL = "plugin_out_failed_events_total"; +const string METRIC_PLUGIN_OUT_KEY_NOT_FOUND_EVENTS_TOTAL = "plugin_out_key_not_found_events_total"; +const string METRIC_PLUGIN_OUT_SUCCESSFUL_EVENTS_TOTAL = "plugin_out_successful_events_total"; /********************************************************** -* processor_parse_apsara_native -* processor_parse_timestamp_native -**********************************************************/ -const std::string METRIC_PLUGIN_HISTORY_FAILURE_TOTAL = "plugin_history_failure_total"; + * processor_parse_apsara_native + * processor_parse_timestamp_native + **********************************************************/ +const string METRIC_PLUGIN_HISTORY_FAILURE_TOTAL = "plugin_history_failure_total"; /********************************************************** -* processor_split_multiline_log_string_native -**********************************************************/ -const std::string METRIC_PLUGIN_SPLIT_MULTILINE_LOG_MATCHED_RECORDS_TOTAL - = "plugin_split_multiline_log_matched_records_total"; -const std::string METRIC_PLUGIN_SPLIT_MULTILINE_LOG_MATCHED_LINES_TOTAL - = "plugin_split_multiline_log_matched_lines_total"; -const std::string METRIC_PLUGIN_SPLIT_MULTILINE_LOG_UNMATCHED_LINES_TOTAL - = "plugin_split_multiline_log_unmatched_lines_total"; + * processor_split_multiline_log_string_native + **********************************************************/ +const string METRIC_PLUGIN_MATCHED_EVENTS_TOTAL = "plugin_matched_events_total"; +const string METRIC_PLUGIN_MATCHED_LINES_TOTAL = "plugin_matched_lines_total"; +const string METRIC_PLUGIN_UNMATCHED_LINES_TOTAL = "plugin_unmatched_lines_total"; /********************************************************** -* processor_merge_multiline_log_native -**********************************************************/ -const std::string METRIC_PLUGIN_MERGE_MULTILINE_LOG_MERGED_RECORDS_TOTAL - = "plugin_merge_multiline_log_merged_records_total"; -const std::string METRIC_PLUGIN_MERGE_MULTILINE_LOG_UNMATCHED_RECORDS_TOTAL - = "plugin_merge_multiline_log_unmatched_records_total"; + * processor_merge_multiline_log_native + **********************************************************/ +const string METRIC_PLUGIN_MERGED_EVENTS_TOTAL = "plugin_merged_events_total"; +const string METRIC_PLUGIN_UNMATCHED_EVENTS_TOTAL = "plugin_unmatched_events_total"; /********************************************************** -* processor_parse_container_log_native -**********************************************************/ -const std::string METRIC_PLUGIN_PARSE_STDERR_TOTAL = "plugin_parse_stderr_total"; -const std::string METRIC_PLUGIN_PARSE_STDOUT_TOTAL = "plugin_parse_stdout_total"; + * processor_parse_container_log_native + **********************************************************/ +const string METRIC_PLUGIN_PARSE_STDERR_TOTAL = "plugin_parse_stderr_total"; +const string METRIC_PLUGIN_PARSE_STDOUT_TOTAL = "plugin_parse_stdout_total"; -} \ No newline at end of file +} // namespace logtail \ No newline at end of file diff --git a/core/monitor/metric_constants/RunnerMetrics.cpp b/core/monitor/metric_constants/RunnerMetrics.cpp index d3bbb8ac03..93803b2638 100644 --- a/core/monitor/metric_constants/RunnerMetrics.cpp +++ b/core/monitor/metric_constants/RunnerMetrics.cpp @@ -12,52 +12,53 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include "../MetricConstants.h" +#include "MetricConstants.h" + +using namespace std; namespace logtail { // label keys -const std::string METRIC_LABEL_KEY_RUNNER_NAME = "runner_name"; +const string METRIC_LABEL_KEY_RUNNER_NAME = "runner_name"; // label values -const std::string METRIC_LABEL_VALUE_RUNNER_NAME_FILE_SERVER = "file_server"; -const std::string METRIC_LABEL_VALUE_RUNNER_NAME_FLUSHER = "flusher_runner"; -const std::string METRIC_LABEL_VALUE_RUNNER_NAME_HTTP_SINK = "http_sink"; -const std::string METRIC_LABEL_VALUE_RUNNER_NAME_PROCESSOR = "processor_runner"; +const string METRIC_LABEL_VALUE_RUNNER_NAME_FILE_SERVER = "file_server"; +const string METRIC_LABEL_VALUE_RUNNER_NAME_FLUSHER = "flusher_runner"; +const string METRIC_LABEL_VALUE_RUNNER_NAME_HTTP_SINK = "http_sink"; +const string METRIC_LABEL_VALUE_RUNNER_NAME_PROCESSOR = "processor_runner"; // metric keys -const std::string METRIC_RUNNER_IN_EVENTS_TOTAL = "runner_in_events_total"; -const std::string METRIC_RUNNER_IN_EVENT_GROUPS_TOTAL = "runner_in_event_groups_total"; -const std::string METRIC_RUNNER_IN_EVENT_GROUP_SIZE_BYTES = "runner_in_event_group_size_bytes"; -const std::string METRIC_RUNNER_IN_ITEMS_TOTAL = "runner_in_items_total"; -const std::string METRIC_RUNNER_IN_ITEM_SIZE_BYTES = "runner_in_item_size_bytes"; -const std::string METRIC_RUNNER_LAST_RUN_TIME = "runner_last_run_time"; -const std::string METRIC_RUNNER_OUT_ITEMS_TOTAL = "runner_out_items_total"; -const std::string METRIC_RUNNER_TOTAL_DELAY_MS = "runner_total_delay_ms"; +const string METRIC_RUNNER_IN_EVENTS_TOTAL = "runner_in_events_total"; +const string METRIC_RUNNER_IN_EVENT_GROUPS_TOTAL = "runner_in_event_groups_total"; +const string METRIC_RUNNER_IN_SIZE_BYTES = "runner_in_size_bytes"; +const string METRIC_RUNNER_IN_ITEMS_TOTAL = "runner_in_items_total"; +const string METRIC_RUNNER_LAST_RUN_TIME = "runner_last_run_time"; +const string METRIC_RUNNER_OUT_ITEMS_TOTAL = "runner_out_items_total"; +const string METRIC_RUNNER_TOTAL_DELAY_MS = "runner_total_delay_ms"; /********************************************************** -* http sink -**********************************************************/ -const std::string METRIC_RUNNER_HTTP_SINK_OUT_SUCCESSFUL_ITEMS_TOTAL = "runner_out_successful_items_total"; -const std::string METRIC_RUNNER_HTTP_SINK_OUT_FAILED_ITEMS_TOTAL = "runner_out_failed_items_total"; -const std::string METRIC_RUNNER_HTTP_SINK_SENDING_ITEMS_TOTAL = "runner_sending_items_total"; -const std::string METRIC_RUNNER_HTTP_SINK_SEND_CONCURRENCY = "runner_send_concurrency"; + * http sink + **********************************************************/ +const string METRIC_RUNNER_HTTP_SINK_OUT_SUCCESSFUL_ITEMS_TOTAL = "runner_out_successful_items_total"; +const string METRIC_RUNNER_HTTP_SINK_OUT_FAILED_ITEMS_TOTAL = "runner_out_failed_items_total"; +const string METRIC_RUNNER_HTTP_SINK_SENDING_ITEMS_TOTAL = "runner_sending_items_total"; +const string METRIC_RUNNER_HTTP_SINK_SEND_CONCURRENCY = "runner_send_concurrency"; /********************************************************** -* flusher runner -**********************************************************/ -const std::string METRIC_RUNNER_FLUSHER_IN_ITEM_RAW_SIZE_BYTES = "runner_in_item_raw_size_bytes"; -const std::string METRIC_RUNNER_FLUSHER_WAITING_ITEMS_TOTAL = "runner_waiting_items_total"; + * flusher runner + **********************************************************/ +const string METRIC_RUNNER_FLUSHER_IN_SIZE_BYTES = "runner_in_size_bytes"; +const string METRIC_RUNNER_FLUSHER_WAITING_ITEMS_TOTAL = "runner_waiting_items_total"; /********************************************************** -* file server -**********************************************************/ -const std::string METRIC_RUNNER_FILE_WATCHED_DIRS_TOTAL = "runner_watched_dirs_total"; -const std::string METRIC_RUNNER_FILE_ACTIVE_READERS_TOTAL = "runner_active_readers_total"; -const std::string METRIC_RUNNER_FILE_ENABLE_FILE_INCLUDED_BY_MULTI_CONFIGS_FLAG + * file server + **********************************************************/ +const string METRIC_RUNNER_FILE_WATCHED_DIRS_TOTAL = "runner_watched_dirs_total"; +const string METRIC_RUNNER_FILE_ACTIVE_READERS_TOTAL = "runner_active_readers_total"; +const string METRIC_RUNNER_FILE_ENABLE_FILE_INCLUDED_BY_MULTI_CONFIGS_FLAG = "runner_enable_file_included_by_multi_configs"; -const std::string METRIC_RUNNER_FILE_POLLING_MODIFY_CACHE_SIZE = "runner_polling_modify_cache_size"; -const std::string METRIC_RUNNER_FILE_POLLING_DIR_CACHE_SIZE = "runner_polling_dir_cache_size"; -const std::string METRIC_RUNNER_FILE_POLLING_FILE_CACHE_SIZE = "runner_polling_file_cache_size"; +const string METRIC_RUNNER_FILE_POLLING_MODIFY_CACHE_SIZE = "runner_polling_modify_cache_size"; +const string METRIC_RUNNER_FILE_POLLING_DIR_CACHE_SIZE = "runner_polling_dir_cache_size"; +const string METRIC_RUNNER_FILE_POLLING_FILE_CACHE_SIZE = "runner_polling_file_cache_size"; -} \ No newline at end of file +} // namespace logtail \ No newline at end of file diff --git a/core/pipeline/Pipeline.cpp b/core/pipeline/Pipeline.cpp index e4574f3653..405a78591a 100644 --- a/core/pipeline/Pipeline.cpp +++ b/core/pipeline/Pipeline.cpp @@ -316,13 +316,13 @@ bool Pipeline::Init(PipelineConfig&& config) { } WriteMetrics::GetInstance()->PrepareMetricsRecordRef( - mMetricsRecordRef, {{METRIC_LABEL_PROJECT, mContext.GetProjectName()}, {METRIC_LABEL_PIPELINE_NAME, mName}}); + mMetricsRecordRef, + {{METRIC_LABEL_KEY_PROJECT, mContext.GetProjectName()}, {METRIC_LABEL_KEY_PIPELINE_NAME, mName}}); mStartTime = mMetricsRecordRef.CreateIntGauge(METRIC_PIPELINE_START_TIME); mProcessorsInEventsTotal = mMetricsRecordRef.CreateCounter(METRIC_PIPELINE_PROCESSORS_IN_EVENTS_TOTAL); mProcessorsInGroupsTotal = mMetricsRecordRef.CreateCounter(METRIC_PIPELINE_PROCESSORS_IN_EVENT_GROUPS_TOTAL); - mProcessorsInGroupDataSizeBytes - = mMetricsRecordRef.CreateCounter(METRIC_PIPELINE_PROCESSORS_IN_EVENT_GROUP_SIZE_BYTES); - mProcessorsTotalDelayMs = mMetricsRecordRef.CreateCounter(METRIC_PIPELINE_PROCESSORS_TOTAL_DELAY_MS); + mProcessorsInSizeBytes = mMetricsRecordRef.CreateCounter(METRIC_PIPELINE_PROCESSORS_IN_SIZE_BYTES); + mProcessorsTotalProcessTimeMs = mMetricsRecordRef.CreateCounter(METRIC_PIPELINE_PROCESSORS_TOTAL_PROCESS_TIME_MS); return true; } @@ -356,7 +356,7 @@ void Pipeline::Start() { void Pipeline::Process(vector& logGroupList, size_t inputIndex) { for (const auto& logGroup : logGroupList) { mProcessorsInEventsTotal->Add(logGroup.GetEvents().size()); - mProcessorsInGroupDataSizeBytes->Add(logGroup.DataSize()); + mProcessorsInSizeBytes->Add(logGroup.DataSize()); } mProcessorsInGroupsTotal->Add(logGroupList.size()); @@ -367,7 +367,7 @@ void Pipeline::Process(vector& logGroupList, size_t inputInd for (auto& p : mProcessorLine) { p->Process(logGroupList); } - mProcessorsTotalDelayMs->Add( + mProcessorsTotalProcessTimeMs->Add( chrono::duration_cast(chrono::system_clock::now() - before).count()); } diff --git a/core/pipeline/Pipeline.h b/core/pipeline/Pipeline.h index ccbfda2ac9..c0ddfa262e 100644 --- a/core/pipeline/Pipeline.h +++ b/core/pipeline/Pipeline.h @@ -89,8 +89,8 @@ class Pipeline { IntGaugePtr mStartTime; CounterPtr mProcessorsInEventsTotal; CounterPtr mProcessorsInGroupsTotal; - CounterPtr mProcessorsInGroupDataSizeBytes; - CounterPtr mProcessorsTotalDelayMs; + CounterPtr mProcessorsInSizeBytes; + CounterPtr mProcessorsTotalProcessTimeMs; #ifdef APSARA_UNIT_TEST_MAIN friend class PipelineMock; diff --git a/core/pipeline/batch/Batcher.h b/core/pipeline/batch/Batcher.h index 69eedd1745..a72a2bd0af 100644 --- a/core/pipeline/batch/Batcher.h +++ b/core/pipeline/batch/Batcher.h @@ -28,7 +28,7 @@ #include "common/ParamExtractor.h" #include "models/PipelineEventGroup.h" #include "monitor/LogtailMetric.h" -#include "monitor/MetricConstants.h" +#include "monitor/metric_constants/MetricConstants.h" #include "pipeline/PipelineContext.h" #include "pipeline/batch/BatchItem.h" #include "pipeline/batch/BatchStatus.h" @@ -100,8 +100,8 @@ class Batcher { mFlusher = flusher; std::vector> labels{ - {METRIC_LABEL_PROJECT, ctx.GetProjectName()}, - {METRIC_LABEL_PIPELINE_NAME, ctx.GetConfigName()}, + {METRIC_LABEL_KEY_PROJECT, ctx.GetProjectName()}, + {METRIC_LABEL_KEY_PIPELINE_NAME, ctx.GetConfigName()}, {METRIC_LABEL_KEY_COMPONENT_NAME, METRIC_LABEL_VALUE_COMPONENT_NAME_BATCHER}, {METRIC_LABEL_KEY_FLUSHER_NODE_ID, flusher->GetNodeID()}}; if (enableGroupBatch) { @@ -111,7 +111,7 @@ class Batcher { } WriteMetrics::GetInstance()->PrepareMetricsRecordRef(mMetricsRecordRef, std::move(labels)); mInEventsTotal = mMetricsRecordRef.CreateCounter(METRIC_COMPONENT_IN_EVENTS_TOTAL); - mInGroupDataSizeBytes = mMetricsRecordRef.CreateCounter(METRIC_COMPONENT_IN_EVENT_GROUP_SIZE_BYTES); + mInGroupDataSizeBytes = mMetricsRecordRef.CreateCounter(METRIC_COMPONENT_IN_SIZE_BYTES); mOutEventsTotal = mMetricsRecordRef.CreateCounter(METRIC_COMPONENT_OUT_EVENTS_TOTAL); mTotalDelayMs = mMetricsRecordRef.CreateCounter(METRIC_COMPONENT_TOTAL_DELAY_MS); mEventBatchItemsTotal = mMetricsRecordRef.CreateIntGauge(METRIC_COMPONENT_BATCHER_EVENT_BATCHES_TOTAL); diff --git a/core/pipeline/plugin/instance/FlusherInstance.cpp b/core/pipeline/plugin/instance/FlusherInstance.cpp index c14e71c47d..45bf74a40e 100644 --- a/core/pipeline/plugin/instance/FlusherInstance.cpp +++ b/core/pipeline/plugin/instance/FlusherInstance.cpp @@ -14,7 +14,7 @@ #include "pipeline/plugin/instance/FlusherInstance.h" -#include "monitor/MetricConstants.h" +#include "monitor/metric_constants/MetricConstants.h" namespace logtail { bool FlusherInstance::Init(const Json::Value& config, PipelineContext& context, Json::Value& optionalGoPipeline) { @@ -26,14 +26,12 @@ bool FlusherInstance::Init(const Json::Value& config, PipelineContext& context, } mInEventsTotal = mPlugin->GetMetricsRecordRef().CreateCounter(METRIC_PLUGIN_IN_EVENTS_TOTAL); - mInEventGroupsTotal = mPlugin->GetMetricsRecordRef().CreateCounter(METRIC_PLUGIN_IN_EVENT_GROUPS_TOTAL); mInSizeBytes = mPlugin->GetMetricsRecordRef().CreateCounter(METRIC_PLUGIN_IN_SIZE_BYTES); return true; } bool FlusherInstance::Send(PipelineEventGroup&& g) { mInEventsTotal->Add(g.GetEvents().size()); - mInEventGroupsTotal->Add(1); mInSizeBytes->Add(g.DataSize()); return mPlugin->Send(std::move(g)); } diff --git a/core/pipeline/plugin/instance/FlusherInstance.h b/core/pipeline/plugin/instance/FlusherInstance.h index 6d15004055..6ce367051d 100644 --- a/core/pipeline/plugin/instance/FlusherInstance.h +++ b/core/pipeline/plugin/instance/FlusherInstance.h @@ -47,7 +47,6 @@ class FlusherInstance : public PluginInstance { std::unique_ptr mPlugin; CounterPtr mInEventsTotal; - CounterPtr mInEventGroupsTotal; CounterPtr mInSizeBytes; }; diff --git a/core/pipeline/plugin/instance/ProcessorInstance.cpp b/core/pipeline/plugin/instance/ProcessorInstance.cpp index 997c0fe205..96a9ac9470 100644 --- a/core/pipeline/plugin/instance/ProcessorInstance.cpp +++ b/core/pipeline/plugin/instance/ProcessorInstance.cpp @@ -20,7 +20,7 @@ #include "common/TimeUtil.h" #include "logger/Logger.h" -#include "monitor/MetricConstants.h" +#include "monitor/metric_constants/MetricConstants.h" using namespace std; @@ -36,8 +36,6 @@ bool ProcessorInstance::Init(const Json::Value& config, PipelineContext& context // should init plugin first， then could GetMetricsRecordRef from plugin mInEventsTotal = mPlugin->GetMetricsRecordRef().CreateCounter(METRIC_PLUGIN_IN_EVENTS_TOTAL); mOutEventsTotal = mPlugin->GetMetricsRecordRef().CreateCounter(METRIC_PLUGIN_OUT_EVENTS_TOTAL); - mInEventGroupsTotal = mPlugin->GetMetricsRecordRef().CreateCounter(METRIC_PLUGIN_IN_EVENT_GROUPS_TOTAL); - mOutEventGroupsTotal = mPlugin->GetMetricsRecordRef().CreateCounter(METRIC_PLUGIN_OUT_EVENT_GROUPS_TOTAL); mInSizeBytes = mPlugin->GetMetricsRecordRef().CreateCounter(METRIC_PLUGIN_IN_SIZE_BYTES); mOutSizeBytes = mPlugin->GetMetricsRecordRef().CreateCounter(METRIC_PLUGIN_OUT_SIZE_BYTES); mTotalProcessTimeMs = mPlugin->GetMetricsRecordRef().CreateCounter(METRIC_PLUGIN_TOTAL_PROCESS_TIME_MS); @@ -45,24 +43,22 @@ bool ProcessorInstance::Init(const Json::Value& config, PipelineContext& context return true; } -void ProcessorInstance::Process(vector& logGroupList) { - if (logGroupList.empty()) { +void ProcessorInstance::Process(vector& eventGroupList) { + if (eventGroupList.empty()) { return; } - for (const auto& logGroup : logGroupList) { - mInEventsTotal->Add(logGroup.GetEvents().size()); - mInEventGroupsTotal->Add(1); - mInSizeBytes->Add(logGroup.DataSize()); + for (const auto& eventGroup : eventGroupList) { + mInEventsTotal->Add(eventGroup.GetEvents().size()); + mInSizeBytes->Add(eventGroup.DataSize()); } auto before = chrono::system_clock::now(); - mPlugin->Process(logGroupList); + mPlugin->Process(eventGroupList); mTotalProcessTimeMs->Add(chrono::duration_cast(chrono::system_clock::now() - before).count()); - for (const auto& logGroup : logGroupList) { - mOutEventsTotal->Add(logGroup.GetEvents().size()); - mOutEventGroupsTotal->Add(1); - mOutSizeBytes->Add(logGroup.DataSize()); + for (const auto& eventGroup : eventGroupList) { + mOutEventsTotal->Add(eventGroup.GetEvents().size()); + mOutSizeBytes->Add(eventGroup.DataSize()); } } diff --git a/core/pipeline/plugin/instance/ProcessorInstance.h b/core/pipeline/plugin/instance/ProcessorInstance.h index 818e78ab8d..eb7f7afb90 100644 --- a/core/pipeline/plugin/instance/ProcessorInstance.h +++ b/core/pipeline/plugin/instance/ProcessorInstance.h @@ -42,8 +42,6 @@ class ProcessorInstance : public PluginInstance { CounterPtr mInEventsTotal; CounterPtr mOutEventsTotal; - CounterPtr mInEventGroupsTotal; - CounterPtr mOutEventGroupsTotal; CounterPtr mInSizeBytes; CounterPtr mOutSizeBytes; CounterPtr mTotalProcessTimeMs; diff --git a/core/pipeline/plugin/interface/Plugin.h b/core/pipeline/plugin/interface/Plugin.h index abebf54671..7d335b1eea 100644 --- a/core/pipeline/plugin/interface/Plugin.h +++ b/core/pipeline/plugin/interface/Plugin.h @@ -21,7 +21,7 @@ #include #include "monitor/LogtailMetric.h" -#include "monitor/MetricConstants.h" +#include "monitor/metric_constants/MetricConstants.h" #include "pipeline/PipelineContext.h" namespace logtail { @@ -41,12 +41,12 @@ class Plugin { const std::string& nodeID, const std::string& childNodeID) { WriteMetrics::GetInstance()->PrepareMetricsRecordRef(mMetricsRecordRef, - {{METRIC_LABEL_PROJECT, mContext->GetProjectName()}, - {METRIC_LABEL_PIPELINE_NAME, mContext->GetConfigName()}, - {METRIC_LABEL_PLUGIN_TYPE, name}, - {METRIC_LABEL_PLUGIN_ID, id}, - {METRIC_LABEL_NODE_ID, nodeID}, - {METRIC_LABEL_CHILD_NODE_ID, childNodeID}}); + {{METRIC_LABEL_KEY_PROJECT, mContext->GetProjectName()}, + {METRIC_LABEL_KEY_PIPELINE_NAME, mContext->GetConfigName()}, + {METRIC_LABEL_KEY_PLUGIN_TYPE, name}, + {METRIC_LABEL_KEY_PLUGIN_ID, id}, + {METRIC_LABEL_KEY_NODE_ID, nodeID}, + {METRIC_LABEL_KEY_CHILD_NODE_ID, childNodeID}}); } protected: diff --git a/core/pipeline/queue/QueueInterface.h b/core/pipeline/queue/QueueInterface.h index 77c3bd96ad..5a079bb49e 100644 --- a/core/pipeline/queue/QueueInterface.h +++ b/core/pipeline/queue/QueueInterface.h @@ -17,7 +17,7 @@ #pragma once #include "monitor/LogtailMetric.h" -#include "monitor/MetricConstants.h" +#include "monitor/metric_constants/MetricConstants.h" #include "pipeline/PipelineContext.h" #include "pipeline/queue/QueueKey.h" @@ -29,12 +29,12 @@ class QueueInterface { QueueInterface(QueueKey key, size_t cap, const PipelineContext& ctx) : mKey(key), mCapacity(cap) { WriteMetrics::GetInstance()->CreateMetricsRecordRef(mMetricsRecordRef, { - {METRIC_LABEL_PROJECT, ctx.GetProjectName()}, - {METRIC_LABEL_PIPELINE_NAME, ctx.GetConfigName()}, + {METRIC_LABEL_KEY_PROJECT, ctx.GetProjectName()}, + {METRIC_LABEL_KEY_PIPELINE_NAME, ctx.GetConfigName()}, }); mInItemsTotal = mMetricsRecordRef.CreateCounter(METRIC_COMPONENT_IN_ITEMS_TOTAL); - mInItemDataSizeBytes = mMetricsRecordRef.CreateCounter(METRIC_COMPONENT_IN_ITEM_SIZE_BYTES); + mInItemDataSizeBytes = mMetricsRecordRef.CreateCounter(METRIC_COMPONENT_IN_SIZE_BYTES); mOutItemsTotal = mMetricsRecordRef.CreateCounter(METRIC_COMPONENT_OUT_ITEMS_TOTAL); mTotalDelayMs = mMetricsRecordRef.CreateCounter(METRIC_COMPONENT_TOTAL_DELAY_MS); mQueueSizeTotal = mMetricsRecordRef.CreateIntGauge(METRIC_COMPONENT_QUEUE_SIZE_TOTAL); diff --git a/core/pipeline/route/Router.cpp b/core/pipeline/route/Router.cpp index f480b060ac..78ff86810e 100644 --- a/core/pipeline/route/Router.cpp +++ b/core/pipeline/route/Router.cpp @@ -15,7 +15,7 @@ #include "pipeline/route/Router.h" #include "common/ParamExtractor.h" -#include "monitor/MetricConstants.h" +#include "monitor/metric_constants/MetricConstants.h" #include "pipeline/Pipeline.h" #include "pipeline/plugin/interface/Flusher.h" @@ -36,11 +36,11 @@ bool Router::Init(std::vector> configs, const P } WriteMetrics::GetInstance()->PrepareMetricsRecordRef(mMetricsRecordRef, - {{METRIC_LABEL_PROJECT, ctx.GetProjectName()}, - {METRIC_LABEL_PIPELINE_NAME, ctx.GetConfigName()}, + {{METRIC_LABEL_KEY_PROJECT, ctx.GetProjectName()}, + {METRIC_LABEL_KEY_PIPELINE_NAME, ctx.GetConfigName()}, {METRIC_LABEL_KEY_COMPONENT_NAME, METRIC_LABEL_VALUE_COMPONENT_NAME_ROUTER}}); mInEventsTotal = mMetricsRecordRef.CreateCounter(METRIC_COMPONENT_IN_EVENTS_TOTAL); - mInGroupDataSizeBytes = mMetricsRecordRef.CreateCounter(METRIC_COMPONENT_IN_EVENT_GROUP_SIZE_BYTES); + mInGroupDataSizeBytes = mMetricsRecordRef.CreateCounter(METRIC_COMPONENT_IN_SIZE_BYTES); return true; } diff --git a/core/pipeline/serializer/Serializer.h b/core/pipeline/serializer/Serializer.h index 0d65693c15..632fdbc011 100644 --- a/core/pipeline/serializer/Serializer.h +++ b/core/pipeline/serializer/Serializer.h @@ -20,7 +20,7 @@ #include #include "models/PipelineEventPtr.h" -#include "monitor/MetricConstants.h" +#include "monitor/metric_constants/MetricConstants.h" #include "pipeline/batch/BatchedEvents.h" #include "pipeline/plugin/interface/Flusher.h" @@ -49,15 +49,15 @@ class Serializer { Serializer(Flusher* f) : mFlusher(f) { WriteMetrics::GetInstance()->PrepareMetricsRecordRef( mMetricsRecordRef, - {{METRIC_LABEL_PROJECT, f->GetContext().GetProjectName()}, - {METRIC_LABEL_PIPELINE_NAME, f->GetContext().GetConfigName()}, + {{METRIC_LABEL_KEY_PROJECT, f->GetContext().GetProjectName()}, + {METRIC_LABEL_KEY_PIPELINE_NAME, f->GetContext().GetConfigName()}, {METRIC_LABEL_KEY_COMPONENT_NAME, METRIC_LABEL_VALUE_COMPONENT_NAME_SERIALIZER}, {METRIC_LABEL_KEY_FLUSHER_NODE_ID, f->GetNodeID()}}); mInItemsTotal = mMetricsRecordRef.CreateCounter(METRIC_COMPONENT_IN_ITEMS_TOTAL); - mInItemSizeBytes = mMetricsRecordRef.CreateCounter(METRIC_COMPONENT_IN_ITEM_SIZE_BYTES); + mInItemSizeBytes = mMetricsRecordRef.CreateCounter(METRIC_COMPONENT_IN_SIZE_BYTES); mOutItemsTotal = mMetricsRecordRef.CreateCounter(METRIC_COMPONENT_OUT_ITEMS_TOTAL); - mOutItemSizeBytes = mMetricsRecordRef.CreateCounter(METRIC_COMPONENT_OUT_ITEM_SIZE_BYTES); - mTotalProcessMs = mMetricsRecordRef.CreateCounter(METRIC_COMPONENT_TOTAL_PROCESS_MS); + mOutItemSizeBytes = mMetricsRecordRef.CreateCounter(METRIC_COMPONENT_OUT_SIZE_BYTES); + mTotalProcessMs = mMetricsRecordRef.CreateCounter(METRIC_COMPONENT_TOTAL_PROCESS_TIME_MS); mDiscardedItemsTotal = mMetricsRecordRef.CreateCounter(METRIC_COMPONENT_DISCARDED_ITEMS_TOTAL); mDiscardedItemSizeBytes = mMetricsRecordRef.CreateCounter(METRIC_COMPONENT_DISCARDED_ITEMS_SIZE_BYTES); } diff --git a/core/plugin/input/InputContainerStdio.cpp b/core/plugin/input/InputContainerStdio.cpp index 82468af869..46a018e067 100644 --- a/core/plugin/input/InputContainerStdio.cpp +++ b/core/plugin/input/InputContainerStdio.cpp @@ -19,7 +19,7 @@ #include "common/LogtailCommonFlags.h" #include "common/ParamExtractor.h" #include "file_server/FileServer.h" -#include "monitor/MetricConstants.h" +#include "monitor/metric_constants/MetricConstants.h" #include "pipeline/Pipeline.h" #include "pipeline/plugin/PluginRegistry.h" #include "plugin/processor/inner/ProcessorMergeMultilineLogNative.h" diff --git a/core/plugin/input/InputFile.cpp b/core/plugin/input/InputFile.cpp index 58a2b4e08d..c5b0039a19 100644 --- a/core/plugin/input/InputFile.cpp +++ b/core/plugin/input/InputFile.cpp @@ -23,7 +23,7 @@ #include "common/ParamExtractor.h" #include "file_server/ConfigManager.h" #include "file_server/FileServer.h" -#include "monitor/MetricConstants.h" +#include "monitor/metric_constants/MetricConstants.h" #include "pipeline/Pipeline.h" #include "pipeline/PipelineManager.h" #include "pipeline/plugin/PluginRegistry.h" diff --git a/core/plugin/processor/ProcessorDesensitizeNative.cpp b/core/plugin/processor/ProcessorDesensitizeNative.cpp index 6ef1656f83..39ba99c9aa 100644 --- a/core/plugin/processor/ProcessorDesensitizeNative.cpp +++ b/core/plugin/processor/ProcessorDesensitizeNative.cpp @@ -18,7 +18,7 @@ #include "common/Constants.h" #include "common/ParamExtractor.h" #include "models/LogEvent.h" -#include "monitor/MetricConstants.h" +#include "monitor/metric_constants/MetricConstants.h" #include "pipeline/plugin/instance/ProcessorInstance.h" #include "sdk/Common.h" diff --git a/core/plugin/processor/ProcessorFilterNative.cpp b/core/plugin/processor/ProcessorFilterNative.cpp index bae09d9d73..0b14998fe5 100644 --- a/core/plugin/processor/ProcessorFilterNative.cpp +++ b/core/plugin/processor/ProcessorFilterNative.cpp @@ -21,7 +21,7 @@ #include "common/ParamExtractor.h" #include "logger/Logger.h" #include "models/LogEvent.h" -#include "monitor/MetricConstants.h" +#include "monitor/metric_constants/MetricConstants.h" namespace logtail { diff --git a/core/plugin/processor/ProcessorParseApsaraNative.cpp b/core/plugin/processor/ProcessorParseApsaraNative.cpp index 2c8e6cb953..43d3854266 100644 --- a/core/plugin/processor/ProcessorParseApsaraNative.cpp +++ b/core/plugin/processor/ProcessorParseApsaraNative.cpp @@ -21,7 +21,7 @@ #include "common/ParamExtractor.h" #include "common/TimeUtil.h" #include "models/LogEvent.h" -#include "monitor/MetricConstants.h" +#include "monitor/metric_constants/MetricConstants.h" #include "pipeline/plugin/instance/ProcessorInstance.h" namespace logtail { diff --git a/core/plugin/processor/ProcessorParseDelimiterNative.cpp b/core/plugin/processor/ProcessorParseDelimiterNative.cpp index 1dbbf3a8d1..4f9b44fdf8 100644 --- a/core/plugin/processor/ProcessorParseDelimiterNative.cpp +++ b/core/plugin/processor/ProcessorParseDelimiterNative.cpp @@ -18,7 +18,7 @@ #include "common/ParamExtractor.h" #include "models/LogEvent.h" -#include "monitor/MetricConstants.h" +#include "monitor/metric_constants/MetricConstants.h" #include "pipeline/plugin/instance/ProcessorInstance.h" namespace logtail { diff --git a/core/plugin/processor/ProcessorParseJsonNative.cpp b/core/plugin/processor/ProcessorParseJsonNative.cpp index 3b21936e2c..90b3d075c5 100644 --- a/core/plugin/processor/ProcessorParseJsonNative.cpp +++ b/core/plugin/processor/ProcessorParseJsonNative.cpp @@ -21,7 +21,7 @@ #include "common/ParamExtractor.h" #include "models/LogEvent.h" -#include "monitor/MetricConstants.h" +#include "monitor/metric_constants/MetricConstants.h" #include "pipeline/plugin/instance/ProcessorInstance.h" namespace logtail { diff --git a/core/plugin/processor/ProcessorParseRegexNative.cpp b/core/plugin/processor/ProcessorParseRegexNative.cpp index 21a1d64c86..f04d8ad0dd 100644 --- a/core/plugin/processor/ProcessorParseRegexNative.cpp +++ b/core/plugin/processor/ProcessorParseRegexNative.cpp @@ -18,7 +18,7 @@ #include "app_config/AppConfig.h" #include "common/ParamExtractor.h" -#include "monitor/MetricConstants.h" +#include "monitor/metric_constants/MetricConstants.h" namespace logtail { diff --git a/core/plugin/processor/ProcessorParseTimestampNative.cpp b/core/plugin/processor/ProcessorParseTimestampNative.cpp index fcd93d8682..88306a3b6b 100644 --- a/core/plugin/processor/ProcessorParseTimestampNative.cpp +++ b/core/plugin/processor/ProcessorParseTimestampNative.cpp @@ -19,7 +19,7 @@ #include "app_config/AppConfig.h" #include "common/LogtailCommonFlags.h" #include "common/ParamExtractor.h" -#include "monitor/MetricConstants.h" +#include "monitor/metric_constants/MetricConstants.h" #include "pipeline/plugin/instance/ProcessorInstance.h" namespace logtail { diff --git a/core/plugin/processor/ProcessorSPL.cpp b/core/plugin/processor/ProcessorSPL.cpp index d3864de236..c2d13caadc 100644 --- a/core/plugin/processor/ProcessorSPL.cpp +++ b/core/plugin/processor/ProcessorSPL.cpp @@ -28,7 +28,7 @@ #include "common/Flags.h" #include "common/ParamExtractor.h" #include "logger/Logger.h" -#include "monitor/MetricConstants.h" +#include "monitor/metric_constants/MetricConstants.h" #include "spl/PipelineEventGroupInput.h" #include "spl/PipelineEventGroupOutput.h" #include "spl/SplConstants.h" diff --git a/core/plugin/processor/inner/ProcessorMergeMultilineLogNative.cpp b/core/plugin/processor/inner/ProcessorMergeMultilineLogNative.cpp index baeace7218..754202979d 100644 --- a/core/plugin/processor/inner/ProcessorMergeMultilineLogNative.cpp +++ b/core/plugin/processor/inner/ProcessorMergeMultilineLogNative.cpp @@ -23,7 +23,7 @@ #include "common/ParamExtractor.h" #include "logger/Logger.h" #include "models/LogEvent.h" -#include "monitor/MetricConstants.h" +#include "monitor/metric_constants/MetricConstants.h" namespace logtail { @@ -75,9 +75,9 @@ bool ProcessorMergeMultilineLogNative::Init(const Json::Value& config) { mSplitLines = &(GetContext().GetProcessProfile().splitLines); - mMergedEventsTotal = GetMetricsRecordRef().CreateCounter(METRIC_PLUGIN_MERGE_MULTILINE_LOG_MERGED_RECORDS_TOTAL); + mMergedEventsTotal = GetMetricsRecordRef().CreateCounter(METRIC_PLUGIN_MERGED_EVENTS_TOTAL); mUnmatchedEventsTotal - = GetMetricsRecordRef().CreateCounter(METRIC_PLUGIN_MERGE_MULTILINE_LOG_UNMATCHED_RECORDS_TOTAL); + = GetMetricsRecordRef().CreateCounter(METRIC_PLUGIN_UNMATCHED_EVENTS_TOTAL); return true; } diff --git a/core/plugin/processor/inner/ProcessorParseContainerLogNative.cpp b/core/plugin/processor/inner/ProcessorParseContainerLogNative.cpp index 5a8536219f..6170321736 100644 --- a/core/plugin/processor/inner/ProcessorParseContainerLogNative.cpp +++ b/core/plugin/processor/inner/ProcessorParseContainerLogNative.cpp @@ -26,7 +26,7 @@ #include "common/JsonUtil.h" #include "common/ParamExtractor.h" #include "models/LogEvent.h" -#include "monitor/MetricConstants.h" +#include "monitor/metric_constants/MetricConstants.h" #include "plugin/processor/inner/ProcessorMergeMultilineLogNative.h" namespace logtail { diff --git a/core/plugin/processor/inner/ProcessorSplitMultilineLogStringNative.cpp b/core/plugin/processor/inner/ProcessorSplitMultilineLogStringNative.cpp index 2cc010cf54..c5970b6dc2 100644 --- a/core/plugin/processor/inner/ProcessorSplitMultilineLogStringNative.cpp +++ b/core/plugin/processor/inner/ProcessorSplitMultilineLogStringNative.cpp @@ -24,7 +24,7 @@ #include "common/ParamExtractor.h" #include "logger/Logger.h" #include "models/LogEvent.h" -#include "monitor/MetricConstants.h" +#include "monitor/metric_constants/MetricConstants.h" #include "pipeline/plugin/instance/ProcessorInstance.h" namespace logtail { @@ -64,9 +64,9 @@ bool ProcessorSplitMultilineLogStringNative::Init(const Json::Value& config) { mContext->GetRegion()); } - mMatchedEventsTotal = GetMetricsRecordRef().CreateCounter(METRIC_PLUGIN_SPLIT_MULTILINE_LOG_MATCHED_RECORDS_TOTAL); - mMatchedLinesTotal = GetMetricsRecordRef().CreateCounter(METRIC_PLUGIN_SPLIT_MULTILINE_LOG_MATCHED_LINES_TOTAL); - mUnmatchedLinesTotal = GetMetricsRecordRef().CreateCounter(METRIC_PLUGIN_SPLIT_MULTILINE_LOG_UNMATCHED_LINES_TOTAL); + mMatchedEventsTotal = GetMetricsRecordRef().CreateCounter(METRIC_PLUGIN_MATCHED_EVENTS_TOTAL); + mMatchedLinesTotal = GetMetricsRecordRef().CreateCounter(METRIC_PLUGIN_MATCHED_LINES_TOTAL); + mUnmatchedLinesTotal = GetMetricsRecordRef().CreateCounter(METRIC_PLUGIN_UNMATCHED_LINES_TOTAL); mSplitLines = &(mContext->GetProcessProfile().splitLines); diff --git a/core/runner/FlusherRunner.cpp b/core/runner/FlusherRunner.cpp index 3f214f4741..630989ea81 100644 --- a/core/runner/FlusherRunner.cpp +++ b/core/runner/FlusherRunner.cpp @@ -44,11 +44,11 @@ bool FlusherRunner::Init() { WriteMetrics::GetInstance()->PrepareMetricsRecordRef(mMetricsRecordRef, {{METRIC_LABEL_KEY_RUNNER_NAME, METRIC_LABEL_VALUE_RUNNER_NAME_FLUSHER}}); mInItemsTotal = mMetricsRecordRef.CreateCounter(METRIC_RUNNER_IN_ITEMS_TOTAL); - mInItemDataSizeBytes = mMetricsRecordRef.CreateCounter(METRIC_RUNNER_IN_ITEM_SIZE_BYTES); + mInItemDataSizeBytes = mMetricsRecordRef.CreateCounter(METRIC_RUNNER_IN_SIZE_BYTES); mOutItemsTotal = mMetricsRecordRef.CreateCounter(METRIC_RUNNER_OUT_ITEMS_TOTAL); mTotalDelayMs = mMetricsRecordRef.CreateCounter(METRIC_RUNNER_TOTAL_DELAY_MS); mLastRunTime = mMetricsRecordRef.CreateIntGauge(METRIC_RUNNER_LAST_RUN_TIME); - mInItemRawDataSizeBytes = mMetricsRecordRef.CreateCounter(METRIC_RUNNER_FLUSHER_IN_ITEM_RAW_SIZE_BYTES); + mInItemRawDataSizeBytes = mMetricsRecordRef.CreateCounter(METRIC_RUNNER_FLUSHER_IN_SIZE_BYTES); mWaitingItemsTotal = mMetricsRecordRef.CreateIntGauge(METRIC_RUNNER_FLUSHER_WAITING_ITEMS_TOTAL); mThreadRes = async(launch::async, &FlusherRunner::Run, this); diff --git a/core/runner/LogProcess.cpp b/core/runner/LogProcess.cpp index 3170afb27b..80962b0ca1 100644 --- a/core/runner/LogProcess.cpp +++ b/core/runner/LogProcess.cpp @@ -19,7 +19,7 @@ #include "go_pipeline/LogtailPlugin.h" #include "monitor/LogFileProfiler.h" #include "monitor/LogtailAlarm.h" -#include "monitor/MetricConstants.h" +#include "monitor/metric_constants/MetricConstants.h" #include "pipeline/PipelineManager.h" #include "pipeline/batch/TimeoutFlushManager.h" #include "pipeline/queue/ExactlyOnceQueueManager.h" @@ -148,7 +148,7 @@ void* LogProcess::ProcessLoop(int32_t threadNo) { sMetricsRecordRef, {{METRIC_LABEL_KEY_RUNNER_NAME, METRIC_LABEL_VALUE_RUNNER_NAME_PROCESSOR}, {"thread_no", ToString(threadNo)}}); sInGroupsCnt = sMetricsRecordRef.CreateCounter(METRIC_RUNNER_IN_EVENT_GROUPS_TOTAL); sInEventsCnt = sMetricsRecordRef.CreateCounter(METRIC_RUNNER_IN_EVENTS_TOTAL); - sInGroupDataSizeBytes = sMetricsRecordRef.CreateCounter(METRIC_RUNNER_IN_EVENT_GROUP_SIZE_BYTES); + sInGroupDataSizeBytes = sMetricsRecordRef.CreateCounter(METRIC_RUNNER_IN_SIZE_BYTES); sLastRunTime = sMetricsRecordRef.CreateIntGauge(METRIC_RUNNER_LAST_RUN_TIME); static int32_t lastMergeTime = 0; diff --git a/core/runner/sink/http/HttpSink.cpp b/core/runner/sink/http/HttpSink.cpp index 63d81fc2e3..7821296ba8 100644 --- a/core/runner/sink/http/HttpSink.cpp +++ b/core/runner/sink/http/HttpSink.cpp @@ -18,7 +18,7 @@ #include "common/StringTools.h" #include "common/http/Curl.h" #include "logger/Logger.h" -#include "monitor/MetricConstants.h" +#include "monitor/metric_constants/MetricConstants.h" #include "pipeline/plugin/interface/HttpFlusher.h" #include "pipeline/queue/QueueKeyManager.h" #include "pipeline/queue/SenderQueueItem.h" diff --git a/core/unittest/batch/BatcherUnittest.cpp b/core/unittest/batch/BatcherUnittest.cpp index b0a3587044..29d15235af 100644 --- a/core/unittest/batch/BatcherUnittest.cpp +++ b/core/unittest/batch/BatcherUnittest.cpp @@ -577,8 +577,8 @@ void BatcherUnittest::TestMetric() { vector res; batch.Add(std::move(g), res); APSARA_TEST_EQUAL(5U, batch.mMetricsRecordRef->GetLabels()->size()); - APSARA_TEST_TRUE(batch.mMetricsRecordRef.HasLabel(METRIC_LABEL_PROJECT, "")); - APSARA_TEST_TRUE(batch.mMetricsRecordRef.HasLabel(METRIC_LABEL_PIPELINE_NAME, "test_config")); + APSARA_TEST_TRUE(batch.mMetricsRecordRef.HasLabel(METRIC_LABEL_KEY_PROJECT, "")); + APSARA_TEST_TRUE(batch.mMetricsRecordRef.HasLabel(METRIC_LABEL_KEY_PIPELINE_NAME, "test_config")); APSARA_TEST_TRUE(batch.mMetricsRecordRef.HasLabel(METRIC_LABEL_KEY_COMPONENT_NAME, METRIC_LABEL_VALUE_COMPONENT_NAME_BATCHER)); APSARA_TEST_TRUE(batch.mMetricsRecordRef.HasLabel(METRIC_LABEL_KEY_FLUSHER_NODE_ID, "1")); APSARA_TEST_TRUE(batch.mMetricsRecordRef.HasLabel("enable_group_batch", "false")); diff --git a/core/unittest/compression/CompressorFactoryUnittest.cpp b/core/unittest/compression/CompressorFactoryUnittest.cpp index 53ac01f852..fd147d4031 100644 --- a/core/unittest/compression/CompressorFactoryUnittest.cpp +++ b/core/unittest/compression/CompressorFactoryUnittest.cpp @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include "monitor/MetricConstants.h" +#include "monitor/metric_constants/MetricConstants.h" #include "common/compression/CompressorFactory.h" #include "unittest/Unittest.h" @@ -96,8 +96,8 @@ void CompressorFactoryUnittest::TestMetric() { auto compressor = CompressorFactory::GetInstance()->Create(Json::Value(), mCtx, "test_plugin", mFlusherId, CompressType::LZ4); APSARA_TEST_EQUAL(4U, compressor->mMetricsRecordRef->GetLabels()->size()); - APSARA_TEST_TRUE(compressor->mMetricsRecordRef.HasLabel(METRIC_LABEL_PROJECT, "")); - APSARA_TEST_TRUE(compressor->mMetricsRecordRef.HasLabel(METRIC_LABEL_PIPELINE_NAME, "test_config")); + APSARA_TEST_TRUE(compressor->mMetricsRecordRef.HasLabel(METRIC_LABEL_KEY_PROJECT, "")); + APSARA_TEST_TRUE(compressor->mMetricsRecordRef.HasLabel(METRIC_LABEL_KEY_PIPELINE_NAME, "test_config")); APSARA_TEST_TRUE(compressor->mMetricsRecordRef.HasLabel(METRIC_LABEL_KEY_COMPONENT_NAME, METRIC_LABEL_VALUE_COMPONENT_NAME_COMPRESSOR)); APSARA_TEST_TRUE(compressor->mMetricsRecordRef.HasLabel(METRIC_LABEL_KEY_FLUSHER_NODE_ID, mFlusherId)); } diff --git a/core/unittest/compression/CompressorUnittest.cpp b/core/unittest/compression/CompressorUnittest.cpp index dbafa5ee6f..a678d692e7 100644 --- a/core/unittest/compression/CompressorUnittest.cpp +++ b/core/unittest/compression/CompressorUnittest.cpp @@ -13,7 +13,7 @@ // limitations under the License. #include "common/compression/LZ4Compressor.h" -#include "monitor/MetricConstants.h" +#include "monitor/metric_constants/MetricConstants.h" #include "unittest/Unittest.h" using namespace std; diff --git a/core/unittest/monitor/PluginMetricManagerUnittest.cpp b/core/unittest/monitor/PluginMetricManagerUnittest.cpp index b6e5d2030a..2e0db37022 100644 --- a/core/unittest/monitor/PluginMetricManagerUnittest.cpp +++ b/core/unittest/monitor/PluginMetricManagerUnittest.cpp @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include "monitor/MetricConstants.h" +#include "monitor/metric_constants/MetricConstants.h" #include "monitor/PluginMetricManager.h" #include "unittest/Unittest.h" @@ -22,12 +22,12 @@ class PluginMetricManagerUnittest : public ::testing::Test { public: void SetUp() { MetricLabelsPtr defaultLabels = std::make_shared(); - defaultLabels->emplace_back(METRIC_LABEL_PROJECT, "default_project"); - defaultLabels->emplace_back(METRIC_LABEL_LOGSTORE, "default_logstore"); - defaultLabels->emplace_back(METRIC_LABEL_REGION, "default_region"); - defaultLabels->emplace_back(METRIC_LABEL_PIPELINE_NAME, "default_config"); - defaultLabels->emplace_back(METRIC_LABEL_PLUGIN_TYPE, "default_plugin"); - defaultLabels->emplace_back(METRIC_LABEL_PLUGIN_ID, "default_id"); + defaultLabels->emplace_back(METRIC_LABEL_KEY_PROJECT, "default_project"); + defaultLabels->emplace_back(METRIC_LABEL_KEY_LOGSTORE, "default_logstore"); + defaultLabels->emplace_back(METRIC_LABEL_KEY_REGION, "default_region"); + defaultLabels->emplace_back(METRIC_LABEL_KEY_PIPELINE_NAME, "default_config"); + defaultLabels->emplace_back(METRIC_LABEL_KEY_PLUGIN_TYPE, "default_plugin"); + defaultLabels->emplace_back(METRIC_LABEL_KEY_PLUGIN_ID, "default_id"); WriteMetrics::GetInstance()->PrepareMetricsRecordRef(mMetricsRecordRef, std::move(*defaultLabels)); std::unordered_map metricKeys; metricKeys.emplace("default_counter", MetricType::METRIC_TYPE_COUNTER); diff --git a/core/unittest/pipeline/PipelineUnittest.cpp b/core/unittest/pipeline/PipelineUnittest.cpp index bdeadd2725..f2cc1ee704 100644 --- a/core/unittest/pipeline/PipelineUnittest.cpp +++ b/core/unittest/pipeline/PipelineUnittest.cpp @@ -113,8 +113,8 @@ void PipelineUnittest::OnSuccessfulInit() const { APSARA_TEST_EQUAL(QueueKeyManager::GetInstance()->GetKey("test_config-flusher_sls-test_project#test_logstore"), pipeline->GetContext().GetLogstoreKey()); APSARA_TEST_EQUAL(2U, pipeline->mMetricsRecordRef->GetLabels()->size()); - APSARA_TEST_TRUE(pipeline->mMetricsRecordRef.HasLabel(METRIC_LABEL_PIPELINE_NAME, configName)); - APSARA_TEST_TRUE(pipeline->mMetricsRecordRef.HasLabel(METRIC_LABEL_PROJECT, "test_project")); + APSARA_TEST_TRUE(pipeline->mMetricsRecordRef.HasLabel(METRIC_LABEL_KEY_PIPELINE_NAME, configName)); + APSARA_TEST_TRUE(pipeline->mMetricsRecordRef.HasLabel(METRIC_LABEL_KEY_PROJECT, "test_project")); // without sls flusher configStr = R"( @@ -2688,11 +2688,14 @@ void PipelineUnittest::TestProcess() const { pipeline.mProcessorLine.emplace_back(std::move(processor)); WriteMetrics::GetInstance()->PrepareMetricsRecordRef(pipeline.mMetricsRecordRef, {}); - pipeline.mProcessorsInEventsTotal = pipeline.mMetricsRecordRef.CreateCounter("pipeline_processors_in_events_total"); - pipeline.mProcessorsInGroupsTotal = pipeline.mMetricsRecordRef.CreateCounter("pipeline_processors_in_event_groups_total"); - pipeline.mProcessorsInGroupDataSizeBytes - = pipeline.mMetricsRecordRef.CreateCounter("processors_in_event_group_size_bytes"); - pipeline.mProcessorsTotalDelayMs = pipeline.mMetricsRecordRef.CreateCounter("processors_total_delay_ms"); + pipeline.mProcessorsInEventsTotal + = pipeline.mMetricsRecordRef.CreateCounter(METRIC_PIPELINE_PROCESSORS_IN_EVENTS_TOTAL); + pipeline.mProcessorsInGroupsTotal + = pipeline.mMetricsRecordRef.CreateCounter(METRIC_PIPELINE_PROCESSORS_IN_EVENT_GROUPS_TOTAL); + pipeline.mProcessorsInSizeBytes + = pipeline.mMetricsRecordRef.CreateCounter(METRIC_PIPELINE_PROCESSORS_IN_SIZE_BYTES); + pipeline.mProcessorsTotalProcessTimeMs + = pipeline.mMetricsRecordRef.CreateCounter(METRIC_PIPELINE_PROCESSORS_TOTAL_PROCESS_TIME_MS); vector groups; groups.emplace_back(make_shared()); @@ -2704,7 +2707,7 @@ void PipelineUnittest::TestProcess() const { APSARA_TEST_EQUAL(1U, static_cast(pipeline.mProcessorLine[0]->mPlugin.get())->mCnt); APSARA_TEST_EQUAL(1U, pipeline.mProcessorsInEventsTotal->GetValue()); APSARA_TEST_EQUAL(1U, pipeline.mProcessorsInGroupsTotal->GetValue()); - APSARA_TEST_EQUAL(size, pipeline.mProcessorsInGroupDataSizeBytes->GetValue()); + APSARA_TEST_EQUAL(size, pipeline.mProcessorsInSizeBytes->GetValue()); } void PipelineUnittest::TestSend() const { diff --git a/core/unittest/queue/BoundedProcessQueueUnittest.cpp b/core/unittest/queue/BoundedProcessQueueUnittest.cpp index ea3672e051..a971045da5 100644 --- a/core/unittest/queue/BoundedProcessQueueUnittest.cpp +++ b/core/unittest/queue/BoundedProcessQueueUnittest.cpp @@ -119,8 +119,8 @@ void BoundedProcessQueueUnittest::TestPop() { void BoundedProcessQueueUnittest::TestMetric() { APSARA_TEST_EQUAL(4U, mQueue->mMetricsRecordRef->GetLabels()->size()); - APSARA_TEST_TRUE(mQueue->mMetricsRecordRef.HasLabel(METRIC_LABEL_PROJECT, "")); - APSARA_TEST_TRUE(mQueue->mMetricsRecordRef.HasLabel(METRIC_LABEL_PIPELINE_NAME, "test_config")); + APSARA_TEST_TRUE(mQueue->mMetricsRecordRef.HasLabel(METRIC_LABEL_KEY_PROJECT, "")); + APSARA_TEST_TRUE(mQueue->mMetricsRecordRef.HasLabel(METRIC_LABEL_KEY_PIPELINE_NAME, "test_config")); APSARA_TEST_TRUE(mQueue->mMetricsRecordRef.HasLabel(METRIC_LABEL_KEY_COMPONENT_NAME, METRIC_LABEL_VALUE_COMPONENT_NAME_PROCESS_QUEUE)); APSARA_TEST_TRUE(mQueue->mMetricsRecordRef.HasLabel(METRIC_LABEL_KEY_QUEUE_TYPE, "bounded")); diff --git a/core/unittest/queue/CircularProcessQueueUnittest.cpp b/core/unittest/queue/CircularProcessQueueUnittest.cpp index 4ef3104644..604f2f7007 100644 --- a/core/unittest/queue/CircularProcessQueueUnittest.cpp +++ b/core/unittest/queue/CircularProcessQueueUnittest.cpp @@ -152,8 +152,8 @@ void CircularProcessQueueUnittest::TestReset() { void CircularProcessQueueUnittest::TestMetric() { APSARA_TEST_EQUAL(4U, mQueue->mMetricsRecordRef->GetLabels()->size()); - APSARA_TEST_TRUE(mQueue->mMetricsRecordRef.HasLabel(METRIC_LABEL_PROJECT, "")); - APSARA_TEST_TRUE(mQueue->mMetricsRecordRef.HasLabel(METRIC_LABEL_PIPELINE_NAME, "test_config")); + APSARA_TEST_TRUE(mQueue->mMetricsRecordRef.HasLabel(METRIC_LABEL_KEY_PROJECT, "")); + APSARA_TEST_TRUE(mQueue->mMetricsRecordRef.HasLabel(METRIC_LABEL_KEY_PIPELINE_NAME, "test_config")); APSARA_TEST_TRUE(mQueue->mMetricsRecordRef.HasLabel(METRIC_LABEL_KEY_COMPONENT_NAME, METRIC_LABEL_VALUE_COMPONENT_NAME_PROCESS_QUEUE)); APSARA_TEST_TRUE(mQueue->mMetricsRecordRef.HasLabel(METRIC_LABEL_KEY_QUEUE_TYPE, "circular")); diff --git a/core/unittest/queue/SenderQueueUnittest.cpp b/core/unittest/queue/SenderQueueUnittest.cpp index 31169eb445..3e798435cd 100644 --- a/core/unittest/queue/SenderQueueUnittest.cpp +++ b/core/unittest/queue/SenderQueueUnittest.cpp @@ -172,8 +172,8 @@ void SenderQueueUnittest::TestGetAllAvailableItems() { void SenderQueueUnittest::TestMetric() { APSARA_TEST_EQUAL(5U, mQueue->mMetricsRecordRef->GetLabels()->size()); - APSARA_TEST_TRUE(mQueue->mMetricsRecordRef.HasLabel(METRIC_LABEL_PROJECT, "")); - APSARA_TEST_TRUE(mQueue->mMetricsRecordRef.HasLabel(METRIC_LABEL_PIPELINE_NAME, "test_config")); + APSARA_TEST_TRUE(mQueue->mMetricsRecordRef.HasLabel(METRIC_LABEL_KEY_PROJECT, "")); + APSARA_TEST_TRUE(mQueue->mMetricsRecordRef.HasLabel(METRIC_LABEL_KEY_PIPELINE_NAME, "test_config")); APSARA_TEST_TRUE(mQueue->mMetricsRecordRef.HasLabel(METRIC_LABEL_KEY_COMPONENT_NAME, METRIC_LABEL_VALUE_COMPONENT_NAME_SENDER_QUEUE)); APSARA_TEST_TRUE(mQueue->mMetricsRecordRef.HasLabel(METRIC_LABEL_KEY_QUEUE_TYPE, "bounded")); APSARA_TEST_TRUE(mQueue->mMetricsRecordRef.HasLabel(METRIC_LABEL_KEY_FLUSHER_NODE_ID, sFlusherId)); diff --git a/core/unittest/route/RouterUnittest.cpp b/core/unittest/route/RouterUnittest.cpp index 5527a8ab5d..3a82eb7d18 100644 --- a/core/unittest/route/RouterUnittest.cpp +++ b/core/unittest/route/RouterUnittest.cpp @@ -13,7 +13,7 @@ // limitations under the License. #include "common/JsonUtil.h" -#include "monitor/MetricConstants.h" +#include "monitor/metric_constants/MetricConstants.h" #include "pipeline/Pipeline.h" #include "pipeline/route/Router.h" #include "unittest/Unittest.h" @@ -151,8 +151,8 @@ void RouterUnittest::TestMetric() { router.Init(configs, ctx); APSARA_TEST_EQUAL(3U, router.mMetricsRecordRef->GetLabels()->size()); - APSARA_TEST_TRUE(router.mMetricsRecordRef.HasLabel(METRIC_LABEL_PROJECT, "")); - APSARA_TEST_TRUE(router.mMetricsRecordRef.HasLabel(METRIC_LABEL_PIPELINE_NAME, "test_config")); + APSARA_TEST_TRUE(router.mMetricsRecordRef.HasLabel(METRIC_LABEL_KEY_PROJECT, "")); + APSARA_TEST_TRUE(router.mMetricsRecordRef.HasLabel(METRIC_LABEL_KEY_PIPELINE_NAME, "test_config")); APSARA_TEST_TRUE(router.mMetricsRecordRef.HasLabel(METRIC_LABEL_KEY_COMPONENT_NAME, METRIC_LABEL_VALUE_COMPONENT_NAME_ROUTER)); PipelineEventGroup g(make_shared()); diff --git a/core/unittest/serializer/SerializerUnittest.cpp b/core/unittest/serializer/SerializerUnittest.cpp index c2dd5603ed..1a7da6d7b7 100644 --- a/core/unittest/serializer/SerializerUnittest.cpp +++ b/core/unittest/serializer/SerializerUnittest.cpp @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include "monitor/MetricConstants.h" +#include "monitor/metric_constants/MetricConstants.h" #include "pipeline/plugin/interface/Flusher.h" #include "pipeline/serializer/Serializer.h" #include "unittest/Unittest.h" diff --git a/pkg/helper/self_metrics_constants.go b/pkg/helper/self_metrics_constants.go index 5ba49209ba..e499e9d607 100644 --- a/pkg/helper/self_metrics_constants.go +++ b/pkg/helper/self_metrics_constants.go @@ -18,13 +18,6 @@ import ( "github.com/alibaba/ilogtail/pkg/pipeline" ) -////////////////////////////////////////////////////////////////////////// -// common -////////////////////////////////////////////////////////////////////////// - -// label keys -const MetricLabelProject = "project" - ////////////////////////////////////////////////////////////////////////// // agent ////////////////////////////////////////////////////////////////////////// @@ -41,13 +34,13 @@ const ( // label keys const ( - MetricLabelLogstore = "logstore" - MetricLabelRegion = "region" - MetricLabelPipelineName = "pipeline_name" - MetricLabelPluginType = "plugin_type" - MetricLabelPluginID = "plugin_id" - MetricLabelNodeID = "node_id" - MetricLabelChildNodeID = "child_node_id" + MetricLabelKeyProject = "project" + MetricLabelKeyLogstore = "logstore" + MetricLabelKeyPipelineName = "pipeline_name" + MetricLabelKeyPluginType = "plugin_type" + MetricLabelKeyPluginID = "plugin_id" + MetricLabelKeyNodeID = "node_id" + MetricLabelKeyChildNodeID = "child_node_id" ) // metric keys @@ -58,7 +51,7 @@ const ( MetricPluginOutEventsTotal = "plugin_out_events_total" MetricPluginOutEventGroupsTotal = "plugin_out_event_groups_total" MetricPluginOutSizeBytes = "plugin_out_size_bytes" - MetricPluginTotalDelayTimeMs = "plugin_total_delay_time_ms" + MetricPluginTotalDelayMs = "plugin_total_delay_ms" MetricPluginTotalProcessTimeMs = "plugin_total_process_time_ms" ) @@ -117,21 +110,21 @@ const ( func GetCommonLabels(context pipeline.Context, pluginMeta *pipeline.PluginMeta) []pipeline.LabelPair { labels := make([]pipeline.LabelPair, 0) - labels = append(labels, pipeline.LabelPair{Key: MetricLabelProject, Value: context.GetProject()}) - labels = append(labels, pipeline.LabelPair{Key: MetricLabelLogstore, Value: context.GetLogstore()}) - labels = append(labels, pipeline.LabelPair{Key: MetricLabelPipelineName, Value: context.GetConfigName()}) + labels = append(labels, pipeline.LabelPair{Key: MetricLabelKeyProject, Value: context.GetProject()}) + labels = append(labels, pipeline.LabelPair{Key: MetricLabelKeyLogstore, Value: context.GetLogstore()}) + labels = append(labels, pipeline.LabelPair{Key: MetricLabelKeyPipelineName, Value: context.GetConfigName()}) if len(pluginMeta.PluginID) > 0 { - labels = append(labels, pipeline.LabelPair{Key: MetricLabelPluginID, Value: pluginMeta.PluginID}) + labels = append(labels, pipeline.LabelPair{Key: MetricLabelKeyPluginID, Value: pluginMeta.PluginID}) } if len(pluginMeta.NodeID) > 0 { - labels = append(labels, pipeline.LabelPair{Key: MetricLabelNodeID, Value: pluginMeta.NodeID}) + labels = append(labels, pipeline.LabelPair{Key: MetricLabelKeyNodeID, Value: pluginMeta.NodeID}) } if len(pluginMeta.ChildNodeID) > 0 { - labels = append(labels, pipeline.LabelPair{Key: MetricLabelChildNodeID, Value: pluginMeta.ChildNodeID}) + labels = append(labels, pipeline.LabelPair{Key: MetricLabelKeyChildNodeID, Value: pluginMeta.ChildNodeID}) } if len(pluginMeta.PluginType) > 0 { - labels = append(labels, pipeline.LabelPair{Key: MetricLabelPluginType, Value: pluginMeta.PluginType}) + labels = append(labels, pipeline.LabelPair{Key: MetricLabelKeyPluginType, Value: pluginMeta.PluginType}) } return labels } diff --git a/pluginmanager/plugin_wrapper.go b/pluginmanager/plugin_wrapper.go index 235c921091..9c0f2c61ab 100644 --- a/pluginmanager/plugin_wrapper.go +++ b/pluginmanager/plugin_wrapper.go @@ -136,5 +136,5 @@ func (wrapper *FlusherWrapper) InitMetricRecord(pluginMeta *pipeline.PluginMeta) wrapper.inEventsTotal = helper.NewCounterMetricAndRegister(wrapper.MetricRecord, helper.MetricPluginInEventsTotal) wrapper.inEventGroupsTotal = helper.NewCounterMetricAndRegister(wrapper.MetricRecord, helper.MetricPluginInEventGroupsTotal) wrapper.inSizeBytes = helper.NewCounterMetricAndRegister(wrapper.MetricRecord, helper.MetricPluginInSizeBytes) - wrapper.totalDelayTimeMs = helper.NewCounterMetricAndRegister(wrapper.MetricRecord, helper.MetricPluginTotalDelayTimeMs) + wrapper.totalDelayTimeMs = helper.NewCounterMetricAndRegister(wrapper.MetricRecord, helper.MetricPluginTotalDelayMs) } diff --git a/pluginmanager/plugin_wrapper_aggregator_v2.go b/pluginmanager/plugin_wrapper_aggregator_v2.go index f2c11b01cf..ac2eed760e 100644 --- a/pluginmanager/plugin_wrapper_aggregator_v2.go +++ b/pluginmanager/plugin_wrapper_aggregator_v2.go @@ -38,7 +38,7 @@ type AggregatorWrapperV2 struct { func (wrapper *AggregatorWrapperV2) Init(pluginMeta *pipeline.PluginMeta) error { wrapper.InitMetricRecord(pluginMeta) - wrapper.totalDelayTimeMs = helper.NewCounterMetricAndRegister(wrapper.MetricRecord, helper.MetricPluginTotalDelayTimeMs) + wrapper.totalDelayTimeMs = helper.NewCounterMetricAndRegister(wrapper.MetricRecord, helper.MetricPluginTotalDelayMs) interval, err := wrapper.Aggregator.Init(wrapper.Config.Context, wrapper) if err != nil {