rename go metrics

core/file_server/reader/LogFileReader.cpp

@@ -215,7 +215,7 @@ void LogFileReader::SetMetrics() {
 
     mOutEventsTotal = mMetricsRecordRef->GetCounter(METRIC_PLUGIN_OUT_EVENTS_TOTAL);
     mOutEventGroupsTotal = mMetricsRecordRef->GetCounter(METRIC_PLUGIN_OUT_EVENT_GROUPS_TOTAL);
-    mOutEventGroupSizeBytes = mMetricsRecordRef->GetCounter(METRIC_PLUGIN_OUT_EVENT_GROUP_SIZE_BYTES);
+    mOutSizeBytes = mMetricsRecordRef->GetCounter(METRIC_PLUGIN_OUT_SIZE_BYTES);
     mSourceSizeBytes = mMetricsRecordRef->GetIntGauge(METRIC_PLUGIN_SOURCE_SIZE_BYTES);
     mSourceReadOffsetBytes = mMetricsRecordRef->GetIntGauge(METRIC_PLUGIN_SOURCE_READ_OFFSET_BYTES);
     mMetricInited = true;
@@ -2136,7 +2136,7 @@ void LogFileReader::ReportMetrics(uint64_t readSize) {
     if (mMetricInited) {
         mOutEventsTotal->Add(1);
         mOutEventGroupsTotal->Add(1);
-        mOutEventGroupSizeBytes->Add(readSize);
+        mOutSizeBytes->Add(readSize);
         mSourceReadOffsetBytes->Set(GetLastFilePos());
         mSourceSizeBytes->Set(GetFileSize());
     }

core/file_server/reader/LogFileReader.h

@@ -539,7 +539,7 @@ class LogFileReader {
     ReentrantMetricsRecordRef mMetricsRecordRef;
     CounterPtr mOutEventsTotal;
     CounterPtr mOutEventGroupsTotal;
-    CounterPtr mOutEventGroupSizeBytes;
+    CounterPtr mOutSizeBytes;
     IntGaugePtr mSourceSizeBytes;
     IntGaugePtr mSourceReadOffsetBytes;
 

core/monitor/MetricConstants.h

@@ -92,10 +92,11 @@ extern const std::string METRIC_LABEL_PLUGIN_TYPE;
 // metric keys
 extern const std::string METRIC_PLUGIN_IN_EVENTS_TOTAL;
 extern const std::string METRIC_PLUGIN_IN_EVENT_GROUPS_TOTAL;
-extern const std::string METRIC_PLUGIN_IN_EVENT_GROUP_SIZE_BYTES;
+extern const std::string METRIC_PLUGIN_IN_SIZE_BYTES;
 extern const std::string METRIC_PLUGIN_OUT_EVENTS_TOTAL;
 extern const std::string METRIC_PLUGIN_OUT_EVENT_GROUPS_TOTAL;
-extern const std::string METRIC_PLUGIN_OUT_EVENT_GROUP_SIZE_BYTES;
+extern const std::string METRIC_PLUGIN_OUT_SIZE_BYTES;
+extern const std::string METRIC_PLUGIN_TOTAL_DELAY_TIME_MS;
 extern const std::string METRIC_PLUGIN_TOTAL_PROCESS_TIME_MS;
 
 /**********************************************************
@@ -111,7 +112,7 @@ extern const std::string METRIC_PLUGIN_SOURCE_READ_OFFSET_BYTES;
 extern const std::string METRIC_PLUGIN_SOURCE_SIZE_BYTES;
 
 /**********************************************************
-*   all processor (parse type)
+*   all processor （所有解析类的处理插件通用指标。Todo：目前统计还不全、不准确）
 **********************************************************/
 extern const std::string METRIC_PLUGIN_DISCARDED_EVENTS_TOTAL;
 extern const std::string METRIC_PLUGIN_OUT_FAILED_EVENTS_TOTAL;

core/monitor/metric_constants/CommonMetrics.cpp

@@ -19,7 +19,7 @@ namespace logtail {
 const std::string METRIC_FIELD_REGION = "region";
 const std::string METRIC_REGION_DEFAULT = "default";
 const std::string METRIC_SLS_LOGSTORE_NAME = "shennong_log_profile";
-const std::string METRIC_TOPIC_TYPE = "logtail_metric";
+const std::string METRIC_TOPIC_TYPE = "loong_collector_metric";
 const std::string METRIC_TOPIC_FIELD_NAME = "__topic__";
 
 const std::string LABEL_PREFIX = "label.";

core/monitor/metric_constants/PluginMetrics.cpp

@@ -25,10 +25,11 @@ const std::string METRIC_LABEL_PLUGIN_TYPE = "plugin_type";
 // metric keys
 const std::string METRIC_PLUGIN_IN_EVENTS_TOTAL = "plugin_in_events_total";
 const std::string METRIC_PLUGIN_IN_EVENT_GROUPS_TOTAL = "plugin_in_event_groups_total";
-const std::string METRIC_PLUGIN_IN_EVENT_GROUP_SIZE_BYTES = "plugin_in_event_group_size_bytes";
+const std::string METRIC_PLUGIN_IN_SIZE_BYTES = "plugin_in_size_bytes";
 const std::string METRIC_PLUGIN_OUT_EVENTS_TOTAL = "plugin_out_events_total";
-const std::string METRIC_PLUGIN_OUT_EVENT_GROUPS_TOTAL = "plugin_in_event_groups_total";
-const std::string METRIC_PLUGIN_OUT_EVENT_GROUP_SIZE_BYTES = "plugin_out_event_group_size_bytes";
+const std::string METRIC_PLUGIN_OUT_EVENT_GROUPS_TOTAL = "plugin_out_event_groups_total";
+const std::string METRIC_PLUGIN_OUT_SIZE_BYTES = "plugin_out_size_bytes";
+const std::string METRIC_PLUGIN_TOTAL_DELAY_TIME_MS = "plugin_total_delay_time_ms";
 const std::string METRIC_PLUGIN_TOTAL_PROCESS_TIME_MS = "plugin_total_process_time_ms";
 
 /**********************************************************
@@ -44,7 +45,7 @@ const std::string METRIC_PLUGIN_SOURCE_READ_OFFSET_BYTES = "plugin_source_read_o
 const std::string METRIC_PLUGIN_SOURCE_SIZE_BYTES = "plugin_source_size_bytes";
 
 /**********************************************************
-*   all processor (parse type)
+*   all processor （所有解析类的处理插件通用指标。Todo：目前统计还不全、不准确）
 **********************************************************/
 const std::string METRIC_PLUGIN_DISCARDED_EVENTS_TOTAL = "plugin_discarded_events_total";
 const std::string METRIC_PLUGIN_OUT_FAILED_EVENTS_TOTAL = "plugin_out_failed_events_total";

core/pipeline/plugin/instance/FlusherInstance.cpp

@@ -27,14 +27,14 @@ bool FlusherInstance::Init(const Json::Value& config, PipelineContext& context,
 
     mInEventsTotal = mPlugin->GetMetricsRecordRef().CreateCounter(METRIC_PLUGIN_IN_EVENTS_TOTAL);
     mInEventGroupsTotal = mPlugin->GetMetricsRecordRef().CreateCounter(METRIC_PLUGIN_IN_EVENT_GROUPS_TOTAL);
-    mInEventGroupSizeBytes = mPlugin->GetMetricsRecordRef().CreateCounter(METRIC_PLUGIN_IN_EVENT_GROUP_SIZE_BYTES);
+    mInSizeBytes = mPlugin->GetMetricsRecordRef().CreateCounter(METRIC_PLUGIN_IN_SIZE_BYTES);
     return true;
 }
 
 bool FlusherInstance::Send(PipelineEventGroup&& g) {
     mInEventsTotal->Add(g.GetEvents().size());
     mInEventGroupsTotal->Add(1);
-    mInEventGroupSizeBytes->Add(g.DataSize());
+    mInSizeBytes->Add(g.DataSize());
     return mPlugin->Send(std::move(g));
 }
 

core/pipeline/plugin/instance/FlusherInstance.h

@@ -48,7 +48,7 @@ class FlusherInstance : public PluginInstance {
 
     CounterPtr mInEventsTotal;
     CounterPtr mInEventGroupsTotal;
-    CounterPtr mInEventGroupSizeBytes;
+    CounterPtr mInSizeBytes;
 };
 
 } // namespace logtail

core/pipeline/plugin/instance/ProcessorInstance.cpp

@@ -38,8 +38,8 @@ bool ProcessorInstance::Init(const Json::Value& config, PipelineContext& context
     mOutEventsTotal = mPlugin->GetMetricsRecordRef().CreateCounter(METRIC_PLUGIN_OUT_EVENTS_TOTAL);
     mInEventGroupsTotal = mPlugin->GetMetricsRecordRef().CreateCounter(METRIC_PLUGIN_IN_EVENT_GROUPS_TOTAL);
     mOutEventGroupsTotal = mPlugin->GetMetricsRecordRef().CreateCounter(METRIC_PLUGIN_OUT_EVENT_GROUPS_TOTAL);
-    mInEventGroupSizeBytes = mPlugin->GetMetricsRecordRef().CreateCounter(METRIC_PLUGIN_IN_EVENT_GROUP_SIZE_BYTES);
-    mOutEventGroupSizeBytes = mPlugin->GetMetricsRecordRef().CreateCounter(METRIC_PLUGIN_OUT_EVENT_GROUP_SIZE_BYTES);
+    mInSizeBytes = mPlugin->GetMetricsRecordRef().CreateCounter(METRIC_PLUGIN_IN_SIZE_BYTES);
+    mOutSizeBytes = mPlugin->GetMetricsRecordRef().CreateCounter(METRIC_PLUGIN_OUT_SIZE_BYTES);
     mCostTimeMS = mPlugin->GetMetricsRecordRef().CreateCounter(METRIC_PLUGIN_TOTAL_PROCESS_TIME_MS);
 
     return true;
@@ -52,7 +52,7 @@ void ProcessorInstance::Process(vector<PipelineEventGroup>& logGroupList) {
     for (const auto& logGroup : logGroupList) {
         mInEventsTotal->Add(logGroup.GetEvents().size());
         mInEventGroupsTotal->Add(1);
-        mInEventGroupSizeBytes->Add(logGroup.DataSize());
+        mInSizeBytes->Add(logGroup.DataSize());
     }
 
     auto before = chrono::system_clock::now();
@@ -62,7 +62,7 @@ void ProcessorInstance::Process(vector<PipelineEventGroup>& logGroupList) {
     for (const auto& logGroup : logGroupList) {
         mOutEventsTotal->Add(logGroup.GetEvents().size());
         mOutEventGroupsTotal->Add(1);
-        mOutEventGroupSizeBytes->Add(logGroup.DataSize());
+        mOutSizeBytes->Add(logGroup.DataSize());
     }    
 }
 

core/pipeline/plugin/instance/ProcessorInstance.h

@@ -44,8 +44,8 @@ class ProcessorInstance : public PluginInstance {
     CounterPtr mOutEventsTotal;
     CounterPtr mInEventGroupsTotal;
     CounterPtr mOutEventGroupsTotal;
-    CounterPtr mInEventGroupSizeBytes;
-    CounterPtr mOutEventGroupSizeBytes;
+    CounterPtr mInSizeBytes;
+    CounterPtr mOutSizeBytes;
     CounterPtr mCostTimeMS;
 
 #ifdef APSARA_UNIT_TEST_MAIN

core/plugin/input/InputContainerStdio.cpp

@@ -163,7 +163,7 @@ bool InputContainerStdio::Init(const Json::Value& config, Json::Value& optionalG
     static const std::unordered_map<std::string, MetricType> inputFileMetricKeys = {
         {METRIC_PLUGIN_OUT_EVENTS_TOTAL, MetricType::METRIC_TYPE_COUNTER},
         {METRIC_PLUGIN_OUT_EVENT_GROUPS_TOTAL, MetricType::METRIC_TYPE_COUNTER},
-        {METRIC_PLUGIN_OUT_EVENT_GROUP_SIZE_BYTES, MetricType::METRIC_TYPE_COUNTER},
+        {METRIC_PLUGIN_OUT_SIZE_BYTES, MetricType::METRIC_TYPE_COUNTER},
         {METRIC_PLUGIN_SOURCE_SIZE_BYTES, MetricType::METRIC_TYPE_INT_GAUGE},
         {METRIC_PLUGIN_SOURCE_READ_OFFSET_BYTES, MetricType::METRIC_TYPE_INT_GAUGE},
     };

core/plugin/input/InputFile.cpp

@@ -161,7 +161,7 @@ bool InputFile::Init(const Json::Value& config, Json::Value& optionalGoPipeline)
     static const std::unordered_map<std::string, MetricType> inputFileMetricKeys = {
         {METRIC_PLUGIN_OUT_EVENTS_TOTAL, MetricType::METRIC_TYPE_COUNTER},
         {METRIC_PLUGIN_OUT_EVENT_GROUPS_TOTAL, MetricType::METRIC_TYPE_COUNTER},
-        {METRIC_PLUGIN_OUT_EVENT_GROUP_SIZE_BYTES, MetricType::METRIC_TYPE_COUNTER},
+        {METRIC_PLUGIN_OUT_SIZE_BYTES, MetricType::METRIC_TYPE_COUNTER},
         {METRIC_PLUGIN_SOURCE_SIZE_BYTES, MetricType::METRIC_TYPE_INT_GAUGE},
         {METRIC_PLUGIN_SOURCE_READ_OFFSET_BYTES, MetricType::METRIC_TYPE_INT_GAUGE},
     };

core/unittest/processor/ProcessorParseApsaraNativeUnittest.cpp

@@ -1162,7 +1162,7 @@ void ProcessorParseApsaraNativeUnittest::TestProcessEventDiscardUnmatch() {
     // discard unmatch, so output is 0
     APSARA_TEST_EQUAL_FATAL(uint64_t(0), processorInstance.mOutEventsTotal->GetValue());
     // event group size is not 0
-    APSARA_TEST_NOT_EQUAL_FATAL(uint64_t(0), processorInstance.mOutEventGroupSizeBytes->GetValue());
+    APSARA_TEST_NOT_EQUAL_FATAL(uint64_t(0), processorInstance.mOutSizeBytes->GetValue());
     APSARA_TEST_EQUAL_FATAL(uint64_t(count), processor.mDiscardedEventsTotal->GetValue());
     APSARA_TEST_EQUAL_FATAL(uint64_t(count), processor.mOutFailedEventsTotal->GetValue());
 }

core/unittest/processor/ProcessorParseDelimiterNativeUnittest.cpp

@@ -2427,7 +2427,7 @@ void ProcessorParseDelimiterNativeUnittest::TestProcessEventDiscardUnmatch() {
     // discard unmatch, so output is 0
     APSARA_TEST_EQUAL_FATAL(uint64_t(0), processorInstance.mOutEventsTotal->GetValue());
     // event group size is not 0
-    APSARA_TEST_NOT_EQUAL_FATAL(uint64_t(0), processorInstance.mOutEventGroupSizeBytes->GetValue());
+    APSARA_TEST_NOT_EQUAL_FATAL(uint64_t(0), processorInstance.mOutSizeBytes->GetValue());
     APSARA_TEST_EQUAL_FATAL(uint64_t(count), processor.mDiscardedEventsTotal->GetValue());
     APSARA_TEST_EQUAL_FATAL(uint64_t(count), processor.mOutFailedEventsTotal->GetValue());
 }

core/unittest/processor/ProcessorParseJsonNativeUnittest.cpp

@@ -689,7 +689,7 @@ void ProcessorParseJsonNativeUnittest::TestProcessEventDiscardUnmatch() {
     // discard unmatch, so output is 0
     APSARA_TEST_EQUAL_FATAL(uint64_t(0), processorInstance.mOutEventsTotal->GetValue());
     // event group size is not 0
-    APSARA_TEST_NOT_EQUAL_FATAL(uint64_t(0), processorInstance.mOutEventGroupSizeBytes->GetValue());
+    APSARA_TEST_NOT_EQUAL_FATAL(uint64_t(0), processorInstance.mOutSizeBytes->GetValue());
     APSARA_TEST_EQUAL_FATAL(uint64_t(count), processor.mDiscardedEventsTotal->GetValue());
     APSARA_TEST_EQUAL_FATAL(uint64_t(count), processor.mOutFailedEventsTotal->GetValue());
 

core/unittest/processor/ProcessorParseRegexNativeUnittest.cpp

@@ -565,7 +565,7 @@ void ProcessorParseRegexNativeUnittest::TestProcessEventDiscardUnmatch() {
     // discard unmatch, so output is 0
     APSARA_TEST_EQUAL_FATAL(0, processorInstance.mOutEventsTotal->GetValue());
     // event group size is not 0
-    APSARA_TEST_NOT_EQUAL_FATAL(0, processorInstance.mOutEventGroupSizeBytes->GetValue());
+    APSARA_TEST_NOT_EQUAL_FATAL(0, processorInstance.mOutSizeBytes->GetValue());
     APSARA_TEST_EQUAL_FATAL(count, processor.mDiscardedEventsTotal->GetValue());
     APSARA_TEST_EQUAL_FATAL(count, processor.mOutFailedEventsTotal->GetValue());
     APSARA_TEST_EQUAL_FATAL(0, processor.mOutKeyNotFoundEventsTotal->GetValue());
@@ -653,7 +653,7 @@ void ProcessorParseRegexNativeUnittest::TestProcessEventKeyCountUnmatch() {
     // discard unmatch, so output is 0
     APSARA_TEST_EQUAL_FATAL(0, processorInstance.mOutEventsTotal->GetValue());
     // event group size is not 0
-    APSARA_TEST_NOT_EQUAL_FATAL(0, processorInstance.mOutEventGroupSizeBytes->GetValue());
+    APSARA_TEST_NOT_EQUAL_FATAL(0, processorInstance.mOutSizeBytes->GetValue());
     APSARA_TEST_EQUAL_FATAL(count, processor.mDiscardedEventsTotal->GetValue());
     // mKeyCountNotMatchErrorTotal should equal count
     APSARA_TEST_EQUAL_FATAL(0, processor.mOutFailedEventsTotal->GetValue());

core/unittest/processor/ProcessorParseTimestampNativeUnittest.cpp

@@ -609,7 +609,7 @@ void ProcessorParseTimestampNativeUnittest::TestProcessHistoryDiscard() {
     // discard history, so output is 0
     APSARA_TEST_EQUAL_FATAL(0UL, processorInstance.mOutEventsTotal->GetValue());
     // event group size is not 0
-    APSARA_TEST_NOT_EQUAL_FATAL(0UL, processorInstance.mOutEventGroupSizeBytes->GetValue());
+    APSARA_TEST_NOT_EQUAL_FATAL(0UL, processorInstance.mOutSizeBytes->GetValue());
     APSARA_TEST_EQUAL_FATAL(2UL, processor.mDiscardedEventsTotal->GetValue());
     APSARA_TEST_EQUAL_FATAL(0UL, processor.mOutFailedEventsTotal->GetValue());
 }

pkg/helper/self_metrics_constants.go

@@ -19,33 +19,17 @@ import (
 )
 
 //////////////////////////////////////////////////////////////////////////
-// labels
+// common
 //////////////////////////////////////////////////////////////////////////
 
-// common
+// label keys
 const MetricLabelProject = "project"
 
-// pipeline
-const (
-	MetricLabelLogstore     = "logstore"
-	MetricLabelRegion       = "region"
-	MetricLabelPipelineName = "pipeline_name"
-)
-
-// plugin
-const (
-	MetricLabelPluginType  = "plugin_type"
-	MetricLabelPluginID    = "plugin_id"
-	MetricLabelNodeID      = "node_id"
-	MetricLabelChildNodeID = "child_node_id"
-	// label for service_mysql/
-	MetricLabelDriver = "driver"
-)
-
 //////////////////////////////////////////////////////////////////////////
 // agent
 //////////////////////////////////////////////////////////////////////////
 
+// metric keys
 const (
 	MetricAgentCPUGo           = "agent_go_cpu_percent"
 	MetricAgentMemoryGo        = "agent_go_memory_used_mb"
@@ -56,26 +40,32 @@ const (
 // plugin
 //////////////////////////////////////////////////////////////////////////
 
-// common metrics
+// label keys
+const (
+	MetricLabelLogstore     = "logstore"
+	MetricLabelRegion       = "region"
+	MetricLabelPipelineName = "pipeline_name"
+	MetricLabelPluginType   = "plugin_type"
+	MetricLabelPluginID     = "plugin_id"
+	MetricLabelNodeID       = "node_id"
+	MetricLabelChildNodeID  = "child_node_id"
+)
+
+// metric keys
 const (
-	MetricPluginCostTimeMs              = "plugin_cost_time_ms"
-	MetricPluginInBufferTotal           = "plugin_in_buffer_total"
-	MetricPluginInBufferSizeBytes       = "plugin_in_buffer_size_bytes"
-	MetricPluginOutBufferTotal          = "plugin_out_buffer_total"
-	MetricPluginOutBufferSizeBytes      = "plugin_out_buffer_size_bytes"
-	MetricPluginInEventsTotal           = "plugin_in_events_total"
-	MetricPluginInEventsSizeBytes       = "plugin_in_events_size_bytes"
-	MetricPluginOutEventsTotal          = "plugin_out_events_total"
-	MetricPluginOutEventsSizeBytes      = "plugin_out_events_size_bytes"
-	MetricPluginInEventGroupsTotal      = "plugin_in_event_groups_total"
-	MetricPluginInEventGroupsSizeBytes  = "plugin_in_event_groups_size_bytes"
-	MetricPluginOutEventGroupsTotal     = "plugin_in_event_groups_total"
-	MetricPluginOutEventGroupsSizeBytes = "plugin_out_event_groups_size_bytes"
-	MetricPluginDiscardEventsTotal      = "plugin_discard_events_total"
-	MetricPluginErrorTotal              = "plugin_error_total"
+	MetricPluginInEventsTotal       = "plugin_in_events_total"
+	MetricPluginInEventGroupsTotal  = "plugin_in_event_groups_total"
+	MetricPluginInSizeBytes         = "plugin_in_size_bytes"
+	MetricPluginOutEventsTotal      = "plugin_out_events_total"
+	MetricPluginOutEventGroupsTotal = "plugin_out_event_groups_total"
+	MetricPluginOutSizeBytes        = "plugin_out_size_bytes"
+	MetricPluginTotalDelayTimeMs    = "plugin_total_delay_time_ms"
+	MetricPluginTotalProcessTimeMs  = "plugin_total_process_time_ms"
 )
 
-// input input_canal custom metrics
+/**********************************************************
+*   input_canal
+**********************************************************/
 const (
 	MetricPluginBinlogRotate     = "plugin_binlog_rotate"
 	MetricPluginBinlogSync       = "plugin_binlog_sync"
@@ -87,21 +77,42 @@ const (
 	MetricPluginBinlogGtid       = "plugin_binlog_gtid"
 )
 
-// input metric_container_info/service_docker_stdout_v2 custom metrics
+/**********************************************************
+*   metric_container_info
+*	service_docker_stdout_v2
+**********************************************************/
 const (
 	MetricPluginContainerTotal       = "plugin_container_total"
 	MetricPluginAddContainerTotal    = "plugin_add_container_total"
 	MetricPluginRemoveContainerTotal = "plugin_remove_container_total"
 	MetricPluginUpdateContainerTotal = "plugin_update_container_total"
 )
 
-// input service_mysql custom metrics
+/**********************************************************
+*   service_mysql
+**********************************************************/
 const (
+	MetricLabelDriver = "driver"
+
 	MetricPluginCollectAvgCostTimeMs = "plugin_collect_avg_cost_time_ms"
 	MetricPluginCollectTotal         = "plugin_collect_total"
 )
 
-// processor processor_anchor/processor_regex/processor_string_replace custom metrics
+/**********************************************************
+*   all processor（所有解析类的处理插件通用指标。Todo：目前统计还不全、不准确）
+**********************************************************/
+const (
+	MetricPluginDiscardedEventsTotal      = "plugin_discarded_events_total"
+	MetricPluginOutFailedEventsTotal      = "plugin_out_failed_events_total"
+	MetricPluginOutKeyNotFoundEventsTotal = "plugin_out_key_not_found_events_total"
+	MetricPluginOutSuccessfulEventsTotal  = "plugin_out_successful_events_total"
+)
+
+/**********************************************************
+*   processor_anchor
+*   processor_regex
+*   processor_string_replace
+**********************************************************/
 const (
 	PluginPairsPerLogTotal = "plugin_pairs_per_log_total"
 )