Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

rename self monitor metrics #1779

Merged
merged 14 commits into from
Sep 29, 2024
Merged

rename self monitor metrics #1779

Show file tree
Hide file tree
Changes from 6 commits
Commits
Show all changes
14 commits
Select commit Hold shift + click to select a range
91cb4b4
c++ part
Takuka0311 Sep 24, 2024
f18fa03
go part
Takuka0311 Sep 24, 2024
8dab792
fix go bug
Takuka0311 Sep 25, 2024
b62db8d
rename cpp metrics
Takuka0311 Sep 25, 2024
8f6e7fe
rename go metrics
Takuka0311 Sep 26, 2024
121b468
change parameters name
Takuka0311 Sep 26, 2024
2b781a5
delete unused metrics
Takuka0311 Sep 27, 2024
0f48b52
change name
Takuka0311 Sep 27, 2024
a451cfc
delete node id
Takuka0311 Sep 27, 2024
366f8ed
change name
Takuka0311 Sep 27, 2024
4a4ee6c
Merge remote-tracking branch 'origin/main' into rename-self-monitor-m…
Takuka0311 Sep 27, 2024
f6885bc
Merge remote-tracking branch 'origin/main' into rename-self-monitor-m…
Takuka0311 Sep 27, 2024
be3e26b
fix build
Takuka0311 Sep 27, 2024
9e965be
change name
Takuka0311 Sep 27, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions core/CMakeLists.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -116,6 +116,7 @@ set(SUB_DIRECTORIES_LIST
application app_config checkpoint container_manager logger go_pipeline monitor profile_sender models
config config/watcher
pipeline pipeline/batch pipeline/limiter pipeline/plugin pipeline/plugin/creator pipeline/plugin/instance pipeline/plugin/interface pipeline/queue pipeline/route pipeline/serializer
monitor/metric_constants
Takuka0311 marked this conversation as resolved.
Show resolved Hide resolved
runner runner/sink/http
protobuf/sls
file_server file_server/event file_server/event_handler file_server/event_listener file_server/reader file_server/polling
Expand Down
16 changes: 8 additions & 8 deletions core/common/compression/Compressor.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,31 +25,31 @@ namespace logtail {
void Compressor::SetMetricRecordRef(MetricLabels&& labels, DynamicMetricLabels&& dynamicLabels) {
WriteMetrics::GetInstance()->PrepareMetricsRecordRef(
mMetricsRecordRef, std::move(labels), std::move(dynamicLabels));
mInItemsCnt = mMetricsRecordRef.CreateCounter(METRIC_COMPONENT_IN_ITEMS_CNT);
mInItemsTotal = mMetricsRecordRef.CreateCounter(METRIC_COMPONENT_IN_ITEMS_TOTAL);
mInItemSizeBytes = mMetricsRecordRef.CreateCounter(METRIC_COMPONENT_IN_ITEM_SIZE_BYTES);
mOutItemsCnt = mMetricsRecordRef.CreateCounter(METRIC_COMPONENT_OUT_ITEMS_CNT);
mOutItemsTotal = mMetricsRecordRef.CreateCounter(METRIC_COMPONENT_OUT_ITEMS_TOTAL);
mOutItemSizeBytes = mMetricsRecordRef.CreateCounter(METRIC_COMPONENT_OUT_ITEM_SIZE_BYTES);
mTotalDelayMs = mMetricsRecordRef.CreateCounter(METRIC_COMPONENT_TOTAL_DELAY_MS);
mDiscardedItemsCnt = mMetricsRecordRef.CreateCounter(METRIC_COMPONENT_DISCARDED_ITEMS_CNT);
mTotalProcessMs = mMetricsRecordRef.CreateCounter(METRIC_COMPONENT_TOTAL_PROCESS_MS);
mDiscardedItemsTotal = mMetricsRecordRef.CreateCounter(METRIC_COMPONENT_DISCARDED_ITEMS_TOTAL);
mDiscardedItemSizeBytes = mMetricsRecordRef.CreateCounter(METRIC_COMPONENT_DISCARDED_ITEMS_SIZE_BYTES);
}

bool Compressor::DoCompress(const string& input, string& output, string& errorMsg) {
if (mMetricsRecordRef != nullptr) {
mInItemsCnt->Add(1);
mInItemsTotal->Add(1);
mInItemSizeBytes->Add(input.size());
}

auto before = chrono::system_clock::now();
auto res = Compress(input, output, errorMsg);

if (mMetricsRecordRef != nullptr) {
mTotalDelayMs->Add(chrono::duration_cast<chrono::milliseconds>(chrono::system_clock::now() - before).count());
mTotalProcessMs->Add(chrono::duration_cast<chrono::milliseconds>(chrono::system_clock::now() - before).count());
if (res) {
mOutItemsCnt->Add(1);
mOutItemsTotal->Add(1);
mOutItemSizeBytes->Add(output.size());
} else {
mDiscardedItemsCnt->Add(1);
mDiscardedItemsTotal->Add(1);
mDiscardedItemSizeBytes->Add(input.size());
}
}
Expand Down
8 changes: 4 additions & 4 deletions core/common/compression/Compressor.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -40,13 +40,13 @@ class Compressor {

protected:
mutable MetricsRecordRef mMetricsRecordRef;
CounterPtr mInItemsCnt;
CounterPtr mInItemsTotal;
CounterPtr mInItemSizeBytes;
CounterPtr mOutItemsCnt;
CounterPtr mOutItemsTotal;
CounterPtr mOutItemSizeBytes;
CounterPtr mDiscardedItemsCnt;
CounterPtr mDiscardedItemsTotal;
CounterPtr mDiscardedItemSizeBytes;
CounterPtr mTotalDelayMs;
CounterPtr mTotalProcessMs;

private:
virtual bool Compress(const std::string& input, std::string& output, std::string& errorMsg) = 0;
Expand Down
4 changes: 2 additions & 2 deletions core/common/compression/CompressorFactory.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -62,8 +62,8 @@ unique_ptr<Compressor> CompressorFactory::Create(const Json::Value& config,
compressor = Create(defaultType);
}
compressor->SetMetricRecordRef({{METRIC_LABEL_PROJECT, ctx.GetProjectName()},
{METRIC_LABEL_CONFIG_NAME, ctx.GetConfigName()},
{METRIC_LABEL_KEY_COMPONENT_NAME, "compressor"},
{METRIC_LABEL_PIPELINE_NAME, ctx.GetConfigName()},
{METRIC_LABEL_KEY_COMPONENT_NAME, METRIC_LABEL_VALUE_COMPONENT_NAME_COMPRESSOR},
{METRIC_LABEL_KEY_FLUSHER_NODE_ID, flusherId}});
return compressor;
}
Expand Down
2 changes: 1 addition & 1 deletion core/file_server/FileServer.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,7 @@ namespace logtail {

FileServer::FileServer() {
WriteMetrics::GetInstance()->PrepareMetricsRecordRef(mMetricsRecordRef,
{{METRIC_LABEL_KEY_RUNNER_NAME, "file_server"}});
{{METRIC_LABEL_KEY_RUNNER_NAME, METRIC_LABEL_VALUE_RUNNER_NAME_FILE_SERVER}});
}

// 启动文件服务，包括加载配置、处理检查点、注册事件等
Expand Down
8 changes: 4 additions & 4 deletions core/file_server/event_handler/LogInput.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -90,8 +90,8 @@ void LogInput::Start() {

mLastRunTime = FileServer::GetInstance()->GetMetricsRecordRef().CreateIntGauge(METRIC_RUNNER_LAST_RUN_TIME);
mAgentOpenFdTotal = LoongCollectorMonitor::GetInstance()->GetIntGauge(METRIC_AGENT_OPEN_FD_TOTAL);
mRegisterdHandlersCnt = FileServer::GetInstance()->GetMetricsRecordRef().CreateIntGauge(METRIC_RUNNER_FILE_WATCHED_DIRS_CNT);
mActiveReadersCnt = FileServer::GetInstance()->GetMetricsRecordRef().CreateIntGauge(METRIC_RUNNER_FILE_ACTIVE_READERS_CNT);
mRegisterdHandlersTotal = FileServer::GetInstance()->GetMetricsRecordRef().CreateIntGauge(METRIC_RUNNER_FILE_WATCHED_DIRS_TOTAL);
mActiveReadersTotal = FileServer::GetInstance()->GetMetricsRecordRef().CreateIntGauge(METRIC_RUNNER_FILE_ACTIVE_READERS_TOTAL);
mEnableFileIncludedByMultiConfigs = FileServer::GetInstance()->GetMetricsRecordRef().CreateIntGauge(METRIC_RUNNER_FILE_ENABLE_FILE_INCLUDED_BY_MULTI_CONFIGS_FLAG);

new Thread([this]() { ProcessLoop(); });
Expand Down Expand Up @@ -353,9 +353,9 @@ void LogInput::UpdateCriticalMetric(int32_t curTime) {
mAgentOpenFdTotal->Set(openFdTotal);
size_t handlerCount = EventDispatcher::GetInstance()->GetHandlerCount();
LogtailMonitor::GetInstance()->UpdateMetric("register_handler", handlerCount);
mRegisterdHandlersCnt->Set(handlerCount);
mRegisterdHandlersTotal->Set(handlerCount);
LogtailMonitor::GetInstance()->UpdateMetric("reader_count", CheckPointManager::Instance()->GetReaderCount());
mActiveReadersCnt->Set(CheckPointManager::Instance()->GetReaderCount());
mActiveReadersTotal->Set(CheckPointManager::Instance()->GetReaderCount());
LogtailMonitor::GetInstance()->UpdateMetric("multi_config", AppConfig::GetInstance()->IsAcceptMultiConfig());
mEventProcessCount = 0;
}
Expand Down
4 changes: 2 additions & 2 deletions core/file_server/event_handler/LogInput.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -82,8 +82,8 @@ class LogInput : public LogRunnable {

IntGaugePtr mLastRunTime;
IntGaugePtr mAgentOpenFdTotal;
IntGaugePtr mRegisterdHandlersCnt;
IntGaugePtr mActiveReadersCnt;
IntGaugePtr mRegisterdHandlersTotal;
IntGaugePtr mActiveReadersTotal;
IntGaugePtr mEnableFileIncludedByMultiConfigs;

std::atomic_int mLastReadEventTime{0};
Expand Down
18 changes: 10 additions & 8 deletions core/file_server/reader/LogFileReader.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -213,10 +213,11 @@ void LogFileReader::SetMetrics() {
return;
}

mInputRecordsSizeBytesCounter = mMetricsRecordRef->GetCounter(METRIC_INPUT_RECORDS_SIZE_BYTES);
mInputReadTotalCounter = mMetricsRecordRef->GetCounter(METRIC_INPUT_READ_TOTAL);
mInputFileSizeBytesGauge = mMetricsRecordRef->GetIntGauge(METRIC_INPUT_FILE_SIZE_BYTES);
mInputFileOffsetBytesGauge = mMetricsRecordRef->GetIntGauge(METRIC_INPUT_FILE_OFFSET_BYTES);
mOutEventsTotal = mMetricsRecordRef->GetCounter(METRIC_PLUGIN_OUT_EVENTS_TOTAL);
mOutEventGroupsTotal = mMetricsRecordRef->GetCounter(METRIC_PLUGIN_OUT_EVENT_GROUPS_TOTAL);
mOutSizeBytes = mMetricsRecordRef->GetCounter(METRIC_PLUGIN_OUT_SIZE_BYTES);
mSourceSizeBytes = mMetricsRecordRef->GetIntGauge(METRIC_PLUGIN_SOURCE_SIZE_BYTES);
mSourceReadOffsetBytes = mMetricsRecordRef->GetIntGauge(METRIC_PLUGIN_SOURCE_READ_OFFSET_BYTES);
mMetricInited = true;
}

Expand Down Expand Up @@ -2133,10 +2134,11 @@ std::unique_ptr<Event> LogFileReader::CreateFlushTimeoutEvent() {

void LogFileReader::ReportMetrics(uint64_t readSize) {
if (mMetricInited) {
mInputReadTotalCounter->Add(1);
mInputRecordsSizeBytesCounter->Add(readSize);
mInputFileOffsetBytesGauge->Set(GetLastFilePos());
mInputFileSizeBytesGauge->Set(GetFileSize());
mOutEventsTotal->Add(1);
mOutEventGroupsTotal->Add(1);
mOutSizeBytes->Add(readSize);
mSourceReadOffsetBytes->Set(GetLastFilePos());
mSourceSizeBytes->Set(GetFileSize());
}
}

Expand Down
9 changes: 5 additions & 4 deletions core/file_server/reader/LogFileReader.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -537,10 +537,11 @@ class LogFileReader {
MetricLabels mMetricLabels;
bool mMetricInited;
ReentrantMetricsRecordRef mMetricsRecordRef;
CounterPtr mInputRecordsSizeBytesCounter;
CounterPtr mInputReadTotalCounter;
IntGaugePtr mInputFileSizeBytesGauge;
IntGaugePtr mInputFileOffsetBytesGauge;
CounterPtr mOutEventsTotal;
CounterPtr mOutEventGroupsTotal;
CounterPtr mOutSizeBytes;
IntGaugePtr mSourceSizeBytes;
IntGaugePtr mSourceReadOffsetBytes;

private:
bool mHasReadContainerBom = false;
Expand Down
Loading
Loading