From 67a0121b66d9d3998f59c6e42a4315ed630b3db2 Mon Sep 17 00:00:00 2001 From: henryzhx8 Date: Sat, 14 Sep 2024 16:30:40 +0800 Subject: [PATCH 1/2] add metric for queue (#1756) --- core/file_server/reader/LogFileReader.cpp | 2 +- core/monitor/LogtailMetric.cpp | 31 ++++- core/monitor/LogtailMetric.h | 21 +++- core/monitor/LoongCollectorMetricTypes.h | 31 +++-- core/monitor/MetricConstants.cpp | 5 + core/monitor/MetricConstants.h | 5 + core/pipeline/Pipeline.cpp | 13 +- .../plugin/instance/FlusherInstance.cpp | 1 + core/pipeline/plugin/interface/Flusher.h | 3 + core/pipeline/queue/BoundedProcessQueue.cpp | 28 ++++- core/pipeline/queue/BoundedProcessQueue.h | 5 +- core/pipeline/queue/BoundedQueueInterface.h | 9 +- .../queue/BoundedSenderQueueInterface.cpp | 9 ++ .../queue/BoundedSenderQueueInterface.h | 11 +- core/pipeline/queue/CircularProcessQueue.cpp | 32 ++++- core/pipeline/queue/CircularProcessQueue.h | 6 +- .../queue/ExactlyOnceQueueManager.cpp | 8 +- core/pipeline/queue/ExactlyOnceQueueManager.h | 2 +- .../pipeline/queue/ExactlyOnceSenderQueue.cpp | 20 ++- core/pipeline/queue/ExactlyOnceSenderQueue.h | 8 +- core/pipeline/queue/ProcessQueueInterface.cpp | 5 + core/pipeline/queue/ProcessQueueInterface.h | 7 +- core/pipeline/queue/ProcessQueueItem.h | 2 + core/pipeline/queue/ProcessQueueManager.cpp | 38 +++--- core/pipeline/queue/ProcessQueueManager.h | 8 +- core/pipeline/queue/QueueInterface.h | 26 +++- core/pipeline/queue/SenderQueue.cpp | 41 +++++- core/pipeline/queue/SenderQueue.h | 8 +- core/pipeline/queue/SenderQueueItem.h | 4 +- core/pipeline/queue/SenderQueueManager.cpp | 11 +- core/pipeline/queue/SenderQueueManager.h | 6 +- .../flusher/blackhole/FlusherBlackHole.cpp | 2 +- core/plugin/flusher/sls/FlusherSLS.cpp | 39 +++--- core/runner/FlusherRunner.cpp | 10 +- .../event_handler/ModifyHandlerUnittest.cpp | 6 +- core/unittest/flusher/FlusherSLSUnittest.cpp | 2 +- core/unittest/plugin/PluginMock.h | 4 +- .../queue/BoundedProcessQueueUnittest.cpp | 45 +++++-- .../queue/CircularProcessQueueUnittest.cpp | 43 ++++++- .../queue/ExactlyOnceQueueManagerUnittest.cpp | 48 +++---- .../queue/ExactlyOnceSenderQueueUnittest.cpp | 4 +- .../queue/ProcessQueueManagerUnittest.cpp | 118 ++++++++++-------- .../queue/SenderQueueManagerUnittest.cpp | 45 ++++--- core/unittest/queue/SenderQueueUnittest.cpp | 61 ++++++++- core/unittest/reader/ForceReadUnittest.cpp | 2 +- .../unittest/sender/FlusherRunnerUnittest.cpp | 6 + 46 files changed, 616 insertions(+), 225 deletions(-) diff --git a/core/file_server/reader/LogFileReader.cpp b/core/file_server/reader/LogFileReader.cpp index 387c274c5f..499fe63658 100644 --- a/core/file_server/reader/LogFileReader.cpp +++ b/core/file_server/reader/LogFileReader.cpp @@ -445,7 +445,7 @@ void LogFileReader::initExactlyOnce(uint32_t concurrency) { mEOOption->fbKey = QueueKeyManager::GetInstance()->GetKey(GetProject() + "-" + mEOOption->primaryCheckpointKey + mEOOption->rangeCheckpointPtrs[0]->data.hash_key()); ExactlyOnceQueueManager::GetInstance()->CreateOrUpdateQueue( - mEOOption->fbKey, ProcessQueueManager::sMaxPriority, mConfigName, mEOOption->rangeCheckpointPtrs); + mEOOption->fbKey, ProcessQueueManager::sMaxPriority, *mReaderConfig.second, mEOOption->rangeCheckpointPtrs); for (auto& cpt : mEOOption->rangeCheckpointPtrs) { cpt->fbKey = mEOOption->fbKey; } diff --git a/core/monitor/LogtailMetric.cpp b/core/monitor/LogtailMetric.cpp index b0211b32a6..d6ddc5e768 100644 --- a/core/monitor/LogtailMetric.cpp +++ b/core/monitor/LogtailMetric.cpp @@ -36,7 +36,7 @@ CounterPtr MetricsRecord::CreateCounter(const std::string& name) { } IntGaugePtr MetricsRecord::CreateIntGauge(const std::string& name) { - IntGaugePtr gaugePtr = std::make_shared>(name); + IntGaugePtr gaugePtr = std::make_shared(name); mIntGauges.emplace_back(gaugePtr); return gaugePtr; } @@ -134,6 +134,21 @@ const MetricsRecord* MetricsRecordRef::operator->() const { return mMetrics; } +void MetricsRecordRef::AddLabels(MetricLabels&& labels) { + mMetrics->GetLabels()->insert(mMetrics->GetLabels()->end(), labels.begin(), labels.end()); +} + +#ifdef APSARA_UNIT_TEST_MAIN +bool MetricsRecordRef::HasLabel(const std::string& key, const std::string& value) const { + for (auto item : *(mMetrics->GetLabels())) { + if (item.first == key && item.second == value) { + return true; + } + } + return false; +} +#endif + // ReentrantMetricsRecord相关操作可以无锁，因为mCounters、mGauges只在初始化时会添加内容，后续只允许Get操作 void ReentrantMetricsRecord::Init(MetricLabels& labels, std::unordered_map& metricKeys) { WriteMetrics::GetInstance()->PrepareMetricsRecordRef(mMetricsRecordRef, std::move(labels)); @@ -220,6 +235,20 @@ void WriteMetrics::PrepareMetricsRecordRef(MetricsRecordRef& ref, mHead = cur; } +void WriteMetrics::CreateMetricsRecordRef(MetricsRecordRef& ref, + MetricLabels&& labels, + DynamicMetricLabels&& dynamicLabels) { + MetricsRecord* cur = new MetricsRecord(std::make_shared(labels), + std::make_shared(dynamicLabels)); + ref.SetMetricsRecord(cur); +} + +void WriteMetrics::CommitMetricsRecordRef(MetricsRecordRef& ref) { + std::lock_guard lock(mMutex); + ref.mMetrics->SetNext(mHead); + mHead = ref.mMetrics; +} + MetricsRecord* WriteMetrics::GetHead() { std::lock_guard lock(mMutex); return mHead; diff --git a/core/monitor/LogtailMetric.h b/core/monitor/LogtailMetric.h index 7c3d6abe98..d6d76666ed 100644 --- a/core/monitor/LogtailMetric.h +++ b/core/monitor/LogtailMetric.h @@ -15,11 +15,15 @@ */ #pragma once + #include +#include #include +#include +#include -#include "LoongCollectorMetricTypes.h" #include "common/Lock.h" +#include "monitor/LoongCollectorMetricTypes.h" #include "protobuf/sls/sls_logs.pb.h" namespace logtail { @@ -53,6 +57,8 @@ class MetricsRecord { }; class MetricsRecordRef { + friend class WriteMetrics; + private: MetricsRecord* mMetrics = nullptr; @@ -70,6 +76,11 @@ class MetricsRecordRef { IntGaugePtr CreateIntGauge(const std::string& name); DoubleGaugePtr CreateDoubleGauge(const std::string& name); const MetricsRecord* operator->() const; + // this is not thread-safe, and should be only used before WriteMetrics::CommitMetricsRecordRef + void AddLabels(MetricLabels&& labels); +#ifdef APSARA_UNIT_TEST_MAIN + bool HasLabel(const std::string& key, const std::string& value) const; +#endif }; class ReentrantMetricsRecord { @@ -110,10 +121,13 @@ class WriteMetrics { const std::string& configName, const std::string& pluginType, const std::string& pluginID, - const std::string& nodeID, + const std::string& nodeID, const std::string& childNodeID, MetricLabels& labels); - void PrepareMetricsRecordRef(MetricsRecordRef& ref, MetricLabels&& labels, DynamicMetricLabels&& dynamicLabels = {}); + void + PrepareMetricsRecordRef(MetricsRecordRef& ref, MetricLabels&& labels, DynamicMetricLabels&& dynamicLabels = {}); + void CreateMetricsRecordRef(MetricsRecordRef& ref, MetricLabels&& labels, DynamicMetricLabels&& dynamicLabels = {}); + void CommitMetricsRecordRef(MetricsRecordRef& ref); MetricsRecord* DoSnapshot(); @@ -144,4 +158,5 @@ class ReadMetrics { friend class ILogtailMetricUnittest; #endif }; + } // namespace logtail diff --git a/core/monitor/LoongCollectorMetricTypes.h b/core/monitor/LoongCollectorMetricTypes.h index e815946ce8..8a94396dde 100644 --- a/core/monitor/LoongCollectorMetricTypes.h +++ b/core/monitor/LoongCollectorMetricTypes.h @@ -15,13 +15,16 @@ */ #pragma once + #include +#include +#include +#include #include +#include -#include "common/Lock.h" #include "protobuf/sls/sls_logs.pb.h" - namespace logtail { enum class MetricType { @@ -45,20 +48,32 @@ class Counter { template class Gauge { -private: - std::string mName; - std::atomic mVal; - public: Gauge(const std::string& name, T val = 0) : mName(name), mVal(val) {} + ~Gauge() = default; + T GetValue() const { return mVal.load(); } const std::string& GetName() const { return mName; } void Set(T val) { mVal.store(val); } Gauge* Collect() { return new Gauge(mName, mVal.load()); } + +protected: + std::string mName; + std::atomic mVal; +}; + +class IntGauge : public Gauge { +public: + IntGauge(const std::string& name, uint64_t val = 0) : Gauge(name, val) {} + ~IntGauge() = default; + + IntGauge* Collect() { return new IntGauge(mName, mVal.load()); } + void Add(uint64_t val) { mVal.fetch_add(val); } + void Sub(uint64_t val) { mVal.fetch_sub(val); } }; using CounterPtr = std::shared_ptr; -using IntGaugePtr = std::shared_ptr>; +using IntGaugePtr = std::shared_ptr; using DoubleGaugePtr = std::shared_ptr>; using MetricLabels = std::vector>; @@ -66,4 +81,4 @@ using MetricLabelsPtr = std::shared_ptr; using DynamicMetricLabels = std::vector>>; using DynamicMetricLabelsPtr = std::shared_ptr; -} // namespace logtail \ No newline at end of file +} // namespace logtail diff --git a/core/monitor/MetricConstants.cpp b/core/monitor/MetricConstants.cpp index 8284b8406a..52c8d1158b 100644 --- a/core/monitor/MetricConstants.cpp +++ b/core/monitor/MetricConstants.cpp @@ -71,6 +71,11 @@ const std::string METRIC_LABEL_PLUGIN_ID = "plugin_id"; const std::string METRIC_LABEL_NODE_ID = "node_id"; const std::string METRIC_LABEL_CHILD_NODE_ID = "child_node_id"; +const std::string METRIC_LABEL_KEY_COMPONENT_NAME = "component_name"; +const std::string METRIC_LABEL_KEY_QUEUE_TYPE = "queue_type"; +const std::string METRIC_LABEL_KEY_EXACTLY_ONCE_FLAG = "is_exactly_once"; +const std::string METRIC_LABEL_KEY_FLUSHER_NODE_ID = "flusher_node_id"; + // input file plugin labels const std::string METRIC_LABEL_FILE_DEV = "file_dev"; const std::string METRIC_LABEL_FILE_INODE = "file_inode"; diff --git a/core/monitor/MetricConstants.h b/core/monitor/MetricConstants.h index 32c6adb601..27f3a762e3 100644 --- a/core/monitor/MetricConstants.h +++ b/core/monitor/MetricConstants.h @@ -78,6 +78,11 @@ extern const std::string METRIC_LABEL_FILE_DEV; extern const std::string METRIC_LABEL_FILE_INODE; extern const std::string METRIC_LABEL_FILE_NAME; +extern const std::string METRIC_LABEL_KEY_COMPONENT_NAME; +extern const std::string METRIC_LABEL_KEY_QUEUE_TYPE; +extern const std::string METRIC_LABEL_KEY_EXACTLY_ONCE_FLAG; +extern const std::string METRIC_LABEL_KEY_FLUSHER_NODE_ID; + // input file metrics extern const std::string METRIC_INPUT_RECORDS_TOTAL; extern const std::string METRIC_INPUT_RECORDS_SIZE_BYTES; diff --git a/core/pipeline/Pipeline.cpp b/core/pipeline/Pipeline.cpp index a36f743e3c..511b642b96 100644 --- a/core/pipeline/Pipeline.cpp +++ b/core/pipeline/Pipeline.cpp @@ -19,17 +19,17 @@ #include #include -#include "pipeline/batch/TimeoutFlushManager.h" #include "common/Flags.h" #include "common/ParamExtractor.h" -#include "plugin/flusher/sls/FlusherSLS.h" #include "go_pipeline/LogtailPlugin.h" -#include "plugin/input/InputFeedbackInterfaceRegistry.h" +#include "pipeline/batch/TimeoutFlushManager.h" #include "pipeline/plugin/PluginRegistry.h" -#include "plugin/processor/ProcessorParseApsaraNative.h" #include "pipeline/queue/ProcessQueueManager.h" #include "pipeline/queue/QueueKeyManager.h" #include "pipeline/queue/SenderQueueManager.h" +#include "plugin/flusher/sls/FlusherSLS.h" +#include "plugin/input/InputFeedbackInterfaceRegistry.h" +#include "plugin/processor/ProcessorParseApsaraNative.h" DECLARE_FLAG_INT32(default_plugin_log_queue_size); @@ -288,10 +288,11 @@ bool Pipeline::Init(PipelineConfig&& config) { ? ProcessQueueManager::sMaxPriority : mContext.GetGlobalConfig().mProcessPriority - 1; if (isInputSupportAck) { - ProcessQueueManager::GetInstance()->CreateOrUpdateBoundedQueue(mContext.GetProcessQueueKey(), priority); + ProcessQueueManager::GetInstance()->CreateOrUpdateBoundedQueue( + mContext.GetProcessQueueKey(), priority, mContext); } else { ProcessQueueManager::GetInstance()->CreateOrUpdateCircularQueue( - mContext.GetProcessQueueKey(), priority, 1024); + mContext.GetProcessQueueKey(), priority, 1024, mContext); } diff --git a/core/pipeline/plugin/instance/FlusherInstance.cpp b/core/pipeline/plugin/instance/FlusherInstance.cpp index a48b3a7f85..9623a96b75 100644 --- a/core/pipeline/plugin/instance/FlusherInstance.cpp +++ b/core/pipeline/plugin/instance/FlusherInstance.cpp @@ -19,6 +19,7 @@ namespace logtail { bool FlusherInstance::Init(const Json::Value& config, PipelineContext& context, Json::Value& optionalGoPipeline) { mPlugin->SetContext(context); + mPlugin->SetNodeID(NodeID()); mPlugin->SetMetricsRecordRef(Name(), PluginID(), NodeID(), ChildNodeID()); if (!mPlugin->Init(config, optionalGoPipeline)) { return false; diff --git a/core/pipeline/plugin/interface/Flusher.h b/core/pipeline/plugin/interface/Flusher.h index 5df95b28b8..d643559611 100644 --- a/core/pipeline/plugin/interface/Flusher.h +++ b/core/pipeline/plugin/interface/Flusher.h @@ -43,6 +43,8 @@ class Flusher : public Plugin { virtual SinkType GetSinkType() { return SinkType::NONE; } QueueKey GetQueueKey() const { return mQueueKey; } + void SetNodeID(const std::string& nodeID) { mNodeID = nodeID; } + const std::string& GetNodeID() const { return mNodeID; } protected: void GenerateQueueKey(const std::string& target); @@ -50,6 +52,7 @@ class Flusher : public Plugin { void DealSenderQueueItemAfterSend(SenderQueueItem* item, bool keep); QueueKey mQueueKey; + std::string mNodeID; #ifdef APSARA_UNIT_TEST_MAIN friend class FlusherInstanceUnittest; diff --git a/core/pipeline/queue/BoundedProcessQueue.cpp b/core/pipeline/queue/BoundedProcessQueue.cpp index 470d134476..ced855e43e 100644 --- a/core/pipeline/queue/BoundedProcessQueue.cpp +++ b/core/pipeline/queue/BoundedProcessQueue.cpp @@ -18,12 +18,31 @@ using namespace std; namespace logtail { +BoundedProcessQueue::BoundedProcessQueue( + size_t cap, size_t low, size_t high, int64_t key, uint32_t priority, const PipelineContext& ctx) + : QueueInterface(key, cap, ctx), + BoundedQueueInterface(key, cap, low, high, ctx), + ProcessQueueInterface(key, cap, priority, ctx) { + if (ctx.IsExactlyOnceEnabled()) { + mMetricsRecordRef.AddLabels({{METRIC_LABEL_KEY_EXACTLY_ONCE_FLAG, "true"}}); + } + WriteMetrics::GetInstance()->CommitMetricsRecordRef(mMetricsRecordRef); +} + bool BoundedProcessQueue::Push(unique_ptr&& item) { if (!IsValidToPush()) { return false; } + item->mEnqueTime = chrono::system_clock::now(); + auto size = item->mEventGroup.DataSize(); mQueue.push(std::move(item)); ChangeStateIfNeededAfterPush(); + + mInItemsCnt->Add(1); + mInItemDataSizeBytes->Add(size); + mQueueSize->Set(Size()); + mQueueDataSizeByte->Add(size); + mValidToPushFlag->Set(IsValidToPush()); return true; } @@ -36,10 +55,17 @@ bool BoundedProcessQueue::Pop(unique_ptr& item) { if (ChangeStateIfNeededAfterPop()) { GiveFeedback(); } + + mOutItemsCnt->Add(1); + mTotalDelayMs->Add( + chrono::duration_cast(chrono::system_clock::now() - item->mEnqueTime).count()); + mQueueSize->Set(Size()); + mQueueDataSizeByte->Sub(item->mEventGroup.DataSize()); + mValidToPushFlag->Set(IsValidToPush()); return true; } -void BoundedProcessQueue::SetUpStreamFeedbacks(std::vector&& feedbacks) { +void BoundedProcessQueue::SetUpStreamFeedbacks(vector&& feedbacks) { mUpStreamFeedbacks.clear(); for (auto& item : feedbacks) { if (item == nullptr) { diff --git a/core/pipeline/queue/BoundedProcessQueue.h b/core/pipeline/queue/BoundedProcessQueue.h index 8b65c17193..78c7772fd0 100644 --- a/core/pipeline/queue/BoundedProcessQueue.h +++ b/core/pipeline/queue/BoundedProcessQueue.h @@ -31,10 +31,7 @@ namespace logtail { class BoundedProcessQueue : public BoundedQueueInterface>, public ProcessQueueInterface { public: - BoundedProcessQueue(size_t cap, size_t low, size_t high, int64_t key, uint32_t priority, const std::string& config) - : QueueInterface(key, cap), - BoundedQueueInterface(key, cap, low, high), - ProcessQueueInterface(key, cap, priority, config) {} + BoundedProcessQueue(size_t cap, size_t low, size_t high, int64_t key, uint32_t priority, const PipelineContext& ctx); bool Push(std::unique_ptr&& item) override; bool Pop(std::unique_ptr& item) override; diff --git a/core/pipeline/queue/BoundedQueueInterface.h b/core/pipeline/queue/BoundedQueueInterface.h index 7b2c0439a5..350e22be37 100644 --- a/core/pipeline/queue/BoundedQueueInterface.h +++ b/core/pipeline/queue/BoundedQueueInterface.h @@ -23,8 +23,11 @@ namespace logtail { template class BoundedQueueInterface : virtual public QueueInterface { public: - BoundedQueueInterface(QueueKey key, size_t cap, size_t low, size_t high) - : QueueInterface(key, cap), mLowWatermark(low), mHighWatermark(high) {} + BoundedQueueInterface(QueueKey key, size_t cap, size_t low, size_t high, const PipelineContext& ctx) + : QueueInterface(key, cap, ctx), mLowWatermark(low), mHighWatermark(high) { + this->mMetricsRecordRef.AddLabels({{METRIC_LABEL_KEY_QUEUE_TYPE, "bounded"}}); + mValidToPushFlag = this->mMetricsRecordRef.CreateIntGauge("valid_to_push"); + } virtual ~BoundedQueueInterface() = default; BoundedQueueInterface(const BoundedQueueInterface& que) = delete; @@ -57,6 +60,8 @@ class BoundedQueueInterface : virtual public QueueInterface { mValidToPush = true; } + IntGaugePtr mValidToPushFlag; + private: virtual void GiveFeedback() const = 0; virtual size_t Size() const = 0; diff --git a/core/pipeline/queue/BoundedSenderQueueInterface.cpp b/core/pipeline/queue/BoundedSenderQueueInterface.cpp index 85bb8b91a8..03e26ebac3 100644 --- a/core/pipeline/queue/BoundedSenderQueueInterface.cpp +++ b/core/pipeline/queue/BoundedSenderQueueInterface.cpp @@ -21,6 +21,15 @@ namespace logtail { FeedbackInterface* BoundedSenderQueueInterface::sFeedback = nullptr; +BoundedSenderQueueInterface::BoundedSenderQueueInterface( + size_t cap, size_t low, size_t high, QueueKey key, const string& flusherId, const PipelineContext& ctx) + : QueueInterface(key, cap, ctx), BoundedQueueInterface>(key, cap, low, high, ctx) { + mMetricsRecordRef.AddLabels({{METRIC_LABEL_KEY_COMPONENT_NAME, "sender_queue"}}); + mMetricsRecordRef.AddLabels({{METRIC_LABEL_KEY_FLUSHER_NODE_ID, flusherId}}); + mExtraBufferSize = mMetricsRecordRef.CreateIntGauge("extra_buffer_size"); + mExtraBufferDataSizeBytes = mMetricsRecordRef.CreateIntGauge("extra_buffer_data_size_bytes"); +} + void BoundedSenderQueueInterface::SetFeedback(FeedbackInterface* feedback) { if (feedback == nullptr) { // should not happen diff --git a/core/pipeline/queue/BoundedSenderQueueInterface.h b/core/pipeline/queue/BoundedSenderQueueInterface.h index 526f7f93f9..5828790ae4 100644 --- a/core/pipeline/queue/BoundedSenderQueueInterface.h +++ b/core/pipeline/queue/BoundedSenderQueueInterface.h @@ -22,11 +22,11 @@ #include #include "common/FeedbackInterface.h" +#include "pipeline/limiter/ConcurrencyLimiter.h" +#include "pipeline/limiter/RateLimiter.h" #include "pipeline/queue/BoundedQueueInterface.h" #include "pipeline/queue/QueueKey.h" #include "pipeline/queue/SenderQueueItem.h" -#include "pipeline/limiter/ConcurrencyLimiter.h" -#include "pipeline/limiter/RateLimiter.h" namespace logtail { @@ -37,8 +37,8 @@ class BoundedSenderQueueInterface : public BoundedQueueInterface>(key, cap, low, high) {} + BoundedSenderQueueInterface( + size_t cap, size_t low, size_t high, QueueKey key, const std::string& flusherId, const PipelineContext& ctx); bool Pop(std::unique_ptr& item) override { return false; } @@ -63,6 +63,9 @@ class BoundedSenderQueueInterface : public BoundedQueueInterface> mConcurrencyLimiters; std::queue> mExtraBuffer; + + IntGaugePtr mExtraBufferSize; + IntGaugePtr mExtraBufferDataSizeBytes; }; } // namespace logtail diff --git a/core/pipeline/queue/CircularProcessQueue.cpp b/core/pipeline/queue/CircularProcessQueue.cpp index 5962bc6c05..c6df833e7f 100644 --- a/core/pipeline/queue/CircularProcessQueue.cpp +++ b/core/pipeline/queue/CircularProcessQueue.cpp @@ -21,17 +21,36 @@ using namespace std; namespace logtail { +CircularProcessQueue::CircularProcessQueue(size_t cap, int64_t key, uint32_t priority, const PipelineContext& ctx) + : QueueInterface>(key, cap, ctx), ProcessQueueInterface(key, cap, priority, ctx) { + mMetricsRecordRef.AddLabels({{METRIC_LABEL_KEY_QUEUE_TYPE, "circular"}}); + mDroppedEventsCnt = mMetricsRecordRef.CreateCounter("dropped_events_cnt"); + WriteMetrics::GetInstance()->CommitMetricsRecordRef(mMetricsRecordRef); +} + bool CircularProcessQueue::Push(unique_ptr&& item) { size_t newCnt = item->mEventGroup.GetEvents().size(); while (!mQueue.empty() && mEventCnt + newCnt > mCapacity) { - mEventCnt -= mQueue.front()->mEventGroup.GetEvents().size(); + auto cnt = mQueue.front()->mEventGroup.GetEvents().size(); + auto size = mQueue.front()->mEventGroup.DataSize(); + mEventCnt -= cnt; mQueue.pop_front(); + mQueueSize->Set(Size()); + mQueueDataSizeByte->Sub(size); + mDroppedEventsCnt->Add(cnt); } if (mEventCnt + newCnt > mCapacity) { return false; } + item->mEnqueTime = chrono::system_clock::now(); + auto size = item->mEventGroup.DataSize(); mQueue.push_back(std::move(item)); mEventCnt += newCnt; + + mInItemsCnt->Add(1); + mInItemDataSizeBytes->Add(size); + mQueueSize->Set(Size()); + mQueueDataSizeByte->Add(size); return true; } @@ -42,12 +61,19 @@ bool CircularProcessQueue::Pop(unique_ptr& item) { item = std::move(mQueue.front()); mQueue.pop_front(); mEventCnt -= item->mEventGroup.GetEvents().size(); + + mOutItemsCnt->Add(1); + mTotalDelayMs->Add( + std::chrono::duration_cast(std::chrono::system_clock::now() - item->mEnqueTime) + .count()); + mQueueSize->Set(Size()); + mQueueDataSizeByte->Sub(item->mEventGroup.DataSize()); return true; } void CircularProcessQueue::Reset(size_t cap) { - // it seems more reasonable to retain extra items and process them immediately, however this contray to current framework design - // so we simply discard extra items, considering that it is a rare case to change capacity + // it seems more reasonable to retain extra items and process them immediately, however this contray to current + // framework design so we simply discard extra items, considering that it is a rare case to change capacity uint32_t cnt = 0; while (!mQueue.empty() && mEventCnt > cap) { mEventCnt -= mQueue.front()->mEventGroup.GetEvents().size(); diff --git a/core/pipeline/queue/CircularProcessQueue.h b/core/pipeline/queue/CircularProcessQueue.h index 45f50a1959..813929c799 100644 --- a/core/pipeline/queue/CircularProcessQueue.h +++ b/core/pipeline/queue/CircularProcessQueue.h @@ -29,9 +29,7 @@ namespace logtail { class CircularProcessQueue : virtual public QueueInterface>, public ProcessQueueInterface { public: - CircularProcessQueue(size_t cap, int64_t key, uint32_t priority, const std::string& config) - : QueueInterface>(key, cap), - ProcessQueueInterface(key, cap, priority, config) {} + CircularProcessQueue(size_t cap, int64_t key, uint32_t priority, const PipelineContext& ctx); bool Push(std::unique_ptr&& item) override; bool Pop(std::unique_ptr& item) override; @@ -44,6 +42,8 @@ class CircularProcessQueue : virtual public QueueInterface> mQueue; size_t mEventCnt = 0; + CounterPtr mDroppedEventsCnt; + #ifdef APSARA_UNIT_TEST_MAIN friend class CircularProcessQueueUnittest; friend class ProcessQueueManagerUnittest; diff --git a/core/pipeline/queue/ExactlyOnceQueueManager.cpp b/core/pipeline/queue/ExactlyOnceQueueManager.cpp index 79433b08de..3c66b636e3 100644 --- a/core/pipeline/queue/ExactlyOnceQueueManager.cpp +++ b/core/pipeline/queue/ExactlyOnceQueueManager.cpp @@ -36,7 +36,7 @@ ExactlyOnceQueueManager::ExactlyOnceQueueManager() : mProcessQueueParam(INT32_FL bool ExactlyOnceQueueManager::CreateOrUpdateQueue(QueueKey key, uint32_t priority, - const string& config, + const PipelineContext& ctx, const vector& checkpoints) { { lock_guard lock(mGCMux); @@ -49,7 +49,7 @@ bool ExactlyOnceQueueManager::CreateOrUpdateQueue(QueueKey key, if (iter != mSenderQueues.end()) { iter->second.Reset(checkpoints); } else { - mSenderQueues.try_emplace(key, checkpoints, key); + mSenderQueues.try_emplace(key, checkpoints, key, ctx); iter = mSenderQueues.find(key); } // limiters are set on first push to the queue @@ -65,7 +65,7 @@ bool ExactlyOnceQueueManager::CreateOrUpdateQueue(QueueKey key, iter->second); iter->second->SetPriority(priority); } - iter->second->SetConfigName(config); + iter->second->SetConfigName(ctx.GetConfigName()); // note: do not reset process queue, to be the same as original implementation } else { // note: Ideally, queue capacity should be the same as checkpoint size. However, since process queue cannot @@ -76,7 +76,7 @@ bool ExactlyOnceQueueManager::CreateOrUpdateQueue(QueueKey key, mProcessQueueParam.GetHighWatermark(), key, priority, - config); + ctx); // mProcessPriorityQueue[priority].emplace_back( // checkpoints.size(), checkpoints.size() - 1, checkpoints.size(), key, priority, config); mProcessQueues[key] = prev(mProcessPriorityQueue[priority].end()); diff --git a/core/pipeline/queue/ExactlyOnceQueueManager.h b/core/pipeline/queue/ExactlyOnceQueueManager.h index 62e4adad3c..7f5e8a70ea 100644 --- a/core/pipeline/queue/ExactlyOnceQueueManager.h +++ b/core/pipeline/queue/ExactlyOnceQueueManager.h @@ -51,7 +51,7 @@ class ExactlyOnceQueueManager { bool CreateOrUpdateQueue(QueueKey key, uint32_t priority, - const std::string& config, + const PipelineContext& ctx, const std::vector& checkpoints); bool DeleteQueue(QueueKey key); diff --git a/core/pipeline/queue/ExactlyOnceSenderQueue.cpp b/core/pipeline/queue/ExactlyOnceSenderQueue.cpp index 7402329d68..71c0547df8 100644 --- a/core/pipeline/queue/ExactlyOnceSenderQueue.cpp +++ b/core/pipeline/queue/ExactlyOnceSenderQueue.cpp @@ -16,14 +16,26 @@ #include -#include "plugin/flusher/sls/FlusherSLS.h" #include "logger/Logger.h" #include "pipeline/queue/SLSSenderQueueItem.h" +#include "plugin/flusher/sls/FlusherSLS.h" using namespace std; namespace logtail { +// mFlusher will be set on first push +ExactlyOnceSenderQueue::ExactlyOnceSenderQueue(const std::vector& checkpoints, + QueueKey key, + const PipelineContext& ctx) + : QueueInterface(key, checkpoints.size(), ctx), + BoundedSenderQueueInterface(checkpoints.size(), checkpoints.size() - 1, checkpoints.size(), key, "", ctx), + mRangeCheckpoints(checkpoints) { + mQueue.resize(checkpoints.size()); + mMetricsRecordRef.AddLabels({{METRIC_LABEL_KEY_EXACTLY_ONCE_FLAG, "true"}}); + WriteMetrics::GetInstance()->CommitMetricsRecordRef(mMetricsRecordRef); +} + bool ExactlyOnceSenderQueue::Push(unique_ptr&& item) { if (item == nullptr) { return false; @@ -46,7 +58,7 @@ bool ExactlyOnceSenderQueue::Push(unique_ptr&& item) { // should not happen return false; } - item->mEnqueTime = time(nullptr); + item->mEnqueTime = chrono::system_clock::now(); mQueue[eo->index] = std::move(item); } else { for (size_t idx = 0; idx < mCapacity; ++idx, ++mWrite) { @@ -54,7 +66,7 @@ bool ExactlyOnceSenderQueue::Push(unique_ptr&& item) { if (mQueue[index] != nullptr) { continue; } - item->mEnqueTime = time(nullptr); + item->mEnqueTime = chrono::system_clock::now(); mQueue[index] = std::move(item); auto& newCpt = mRangeCheckpoints[index]; newCpt->data.set_read_offset(eo->data.read_offset()); @@ -65,7 +77,7 @@ bool ExactlyOnceSenderQueue::Push(unique_ptr&& item) { break; } if (!eo->IsComplete()) { - item->mEnqueTime = time(nullptr); + item->mEnqueTime = chrono::system_clock::now(); mExtraBuffer.push(std::move(item)); return true; } diff --git a/core/pipeline/queue/ExactlyOnceSenderQueue.h b/core/pipeline/queue/ExactlyOnceSenderQueue.h index 6f5cb3ada0..7187bb0845 100644 --- a/core/pipeline/queue/ExactlyOnceSenderQueue.h +++ b/core/pipeline/queue/ExactlyOnceSenderQueue.h @@ -30,13 +30,7 @@ namespace logtail { // not thread-safe, should be protected explicitly by queue manager class ExactlyOnceSenderQueue : public BoundedSenderQueueInterface { public: - // mFlusher will be set on first push - ExactlyOnceSenderQueue(const std::vector& checkpoints, QueueKey key) - : QueueInterface(key, checkpoints.size()), - BoundedSenderQueueInterface(checkpoints.size(), checkpoints.size() - 1, checkpoints.size(), key), - mRangeCheckpoints(checkpoints) { - mQueue.resize(checkpoints.size()); - } + ExactlyOnceSenderQueue(const std::vector& checkpoints, QueueKey key, const PipelineContext& ctx); bool Push(std::unique_ptr&& item) override; bool Remove(SenderQueueItem* item) override; diff --git a/core/pipeline/queue/ProcessQueueInterface.cpp b/core/pipeline/queue/ProcessQueueInterface.cpp index afc78620ea..edf8aabf49 100644 --- a/core/pipeline/queue/ProcessQueueInterface.cpp +++ b/core/pipeline/queue/ProcessQueueInterface.cpp @@ -20,6 +20,11 @@ using namespace std; namespace logtail { +ProcessQueueInterface::ProcessQueueInterface(int64_t key, size_t cap, uint32_t priority, const PipelineContext& ctx) + : QueueInterface(key, cap, ctx), mPriority(priority), mConfigName(ctx.GetConfigName()) { + mMetricsRecordRef.AddLabels({{METRIC_LABEL_KEY_COMPONENT_NAME, "process_queue"}}); +} + void ProcessQueueInterface::SetDownStreamQueues(vector&& ques) { mDownStreamQueues.clear(); for (auto& item : ques) { diff --git a/core/pipeline/queue/ProcessQueueInterface.h b/core/pipeline/queue/ProcessQueueInterface.h index 363bc55f27..6a3b8833c8 100644 --- a/core/pipeline/queue/ProcessQueueInterface.h +++ b/core/pipeline/queue/ProcessQueueInterface.h @@ -31,8 +31,7 @@ class BoundedSenderQueueInterface; // not thread-safe, should be protected explicitly by queue manager class ProcessQueueInterface : virtual public QueueInterface> { public: - ProcessQueueInterface(int64_t key, size_t cap, uint32_t priority, const std::string& config) - : QueueInterface(key, cap), mPriority(priority), mConfigName(config) {} + ProcessQueueInterface(int64_t key, size_t cap, uint32_t priority, const PipelineContext& ctx); virtual ~ProcessQueueInterface() = default; void SetPriority(uint32_t priority) { mPriority = priority; } @@ -46,9 +45,7 @@ class ProcessQueueInterface : virtual public QueueInterface #include #include "models/PipelineEventGroup.h" @@ -28,6 +29,7 @@ struct ProcessQueueItem { PipelineEventGroup mEventGroup; std::shared_ptr mPipeline; // not null only during pipeline update size_t mInputIndex = 0; // index of the input in the pipeline + std::chrono::system_clock::time_point mEnqueTime; ProcessQueueItem(PipelineEventGroup&& group, size_t index) : mEventGroup(std::move(group)), mInputIndex(index) {} }; diff --git a/core/pipeline/queue/ProcessQueueManager.cpp b/core/pipeline/queue/ProcessQueueManager.cpp index 0fc012ec24..0b66f6769b 100644 --- a/core/pipeline/queue/ProcessQueueManager.cpp +++ b/core/pipeline/queue/ProcessQueueManager.cpp @@ -32,7 +32,7 @@ ProcessQueueManager::ProcessQueueManager() : mBoundedQueueParam(INT32_FLAG(bound ResetCurrentQueueIndex(); } -bool ProcessQueueManager::CreateOrUpdateBoundedQueue(QueueKey key, uint32_t priority) { +bool ProcessQueueManager::CreateOrUpdateBoundedQueue(QueueKey key, uint32_t priority, const PipelineContext& ctx) { lock_guard lock(mQueueMux); auto iter = mQueues.find(key); if (iter != mQueues.end()) { @@ -40,7 +40,7 @@ bool ProcessQueueManager::CreateOrUpdateBoundedQueue(QueueKey key, uint32_t prio // queue type change only happen when all input plugin types are changed. in such case, old input data not // been processed can be discarded since whole pipeline is actually changed. DeleteQueueEntity(iter->second.first); - CreateBoundedQueue(key, priority); + CreateBoundedQueue(key, priority, ctx); } else { if ((*iter->second.first)->GetPriority() == priority) { return false; @@ -48,7 +48,7 @@ bool ProcessQueueManager::CreateOrUpdateBoundedQueue(QueueKey key, uint32_t prio AdjustQueuePriority(iter->second.first, priority); } } else { - CreateBoundedQueue(key, priority); + CreateBoundedQueue(key, priority, ctx); } if (mCurrentQueueIndex.second == mPriorityQueue[mCurrentQueueIndex.first].end()) { mCurrentQueueIndex.second = mPriorityQueue[mCurrentQueueIndex.first].begin(); @@ -56,7 +56,10 @@ bool ProcessQueueManager::CreateOrUpdateBoundedQueue(QueueKey key, uint32_t prio return true; } -bool ProcessQueueManager::CreateOrUpdateCircularQueue(QueueKey key, uint32_t priority, size_t capacity) { +bool ProcessQueueManager::CreateOrUpdateCircularQueue(QueueKey key, + uint32_t priority, + size_t capacity, + const PipelineContext& ctx) { lock_guard lock(mQueueMux); auto iter = mQueues.find(key); if (iter != mQueues.end()) { @@ -64,7 +67,7 @@ bool ProcessQueueManager::CreateOrUpdateCircularQueue(QueueKey key, uint32_t pri // queue type change only happen when all input plugin types are changed. in such case, old input data not // been processed can be discarded since whole pipeline is actually changed. DeleteQueueEntity(iter->second.first); - CreateCircularQueue(key, priority, capacity); + CreateCircularQueue(key, priority, capacity, ctx); } else { static_cast(iter->second.first->get())->Reset(capacity); if ((*iter->second.first)->GetPriority() == priority) { @@ -73,7 +76,7 @@ bool ProcessQueueManager::CreateOrUpdateCircularQueue(QueueKey key, uint32_t pri AdjustQueuePriority(iter->second.first, priority); } } else { - CreateCircularQueue(key, priority, capacity); + CreateCircularQueue(key, priority, capacity, ctx); } if (mCurrentQueueIndex.second == mPriorityQueue[mCurrentQueueIndex.first].end()) { mCurrentQueueIndex.second = mPriorityQueue[mCurrentQueueIndex.first].begin(); @@ -271,20 +274,21 @@ void ProcessQueueManager::Trigger() { mCond.notify_one(); } -void ProcessQueueManager::CreateBoundedQueue(QueueKey key, uint32_t priority) { - mPriorityQueue[priority].emplace_back( - make_unique(mBoundedQueueParam.GetCapacity(), - mBoundedQueueParam.GetLowWatermark(), - mBoundedQueueParam.GetHighWatermark(), - key, - priority, - QueueKeyManager::GetInstance()->GetName(key))); +void ProcessQueueManager::CreateBoundedQueue(QueueKey key, uint32_t priority, const PipelineContext& ctx) { + mPriorityQueue[priority].emplace_back(make_unique(mBoundedQueueParam.GetCapacity(), + mBoundedQueueParam.GetLowWatermark(), + mBoundedQueueParam.GetHighWatermark(), + key, + priority, + ctx)); mQueues[key] = make_pair(prev(mPriorityQueue[priority].end()), QueueType::BOUNDED); } -void ProcessQueueManager::CreateCircularQueue(QueueKey key, uint32_t priority, size_t capacity) { - mPriorityQueue[priority].emplace_back( - make_unique(capacity, key, priority, QueueKeyManager::GetInstance()->GetName(key))); +void ProcessQueueManager::CreateCircularQueue(QueueKey key, + uint32_t priority, + size_t capacity, + const PipelineContext& ctx) { + mPriorityQueue[priority].emplace_back(make_unique(capacity, key, priority, ctx)); mQueues[key] = make_pair(prev(mPriorityQueue[priority].end()), QueueType::CIRCULAR); } diff --git a/core/pipeline/queue/ProcessQueueManager.h b/core/pipeline/queue/ProcessQueueManager.h index e3d6b464b3..dd0198c4b4 100644 --- a/core/pipeline/queue/ProcessQueueManager.h +++ b/core/pipeline/queue/ProcessQueueManager.h @@ -52,8 +52,8 @@ class ProcessQueueManager : public FeedbackInterface { void Feedback(QueueKey key) override { Trigger(); } - bool CreateOrUpdateBoundedQueue(QueueKey key, uint32_t priority); - bool CreateOrUpdateCircularQueue(QueueKey key, uint32_t priority, size_t capacity); + bool CreateOrUpdateBoundedQueue(QueueKey key, uint32_t priority, const PipelineContext& ctx); + bool CreateOrUpdateCircularQueue(QueueKey key, uint32_t priority, size_t capacity, const PipelineContext& ctx); bool DeleteQueue(QueueKey key); bool IsValidToPush(QueueKey key) const; // 0: success, 1: queue is full, 2: queue not found @@ -76,8 +76,8 @@ class ProcessQueueManager : public FeedbackInterface { ProcessQueueManager(); ~ProcessQueueManager() = default; - void CreateBoundedQueue(QueueKey key, uint32_t priority); - void CreateCircularQueue(QueueKey key, uint32_t priority, size_t capacity); + void CreateBoundedQueue(QueueKey key, uint32_t priority, const PipelineContext& ctx); + void CreateCircularQueue(QueueKey key, uint32_t priority, size_t capacity, const PipelineContext& ctx); void AdjustQueuePriority(const ProcessQueueIterator& iter, uint32_t priority); void DeleteQueueEntity(const ProcessQueueIterator& iter); void ResetCurrentQueueIndex(); diff --git a/core/pipeline/queue/QueueInterface.h b/core/pipeline/queue/QueueInterface.h index 12346f98d9..5c2e3b82cb 100644 --- a/core/pipeline/queue/QueueInterface.h +++ b/core/pipeline/queue/QueueInterface.h @@ -16,6 +16,9 @@ #pragma once +#include "monitor/LogtailMetric.h" +#include "monitor/MetricConstants.h" +#include "pipeline/PipelineContext.h" #include "pipeline/queue/QueueKey.h" namespace logtail { @@ -23,7 +26,20 @@ namespace logtail { template class QueueInterface { public: - QueueInterface(QueueKey key, size_t cap) : mKey(key), mCapacity(cap) {} + QueueInterface(QueueKey key, size_t cap, const PipelineContext& ctx) : mKey(key), mCapacity(cap) { + WriteMetrics::GetInstance()->CreateMetricsRecordRef(mMetricsRecordRef, + { + {METRIC_LABEL_PROJECT, ctx.GetProjectName()}, + {METRIC_LABEL_CONFIG_NAME, ctx.GetConfigName()}, + }); + + mInItemsCnt = mMetricsRecordRef.CreateCounter("in_items_cnt"); + mInItemDataSizeBytes = mMetricsRecordRef.CreateCounter("in_item_data_size_bytes"); + mOutItemsCnt = mMetricsRecordRef.CreateCounter("out_items_cnt"); + mTotalDelayMs = mMetricsRecordRef.CreateCounter("total_delay_ms"); + mQueueSize = mMetricsRecordRef.CreateIntGauge("queue_size"); + mQueueDataSizeByte = mMetricsRecordRef.CreateIntGauge("queue_data_size_bytes"); + } virtual ~QueueInterface() = default; QueueInterface(const QueueInterface& que) = delete; @@ -42,6 +58,14 @@ class QueueInterface { const QueueKey mKey; size_t mCapacity = 0; + mutable MetricsRecordRef mMetricsRecordRef; + CounterPtr mInItemsCnt; + CounterPtr mInItemDataSizeBytes; + CounterPtr mOutItemsCnt; + CounterPtr mTotalDelayMs; + IntGaugePtr mQueueSize; + IntGaugePtr mQueueDataSizeByte; + private: virtual size_t Size() const = 0; }; diff --git a/core/pipeline/queue/SenderQueue.cpp b/core/pipeline/queue/SenderQueue.cpp index ad4d837aba..fda5330f56 100644 --- a/core/pipeline/queue/SenderQueue.cpp +++ b/core/pipeline/queue/SenderQueue.cpp @@ -20,10 +20,25 @@ using namespace std; namespace logtail { +SenderQueue::SenderQueue( + size_t cap, size_t low, size_t high, QueueKey key, const string& flusherId, const PipelineContext& ctx) + : QueueInterface(key, cap, ctx), BoundedSenderQueueInterface(cap, low, high, key, flusherId, ctx) { + mQueue.resize(cap); + WriteMetrics::GetInstance()->CommitMetricsRecordRef(mMetricsRecordRef); +} + bool SenderQueue::Push(unique_ptr&& item) { + item->mEnqueTime = chrono::system_clock::now(); + auto size = item->mData.size(); + + mInItemsCnt->Add(1); + mInItemDataSizeBytes->Add(size); + if (Full()) { - item->mEnqueTime = time(nullptr); mExtraBuffer.push(std::move(item)); + + mExtraBufferSize->Set(mExtraBuffer.size()); + mExtraBufferDataSizeBytes->Add(size); return true; } @@ -33,20 +48,30 @@ bool SenderQueue::Push(unique_ptr&& item) { break; } } - item->mEnqueTime = time(nullptr); mQueue[index % mCapacity] = std::move(item); if (index == mWrite) { ++mWrite; } ++mSize; ChangeStateIfNeededAfterPush(); + + mQueueSize->Set(Size()); + mQueueDataSizeByte->Add(size); + mValidToPushFlag->Set(IsValidToPush()); return true; } bool SenderQueue::Remove(SenderQueueItem* item) { + if (item == nullptr) { + return false; + } + size_t size = 0; + chrono::system_clock::time_point enQueuTime; auto index = mRead; for (; index < mWrite; ++index) { if (mQueue[index % mCapacity].get() == item) { + size = item->mData.size(); + enQueuTime = item->mEnqueTime; mQueue[index % mCapacity].reset(); break; } @@ -58,14 +83,26 @@ bool SenderQueue::Remove(SenderQueueItem* item) { ++mRead; } --mSize; + + mOutItemsCnt->Add(1); + mTotalDelayMs->Add(chrono::duration_cast(chrono::system_clock::now() - enQueuTime).count()); + mQueueDataSizeByte->Sub(size); + if (!mExtraBuffer.empty()) { + auto newSize = mExtraBuffer.front()->mData.size(); Push(std::move(mExtraBuffer.front())); mExtraBuffer.pop(); + + mExtraBufferSize->Set(mExtraBuffer.size()); + mExtraBufferDataSizeBytes->Sub(newSize); return true; } if (ChangeStateIfNeededAfterPop()) { GiveFeedback(); } + + mQueueSize->Set(Size()); + mValidToPushFlag->Set(IsValidToPush()); return true; } diff --git a/core/pipeline/queue/SenderQueue.h b/core/pipeline/queue/SenderQueue.h index 310c981d96..1fca5ab85b 100644 --- a/core/pipeline/queue/SenderQueue.h +++ b/core/pipeline/queue/SenderQueue.h @@ -19,8 +19,8 @@ #include #include -#include "pipeline/queue/QueueKey.h" #include "pipeline/queue/BoundedSenderQueueInterface.h" +#include "pipeline/queue/QueueKey.h" #include "pipeline/queue/SenderQueueItem.h" namespace logtail { @@ -30,10 +30,8 @@ class Flusher; // not thread-safe, should be protected explicitly by queue manager class SenderQueue : public BoundedSenderQueueInterface { public: - SenderQueue(size_t cap, size_t low, size_t high, QueueKey key) - : QueueInterface(key, cap), BoundedSenderQueueInterface(cap, low, high, key) { - mQueue.resize(cap); - } + SenderQueue( + size_t cap, size_t low, size_t high, QueueKey key, const std::string& flusherId, const PipelineContext& ctx); bool Push(std::unique_ptr&& item) override; bool Remove(SenderQueueItem* item) override; diff --git a/core/pipeline/queue/SenderQueueItem.h b/core/pipeline/queue/SenderQueueItem.h index 9e3b83c979..914f98e971 100644 --- a/core/pipeline/queue/SenderQueueItem.h +++ b/core/pipeline/queue/SenderQueueItem.h @@ -16,8 +16,8 @@ #pragma once +#include #include -#include #include #include @@ -41,7 +41,7 @@ struct SenderQueueItem { QueueKey mQueueKey; SendingStatus mStatus = SendingStatus::IDLE; - time_t mEnqueTime = 0; + std::chrono::system_clock::time_point mEnqueTime; time_t mLastSendTime = 0; uint32_t mTryCnt = 1; diff --git a/core/pipeline/queue/SenderQueueManager.cpp b/core/pipeline/queue/SenderQueueManager.cpp index 20987e0b3f..ec132e943d 100644 --- a/core/pipeline/queue/SenderQueueManager.cpp +++ b/core/pipeline/queue/SenderQueueManager.cpp @@ -29,13 +29,20 @@ SenderQueueManager::SenderQueueManager() : mQueueParam(INT32_FLAG(sender_queue_c } bool SenderQueueManager::CreateQueue(QueueKey key, + const string& flusherId, + const PipelineContext& ctx, vector>&& concurrencyLimiters, uint32_t maxRate) { lock_guard lock(mQueueMux); auto iter = mQueues.find(key); if (iter == mQueues.end()) { - mQueues.try_emplace( - key, mQueueParam.GetCapacity(), mQueueParam.GetLowWatermark(), mQueueParam.GetHighWatermark(), key); + mQueues.try_emplace(key, + mQueueParam.GetCapacity(), + mQueueParam.GetLowWatermark(), + mQueueParam.GetHighWatermark(), + key, + flusherId, + ctx); iter = mQueues.find(key); } iter->second.SetConcurrencyLimiters(std::move(concurrencyLimiters)); diff --git a/core/pipeline/queue/SenderQueueManager.h b/core/pipeline/queue/SenderQueueManager.h index 08b5508794..0755a0b946 100644 --- a/core/pipeline/queue/SenderQueueManager.h +++ b/core/pipeline/queue/SenderQueueManager.h @@ -24,11 +24,11 @@ #include #include "common/FeedbackInterface.h" +#include "pipeline/limiter/ConcurrencyLimiter.h" +#include "pipeline/limiter/RateLimiter.h" #include "pipeline/queue/QueueParam.h" #include "pipeline/queue/SenderQueue.h" #include "pipeline/queue/SenderQueueItem.h" -#include "pipeline/limiter/ConcurrencyLimiter.h" -#include "pipeline/limiter/RateLimiter.h" namespace logtail { @@ -47,6 +47,8 @@ class SenderQueueManager : public FeedbackInterface { void Feedback(QueueKey key) override { Trigger(); } bool CreateQueue(QueueKey key, + const std::string& flusherId, + const PipelineContext& ctx, std::vector>&& concurrencyLimiters = std::vector>(), uint32_t maxRate = 0); diff --git a/core/plugin/flusher/blackhole/FlusherBlackHole.cpp b/core/plugin/flusher/blackhole/FlusherBlackHole.cpp index 515beb0dd4..2f3a6510b5 100644 --- a/core/plugin/flusher/blackhole/FlusherBlackHole.cpp +++ b/core/plugin/flusher/blackhole/FlusherBlackHole.cpp @@ -25,7 +25,7 @@ const string FlusherBlackHole::sName = "flusher_blackhole"; bool FlusherBlackHole::Init(const Json::Value& config, Json::Value& optionalGoPipeline) { static uint32_t cnt = 0; GenerateQueueKey(to_string(++cnt)); - SenderQueueManager::GetInstance()->CreateQueue(mQueueKey); + SenderQueueManager::GetInstance()->CreateQueue(mQueueKey, mNodeID, *mContext); return true; } diff --git a/core/plugin/flusher/sls/FlusherSLS.cpp b/core/plugin/flusher/sls/FlusherSLS.cpp index acb0f4da66..5df36854c9 100644 --- a/core/plugin/flusher/sls/FlusherSLS.cpp +++ b/core/plugin/flusher/sls/FlusherSLS.cpp @@ -18,28 +18,28 @@ #include "config/provider/EnterpriseConfigProvider.h" #endif #include "app_config/AppConfig.h" -#include "pipeline/batch/FlushStrategy.h" #include "common/EndpointUtil.h" #include "common/HashUtil.h" #include "common/LogtailCommonFlags.h" #include "common/ParamExtractor.h" #include "common/TimeUtil.h" +#include "pipeline/Pipeline.h" +#include "pipeline/batch/FlushStrategy.h" #include "pipeline/compression/CompressorFactory.h" +#include "pipeline/queue/QueueKeyManager.h" +#include "pipeline/queue/SLSSenderQueueItem.h" +#include "pipeline/queue/SenderQueueManager.h" #include "plugin/flusher/sls/PackIdManager.h" #include "plugin/flusher/sls/SLSClientManager.h" #include "plugin/flusher/sls/SLSResponse.h" #include "plugin/flusher/sls/SendResult.h" -#include "pipeline/Pipeline.h" #include "profile_sender/ProfileSender.h" -#include "pipeline/queue/QueueKeyManager.h" -#include "pipeline/queue/SLSSenderQueueItem.h" -#include "pipeline/queue/SenderQueueManager.h" -#include "sdk/Common.h" #include "runner/FlusherRunner.h" +#include "sdk/Common.h" #include "sls_control/SLSControl.h" // TODO: temporarily used here -#include "plugin/flusher/sls/DiskBufferWriter.h" #include "pipeline/PipelineManager.h" +#include "plugin/flusher/sls/DiskBufferWriter.h" using namespace std; @@ -470,6 +470,8 @@ bool FlusherSLS::Init(const Json::Value& config, Json::Value& optionalGoPipeline GenerateQueueKey(mProject + "#" + mLogstore); SenderQueueManager::GetInstance()->CreateQueue( mQueueKey, + mNodeID, + *mContext, vector>{GetRegionConcurrencyLimiter(mRegion), GetProjectConcurrencyLimiter(mProject)}, mMaxSendRate); @@ -610,6 +612,7 @@ void FlusherSLS::OnSendDone(const HttpResponse& response, SenderQueueItem* item) string configName = HasContext() ? GetContext().GetConfigName() : ""; bool isProfileData = ProfileSender::GetInstance()->IsProfileData(mRegion, mProject, data->mLogstore); int32_t curTime = time(NULL); + auto curSystemTime = chrono::system_clock::now(); if (slsResponse.mStatusCode == 200) { auto& cpt = data->mExactlyOnceCheckpoint; if (cpt) { @@ -617,10 +620,13 @@ void FlusherSLS::OnSendDone(const HttpResponse& response, SenderQueueItem* item) cpt->IncreaseSequenceID(); } LOG_DEBUG(sLogger, - ("send data to sls succeeded, item address", item)("request id", slsResponse.mRequestId)( - "config", configName)("region", mRegion)("project", mProject)("logstore", data->mLogstore)( - "response time", curTime - data->mLastSendTime)("total send time", curTime - data->mEnqueTime)( - "try cnt", data->mTryCnt)("endpoint", data->mCurrentEndpoint)("is profile data", isProfileData)); + ("send data to sls succeeded, item address", + item)("request id", slsResponse.mRequestId)("config", configName)("region", mRegion)( + "project", mProject)("logstore", data->mLogstore)("response time", curTime - data->mLastSendTime)( + "total send time", + ToString(chrono::duration_cast(curSystemTime - item->mEnqueTime).count()) + + "ms")("try cnt", data->mTryCnt)("endpoint", data->mCurrentEndpoint)("is profile data", + isProfileData)); GetRegionConcurrencyLimiter(mRegion)->OnSuccess(); DealSenderQueueItemAfterSend(item, false); } else { @@ -747,7 +753,8 @@ void FlusherSLS::OnSendDone(const HttpResponse& response, SenderQueueItem* item) // when retry times > unknow_error_try_max, we will drop this data operation = DefaultOperation(item->mTryCnt); } - if (curTime - data->mEnqueTime > INT32_FLAG(discard_send_fail_interval)) { + if (chrono::duration_cast(curSystemTime - item->mEnqueTime).count() + > INT32_FLAG(discard_send_fail_interval)) { operation = OperationOnFail::DISCARD; } if (isProfileData && data->mTryCnt >= static_cast(INT32_FLAG(profile_data_send_retrytimes))) { @@ -760,8 +767,9 @@ void FlusherSLS::OnSendDone(const HttpResponse& response, SenderQueueItem* item) "status code", slsResponse.mStatusCode)("error code", slsResponse.mErrorCode)( \ "errMsg", slsResponse.mErrorMsg)("config", configName)("region", mRegion)("project", mProject)( \ "logstore", data->mLogstore)("try cnt", data->mTryCnt)("response time", curTime - data->mLastSendTime)( \ - "total send time", curTime - data->mEnqueTime)("endpoint", data->mCurrentEndpoint)("is profile data", \ - isProfileData) + "total send time", \ + ToString(chrono::duration_cast(curSystemTime - data->mEnqueTime).count()) \ + + "ms")("endpoint", data->mCurrentEndpoint)("is profile data", isProfileData) switch (operation) { case OperationOnFail::RETRY_IMMEDIATELY: @@ -821,7 +829,8 @@ bool FlusherSLS::Send(string&& data, const string& shardHashKey, const string& l if (!HasContext()) { key = QueueKeyManager::GetInstance()->GetKey(mProject + "-" + mLogstore); if (SenderQueueManager::GetInstance()->GetQueue(key) == nullptr) { - SenderQueueManager::GetInstance()->CreateQueue(key, vector>()); + PipelineContext ctx; + SenderQueueManager::GetInstance()->CreateQueue(key, "", ctx, vector>()); } } return Flusher::PushToQueue(make_unique(std::move(compressedData), diff --git a/core/runner/FlusherRunner.cpp b/core/runner/FlusherRunner.cpp index 88316bce20..16af0a51df 100644 --- a/core/runner/FlusherRunner.cpp +++ b/core/runner/FlusherRunner.cpp @@ -18,14 +18,14 @@ #include "application/Application.h" #include "common/LogtailCommonFlags.h" #include "common/StringTools.h" -#include "plugin/flusher/sls/DiskBufferWriter.h" +#include "common/http/HttpRequest.h" #include "logger/Logger.h" #include "monitor/LogtailAlarm.h" #include "pipeline/plugin/interface/HttpFlusher.h" #include "pipeline/queue/QueueKeyManager.h" #include "pipeline/queue/SenderQueueItem.h" #include "pipeline/queue/SenderQueueManager.h" -#include "common/http/HttpRequest.h" +#include "plugin/flusher/sls/DiskBufferWriter.h" #include "runner/sink/http/HttpSink.h" // TODO: temporarily used here #include "plugin/flusher/sls/PackIdManager.h" @@ -98,7 +98,7 @@ void FlusherRunner::PushToHttpSink(SenderQueueItem* item, bool withLimit) { void FlusherRunner::Run() { LOG_INFO(sLogger, ("flusher runner", "started")); while (true) { - int32_t curTime = time(NULL); + auto curTime = chrono::system_clock::now(); vector items; SenderQueueManager::GetInstance()->GetAllAvailableItems(items, !Application::GetInstance()->IsExiting()); @@ -121,11 +121,11 @@ void FlusherRunner::Run() { } for (auto itr = items.begin(); itr != items.end(); ++itr) { - int32_t waitTime = curTime - (*itr)->mEnqueTime; + auto waitTime = chrono::duration_cast(curTime - (*itr)->mEnqueTime); LOG_DEBUG(sLogger, ("got item from sender queue, item address", *itr)("config-flusher-dst", QueueKeyManager::GetInstance()->GetName((*itr)->mQueueKey))( - "wait time", ToString(waitTime))("try cnt", ToString((*itr)->mTryCnt))); + "wait time", ToString(waitTime.count()) + "ms")("try cnt", ToString((*itr)->mTryCnt))); if (!Application::GetInstance()->IsExiting() && AppConfig::GetInstance()->IsSendFlowControl()) { RateLimiter::FlowControl((*itr)->mRawSize, mSendLastTime, mSendLastByte, true); diff --git a/core/unittest/event_handler/ModifyHandlerUnittest.cpp b/core/unittest/event_handler/ModifyHandlerUnittest.cpp index ea26051661..490ed7890a 100644 --- a/core/unittest/event_handler/ModifyHandlerUnittest.cpp +++ b/core/unittest/event_handler/ModifyHandlerUnittest.cpp @@ -23,12 +23,12 @@ #include "common/Flags.h" #include "common/JsonUtil.h" #include "config/PipelineConfig.h" +#include "file_server/FileServer.h" #include "file_server/event/Event.h" #include "file_server/event_handler/EventHandler.h" -#include "file_server/FileServer.h" +#include "file_server/reader/LogFileReader.h" #include "pipeline/Pipeline.h" #include "pipeline/queue/ProcessQueueManager.h" -#include "file_server/reader/LogFileReader.h" #include "unittest/Unittest.h" using namespace std; @@ -115,7 +115,7 @@ class ModifyHandlerUnittest : public ::testing::Test { FileServer::GetInstance()->AddFileDiscoveryConfig(mConfigName, &discoveryOpts, &ctx); FileServer::GetInstance()->AddFileReaderConfig(mConfigName, &readerOpts, &ctx); FileServer::GetInstance()->AddMultilineConfig(mConfigName, &multilineOpts, &ctx); - ProcessQueueManager::GetInstance()->CreateOrUpdateBoundedQueue(0, 0); + ProcessQueueManager::GetInstance()->CreateOrUpdateBoundedQueue(0, 0, ctx); // build a reader mReaderPtr = std::make_shared( diff --git a/core/unittest/flusher/FlusherSLSUnittest.cpp b/core/unittest/flusher/FlusherSLSUnittest.cpp index 3edd9db556..52a3d2f510 100644 --- a/core/unittest/flusher/FlusherSLSUnittest.cpp +++ b/core/unittest/flusher/FlusherSLSUnittest.cpp @@ -581,7 +581,7 @@ void FlusherSLSUnittest::TestSend() { } QueueKey eooKey = QueueKeyManager::GetInstance()->GetKey("eoo"); ExactlyOnceQueueManager::GetInstance()->CreateOrUpdateQueue( - eooKey, ProcessQueueManager::sMaxPriority, flusher.GetContext().GetConfigName(), checkpoints); + eooKey, ProcessQueueManager::sMaxPriority, flusher.GetContext(), checkpoints); { // replayed group diff --git a/core/unittest/plugin/PluginMock.h b/core/unittest/plugin/PluginMock.h index a2c5ccfbc5..f94bf8f044 100644 --- a/core/unittest/plugin/PluginMock.h +++ b/core/unittest/plugin/PluginMock.h @@ -94,7 +94,7 @@ class FlusherMock : public Flusher { const std::string& Name() const override { return sName; } bool Init(const Json::Value& config, Json::Value& optionalGoPipeline) override { GenerateQueueKey("mock"); - SenderQueueManager::GetInstance()->CreateQueue(mQueueKey); + SenderQueueManager::GetInstance()->CreateQueue(mQueueKey, mNodeID, *mContext); return true; } bool Send(PipelineEventGroup&& g) override { return mIsValid; } @@ -117,7 +117,7 @@ class FlusherHttpMock : public HttpFlusher { const std::string& Name() const override { return sName; } bool Init(const Json::Value& config, Json::Value& optionalGoPipeline) override { GenerateQueueKey("mock"); - SenderQueueManager::GetInstance()->CreateQueue(mQueueKey); + SenderQueueManager::GetInstance()->CreateQueue(mQueueKey, mNodeID, *mContext); return true; } bool Send(PipelineEventGroup&& g) override { return mIsValid; } diff --git a/core/unittest/queue/BoundedProcessQueueUnittest.cpp b/core/unittest/queue/BoundedProcessQueueUnittest.cpp index c36299ad86..e2228108d5 100644 --- a/core/unittest/queue/BoundedProcessQueueUnittest.cpp +++ b/core/unittest/queue/BoundedProcessQueueUnittest.cpp @@ -29,15 +29,16 @@ class BoundedProcessQueueUnittest : public testing::Test { public: void TestPush(); void TestPop(); + void TestMetric(); protected: - static void SetUpTestCase() { sEventGroup.reset(new PipelineEventGroup(make_shared())); } + static void SetUpTestCase() { sCtx.SetConfigName("test_config"); } void SetUp() override { - mQueue.reset(new BoundedProcessQueue(sCap, sLowWatermark, sHighWatermark, sKey, 1, "test_config")); + mQueue.reset(new BoundedProcessQueue(sCap, sLowWatermark, sHighWatermark, sKey, 1, sCtx)); - mSenderQueue1.reset(new SenderQueue(10, 0, 10, 0)); - mSenderQueue2.reset(new SenderQueue(10, 0, 10, 0)); + mSenderQueue1.reset(new SenderQueue(10, 0, 10, 0, "", sCtx)); + mSenderQueue2.reset(new SenderQueue(10, 0, 10, 0, "", sCtx)); mQueue->SetDownStreamQueues(vector{mSenderQueue1.get(), mSenderQueue2.get()}); mFeedback1.reset(new FeedbackInterfaceMock); @@ -46,13 +47,16 @@ class BoundedProcessQueueUnittest : public testing::Test { } private: - static unique_ptr sEventGroup; + static PipelineContext sCtx; static const QueueKey sKey = 0; static const size_t sCap = 6; static const size_t sLowWatermark = 2; static const size_t sHighWatermark = 4; - unique_ptr GenerateItem() { return make_unique(std::move(*sEventGroup), 0); } + unique_ptr GenerateItem() { + PipelineEventGroup g(make_shared()); + return make_unique(std::move(g), 0); + } unique_ptr mQueue; unique_ptr mFeedback1; @@ -61,7 +65,7 @@ class BoundedProcessQueueUnittest : public testing::Test { unique_ptr mSenderQueue2; }; -unique_ptr BoundedProcessQueueUnittest::sEventGroup; +PipelineContext BoundedProcessQueueUnittest::sCtx; void BoundedProcessQueueUnittest::TestPush() { // push first @@ -110,8 +114,35 @@ void BoundedProcessQueueUnittest::TestPop() { APSARA_TEST_TRUE(static_cast(mFeedback2.get())->HasFeedback(sKey)); } +void BoundedProcessQueueUnittest::TestMetric() { + APSARA_TEST_EQUAL(4U, mQueue->mMetricsRecordRef->GetLabels()->size()); + APSARA_TEST_TRUE(mQueue->mMetricsRecordRef.HasLabel(METRIC_LABEL_PROJECT, "")); + APSARA_TEST_TRUE(mQueue->mMetricsRecordRef.HasLabel(METRIC_LABEL_CONFIG_NAME, "test_config")); + APSARA_TEST_TRUE(mQueue->mMetricsRecordRef.HasLabel(METRIC_LABEL_KEY_COMPONENT_NAME, "process_queue")); + APSARA_TEST_TRUE(mQueue->mMetricsRecordRef.HasLabel(METRIC_LABEL_KEY_QUEUE_TYPE, "bounded")); + + auto item = GenerateItem(); + auto e = item->mEventGroup.AddLogEvent(); + e->SetContent(string("key"), string("value")); + auto dataSize = item->mEventGroup.DataSize(); + mQueue->Push(std::move(item)); + + APSARA_TEST_EQUAL(1U, mQueue->mInItemsCnt->GetValue()); + APSARA_TEST_EQUAL(dataSize, mQueue->mInItemDataSizeBytes->GetValue()); + APSARA_TEST_EQUAL(1U, mQueue->mQueueSize->GetValue()); + APSARA_TEST_EQUAL(dataSize, mQueue->mQueueDataSizeByte->GetValue()); + APSARA_TEST_EQUAL(1U, mQueue->mValidToPushFlag->GetValue()); + + mQueue->Pop(item); + APSARA_TEST_EQUAL(1U, mQueue->mOutItemsCnt->GetValue()); + APSARA_TEST_EQUAL(0U, mQueue->mQueueSize->GetValue()); + APSARA_TEST_EQUAL(0U, mQueue->mQueueDataSizeByte->GetValue()); + APSARA_TEST_EQUAL(1U, mQueue->mValidToPushFlag->GetValue()); +} + UNIT_TEST_CASE(BoundedProcessQueueUnittest, TestPush) UNIT_TEST_CASE(BoundedProcessQueueUnittest, TestPop) +UNIT_TEST_CASE(BoundedProcessQueueUnittest, TestMetric) } // namespace logtail diff --git a/core/unittest/queue/CircularProcessQueueUnittest.cpp b/core/unittest/queue/CircularProcessQueueUnittest.cpp index 1bed2ded03..7b5ef9d0d3 100644 --- a/core/unittest/queue/CircularProcessQueueUnittest.cpp +++ b/core/unittest/queue/CircularProcessQueueUnittest.cpp @@ -27,17 +27,21 @@ class CircularProcessQueueUnittest : public testing::Test { void TestPush(); void TestPop(); void TestReset(); + void TestMetric(); protected: + static void SetUpTestCase() { sCtx.SetConfigName("test_config"); } + void SetUp() override { - mQueue.reset(new CircularProcessQueue(sCap, sKey, 1, "test_config")); + mQueue.reset(new CircularProcessQueue(sCap, sKey, 1, sCtx)); - mSenderQueue1.reset(new SenderQueue(10, 0, 10, 0)); - mSenderQueue2.reset(new SenderQueue(10, 0, 10, 0)); + mSenderQueue1.reset(new SenderQueue(10, 0, 10, 0, "", sCtx)); + mSenderQueue2.reset(new SenderQueue(10, 0, 10, 0, "", sCtx)); mQueue->SetDownStreamQueues(vector{mSenderQueue1.get(), mSenderQueue2.get()}); } private: + static PipelineContext sCtx; static const QueueKey sKey = 0; static const size_t sCap = 2; @@ -54,6 +58,8 @@ class CircularProcessQueueUnittest : public testing::Test { unique_ptr mSenderQueue2; }; +PipelineContext CircularProcessQueueUnittest::sCtx; + void CircularProcessQueueUnittest::TestPush() { unique_ptr res; { @@ -141,9 +147,40 @@ void CircularProcessQueueUnittest::TestReset() { } } +void CircularProcessQueueUnittest::TestMetric() { + APSARA_TEST_EQUAL(4U, mQueue->mMetricsRecordRef->GetLabels()->size()); + APSARA_TEST_TRUE(mQueue->mMetricsRecordRef.HasLabel(METRIC_LABEL_PROJECT, "")); + APSARA_TEST_TRUE(mQueue->mMetricsRecordRef.HasLabel(METRIC_LABEL_CONFIG_NAME, "test_config")); + APSARA_TEST_TRUE(mQueue->mMetricsRecordRef.HasLabel(METRIC_LABEL_KEY_COMPONENT_NAME, "process_queue")); + APSARA_TEST_TRUE(mQueue->mMetricsRecordRef.HasLabel(METRIC_LABEL_KEY_QUEUE_TYPE, "circular")); + + auto item = GenerateItem(2); + auto dataSize1 = item->mEventGroup.DataSize(); + mQueue->Push(std::move(item)); + APSARA_TEST_EQUAL(1U, mQueue->mInItemsCnt->GetValue()); + APSARA_TEST_EQUAL(dataSize1, mQueue->mInItemDataSizeBytes->GetValue()); + APSARA_TEST_EQUAL(2U, mQueue->mQueueSize->GetValue()); + APSARA_TEST_EQUAL(dataSize1, mQueue->mQueueDataSizeByte->GetValue()); + + item = GenerateItem(1); + auto dataSize2 = item->mEventGroup.DataSize(); + mQueue->Push(std::move(item)); + APSARA_TEST_EQUAL(2U, mQueue->mInItemsCnt->GetValue()); + APSARA_TEST_EQUAL(dataSize1 + dataSize2, mQueue->mInItemDataSizeBytes->GetValue()); + APSARA_TEST_EQUAL(1U, mQueue->mQueueSize->GetValue()); + APSARA_TEST_EQUAL(dataSize2, mQueue->mQueueDataSizeByte->GetValue()); + APSARA_TEST_EQUAL(2U, mQueue->mDroppedEventsCnt->GetValue()); + + mQueue->Pop(item); + APSARA_TEST_EQUAL(1U, mQueue->mOutItemsCnt->GetValue()); + APSARA_TEST_EQUAL(0U, mQueue->mQueueSize->GetValue()); + APSARA_TEST_EQUAL(0U, mQueue->mQueueDataSizeByte->GetValue()); +} + UNIT_TEST_CASE(CircularProcessQueueUnittest, TestPush) UNIT_TEST_CASE(CircularProcessQueueUnittest, TestPop) UNIT_TEST_CASE(CircularProcessQueueUnittest, TestReset) +UNIT_TEST_CASE(CircularProcessQueueUnittest, TestMetric) } // namespace logtail diff --git a/core/unittest/queue/ExactlyOnceQueueManagerUnittest.cpp b/core/unittest/queue/ExactlyOnceQueueManagerUnittest.cpp index 4adaffeccb..5e929eb477 100644 --- a/core/unittest/queue/ExactlyOnceQueueManagerUnittest.cpp +++ b/core/unittest/queue/ExactlyOnceQueueManagerUnittest.cpp @@ -14,12 +14,12 @@ #include -#include "plugin/flusher/sls/FlusherSLS.h" -#include "plugin/input/InputFeedbackInterfaceRegistry.h" #include "models/PipelineEventGroup.h" #include "pipeline/queue/ExactlyOnceQueueManager.h" #include "pipeline/queue/QueueKeyManager.h" #include "pipeline/queue/SLSSenderQueueItem.h" +#include "plugin/flusher/sls/FlusherSLS.h" +#include "plugin/input/InputFeedbackInterfaceRegistry.h" #include "unittest/Unittest.h" DECLARE_FLAG_INT32(logtail_queue_gc_threshold_sec); @@ -43,7 +43,6 @@ class ExactlyOnceQueueManagerUnittest : public testing::Test { protected: static void SetUpTestCase() { InputFeedbackInterfaceRegistry::GetInstance()->LoadFeedbackInterfaces(); - sEventGroup.reset(new PipelineEventGroup(make_shared())); for (size_t i = 0; i < 5; ++i) { auto cpt = make_shared(); cpt->index = i; @@ -68,9 +67,9 @@ class ExactlyOnceQueueManagerUnittest : public testing::Test { private: static const size_t sDataSize = 10; - static unique_ptr sEventGroup; static ExactlyOnceQueueManager* sManager; static vector sCheckpoints; + static PipelineContext sCtx; unique_ptr GenerateProcessItem(); unique_ptr GenerateSenderItem(); @@ -80,16 +79,16 @@ class ExactlyOnceQueueManagerUnittest : public testing::Test { }; const size_t ExactlyOnceQueueManagerUnittest::sDataSize; -unique_ptr ExactlyOnceQueueManagerUnittest::sEventGroup; ExactlyOnceQueueManager* ExactlyOnceQueueManagerUnittest::sManager; vector ExactlyOnceQueueManagerUnittest::sCheckpoints; +PipelineContext ExactlyOnceQueueManagerUnittest::sCtx; void ExactlyOnceQueueManagerUnittest::TestUpdateQueue() { QueueKey key = 0; { // create queue size_t queueSize = 5; - APSARA_TEST_TRUE(sManager->CreateOrUpdateQueue(key, 0, "test_config", vector(queueSize))); + APSARA_TEST_TRUE(sManager->CreateOrUpdateQueue(key, 0, sCtx, vector(queueSize))); APSARA_TEST_EQUAL(1U, sManager->mSenderQueues.size()); auto& senderQue = sManager->mSenderQueues.at(key); APSARA_TEST_EQUAL(queueSize, senderQue.mCapacity); @@ -106,7 +105,7 @@ void ExactlyOnceQueueManagerUnittest::TestUpdateQueue() { { // update queue with the same priority size_t queueSize = 8; - APSARA_TEST_TRUE(sManager->CreateOrUpdateQueue(key, 0, "test_config", vector(queueSize))); + APSARA_TEST_TRUE(sManager->CreateOrUpdateQueue(key, 0, sCtx, vector(queueSize))); APSARA_TEST_EQUAL(1U, sManager->mSenderQueues.size()); auto& senderQue = sManager->mSenderQueues.at(key); APSARA_TEST_EQUAL(queueSize, senderQue.mCapacity); @@ -122,7 +121,7 @@ void ExactlyOnceQueueManagerUnittest::TestUpdateQueue() { { // update queue with different priority size_t queueSize = 6; - APSARA_TEST_TRUE(sManager->CreateOrUpdateQueue(key, 1, "test_config", vector(queueSize))); + APSARA_TEST_TRUE(sManager->CreateOrUpdateQueue(key, 1, sCtx, vector(queueSize))); APSARA_TEST_EQUAL(1U, sManager->mProcessQueues.size()); APSARA_TEST_EQUAL(0U, sManager->mProcessPriorityQueue[0].size()); APSARA_TEST_EQUAL(1U, sManager->mProcessPriorityQueue[1].size()); @@ -141,8 +140,8 @@ void ExactlyOnceQueueManagerUnittest::TestDeleteQueue() { QueueKey key1 = QueueKeyManager::GetInstance()->GetKey("name_1"); QueueKey key2 = QueueKeyManager::GetInstance()->GetKey("name_2"); - sManager->CreateOrUpdateQueue(key1, 1, "test_config_1", sCheckpoints); - sManager->CreateOrUpdateQueue(key2, 1, "test_config_2", sCheckpoints); + sManager->CreateOrUpdateQueue(key1, 1, sCtx, sCheckpoints); + sManager->CreateOrUpdateQueue(key2, 1, sCtx, sCheckpoints); sManager->PushProcessQueue(key2, GenerateProcessItem()); // queue exists and not marked deleted @@ -162,12 +161,12 @@ void ExactlyOnceQueueManagerUnittest::TestDeleteQueue() { APSARA_TEST_EQUAL("", QueueKeyManager::GetInstance()->GetName(key1)); // update queue will remove the queue from gc queue - sManager->CreateOrUpdateQueue(key2, 0, "test_config_2", sCheckpoints); + sManager->CreateOrUpdateQueue(key2, 0, sCtx, sCheckpoints); APSARA_TEST_EQUAL(0U, sManager->mQueueDeletionTimeMap.size()); } void ExactlyOnceQueueManagerUnittest::TestPushProcessQueue() { - sManager->CreateOrUpdateQueue(0, 0, "test_config", sCheckpoints); + sManager->CreateOrUpdateQueue(0, 0, sCtx, sCheckpoints); // queue exists APSARA_TEST_TRUE(sManager->IsValidToPushProcessQueue(0)); @@ -184,8 +183,8 @@ void ExactlyOnceQueueManagerUnittest::TestPushProcessQueue() { } void ExactlyOnceQueueManagerUnittest::TestIsAllProcessQueueEmpty() { - sManager->CreateOrUpdateQueue(0, 0, "test_config_1", sCheckpoints); - sManager->CreateOrUpdateQueue(1, 2, "test_config_2", sCheckpoints); + sManager->CreateOrUpdateQueue(0, 0, sCtx, sCheckpoints); + sManager->CreateOrUpdateQueue(1, 2, sCtx, sCheckpoints); APSARA_TEST_TRUE(sManager->IsAllProcessQueueEmpty()); sManager->PushProcessQueue(0, GenerateProcessItem()); @@ -193,7 +192,7 @@ void ExactlyOnceQueueManagerUnittest::TestIsAllProcessQueueEmpty() { } void ExactlyOnceQueueManagerUnittest::TestPushSenderQueue() { - sManager->CreateOrUpdateQueue(0, 0, "test_config", sCheckpoints); + sManager->CreateOrUpdateQueue(0, 0, sCtx, sCheckpoints); // queue exists APSARA_TEST_EQUAL(0, sManager->PushSenderQueue(0, GenerateSenderItem())); @@ -211,7 +210,7 @@ void ExactlyOnceQueueManagerUnittest::TestGetAllAvailableSenderQueueItems() { cpt->data.set_sequence_id(0); checkpoints1.emplace_back(cpt); } - sManager->CreateOrUpdateQueue(0, 0, "test_config_1", checkpoints1); + sManager->CreateOrUpdateQueue(0, 0, sCtx, checkpoints1); vector checkpoints2; for (size_t i = 0; i < 2; ++i) { @@ -221,7 +220,7 @@ void ExactlyOnceQueueManagerUnittest::TestGetAllAvailableSenderQueueItems() { cpt->data.set_sequence_id(0); checkpoints2.emplace_back(cpt); } - sManager->CreateOrUpdateQueue(1, 2, "test_config_2", checkpoints2); + sManager->CreateOrUpdateQueue(1, 2, sCtx, checkpoints2); for (size_t i = 0; i <= 2; ++i) { sManager->PushSenderQueue(0, GenerateSenderItem()); @@ -250,7 +249,7 @@ void ExactlyOnceQueueManagerUnittest::TestGetAllAvailableSenderQueueItems() { } void ExactlyOnceQueueManagerUnittest::TestRemoveSenderItem() { - sManager->CreateOrUpdateQueue(1, 0, "test_config", sCheckpoints); + sManager->CreateOrUpdateQueue(1, 0, sCtx, sCheckpoints); { // queue exists auto item = GenerateSenderItem(); @@ -268,8 +267,8 @@ void ExactlyOnceQueueManagerUnittest::TestRemoveSenderItem() { } void ExactlyOnceQueueManagerUnittest::TestIsAllSenderQueueEmpty() { - sManager->CreateOrUpdateQueue(0, 0, "test_config_1", sCheckpoints); - sManager->CreateOrUpdateQueue(1, 2, "test_config_2", sCheckpoints); + sManager->CreateOrUpdateQueue(0, 0, sCtx, sCheckpoints); + sManager->CreateOrUpdateQueue(1, 2, sCtx, sCheckpoints); APSARA_TEST_TRUE(sManager->IsAllSenderQueueEmpty()); sManager->PushSenderQueue(0, GenerateSenderItem()); @@ -277,8 +276,10 @@ void ExactlyOnceQueueManagerUnittest::TestIsAllSenderQueueEmpty() { } void ExactlyOnceQueueManagerUnittest::OnPipelineUpdate() { - sManager->CreateOrUpdateQueue(0, 0, "test_config", sCheckpoints); - sManager->CreateOrUpdateQueue(1, 0, "test_config", sCheckpoints); + PipelineContext ctx; + ctx.SetConfigName("test_config"); + sManager->CreateOrUpdateQueue(0, 0, ctx, sCheckpoints); + sManager->CreateOrUpdateQueue(1, 0, ctx, sCheckpoints); sManager->InvalidatePopProcessQueue("test_config"); APSARA_TEST_FALSE(sManager->mProcessQueues[0]->mValidToPop); @@ -290,7 +291,8 @@ void ExactlyOnceQueueManagerUnittest::OnPipelineUpdate() { } unique_ptr ExactlyOnceQueueManagerUnittest::GenerateProcessItem() { - return make_unique(std::move(*sEventGroup), 0); + PipelineEventGroup g(make_shared()); + return make_unique(std::move(g), 0); } unique_ptr ExactlyOnceQueueManagerUnittest::GenerateSenderItem() { diff --git a/core/unittest/queue/ExactlyOnceSenderQueueUnittest.cpp b/core/unittest/queue/ExactlyOnceSenderQueueUnittest.cpp index 21f670a4ef..16dba46c8f 100644 --- a/core/unittest/queue/ExactlyOnceSenderQueueUnittest.cpp +++ b/core/unittest/queue/ExactlyOnceSenderQueueUnittest.cpp @@ -41,7 +41,7 @@ class ExactlyOnceSenderQueueUnittest : public testing::Test { } void SetUp() override { - mQueue.reset(new ExactlyOnceSenderQueue(sCheckpoints, sKey)); + mQueue.reset(new ExactlyOnceSenderQueue(sCheckpoints, sKey, sCtx)); mQueue->SetFeedback(&sFeedback); mFlusher.mMaxSendRate = 100; mFlusher.mRegion = "region"; @@ -51,6 +51,7 @@ class ExactlyOnceSenderQueueUnittest : public testing::Test { void TearDown() override { sFeedback.Clear(); } private: + static PipelineContext sCtx; static const QueueKey sKey = 0; static const size_t sDataSize = 10; @@ -64,6 +65,7 @@ class ExactlyOnceSenderQueueUnittest : public testing::Test { unique_ptr mQueue; }; +PipelineContext ExactlyOnceSenderQueueUnittest::sCtx; const size_t ExactlyOnceSenderQueueUnittest::sDataSize; FeedbackInterfaceMock ExactlyOnceSenderQueueUnittest::sFeedback; vector ExactlyOnceSenderQueueUnittest::sCheckpoints; diff --git a/core/unittest/queue/ProcessQueueManagerUnittest.cpp b/core/unittest/queue/ProcessQueueManagerUnittest.cpp index f283f16342..d87bf88e96 100644 --- a/core/unittest/queue/ProcessQueueManagerUnittest.cpp +++ b/core/unittest/queue/ProcessQueueManagerUnittest.cpp @@ -37,10 +37,7 @@ class ProcessQueueManagerUnittest : public testing::Test { void OnPipelineUpdate(); protected: - static void SetUpTestCase() { - sEventGroup.reset(new PipelineEventGroup(make_shared())); - sProcessQueueManager = ProcessQueueManager::GetInstance(); - } + static void SetUpTestCase() { sProcessQueueManager = ProcessQueueManager::GetInstance(); } void TearDown() override { QueueKeyManager::GetInstance()->Clear(); @@ -49,18 +46,26 @@ class ProcessQueueManagerUnittest : public testing::Test { } private: - static unique_ptr sEventGroup; static ProcessQueueManager* sProcessQueueManager; + static PipelineContext sCtx; + + unique_ptr GenerateItem() { + PipelineEventGroup g(make_shared()); + return make_unique(std::move(g), 0); + } }; -unique_ptr ProcessQueueManagerUnittest::sEventGroup; ProcessQueueManager* ProcessQueueManagerUnittest::sProcessQueueManager; +PipelineContext ProcessQueueManagerUnittest::sCtx; void ProcessQueueManagerUnittest::TestUpdateSameTypeQueue() { // create queue // and current index is invalid before creation QueueKey key = QueueKeyManager::GetInstance()->GetKey("test_config_1"); - APSARA_TEST_TRUE(sProcessQueueManager->CreateOrUpdateBoundedQueue(key, 0)); + PipelineContext ctx; + ctx.SetConfigName("test_config_1"); + ctx.SetProcessQueueKey(key); + APSARA_TEST_TRUE(sProcessQueueManager->CreateOrUpdateBoundedQueue(key, 0, ctx)); APSARA_TEST_EQUAL(1U, sProcessQueueManager->mQueues.size()); APSARA_TEST_EQUAL(1U, sProcessQueueManager->mPriorityQueue[0].size()); auto iter = sProcessQueueManager->mQueues[key].first; @@ -75,23 +80,23 @@ void ProcessQueueManagerUnittest::TestUpdateSameTypeQueue() { // create queue // and current index is valid before creation - APSARA_TEST_TRUE(sProcessQueueManager->CreateOrUpdateBoundedQueue(1, 0)); + APSARA_TEST_TRUE(sProcessQueueManager->CreateOrUpdateBoundedQueue(1, 0, sCtx)); APSARA_TEST_EQUAL(2U, sProcessQueueManager->mQueues.size()); APSARA_TEST_EQUAL(2U, sProcessQueueManager->mPriorityQueue[0].size()); APSARA_TEST_TRUE(sProcessQueueManager->mQueues[1].first == prev(sProcessQueueManager->mPriorityQueue[0].end())); APSARA_TEST_TRUE(sProcessQueueManager->mCurrentQueueIndex.second == sProcessQueueManager->mQueues[0].first); // add more queue - APSARA_TEST_TRUE(sProcessQueueManager->CreateOrUpdateBoundedQueue(2, 0)); - APSARA_TEST_TRUE(sProcessQueueManager->CreateOrUpdateBoundedQueue(3, 0)); + APSARA_TEST_TRUE(sProcessQueueManager->CreateOrUpdateBoundedQueue(2, 0, sCtx)); + APSARA_TEST_TRUE(sProcessQueueManager->CreateOrUpdateBoundedQueue(3, 0, sCtx)); sProcessQueueManager->mCurrentQueueIndex.second = sProcessQueueManager->mQueues[2].first; // update queue with same priority - APSARA_TEST_FALSE(sProcessQueueManager->CreateOrUpdateBoundedQueue(0, 0)); + APSARA_TEST_FALSE(sProcessQueueManager->CreateOrUpdateBoundedQueue(0, 0, sCtx)); // update queue with different priority // and current index not equal to the updated queue - APSARA_TEST_TRUE(sProcessQueueManager->CreateOrUpdateBoundedQueue(0, 1)); + APSARA_TEST_TRUE(sProcessQueueManager->CreateOrUpdateBoundedQueue(0, 1, sCtx)); APSARA_TEST_EQUAL(4U, sProcessQueueManager->mQueues.size()); APSARA_TEST_EQUAL(3U, sProcessQueueManager->mPriorityQueue[0].size()); APSARA_TEST_EQUAL(1U, sProcessQueueManager->mPriorityQueue[1].size()); @@ -101,7 +106,7 @@ void ProcessQueueManagerUnittest::TestUpdateSameTypeQueue() { // update queue with different priority // and current index equals to the updated queue // and the updated queue is not the last in the list - APSARA_TEST_TRUE(sProcessQueueManager->CreateOrUpdateBoundedQueue(2, 1)); + APSARA_TEST_TRUE(sProcessQueueManager->CreateOrUpdateBoundedQueue(2, 1, sCtx)); APSARA_TEST_EQUAL(4U, sProcessQueueManager->mQueues.size()); APSARA_TEST_EQUAL(2U, sProcessQueueManager->mPriorityQueue[0].size()); APSARA_TEST_EQUAL(2U, sProcessQueueManager->mPriorityQueue[1].size()); @@ -112,7 +117,7 @@ void ProcessQueueManagerUnittest::TestUpdateSameTypeQueue() { // and current index equals to the updated queue // and the updated queue is the last in the list // and more queues exist - APSARA_TEST_TRUE(sProcessQueueManager->CreateOrUpdateBoundedQueue(3, 1)); + APSARA_TEST_TRUE(sProcessQueueManager->CreateOrUpdateBoundedQueue(3, 1, sCtx)); APSARA_TEST_EQUAL(4U, sProcessQueueManager->mQueues.size()); APSARA_TEST_EQUAL(1U, sProcessQueueManager->mPriorityQueue[0].size()); APSARA_TEST_EQUAL(3U, sProcessQueueManager->mPriorityQueue[1].size()); @@ -123,7 +128,7 @@ void ProcessQueueManagerUnittest::TestUpdateSameTypeQueue() { // and current index equals to the updated queue // and the updated queue is the last in the list // and no more queue exists - APSARA_TEST_TRUE(sProcessQueueManager->CreateOrUpdateBoundedQueue(1, 1)); + APSARA_TEST_TRUE(sProcessQueueManager->CreateOrUpdateBoundedQueue(1, 1, sCtx)); APSARA_TEST_EQUAL(4U, sProcessQueueManager->mQueues.size()); APSARA_TEST_EQUAL(0U, sProcessQueueManager->mPriorityQueue[0].size()); APSARA_TEST_EQUAL(4U, sProcessQueueManager->mPriorityQueue[1].size()); @@ -132,7 +137,7 @@ void ProcessQueueManagerUnittest::TestUpdateSameTypeQueue() { // update queue with different priority // and current index is invalid before update - APSARA_TEST_TRUE(sProcessQueueManager->CreateOrUpdateBoundedQueue(0, 0)); + APSARA_TEST_TRUE(sProcessQueueManager->CreateOrUpdateBoundedQueue(0, 0, sCtx)); APSARA_TEST_EQUAL(4U, sProcessQueueManager->mQueues.size()); APSARA_TEST_EQUAL(1U, sProcessQueueManager->mPriorityQueue[0].size()); APSARA_TEST_EQUAL(3U, sProcessQueueManager->mPriorityQueue[1].size()); @@ -141,11 +146,11 @@ void ProcessQueueManagerUnittest::TestUpdateSameTypeQueue() { } void ProcessQueueManagerUnittest::TestUpdateDifferentTypeQueue() { - sProcessQueueManager->CreateOrUpdateBoundedQueue(0, 0); - sProcessQueueManager->CreateOrUpdateBoundedQueue(1, 0); + sProcessQueueManager->CreateOrUpdateBoundedQueue(0, 0, sCtx); + sProcessQueueManager->CreateOrUpdateBoundedQueue(1, 0, sCtx); // current index not equal to the updated queue - APSARA_TEST_TRUE(sProcessQueueManager->CreateOrUpdateCircularQueue(1, 0, 100)); + APSARA_TEST_TRUE(sProcessQueueManager->CreateOrUpdateCircularQueue(1, 0, 100, sCtx)); APSARA_TEST_EQUAL(2U, sProcessQueueManager->mQueues.size()); APSARA_TEST_EQUAL(2U, sProcessQueueManager->mPriorityQueue[0].size()); APSARA_TEST_TRUE(sProcessQueueManager->mQueues[1].first == prev(sProcessQueueManager->mPriorityQueue[0].end())); @@ -153,7 +158,7 @@ void ProcessQueueManagerUnittest::TestUpdateDifferentTypeQueue() { // current index equals to the updated queue // and the updated queue is not the last in the list - APSARA_TEST_TRUE(sProcessQueueManager->CreateOrUpdateCircularQueue(0, 0, 100)); + APSARA_TEST_TRUE(sProcessQueueManager->CreateOrUpdateCircularQueue(0, 0, 100, sCtx)); APSARA_TEST_EQUAL(2U, sProcessQueueManager->mQueues.size()); APSARA_TEST_EQUAL(2U, sProcessQueueManager->mPriorityQueue[0].size()); APSARA_TEST_TRUE(sProcessQueueManager->mQueues[0].first == prev(sProcessQueueManager->mPriorityQueue[0].end())); @@ -162,7 +167,7 @@ void ProcessQueueManagerUnittest::TestUpdateDifferentTypeQueue() { // current index equals to the update queue // and the updated queue is the last in the list sProcessQueueManager->mCurrentQueueIndex.second = prev(sProcessQueueManager->mPriorityQueue[0].end()); - APSARA_TEST_TRUE(sProcessQueueManager->CreateOrUpdateBoundedQueue(0, 0)); + APSARA_TEST_TRUE(sProcessQueueManager->CreateOrUpdateBoundedQueue(0, 0, sCtx)); APSARA_TEST_EQUAL(2U, sProcessQueueManager->mQueues.size()); APSARA_TEST_EQUAL(2U, sProcessQueueManager->mPriorityQueue[0].size()); APSARA_TEST_TRUE(sProcessQueueManager->mQueues[0].first == prev(sProcessQueueManager->mPriorityQueue[0].end())); @@ -174,10 +179,10 @@ void ProcessQueueManagerUnittest::TestDeleteQueue() { QueueKey key2 = QueueKeyManager::GetInstance()->GetKey("test_config_2"); QueueKey key3 = QueueKeyManager::GetInstance()->GetKey("test_config_3"); QueueKey key4 = QueueKeyManager::GetInstance()->GetKey("test_config_4"); - sProcessQueueManager->CreateOrUpdateBoundedQueue(key1, 0); - sProcessQueueManager->CreateOrUpdateBoundedQueue(key2, 0); - sProcessQueueManager->CreateOrUpdateBoundedQueue(key3, 0); - sProcessQueueManager->CreateOrUpdateBoundedQueue(key4, 0); + sProcessQueueManager->CreateOrUpdateBoundedQueue(key1, 0, sCtx); + sProcessQueueManager->CreateOrUpdateBoundedQueue(key2, 0, sCtx); + sProcessQueueManager->CreateOrUpdateBoundedQueue(key3, 0, sCtx); + sProcessQueueManager->CreateOrUpdateBoundedQueue(key4, 0, sCtx); sProcessQueueManager->mCurrentQueueIndex.second = sProcessQueueManager->mQueues[key3].first; // current index not equal to the deleted queue @@ -218,7 +223,7 @@ void ProcessQueueManagerUnittest::TestDeleteQueue() { } void ProcessQueueManagerUnittest::TestSetQueueUpstreamAndDownStream() { - sProcessQueueManager->CreateOrUpdateBoundedQueue(0, 0); + sProcessQueueManager->CreateOrUpdateBoundedQueue(0, 0, sCtx); // queue exists APSARA_TEST_TRUE(sProcessQueueManager->SetDownStreamQueues(0, vector())); @@ -229,47 +234,53 @@ void ProcessQueueManagerUnittest::TestSetQueueUpstreamAndDownStream() { } void ProcessQueueManagerUnittest::TestPushQueue() { - sProcessQueueManager->CreateOrUpdateBoundedQueue(0, 0); - ExactlyOnceQueueManager::GetInstance()->CreateOrUpdateQueue(1, 0, "test_config", vector(5)); + sProcessQueueManager->CreateOrUpdateBoundedQueue(0, 0, sCtx); + ExactlyOnceQueueManager::GetInstance()->CreateOrUpdateQueue(1, 0, sCtx, vector(5)); // queue belongs to normal process queue APSARA_TEST_TRUE(sProcessQueueManager->IsValidToPush(0)); - APSARA_TEST_EQUAL(0, sProcessQueueManager->PushQueue(0, make_unique(std::move(*sEventGroup), 0))); + APSARA_TEST_EQUAL(0, sProcessQueueManager->PushQueue(0, GenerateItem())); // queue belongs to exactly once process queue APSARA_TEST_TRUE(sProcessQueueManager->IsValidToPush(1)); - APSARA_TEST_EQUAL(0, sProcessQueueManager->PushQueue(1, make_unique(std::move(*sEventGroup), 0))); + APSARA_TEST_EQUAL(0, sProcessQueueManager->PushQueue(1, GenerateItem())); // no queue exists APSARA_TEST_FALSE(sProcessQueueManager->IsValidToPush(2)); - APSARA_TEST_EQUAL(2, sProcessQueueManager->PushQueue(2, make_unique(std::move(*sEventGroup), 0))); + APSARA_TEST_EQUAL(2, sProcessQueueManager->PushQueue(2, GenerateItem())); // invalid to push static_cast(sProcessQueueManager->mQueues[0].first->get())->mValidToPush = false; APSARA_TEST_FALSE(sProcessQueueManager->IsValidToPush(0)); - APSARA_TEST_EQUAL(1, sProcessQueueManager->PushQueue(0, make_unique(std::move(*sEventGroup), 0))); + APSARA_TEST_EQUAL(1, sProcessQueueManager->PushQueue(0, GenerateItem())); ExactlyOnceQueueManager::GetInstance()->mProcessQueues[1]->mValidToPush = false; APSARA_TEST_FALSE(sProcessQueueManager->IsValidToPush(1)); - APSARA_TEST_EQUAL(1, sProcessQueueManager->PushQueue(1, make_unique(std::move(*sEventGroup), 0))); + APSARA_TEST_EQUAL(1, sProcessQueueManager->PushQueue(1, GenerateItem())); } void ProcessQueueManagerUnittest::TestPopItem() { unique_ptr item; string configName; + PipelineContext ctx; + ctx.SetConfigName("test_config_1"); QueueKey key1 = QueueKeyManager::GetInstance()->GetKey("test_config_1"); + sProcessQueueManager->CreateOrUpdateBoundedQueue(key1, 0, ctx); + ctx.SetConfigName("test_config_2"); QueueKey key2 = QueueKeyManager::GetInstance()->GetKey("test_config_2"); + sProcessQueueManager->CreateOrUpdateBoundedQueue(key2, 1, ctx); + ctx.SetConfigName("test_config_3"); QueueKey key3 = QueueKeyManager::GetInstance()->GetKey("test_config_3"); + sProcessQueueManager->CreateOrUpdateBoundedQueue(key3, 1, ctx); + ctx.SetConfigName("test_config_4"); QueueKey key4 = QueueKeyManager::GetInstance()->GetKey("test_config_4"); - sProcessQueueManager->CreateOrUpdateBoundedQueue(key1, 0); - sProcessQueueManager->CreateOrUpdateBoundedQueue(key2, 1); - sProcessQueueManager->CreateOrUpdateBoundedQueue(key3, 1); - sProcessQueueManager->CreateOrUpdateBoundedQueue(key4, 1); - ExactlyOnceQueueManager::GetInstance()->CreateOrUpdateQueue(5, 0, "test_config_5", vector(5)); - - sProcessQueueManager->PushQueue(key2, make_unique(std::move(*sEventGroup), 0)); - sProcessQueueManager->PushQueue(key3, make_unique(std::move(*sEventGroup), 0)); + sProcessQueueManager->CreateOrUpdateBoundedQueue(key4, 1, ctx); + ctx.SetConfigName("test_config_5"); + ExactlyOnceQueueManager::GetInstance()->CreateOrUpdateQueue(5, 0, ctx, vector(5)); + + sProcessQueueManager->PushQueue(key2, GenerateItem()); + sProcessQueueManager->PushQueue(key3, GenerateItem()); sProcessQueueManager->mCurrentQueueIndex = {1, prev(prev(sProcessQueueManager->mPriorityQueue[1].end()))}; // the item comes from the queue between current index and queue list end @@ -284,7 +295,7 @@ void ProcessQueueManagerUnittest::TestPopItem() { APSARA_TEST_EQUAL(1U, sProcessQueueManager->mCurrentQueueIndex.first); APSARA_TEST_TRUE(sProcessQueueManager->mCurrentQueueIndex.second == sProcessQueueManager->mQueues[key3].first); - sProcessQueueManager->PushQueue(key1, make_unique(std::move(*sEventGroup), 0)); + sProcessQueueManager->PushQueue(key1, GenerateItem()); // the item comes from queue list other than the one pointed by current index APSARA_TEST_TRUE(sProcessQueueManager->PopItem(0, item, configName)); APSARA_TEST_EQUAL("test_config_1", configName); @@ -292,7 +303,7 @@ void ProcessQueueManagerUnittest::TestPopItem() { APSARA_TEST_TRUE(sProcessQueueManager->mCurrentQueueIndex.second == sProcessQueueManager->mQueues[key1].first); sProcessQueueManager->mCurrentQueueIndex = {1, prev(sProcessQueueManager->mPriorityQueue[1].end())}; - sProcessQueueManager->PushQueue(5, make_unique(std::move(*sEventGroup), 0)); + sProcessQueueManager->PushQueue(5, GenerateItem()); // the item comes from exactly once queue APSARA_TEST_TRUE(sProcessQueueManager->PopItem(0, item, configName)); APSARA_TEST_EQUAL("test_config_5", configName); @@ -307,14 +318,14 @@ void ProcessQueueManagerUnittest::TestPopItem() { } void ProcessQueueManagerUnittest::TestIsAllQueueEmpty() { - sProcessQueueManager->CreateOrUpdateBoundedQueue(0, 0); - sProcessQueueManager->CreateOrUpdateBoundedQueue(1, 1); - ExactlyOnceQueueManager::GetInstance()->CreateOrUpdateQueue(2, 0, "test_config_1", vector(5)); - ExactlyOnceQueueManager::GetInstance()->CreateOrUpdateQueue(3, 2, "test_config_2", vector(5)); + sProcessQueueManager->CreateOrUpdateBoundedQueue(0, 0, sCtx); + sProcessQueueManager->CreateOrUpdateBoundedQueue(1, 1, sCtx); + ExactlyOnceQueueManager::GetInstance()->CreateOrUpdateQueue(2, 0, sCtx, vector(5)); + ExactlyOnceQueueManager::GetInstance()->CreateOrUpdateQueue(3, 2, sCtx, vector(5)); APSARA_TEST_TRUE(sProcessQueueManager->IsAllQueueEmpty()); // non empty normal process queue - sProcessQueueManager->PushQueue(0, make_unique(std::move(*sEventGroup), 0)); + sProcessQueueManager->PushQueue(0, GenerateItem()); APSARA_TEST_FALSE(sProcessQueueManager->IsAllQueueEmpty()); unique_ptr item; @@ -323,7 +334,7 @@ void ProcessQueueManagerUnittest::TestIsAllQueueEmpty() { APSARA_TEST_TRUE(sProcessQueueManager->IsAllQueueEmpty()); // non empty exactly once process queue - sProcessQueueManager->PushQueue(2, make_unique(std::move(*sEventGroup), 0)); + sProcessQueueManager->PushQueue(2, GenerateItem()); APSARA_TEST_FALSE(sProcessQueueManager->IsAllQueueEmpty()); sProcessQueueManager->PopItem(0, item, configName); @@ -331,10 +342,13 @@ void ProcessQueueManagerUnittest::TestIsAllQueueEmpty() { } void ProcessQueueManagerUnittest::OnPipelineUpdate() { + PipelineContext ctx; + ctx.SetConfigName("test_config_1"); QueueKey key = QueueKeyManager::GetInstance()->GetKey("test_config_1"); - sProcessQueueManager->CreateOrUpdateBoundedQueue(key, 0); - ExactlyOnceQueueManager::GetInstance()->CreateOrUpdateQueue(1, 0, "test_config_2", vector(5)); - ExactlyOnceQueueManager::GetInstance()->CreateOrUpdateQueue(2, 0, "test_config_2", vector(5)); + sProcessQueueManager->CreateOrUpdateBoundedQueue(key, 0, ctx); + ctx.SetConfigName("test_config_2"); + ExactlyOnceQueueManager::GetInstance()->CreateOrUpdateQueue(1, 0, ctx, vector(5)); + ExactlyOnceQueueManager::GetInstance()->CreateOrUpdateQueue(2, 0, ctx, vector(5)); sProcessQueueManager->InvalidatePop("test_config_1"); APSARA_TEST_FALSE((*sProcessQueueManager->mQueues[key].first)->mValidToPop); diff --git a/core/unittest/queue/SenderQueueManagerUnittest.cpp b/core/unittest/queue/SenderQueueManagerUnittest.cpp index c238327fb3..25b8d9ee74 100644 --- a/core/unittest/queue/SenderQueueManagerUnittest.cpp +++ b/core/unittest/queue/SenderQueueManagerUnittest.cpp @@ -12,12 +12,12 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include "plugin/flusher/sls/FlusherSLS.h" #include "pipeline/queue/ExactlyOnceQueueManager.h" #include "pipeline/queue/QueueKeyManager.h" #include "pipeline/queue/QueueParam.h" #include "pipeline/queue/SLSSenderQueueItem.h" #include "pipeline/queue/SenderQueueManager.h" +#include "plugin/flusher/sls/FlusherSLS.h" #include "unittest/Unittest.h" DECLARE_FLAG_INT32(sender_queue_gc_threshold_sec); @@ -72,6 +72,8 @@ class SenderQueueManagerUnittest : public testing::Test { static SenderQueueManager* sManager; static shared_ptr sConcurrencyLimiter; static vector sCheckpoints; + static PipelineContext sCtx; + static string sFlusherId; unique_ptr GenerateItem(bool isSLS = false); @@ -83,13 +85,15 @@ const size_t SenderQueueManagerUnittest::sDataSize; SenderQueueManager* SenderQueueManagerUnittest::sManager; shared_ptr SenderQueueManagerUnittest::sConcurrencyLimiter; vector SenderQueueManagerUnittest::sCheckpoints; +PipelineContext SenderQueueManagerUnittest::sCtx; +string SenderQueueManagerUnittest::sFlusherId; void SenderQueueManagerUnittest::TestCreateQueue() { { // new queue uint32_t maxRate = 100U; - APSARA_TEST_TRUE( - sManager->CreateQueue(0, vector>{sConcurrencyLimiter}, maxRate)); + APSARA_TEST_TRUE(sManager->CreateQueue( + 0, sFlusherId, sCtx, vector>{sConcurrencyLimiter}, maxRate)); APSARA_TEST_EQUAL(1U, sManager->mQueues.size()); auto& queue = sManager->mQueues.at(0); APSARA_TEST_EQUAL(sManager->mQueueParam.GetCapacity(), queue.mCapacity); @@ -104,7 +108,8 @@ void SenderQueueManagerUnittest::TestCreateQueue() { // resued queue shared_ptr newLimiter = make_shared(); uint32_t maxRate = 10U; - APSARA_TEST_TRUE(sManager->CreateQueue(0, vector>{newLimiter}, maxRate)); + APSARA_TEST_TRUE( + sManager->CreateQueue(0, sFlusherId, sCtx, vector>{newLimiter}, maxRate)); APSARA_TEST_EQUAL(1U, sManager->mQueues.size()); auto& queue = sManager->mQueues.at(0); APSARA_TEST_EQUAL(1U, queue.mConcurrencyLimiters.size()); @@ -122,8 +127,10 @@ void SenderQueueManagerUnittest::TestDeleteQueue() { QueueKey key1 = QueueKeyManager::GetInstance()->GetKey("name_1"); QueueKey key2 = QueueKeyManager::GetInstance()->GetKey("name_2"); - sManager->CreateQueue(key1, vector>{sConcurrencyLimiter}, sMaxRate); - sManager->CreateQueue(key2, vector>{sConcurrencyLimiter}, sMaxRate); + sManager->CreateQueue( + key1, sFlusherId, sCtx, vector>{sConcurrencyLimiter}, sMaxRate); + sManager->CreateQueue( + key2, sFlusherId, sCtx, vector>{sConcurrencyLimiter}, sMaxRate); sManager->PushQueue(key2, GenerateItem()); // queue exists and not marked deleted @@ -150,13 +157,13 @@ void SenderQueueManagerUnittest::TestGetQueue() { APSARA_TEST_EQUAL(nullptr, sManager->GetQueue(0)); // queue existed - sManager->CreateQueue(0, vector>{sConcurrencyLimiter}, sMaxRate); + sManager->CreateQueue(0, sFlusherId, sCtx, vector>{sConcurrencyLimiter}, sMaxRate); APSARA_TEST_NOT_EQUAL(nullptr, sManager->GetQueue(0)); } void SenderQueueManagerUnittest::TestPushQueue() { - sManager->CreateQueue(0, vector>{sConcurrencyLimiter}, sMaxRate); - ExactlyOnceQueueManager::GetInstance()->CreateOrUpdateQueue(1, 0, "test_config", sCheckpoints); + sManager->CreateQueue(0, sFlusherId, sCtx, vector>{sConcurrencyLimiter}, sMaxRate); + ExactlyOnceQueueManager::GetInstance()->CreateOrUpdateQueue(1, 0, sCtx, sCheckpoints); // queue belongs to normal queue APSARA_TEST_TRUE(sManager->IsValidToPush(0)); @@ -178,7 +185,11 @@ void SenderQueueManagerUnittest::TestPushQueue() { void SenderQueueManagerUnittest::TestGetAllAvailableItems() { // prepare nomal queue sManager->CreateQueue( - 0, vector>{FlusherSLS::GetRegionConcurrencyLimiter(mFlusher.mRegion)}, sMaxRate); + 0, + sFlusherId, + sCtx, + vector>{FlusherSLS::GetRegionConcurrencyLimiter(mFlusher.mRegion)}, + sMaxRate); for (size_t i = 0; i <= sManager->mQueueParam.GetCapacity(); ++i) { sManager->PushQueue(0, GenerateItem()); } @@ -192,7 +203,7 @@ void SenderQueueManagerUnittest::TestGetAllAvailableItems() { cpt->data.set_sequence_id(0); checkpoints.emplace_back(cpt); } - ExactlyOnceQueueManager::GetInstance()->CreateOrUpdateQueue(1, 0, "test_config", checkpoints); + ExactlyOnceQueueManager::GetInstance()->CreateOrUpdateQueue(1, 0, sCtx, checkpoints); for (size_t i = 0; i <= 2; ++i) { ExactlyOnceQueueManager::GetInstance()->PushSenderQueue(1, GenerateItem(true)); } @@ -218,8 +229,8 @@ void SenderQueueManagerUnittest::TestGetAllAvailableItems() { } void SenderQueueManagerUnittest::TestRemoveItem() { - sManager->CreateQueue(0, vector>{sConcurrencyLimiter}, sMaxRate); - ExactlyOnceQueueManager::GetInstance()->CreateOrUpdateQueue(1, 0, "test_config", sCheckpoints); + sManager->CreateQueue(0, sFlusherId, sCtx, vector>{sConcurrencyLimiter}, sMaxRate); + ExactlyOnceQueueManager::GetInstance()->CreateOrUpdateQueue(1, 0, sCtx, sCheckpoints); { // normal queue auto item = GenerateItem(); @@ -245,10 +256,10 @@ void SenderQueueManagerUnittest::TestRemoveItem() { } void SenderQueueManagerUnittest::TestIsAllQueueEmpty() { - sManager->CreateQueue(0, vector>{sConcurrencyLimiter}, sMaxRate); - sManager->CreateQueue(1, vector>{sConcurrencyLimiter}, sMaxRate); - ExactlyOnceQueueManager::GetInstance()->CreateOrUpdateQueue(2, 0, "test_config_1", sCheckpoints); - ExactlyOnceQueueManager::GetInstance()->CreateOrUpdateQueue(3, 2, "test_config_2", sCheckpoints); + sManager->CreateQueue(0, sFlusherId, sCtx, vector>{sConcurrencyLimiter}, sMaxRate); + sManager->CreateQueue(1, sFlusherId, sCtx, vector>{sConcurrencyLimiter}, sMaxRate); + ExactlyOnceQueueManager::GetInstance()->CreateOrUpdateQueue(2, 0, sCtx, sCheckpoints); + ExactlyOnceQueueManager::GetInstance()->CreateOrUpdateQueue(3, 2, sCtx, sCheckpoints); APSARA_TEST_TRUE(sManager->IsAllQueueEmpty()); { // non-empty normal queue diff --git a/core/unittest/queue/SenderQueueUnittest.cpp b/core/unittest/queue/SenderQueueUnittest.cpp index 181f24d386..92a4dffea4 100644 --- a/core/unittest/queue/SenderQueueUnittest.cpp +++ b/core/unittest/queue/SenderQueueUnittest.cpp @@ -25,12 +25,16 @@ class SenderQueueUnittest : public testing::Test { void TestPush(); void TestRemove(); void TestGetAllAvailableItems(); + void TestMetric(); protected: - static void SetUpTestCase() { sConcurrencyLimiter = make_shared(); } + static void SetUpTestCase() { + sConcurrencyLimiter = make_shared(); + sCtx.SetConfigName("test_config"); + } void SetUp() override { - mQueue.reset(new SenderQueue(sCap, sLowWatermark, sHighWatermark, sKey)); + mQueue.reset(new SenderQueue(sCap, sLowWatermark, sHighWatermark, sKey, sFlusherId, sCtx)); mQueue->SetConcurrencyLimiters(vector>{sConcurrencyLimiter}); mQueue->mRateLimiter = RateLimiter(100); mQueue->SetFeedback(&sFeedback); @@ -42,7 +46,9 @@ class SenderQueueUnittest : public testing::Test { } private: + static PipelineContext sCtx; static const QueueKey sKey = 0; + static const string sFlusherId; static const size_t sCap = 2; static const size_t sLowWatermark = 1; static const size_t sHighWatermark = 2; @@ -57,7 +63,9 @@ class SenderQueueUnittest : public testing::Test { unique_ptr mQueue; }; +PipelineContext SenderQueueUnittest::sCtx; const QueueKey SenderQueueUnittest::sKey; +const string SenderQueueUnittest::sFlusherId = "1"; const size_t SenderQueueUnittest::sDataSize; shared_ptr SenderQueueUnittest::sConcurrencyLimiter; FeedbackInterfaceMock SenderQueueUnittest::sFeedback; @@ -162,6 +170,54 @@ void SenderQueueUnittest::TestGetAllAvailableItems() { } } +void SenderQueueUnittest::TestMetric() { + APSARA_TEST_EQUAL(5U, mQueue->mMetricsRecordRef->GetLabels()->size()); + APSARA_TEST_TRUE(mQueue->mMetricsRecordRef.HasLabel(METRIC_LABEL_PROJECT, "")); + APSARA_TEST_TRUE(mQueue->mMetricsRecordRef.HasLabel(METRIC_LABEL_CONFIG_NAME, "test_config")); + APSARA_TEST_TRUE(mQueue->mMetricsRecordRef.HasLabel(METRIC_LABEL_KEY_COMPONENT_NAME, "sender_queue")); + APSARA_TEST_TRUE(mQueue->mMetricsRecordRef.HasLabel(METRIC_LABEL_KEY_QUEUE_TYPE, "bounded")); + APSARA_TEST_TRUE(mQueue->mMetricsRecordRef.HasLabel(METRIC_LABEL_KEY_FLUSHER_NODE_ID, sFlusherId)); + + auto item1 = GenerateItem(); + auto dataSize = item1->mData.size(); + auto ptr1 = item1.get(); + mQueue->Push(std::move(item1)); + + APSARA_TEST_EQUAL(1U, mQueue->mInItemsCnt->GetValue()); + APSARA_TEST_EQUAL(dataSize, mQueue->mInItemDataSizeBytes->GetValue()); + APSARA_TEST_EQUAL(1U, mQueue->mQueueSize->GetValue()); + APSARA_TEST_EQUAL(dataSize, mQueue->mQueueDataSizeByte->GetValue()); + APSARA_TEST_EQUAL(1U, mQueue->mValidToPushFlag->GetValue()); + + auto item2 = GenerateItem(); + auto ptr2 = item2.get(); + mQueue->Push(std::move(item2)); + + mQueue->Push(GenerateItem()); + + APSARA_TEST_EQUAL(3U, mQueue->mInItemsCnt->GetValue()); + APSARA_TEST_EQUAL(dataSize * 3, mQueue->mInItemDataSizeBytes->GetValue()); + APSARA_TEST_EQUAL(2U, mQueue->mQueueSize->GetValue()); + APSARA_TEST_EQUAL(dataSize * 2, mQueue->mQueueDataSizeByte->GetValue()); + APSARA_TEST_EQUAL(0U, mQueue->mValidToPushFlag->GetValue()); + APSARA_TEST_EQUAL(1U, mQueue->mExtraBufferSize->GetValue()); + APSARA_TEST_EQUAL(dataSize, mQueue->mExtraBufferDataSizeBytes->GetValue()); + + mQueue->Remove(ptr1); + APSARA_TEST_EQUAL(1U, mQueue->mOutItemsCnt->GetValue()); + APSARA_TEST_EQUAL(2U, mQueue->mQueueSize->GetValue()); + APSARA_TEST_EQUAL(dataSize * 2, mQueue->mQueueDataSizeByte->GetValue()); + APSARA_TEST_EQUAL(0U, mQueue->mValidToPushFlag->GetValue()); + APSARA_TEST_EQUAL(0U, mQueue->mExtraBufferSize->GetValue()); + APSARA_TEST_EQUAL(0U, mQueue->mExtraBufferDataSizeBytes->GetValue()); + + mQueue->Remove(ptr2); + APSARA_TEST_EQUAL(2U, mQueue->mOutItemsCnt->GetValue()); + APSARA_TEST_EQUAL(1U, mQueue->mQueueSize->GetValue()); + APSARA_TEST_EQUAL(dataSize, mQueue->mQueueDataSizeByte->GetValue()); + APSARA_TEST_EQUAL(1U, mQueue->mValidToPushFlag->GetValue()); +} + unique_ptr SenderQueueUnittest::GenerateItem() { return make_unique("content", sDataSize, nullptr, sKey); } @@ -169,6 +225,7 @@ unique_ptr SenderQueueUnittest::GenerateItem() { UNIT_TEST_CASE(SenderQueueUnittest, TestPush) UNIT_TEST_CASE(SenderQueueUnittest, TestRemove) UNIT_TEST_CASE(SenderQueueUnittest, TestGetAllAvailableItems) +UNIT_TEST_CASE(SenderQueueUnittest, TestMetric) } // namespace logtail diff --git a/core/unittest/reader/ForceReadUnittest.cpp b/core/unittest/reader/ForceReadUnittest.cpp index cd3a750f6f..aac74460f1 100644 --- a/core/unittest/reader/ForceReadUnittest.cpp +++ b/core/unittest/reader/ForceReadUnittest.cpp @@ -118,7 +118,7 @@ class ForceReadUnittest : public testing::Test { FileServer::GetInstance()->AddFileDiscoveryConfig(mConfigName, &discoveryOpts, &ctx); FileServer::GetInstance()->AddFileReaderConfig(mConfigName, &readerOpts, &ctx); FileServer::GetInstance()->AddMultilineConfig(mConfigName, &multilineOpts, &ctx); - ProcessQueueManager::GetInstance()->CreateOrUpdateBoundedQueue(0, 0); + ProcessQueueManager::GetInstance()->CreateOrUpdateBoundedQueue(0, 0, ctx); } void TearDown() override { remove(utf8File.c_str()); } diff --git a/core/unittest/sender/FlusherRunnerUnittest.cpp b/core/unittest/sender/FlusherRunnerUnittest.cpp index 4ddf83f2cb..2940cd9fad 100644 --- a/core/unittest/sender/FlusherRunnerUnittest.cpp +++ b/core/unittest/sender/FlusherRunnerUnittest.cpp @@ -33,6 +33,9 @@ void FlusherRunnerUnittest::TestDispatch() { // http auto flusher = make_unique(); Json::Value tmp; + PipelineContext ctx; + flusher->SetContext(ctx); + flusher->SetMetricsRecordRef("name", "pluginId", "nodeId", "childNodeId"); flusher->Init(Json::Value(), tmp); auto item = make_unique("content", 10, flusher.get(), flusher->GetQueueKey()); @@ -49,6 +52,9 @@ void FlusherRunnerUnittest::TestDispatch() { // unknown auto flusher = make_unique(); Json::Value tmp; + PipelineContext ctx; + flusher->SetContext(ctx); + flusher->SetMetricsRecordRef("name", "pluginId", "nodeId", "childNodeId"); flusher->Init(Json::Value(), tmp); auto item = make_unique("content", 10, flusher.get(), flusher->GetQueueKey()); From 38bc1fe8e9d4601e6a5879e7763fe4755a89c988 Mon Sep 17 00:00:00 2001 From: henryzhx8 Date: Thu, 19 Sep 2024 09:30:40 +0800 Subject: [PATCH 2/2] add metric for flusher component (#1762) --- core/CMakeLists.txt | 2 +- core/common/common.cmake | 2 +- .../compression/CompressType.h | 10 +- core/common/compression/Compressor.cpp | 59 +++++++++ .../compression/Compressor.h | 23 +++- .../compression/CompressorFactory.cpp | 30 +++-- .../compression/CompressorFactory.h | 8 +- .../compression/LZ4Compressor.cpp | 2 +- .../compression/LZ4Compressor.h | 9 +- .../compression/ZstdCompressor.cpp | 2 +- .../compression/ZstdCompressor.h | 8 +- core/go_pipeline/LogtailPlugin.cpp | 16 ++- core/monitor/LogtailMetric.h | 18 +++ core/monitor/MetricConstants.cpp | 12 +- core/monitor/MetricConstants.h | 12 +- core/pipeline/Pipeline.cpp | 26 +++- core/pipeline/Pipeline.h | 14 ++- core/pipeline/batch/BatchItem.h | 34 +++++- core/pipeline/batch/BatchStatus.h | 2 +- core/pipeline/batch/BatchedEvents.h | 10 +- core/pipeline/batch/Batcher.h | 88 +++++++++++++- .../plugin/instance/FlusherInstance.cpp | 8 +- .../plugin/instance/FlusherInstance.h | 4 +- .../plugin/instance/ProcessorInstance.cpp | 6 +- core/pipeline/queue/QueueInterface.h | 8 +- core/pipeline/route/Router.cpp | 11 ++ core/pipeline/route/Router.h | 5 + core/pipeline/serializer/SLSSerializer.cpp | 41 +++++-- core/pipeline/serializer/SLSSerializer.h | 5 + core/pipeline/serializer/Serializer.h | 73 ++++++++++- core/plugin/flusher/sls/FlusherSLS.cpp | 16 +-- core/plugin/flusher/sls/FlusherSLS.h | 6 +- core/profile_sender/ProfileSender.cpp | 6 +- core/unittest/batch/BatchItemUnittest.cpp | 31 ++++- core/unittest/batch/BatchStatusUnittest.cpp | 5 +- core/unittest/batch/BatcherUnittest.cpp | 63 +++++++++- core/unittest/compression/CMakeLists.txt | 4 + .../compression/CompressorFactoryUnittest.cpp | 42 +++++-- .../compression/CompressorUnittest.cpp | 79 ++++++++++++ .../compression/LZ4CompressorUnittest.cpp | 4 +- .../compression/ZstdCompressorUnittest.cpp | 4 +- core/unittest/flusher/FlusherSLSUnittest.cpp | 22 ++-- core/unittest/pipeline/PipelineUnittest.cpp | 29 +++-- core/unittest/route/RouterUnittest.cpp | 41 +++++++ core/unittest/serializer/CMakeLists.txt | 4 + .../serializer/SLSSerializerUnittest.cpp | 18 +-- .../serializer/SerializerUnittest.cpp | 114 ++++++++++++++++++ 47 files changed, 882 insertions(+), 154 deletions(-) rename core/{pipeline => common}/compression/CompressType.h (85%) create mode 100644 core/common/compression/Compressor.cpp rename core/{pipeline => common}/compression/Compressor.h (64%) rename core/{pipeline => common}/compression/CompressorFactory.cpp (71%) rename core/{pipeline => common}/compression/CompressorFactory.h (87%) rename core/{pipeline => common}/compression/LZ4Compressor.cpp (97%) rename core/{pipeline => common}/compression/LZ4Compressor.h (89%) rename core/{pipeline => common}/compression/ZstdCompressor.cpp (97%) rename core/{pipeline => common}/compression/ZstdCompressor.h (92%) create mode 100644 core/unittest/compression/CompressorUnittest.cpp create mode 100644 core/unittest/serializer/SerializerUnittest.cpp diff --git a/core/CMakeLists.txt b/core/CMakeLists.txt index 162ed8a82b..6c36c26eeb 100644 --- a/core/CMakeLists.txt +++ b/core/CMakeLists.txt @@ -113,7 +113,7 @@ include(${CMAKE_CURRENT_SOURCE_DIR}/plugin/flusher/links.cmake) set(SUB_DIRECTORIES_LIST application app_config checkpoint container_manager logger go_pipeline monitor profile_sender models config config/feedbacker config/provider config/watcher - pipeline pipeline/batch pipeline/compression pipeline/limiter pipeline/plugin pipeline/plugin/creator pipeline/plugin/instance pipeline/plugin/interface pipeline/queue pipeline/route pipeline/serializer + pipeline pipeline/batch pipeline/limiter pipeline/plugin pipeline/plugin/creator pipeline/plugin/instance pipeline/plugin/interface pipeline/queue pipeline/route pipeline/serializer runner runner/sink/http protobuf/config_server/v1 protobuf/config_server/v2 protobuf/sls file_server file_server/event file_server/event_handler file_server/event_listener file_server/reader file_server/polling diff --git a/core/common/common.cmake b/core/common/common.cmake index e905a95fa3..caf8e45b98 100644 --- a/core/common/common.cmake +++ b/core/common/common.cmake @@ -30,7 +30,7 @@ list(APPEND THIS_SOURCE_FILES_LIST ${XX_HASH_SOURCE_FILES}) list(APPEND THIS_SOURCE_FILES_LIST ${CMAKE_SOURCE_DIR}/common/memory/SourceBuffer.h) list(APPEND THIS_SOURCE_FILES_LIST ${CMAKE_SOURCE_DIR}/common/http/AsynCurlRunner.cpp ${CMAKE_SOURCE_DIR}/common/http/Curl.cpp ${CMAKE_SOURCE_DIR}/common/http/HttpResponse.cpp) list(APPEND THIS_SOURCE_FILES_LIST ${CMAKE_SOURCE_DIR}/common/timer/Timer.cpp ${CMAKE_SOURCE_DIR}/common/timer/HttpRequestTimerEvent.cpp) - +list(APPEND THIS_SOURCE_FILES_LIST ${CMAKE_SOURCE_DIR}/common/compression/Compressor.cpp ${CMAKE_SOURCE_DIR}/common/compression/CompressorFactory.cpp ${CMAKE_SOURCE_DIR}/common/compression/LZ4Compressor.cpp ${CMAKE_SOURCE_DIR}/common/compression/ZstdCompressor.cpp) # remove several files in common list(REMOVE_ITEM THIS_SOURCE_FILES_LIST ${CMAKE_SOURCE_DIR}/common/BoostRegexValidator.cpp ${CMAKE_SOURCE_DIR}/common/GetUUID.cpp) diff --git a/core/pipeline/compression/CompressType.h b/core/common/compression/CompressType.h similarity index 85% rename from core/pipeline/compression/CompressType.h rename to core/common/compression/CompressType.h index 96f996ee9a..1580bb5967 100644 --- a/core/pipeline/compression/CompressType.h +++ b/core/common/compression/CompressType.h @@ -18,6 +18,14 @@ namespace logtail { -enum class CompressType { NONE, LZ4, ZSTD }; +enum class CompressType { + NONE, + LZ4, + ZSTD +#ifdef APSARA_UNIT_TEST_MAIN + , + MOCK +#endif +}; } // namespace logtail diff --git a/core/common/compression/Compressor.cpp b/core/common/compression/Compressor.cpp new file mode 100644 index 0000000000..39a7af4b57 --- /dev/null +++ b/core/common/compression/Compressor.cpp @@ -0,0 +1,59 @@ +// Copyright 2024 iLogtail Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "common/compression/Compressor.h" + +#include + +#include "monitor/MetricConstants.h" + +using namespace std; + +namespace logtail { + +void Compressor::SetMetricRecordRef(MetricLabels&& labels, DynamicMetricLabels&& dynamicLabels) { + WriteMetrics::GetInstance()->PrepareMetricsRecordRef( + mMetricsRecordRef, std::move(labels), std::move(dynamicLabels)); + mInItemsCnt = mMetricsRecordRef.CreateCounter(METRIC_IN_ITEMS_CNT); + mInItemSizeBytes = mMetricsRecordRef.CreateCounter(METRIC_IN_ITEM_SIZE_BYTES); + mOutItemsCnt = mMetricsRecordRef.CreateCounter(METRIC_OUT_ITEMS_CNT); + mOutItemSizeBytes = mMetricsRecordRef.CreateCounter(METRIC_OUT_ITEM_SIZE_BYTES); + mDiscardedItemsCnt = mMetricsRecordRef.CreateCounter("discarded_items_cnt"); + mDiscardedItemSizeBytes = mMetricsRecordRef.CreateCounter("discarded_item_size_bytes"); + mTotalDelayMs = mMetricsRecordRef.CreateCounter(METRIC_TOTAL_DELAY_MS); +} + +bool Compressor::DoCompress(const string& input, string& output, string& errorMsg) { + if (mMetricsRecordRef != nullptr) { + mInItemsCnt->Add(1); + mInItemSizeBytes->Add(input.size()); + } + + auto before = chrono::system_clock::now(); + auto res = Compress(input, output, errorMsg); + + if (mMetricsRecordRef != nullptr) { + mTotalDelayMs->Add(chrono::duration_cast(chrono::system_clock::now() - before).count()); + if (res) { + mOutItemsCnt->Add(1); + mOutItemSizeBytes->Add(output.size()); + } else { + mDiscardedItemsCnt->Add(1); + mDiscardedItemSizeBytes->Add(input.size()); + } + } + return res; +} + +} // namespace logtail diff --git a/core/pipeline/compression/Compressor.h b/core/common/compression/Compressor.h similarity index 64% rename from core/pipeline/compression/Compressor.h rename to core/common/compression/Compressor.h index 1694f6e6ad..0c32358ca5 100644 --- a/core/pipeline/compression/Compressor.h +++ b/core/common/compression/Compressor.h @@ -18,7 +18,8 @@ #include -#include "pipeline/compression/CompressType.h" +#include "monitor/LogtailMetric.h" +#include "common/compression/CompressType.h" namespace logtail { @@ -27,7 +28,7 @@ class Compressor { Compressor(CompressType type) : mType(type) {} virtual ~Compressor() = default; - virtual bool Compress(const std::string& input, std::string& output, std::string& errorMsg) = 0; + bool DoCompress(const std::string& input, std::string& output, std::string& errorMsg); #ifdef APSARA_UNIT_TEST_MAIN // buffer shoudl be reserved for output before calling this function @@ -35,9 +36,27 @@ class Compressor { #endif CompressType GetCompressType() const { return mType; } + void SetMetricRecordRef(MetricLabels&& labels, DynamicMetricLabels&& dynamicLabels = {}); + +protected: + mutable MetricsRecordRef mMetricsRecordRef; + CounterPtr mInItemsCnt; + CounterPtr mInItemSizeBytes; + CounterPtr mOutItemsCnt; + CounterPtr mOutItemSizeBytes; + CounterPtr mDiscardedItemsCnt; + CounterPtr mDiscardedItemSizeBytes; + CounterPtr mTotalDelayMs; private: + virtual bool Compress(const std::string& input, std::string& output, std::string& errorMsg) = 0; + CompressType mType = CompressType::NONE; + +#ifdef APSARA_UNIT_TEST_MAIN + friend class CompressorUnittest; + friend class CompressorFactoryUnittest; +#endif }; } // namespace logtail diff --git a/core/pipeline/compression/CompressorFactory.cpp b/core/common/compression/CompressorFactory.cpp similarity index 71% rename from core/pipeline/compression/CompressorFactory.cpp rename to core/common/compression/CompressorFactory.cpp index 1c91f24035..0e3b27197b 100644 --- a/core/pipeline/compression/CompressorFactory.cpp +++ b/core/common/compression/CompressorFactory.cpp @@ -12,21 +12,24 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include "pipeline/compression/CompressorFactory.h" +#include "common/compression/CompressorFactory.h" #include "common/ParamExtractor.h" -#include "pipeline/compression/LZ4Compressor.h" -#include "pipeline/compression/ZstdCompressor.h" +#include "monitor/MetricConstants.h" +#include "common/compression/LZ4Compressor.h" +#include "common/compression/ZstdCompressor.h" using namespace std; namespace logtail { unique_ptr CompressorFactory::Create(const Json::Value& config, - const PipelineContext& ctx, - const string& pluginType, - CompressType defaultType) { + const PipelineContext& ctx, + const string& pluginType, + const string& flusherId, + CompressType defaultType) { string compressType, errorMsg; + unique_ptr compressor; if (!GetOptionalStringParam(config, "CompressType", compressType, errorMsg)) { PARAM_WARNING_DEFAULT(ctx.GetLogger(), ctx.GetAlarm(), @@ -37,11 +40,11 @@ unique_ptr CompressorFactory::Create(const Json::Value& config, ctx.GetProjectName(), ctx.GetLogstoreName(), ctx.GetRegion()); - return Create(defaultType); + compressor = Create(defaultType); } else if (compressType == "lz4") { - return Create(CompressType::LZ4); + compressor = Create(CompressType::LZ4); } else if (compressType == "zstd") { - return Create(CompressType::ZSTD); + compressor = Create(CompressType::ZSTD); } else if (compressType == "none") { return nullptr; } else if (!compressType.empty()) { @@ -54,10 +57,15 @@ unique_ptr CompressorFactory::Create(const Json::Value& config, ctx.GetProjectName(), ctx.GetLogstoreName(), ctx.GetRegion()); - return Create(defaultType); + compressor = Create(defaultType); } else { - return Create(defaultType); + compressor = Create(defaultType); } + compressor->SetMetricRecordRef({{METRIC_LABEL_PROJECT, ctx.GetProjectName()}, + {METRIC_LABEL_CONFIG_NAME, ctx.GetConfigName()}, + {METRIC_LABEL_KEY_COMPONENT_NAME, "compressor"}, + {METRIC_LABEL_KEY_FLUSHER_NODE_ID, flusherId}}); + return compressor; } unique_ptr CompressorFactory::Create(CompressType type) { diff --git a/core/pipeline/compression/CompressorFactory.h b/core/common/compression/CompressorFactory.h similarity index 87% rename from core/pipeline/compression/CompressorFactory.h rename to core/common/compression/CompressorFactory.h index dbdc826f16..8311332b7a 100644 --- a/core/pipeline/compression/CompressorFactory.h +++ b/core/common/compression/CompressorFactory.h @@ -21,9 +21,9 @@ #include #include -#include "pipeline/compression/CompressType.h" -#include "pipeline/compression/Compressor.h" #include "pipeline/PipelineContext.h" +#include "common/compression/CompressType.h" +#include "common/compression/Compressor.h" namespace logtail { @@ -42,13 +42,13 @@ class CompressorFactory { std::unique_ptr Create(const Json::Value& config, const PipelineContext& ctx, const std::string& pluginType, + const std::string& flusherId, CompressType defaultType); + std::unique_ptr Create(CompressType type); private: CompressorFactory() = default; ~CompressorFactory() = default; - - std::unique_ptr Create(CompressType defaultType); }; } // namespace logtail diff --git a/core/pipeline/compression/LZ4Compressor.cpp b/core/common/compression/LZ4Compressor.cpp similarity index 97% rename from core/pipeline/compression/LZ4Compressor.cpp rename to core/common/compression/LZ4Compressor.cpp index 7063812610..ec54c1c4e4 100644 --- a/core/pipeline/compression/LZ4Compressor.cpp +++ b/core/common/compression/LZ4Compressor.cpp @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include "pipeline/compression/LZ4Compressor.h" +#include "common/compression/LZ4Compressor.h" #include diff --git a/core/pipeline/compression/LZ4Compressor.h b/core/common/compression/LZ4Compressor.h similarity index 89% rename from core/pipeline/compression/LZ4Compressor.h rename to core/common/compression/LZ4Compressor.h index 6a64ea2af6..9fd453c163 100644 --- a/core/pipeline/compression/LZ4Compressor.h +++ b/core/common/compression/LZ4Compressor.h @@ -16,19 +16,20 @@ #pragma once -#include "pipeline/compression/Compressor.h" +#include "common/compression/Compressor.h" namespace logtail { class LZ4Compressor : public Compressor { public: - LZ4Compressor(CompressType type) : Compressor(type){}; + LZ4Compressor(CompressType type) : Compressor(type) {}; - bool Compress(const std::string& input, std::string& output, std::string& errorMsg) override; - #ifdef APSARA_UNIT_TEST_MAIN bool UnCompress(const std::string& input, std::string& output, std::string& errorMsg) override; #endif + +private: + bool Compress(const std::string& input, std::string& output, std::string& errorMsg) override; }; } // namespace logtail diff --git a/core/pipeline/compression/ZstdCompressor.cpp b/core/common/compression/ZstdCompressor.cpp similarity index 97% rename from core/pipeline/compression/ZstdCompressor.cpp rename to core/common/compression/ZstdCompressor.cpp index 6e47985b5e..c482c569a1 100644 --- a/core/pipeline/compression/ZstdCompressor.cpp +++ b/core/common/compression/ZstdCompressor.cpp @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include "pipeline/compression/ZstdCompressor.h" +#include "common/compression/ZstdCompressor.h" #include diff --git a/core/pipeline/compression/ZstdCompressor.h b/core/common/compression/ZstdCompressor.h similarity index 92% rename from core/pipeline/compression/ZstdCompressor.h rename to core/common/compression/ZstdCompressor.h index 1e910a2b0d..bf53c7d96d 100644 --- a/core/pipeline/compression/ZstdCompressor.h +++ b/core/common/compression/ZstdCompressor.h @@ -16,21 +16,21 @@ #pragma once -#include "pipeline/compression/Compressor.h" +#include "common/compression/Compressor.h" namespace logtail { class ZstdCompressor : public Compressor { public: - ZstdCompressor(CompressType type, int32_t level = 1) : Compressor(type), mCompressionLevel(level){}; - - bool Compress(const std::string& input, std::string& output, std::string& errorMsg) override; + ZstdCompressor(CompressType type, int32_t level = 1) : Compressor(type), mCompressionLevel(level) {}; #ifdef APSARA_UNIT_TEST_MAIN bool UnCompress(const std::string& input, std::string& output, std::string& errorMsg) override; #endif private: + bool Compress(const std::string& input, std::string& output, std::string& errorMsg) override; + int32_t mCompressionLevel = 1; }; diff --git a/core/go_pipeline/LogtailPlugin.cpp b/core/go_pipeline/LogtailPlugin.cpp index a84ff893c9..bb90dcdecc 100644 --- a/core/go_pipeline/LogtailPlugin.cpp +++ b/core/go_pipeline/LogtailPlugin.cpp @@ -22,15 +22,15 @@ #include "common/JsonUtil.h" #include "common/LogtailCommonFlags.h" #include "common/TimeUtil.h" -#include "pipeline/compression/CompressorFactory.h" +#include "common/compression/CompressorFactory.h" #include "container_manager/ConfigContainerInfoUpdateCmd.h" #include "file_server/ConfigManager.h" #include "logger/Logger.h" #include "monitor/LogFileProfiler.h" #include "monitor/LogtailAlarm.h" #include "pipeline/PipelineManager.h" -#include "profile_sender/ProfileSender.h" #include "pipeline/queue/SenderQueueManager.h" +#include "profile_sender/ProfileSender.h" DEFINE_FLAG_BOOL(enable_sls_metrics_format, "if enable format metrics in SLS metricstore log pattern", false); DEFINE_FLAG_BOOL(enable_containerd_upper_dir_detect, @@ -53,16 +53,13 @@ LogtailPlugin::LogtailPlugin() { mPluginValid = false; mPluginAlarmConfig.mLogstore = "logtail_alarm"; mPluginAlarmConfig.mAliuid = STRING_FLAG(logtail_profile_aliuid); - mPluginAlarmConfig.mCompressor - = CompressorFactory::GetInstance()->Create(Json::Value(), PipelineContext(), "flusher_sls", CompressType::ZSTD); + mPluginAlarmConfig.mCompressor = CompressorFactory::GetInstance()->Create(CompressType::ZSTD); mPluginProfileConfig.mLogstore = "shennong_log_profile"; mPluginProfileConfig.mAliuid = STRING_FLAG(logtail_profile_aliuid); - mPluginProfileConfig.mCompressor - = CompressorFactory::GetInstance()->Create(Json::Value(), PipelineContext(), "flusher_sls", CompressType::ZSTD); + mPluginProfileConfig.mCompressor = CompressorFactory::GetInstance()->Create(CompressType::ZSTD); mPluginContainerConfig.mLogstore = "logtail_containers"; mPluginContainerConfig.mAliuid = STRING_FLAG(logtail_profile_aliuid); - mPluginContainerConfig.mCompressor - = CompressorFactory::GetInstance()->Create(Json::Value(), PipelineContext(), "flusher_sls", CompressType::ZSTD); + mPluginContainerConfig.mCompressor = CompressorFactory::GetInstance()->Create(CompressType::ZSTD); mPluginCfg["LogtailSysConfDir"] = AppConfig::GetInstance()->GetLogtailSysConfDir(); mPluginCfg["HostIP"] = LogFileProfiler::mIpAddr; @@ -470,7 +467,8 @@ void LogtailPlugin::ProcessLogGroup(const std::string& configName, } } -void LogtailPlugin::GetGoMetrics(std::vector>& metircsList, const string& metricType) { +void LogtailPlugin::GetGoMetrics(std::vector>& metircsList, + const string& metricType) { if (mGetGoMetricsFun != nullptr) { GoString type; type.n = metricType.size(); diff --git a/core/monitor/LogtailMetric.h b/core/monitor/LogtailMetric.h index d6d76666ed..6c98081b2a 100644 --- a/core/monitor/LogtailMetric.h +++ b/core/monitor/LogtailMetric.h @@ -58,6 +58,8 @@ class MetricsRecord { class MetricsRecordRef { friend class WriteMetrics; + friend bool operator==(const MetricsRecordRef& lhs, std::nullptr_t rhs); + friend bool operator==(std::nullptr_t rhs, const MetricsRecordRef& lhs); private: MetricsRecord* mMetrics = nullptr; @@ -83,6 +85,22 @@ class MetricsRecordRef { #endif }; +inline bool operator==(const MetricsRecordRef& lhs, std::nullptr_t rhs) { + return lhs.mMetrics == rhs; +} + +inline bool operator==(std::nullptr_t lhs, const MetricsRecordRef& rhs) { + return lhs == rhs.mMetrics; +} + +inline bool operator!=(const MetricsRecordRef& lhs, std::nullptr_t rhs) { + return !(lhs == rhs); +} + +inline bool operator!=(std::nullptr_t lhs, const MetricsRecordRef& rhs) { + return !(lhs == rhs); +} + class ReentrantMetricsRecord { private: MetricsRecordRef mMetricsRecordRef; diff --git a/core/monitor/MetricConstants.cpp b/core/monitor/MetricConstants.cpp index 52c8d1158b..24212c9d3c 100644 --- a/core/monitor/MetricConstants.cpp +++ b/core/monitor/MetricConstants.cpp @@ -137,8 +137,6 @@ const std::string METRIC_PROC_PARSE_STDOUT_TOTAL = "proc_parse_stdout_total"; const std::string METRIC_PROC_PARSE_STDERR_TOTAL = "proc_parse_stderr_total"; // flusher common metrics -const std::string METRIC_FLUSHER_IN_RECORDS_TOTAL = "flusher_in_records_total"; -const std::string METRIC_FLUSHER_IN_RECORDS_SIZE_BYTES = "flusher_in_records_size_bytes"; const std::string METRIC_FLUSHER_ERROR_TOTAL = "flusher_error_total"; const std::string METRIC_FLUSHER_DISCARD_RECORDS_TOTAL = "flusher_discard_records_total"; const std::string METRIC_FLUSHER_SUCCESS_RECORDS_TOTAL = "flusher_success_records_total"; @@ -151,4 +149,14 @@ const std::string METRIC_FLUSHER_QUOTA_ERROR_TOTAL = "flusher_quota_error_total" const std::string METRIC_FLUSHER_RETRIES_TOTAL = "flusher_retries_total"; const std::string METRIC_FLUSHER_RETRIES_ERROR_TOTAL = "flusher_retries_error_total"; +const std::string METRIC_IN_EVENTS_CNT = "in_events_cnt"; +const std::string METRIC_IN_ITEMS_CNT = "in_items_cnt"; +const std::string METRIC_IN_EVENT_GROUP_SIZE_BYTES = "in_event_group_data_size_bytes"; +const std::string METRIC_IN_ITEM_SIZE_BYTES = "in_item_data_size_bytes"; +const std::string METRIC_OUT_EVENTS_CNT = "out_events_cnt"; +const std::string METRIC_OUT_ITEMS_CNT = "out_items_cnt"; +const std::string METRIC_OUT_EVENT_GROUP_SIZE_BYTES = "out_event_group_data_size_bytes"; +const std::string METRIC_OUT_ITEM_SIZE_BYTES = "out_item_data_size_bytes"; +const std::string METRIC_TOTAL_DELAY_MS = "total_delay_ms"; + } // namespace logtail diff --git a/core/monitor/MetricConstants.h b/core/monitor/MetricConstants.h index 27f3a762e3..3b78a5eae0 100644 --- a/core/monitor/MetricConstants.h +++ b/core/monitor/MetricConstants.h @@ -130,8 +130,6 @@ extern const std::string METRIC_PROC_PARSE_STDOUT_TOTAL; extern const std::string METRIC_PROC_PARSE_STDERR_TOTAL; // flusher common metrics -extern const std::string METRIC_FLUSHER_IN_RECORDS_TOTAL; -extern const std::string METRIC_FLUSHER_IN_RECORDS_SIZE_BYTES; extern const std::string METRIC_FLUSHER_ERROR_TOTAL; extern const std::string METRIC_FLUSHER_DISCARD_RECORDS_TOTAL; extern const std::string METRIC_FLUSHER_SUCCESS_RECORDS_TOTAL; @@ -144,4 +142,14 @@ extern const std::string METRIC_FLUSHER_QUOTA_ERROR_TOTAL; extern const std::string METRIC_FLUSHER_RETRIES_TOTAL; extern const std::string METRIC_FLUSHER_RETRIES_ERROR_TOTAL; +extern const std::string METRIC_IN_EVENTS_CNT; +extern const std::string METRIC_IN_ITEMS_CNT; +extern const std::string METRIC_IN_EVENT_GROUP_SIZE_BYTES; +extern const std::string METRIC_IN_ITEM_SIZE_BYTES; +extern const std::string METRIC_OUT_EVENTS_CNT; +extern const std::string METRIC_OUT_ITEMS_CNT; +extern const std::string METRIC_OUT_EVENT_GROUP_SIZE_BYTES; +extern const std::string METRIC_OUT_ITEM_SIZE_BYTES; +extern const std::string METRIC_TOTAL_DELAY_MS; + } // namespace logtail diff --git a/core/pipeline/Pipeline.cpp b/core/pipeline/Pipeline.cpp index 511b642b96..7d38880084 100644 --- a/core/pipeline/Pipeline.cpp +++ b/core/pipeline/Pipeline.cpp @@ -16,6 +16,7 @@ #include "pipeline/Pipeline.h" +#include #include #include @@ -314,10 +315,19 @@ bool Pipeline::Init(PipelineConfig&& config) { ProcessQueueManager::GetInstance()->SetDownStreamQueues(mContext.GetProcessQueueKey(), std::move(senderQueues)); } + WriteMetrics::GetInstance()->PrepareMetricsRecordRef( + mMetricsRecordRef, {{METRIC_LABEL_PROJECT, mContext.GetProjectName()}, {METRIC_LABEL_CONFIG_NAME, mName}}); + mStartTime = mMetricsRecordRef.CreateIntGauge("start_time"); + mProcessorsInEventsCnt = mMetricsRecordRef.CreateCounter("processors_in_events_cnt"); + mProcessorsInGroupsCnt = mMetricsRecordRef.CreateCounter("processors_in_event_groups_cnt"); + mProcessorsInGroupDataSizeBytes = mMetricsRecordRef.CreateCounter("processors_in_event_group_data_size_bytes"); + mProcessorsTotalDelayMs = mMetricsRecordRef.CreateCounter("processors_total_delay_ms"); + return true; } void Pipeline::Start() { +#ifndef APSARA_UNIT_TEST_MAIN // TODO: 应该保证指定时间内返回，如果无法返回，将配置放入startDisabled里 for (const auto& flusher : mFlushers) { flusher->Start(); @@ -337,16 +347,27 @@ void Pipeline::Start() { input->Start(); } - LOG_INFO(sLogger, ("pipeline start", "succeeded")("config", mName)); + mStartTime->Set(chrono::duration_cast(chrono::system_clock::now().time_since_epoch()).count()); +#endif + LOG_INFO(sLogger, ("pipeline start", "succeeded")("config", mName)("ptr", mStartTime.get())); } void Pipeline::Process(vector& logGroupList, size_t inputIndex) { + for (const auto& logGroup : logGroupList) { + mProcessorsInEventsCnt->Add(logGroup.GetEvents().size()); + mProcessorsInGroupDataSizeBytes->Add(logGroup.DataSize()); + } + mProcessorsInGroupsCnt->Add(logGroupList.size()); + + auto before = chrono::system_clock::now(); for (auto& p : mInputs[inputIndex]->GetInnerProcessors()) { p->Process(logGroupList); } for (auto& p : mProcessorLine) { p->Process(logGroupList); } + mProcessorsTotalDelayMs->Add( + chrono::duration_cast(chrono::system_clock::now() - before).count()); } bool Pipeline::Send(vector&& groupList) { @@ -381,6 +402,7 @@ bool Pipeline::FlushBatch() { } void Pipeline::Stop(bool isRemoving) { +#ifndef APSARA_UNIT_TEST_MAIN // TODO: 应该保证指定时间内返回，如果无法返回，将配置放入stopDisabled里 for (const auto& input : mInputs) { input->Stop(isRemoving); @@ -403,7 +425,7 @@ void Pipeline::Stop(bool isRemoving) { for (const auto& flusher : mFlushers) { flusher->Stop(isRemoving); } - +#endif LOG_INFO(sLogger, ("pipeline stop", "succeeded")("config", mName)); } diff --git a/core/pipeline/Pipeline.h b/core/pipeline/Pipeline.h index 90bc5ada01..e3775f7b48 100644 --- a/core/pipeline/Pipeline.h +++ b/core/pipeline/Pipeline.h @@ -24,14 +24,15 @@ #include #include "config/PipelineConfig.h" -#include "plugin/input/InputContainerStdio.h" -#include "plugin/input/InputFile.h" #include "models/PipelineEventGroup.h" +#include "monitor/LogtailMetric.h" #include "pipeline/PipelineContext.h" #include "pipeline/plugin/instance/FlusherInstance.h" #include "pipeline/plugin/instance/InputInstance.h" #include "pipeline/plugin/instance/ProcessorInstance.h" #include "pipeline/route/Router.h" +#include "plugin/input/InputContainerStdio.h" +#include "plugin/input/InputFile.h" namespace logtail { @@ -84,8 +85,15 @@ class Pipeline { std::unique_ptr mConfig; std::atomic_uint16_t mPluginID; + mutable MetricsRecordRef mMetricsRecordRef; + IntGaugePtr mStartTime; + CounterPtr mProcessorsInEventsCnt; + CounterPtr mProcessorsInGroupsCnt; + CounterPtr mProcessorsInGroupDataSizeBytes; + CounterPtr mProcessorsTotalDelayMs; + #ifdef APSARA_UNIT_TEST_MAIN - friend class PipelineMock; + friend class PipelineMock; friend class PipelineUnittest; friend class InputContainerStdioUnittest; friend class InputFileUnittest; diff --git a/core/pipeline/batch/BatchItem.h b/core/pipeline/batch/BatchItem.h index b27fa0f5e8..9fa956dbef 100644 --- a/core/pipeline/batch/BatchItem.h +++ b/core/pipeline/batch/BatchItem.h @@ -20,11 +20,11 @@ #include #include +#include "models/PipelineEventGroup.h" +#include "models/StringView.h" #include "pipeline/batch/BatchStatus.h" #include "pipeline/batch/BatchedEvents.h" #include "pipeline/batch/FlushStrategy.h" -#include "models/PipelineEventGroup.h" -#include "models/StringView.h" namespace logtail { @@ -32,7 +32,9 @@ class GroupBatchItem { public: GroupBatchItem() { mStatus.Reset(); } - void Add(BatchedEvents&& g) { + void Add(BatchedEvents&& g, int64_t totalEnqueTimeMs) { + mEventsCnt += g.mEvents.size(); + mTotalEnqueTimeMs += totalEnqueTimeMs; mGroups.emplace_back(std::move(g)); mStatus.Update(mGroups.back()); } @@ -56,6 +58,10 @@ class GroupBatchItem { } GroupBatchStatus& GetStatus() { return mStatus; } + size_t GroupSize() const { return mGroups.size(); } + size_t EventSize() const { return mEventsCnt; } + size_t DataSize() const { return mStatus.GetSize(); } + int64_t TotalEnqueTimeMs() const { return mTotalEnqueTimeMs; } bool IsEmpty() { return mGroups.empty(); } @@ -63,11 +69,17 @@ class GroupBatchItem { void Clear() { mGroups.clear(); mStatus.Reset(); + mEventsCnt = 0; + mTotalEnqueTimeMs = 0; } std::vector mGroups; GroupBatchStatus mStatus; + size_t mEventsCnt = 0; + // if more than 10^6 events are contained in the batch, the value may overflow + // however, this is almost impossible in practice + int64_t mTotalEnqueTimeMs = 0; #ifdef APSARA_UNIT_TEST_MAIN friend class EventBatchItemUnittest; @@ -82,6 +94,9 @@ class EventBatchItem { void Add(PipelineEventPtr&& e) { mBatch.mEvents.emplace_back(std::move(e)); mStatus.Update(mBatch.mEvents.back()); + mTotalEnqueTimeMs += std::chrono::time_point_cast(std::chrono::system_clock::now()) + .time_since_epoch() + .count(); } void Flush(GroupBatchItem& res) { @@ -91,7 +106,8 @@ class EventBatchItem { if (mBatch.mExactlyOnceCheckpoint) { UpdateExactlyOnceLogPosition(); } - res.Add(std::move(mBatch)); + mBatch.mSizeBytes = DataSize(); + res.Add(std::move(mBatch), mTotalEnqueTimeMs); Clear(); } @@ -102,6 +118,7 @@ class EventBatchItem { if (mBatch.mExactlyOnceCheckpoint) { UpdateExactlyOnceLogPosition(); } + mBatch.mSizeBytes = DataSize(); res.emplace_back(std::move(mBatch)); Clear(); } @@ -114,6 +131,7 @@ class EventBatchItem { if (mBatch.mExactlyOnceCheckpoint) { UpdateExactlyOnceLogPosition(); } + mBatch.mSizeBytes = DataSize(); res.back().emplace_back(std::move(mBatch)); Clear(); } @@ -141,11 +159,16 @@ class EventBatchItem { bool IsEmpty() { return mBatch.mEvents.empty(); } + size_t DataSize() const { return sizeof(decltype(mBatch.mEvents)) + mStatus.GetSize() + mBatch.mTags.DataSize(); } + size_t EventSize() const { return mBatch.mEvents.size(); } + int64_t TotalEnqueTimeMs() const { return mTotalEnqueTimeMs; } + private: void Clear() { mBatch.Clear(); mSourceBuffers.clear(); mStatus.Reset(); + mTotalEnqueTimeMs = 0; } void UpdateExactlyOnceLogPosition() { @@ -159,6 +182,9 @@ class EventBatchItem { BatchedEvents mBatch; std::unordered_set mSourceBuffers; T mStatus; + // if more than 10^6 events are contained in the batch, the value may overflow + // however, this is almost impossible in practice + int64_t mTotalEnqueTimeMs = 0; #ifdef APSARA_UNIT_TEST_MAIN friend class EventBatchItemUnittest; diff --git a/core/pipeline/batch/BatchStatus.h b/core/pipeline/batch/BatchStatus.h index 18c2b7db6c..224a83dc78 100644 --- a/core/pipeline/batch/BatchStatus.h +++ b/core/pipeline/batch/BatchStatus.h @@ -67,7 +67,7 @@ class GroupBatchStatus { if (mCreateTime == 0) { mCreateTime = time(nullptr); } - mSizeBytes += g.DataSize(); + mSizeBytes += g.mSizeBytes; } uint32_t GetSize() const { return mSizeBytes; } diff --git a/core/pipeline/batch/BatchedEvents.h b/core/pipeline/batch/BatchedEvents.h index 9ec5d0ff37..e16d15be15 100644 --- a/core/pipeline/batch/BatchedEvents.h +++ b/core/pipeline/batch/BatchedEvents.h @@ -28,6 +28,7 @@ struct BatchedEvents { EventsContainer mEvents; SizedMap mTags; std::vector> mSourceBuffers; + size_t mSizeBytes = 0; // only set on completion // for flusher_sls only RangeCheckpointPtr mExactlyOnceCheckpoint; StringView mPackIdPrefix; @@ -45,20 +46,17 @@ struct BatchedEvents { mExactlyOnceCheckpoint(std::move(eoo)), mPackIdPrefix(packIdPrefix) { mSourceBuffers.emplace_back(std::move(sourceBuffer)); - } - - size_t DataSize() const { - size_t eventsSize = sizeof(decltype(mEvents)); + mSizeBytes = sizeof(decltype(mEvents)) + mTags.DataSize(); for (const auto& item : mEvents) { - eventsSize += item->DataSize(); + mSizeBytes += item->DataSize(); } - return eventsSize + mTags.DataSize(); } void Clear() { mEvents.clear(); mTags.Clear(); mSourceBuffers.clear(); + mSizeBytes = 0; mExactlyOnceCheckpoint.reset(); mPackIdPrefix = StringView(); } diff --git a/core/pipeline/batch/Batcher.h b/core/pipeline/batch/Batcher.h index b165bab370..ca01986ca5 100644 --- a/core/pipeline/batch/Batcher.h +++ b/core/pipeline/batch/Batcher.h @@ -24,14 +24,16 @@ #include #include -#include "pipeline/batch/BatchItem.h" -#include "pipeline/batch/BatchStatus.h" -#include "pipeline/batch/FlushStrategy.h" -#include "pipeline/batch/TimeoutFlushManager.h" #include "common/Flags.h" #include "common/ParamExtractor.h" #include "models/PipelineEventGroup.h" +#include "monitor/LogtailMetric.h" +#include "monitor/MetricConstants.h" #include "pipeline/PipelineContext.h" +#include "pipeline/batch/BatchItem.h" +#include "pipeline/batch/BatchStatus.h" +#include "pipeline/batch/FlushStrategy.h" +#include "pipeline/batch/TimeoutFlushManager.h" namespace logtail { @@ -97,6 +99,26 @@ class Batcher { mFlusher = flusher; + std::vector> labels{ + {METRIC_LABEL_PROJECT, ctx.GetProjectName()}, + {METRIC_LABEL_CONFIG_NAME, ctx.GetConfigName()}, + {METRIC_LABEL_KEY_COMPONENT_NAME, "batcher"}, + {METRIC_LABEL_KEY_FLUSHER_NODE_ID, flusher->GetNodeID()}}; + if (enableGroupBatch) { + labels.emplace_back("enable_group_batch", "true"); + } else { + labels.emplace_back("enable_group_batch", "false"); + } + WriteMetrics::GetInstance()->PrepareMetricsRecordRef(mMetricsRecordRef, std::move(labels)); + mInEventsCnt = mMetricsRecordRef.CreateCounter(METRIC_IN_EVENTS_CNT); + mInGroupDataSizeBytes = mMetricsRecordRef.CreateCounter(METRIC_IN_EVENT_GROUP_SIZE_BYTES); + mOutEventsCnt = mMetricsRecordRef.CreateCounter(METRIC_OUT_EVENTS_CNT); + mTotalDelayMs = mMetricsRecordRef.CreateCounter(METRIC_TOTAL_DELAY_MS); + mEventBatchItemsCnt = mMetricsRecordRef.CreateIntGauge("event_batches_cnt"); + mBufferedGroupsCnt = mMetricsRecordRef.CreateIntGauge("buffered_groups_cnt"); + mBufferedEventsCnt = mMetricsRecordRef.CreateIntGauge("buffered_events_cnt"); + mBufferedDataSizeByte = mMetricsRecordRef.CreateIntGauge("buffered_data_size_bytes"); + return true; } @@ -105,15 +127,20 @@ class Batcher { std::lock_guard lock(mMux); size_t key = g.GetTagsHash(); EventBatchItem& item = mEventQueueMap[key]; + mInEventsCnt->Add(g.GetEvents().size()); + mInGroupDataSizeBytes->Add(g.DataSize()); + mEventBatchItemsCnt->Set(mEventQueueMap.size()); size_t eventsSize = g.GetEvents().size(); for (size_t i = 0; i < eventsSize; ++i) { PipelineEventPtr& e = g.MutableEvents()[i]; if (!item.IsEmpty() && mEventFlushStrategy.NeedFlushByTime(item.GetStatus(), e)) { if (!mGroupQueue) { + UpdateMetricsOnFlushingEventQueue(item); item.Flush(res); } else { if (!mGroupQueue->IsEmpty() && mGroupFlushStrategy->NeedFlushByTime(mGroupQueue->GetStatus())) { + UpdateMetricsOnFlushingGroupQueue(); mGroupQueue->Flush(res); } if (mGroupQueue->IsEmpty()) { @@ -125,6 +152,7 @@ class Batcher { } item.Flush(mGroupQueue.value()); if (mGroupFlushStrategy->NeedFlushBySize(mGroupQueue->GetStatus())) { + UpdateMetricsOnFlushingGroupQueue(); mGroupQueue->Flush(res); } } @@ -136,12 +164,17 @@ class Batcher { g.GetMetadata(EventGroupMetaKey::SOURCE_ID)); TimeoutFlushManager::GetInstance()->UpdateRecord( mFlusher->GetContext().GetConfigName(), 0, key, mEventFlushStrategy.GetTimeoutSecs(), mFlusher); + mBufferedGroupsCnt->Add(1); + mBufferedDataSizeByte->Add(item.DataSize()); } else if (i == 0) { item.AddSourceBuffer(g.GetSourceBuffer()); } + mBufferedEventsCnt->Add(1); + mBufferedDataSizeByte->Add(e->DataSize()); item.Add(std::move(e)); if (mEventFlushStrategy.NeedFlushBySize(item.GetStatus()) || mEventFlushStrategy.NeedFlushByCnt(item.GetStatus())) { + UpdateMetricsOnFlushingEventQueue(item); item.Flush(res); } } @@ -155,6 +188,7 @@ class Batcher { if (!mGroupQueue) { return; } + UpdateMetricsOnFlushingGroupQueue(); return mGroupQueue->Flush(res); } @@ -164,12 +198,15 @@ class Batcher { } if (!mGroupQueue) { + UpdateMetricsOnFlushingEventQueue(iter->second); iter->second.Flush(res); mEventQueueMap.erase(iter); + mEventBatchItemsCnt->Set(mEventQueueMap.size()); return; } if (!mGroupQueue->IsEmpty() && mGroupFlushStrategy->NeedFlushByTime(mGroupQueue->GetStatus())) { + UpdateMetricsOnFlushingGroupQueue(); mGroupQueue->Flush(res); } if (mGroupQueue->IsEmpty()) { @@ -178,7 +215,9 @@ class Batcher { } iter->second.Flush(mGroupQueue.value()); mEventQueueMap.erase(iter); + mEventBatchItemsCnt->Set(mEventQueueMap.size()); if (mGroupFlushStrategy->NeedFlushBySize(mGroupQueue->GetStatus())) { + UpdateMetricsOnFlushingGroupQueue(); mGroupQueue->Flush(res); } } @@ -187,20 +226,25 @@ class Batcher { std::lock_guard lock(mMux); for (auto& item : mEventQueueMap) { if (!mGroupQueue) { + UpdateMetricsOnFlushingEventQueue(item.second); item.second.Flush(res); } else { if (!mGroupQueue->IsEmpty() && mGroupFlushStrategy->NeedFlushByTime(mGroupQueue->GetStatus())) { + UpdateMetricsOnFlushingGroupQueue(); mGroupQueue->Flush(res); } item.second.Flush(mGroupQueue.value()); if (mGroupFlushStrategy->NeedFlushBySize(mGroupQueue->GetStatus())) { + UpdateMetricsOnFlushingGroupQueue(); mGroupQueue->Flush(res); } } } if (mGroupQueue) { + UpdateMetricsOnFlushingGroupQueue(); mGroupQueue->Flush(res); } + mEventBatchItemsCnt->Set(0); mEventQueueMap.clear(); } @@ -210,6 +254,32 @@ class Batcher { #endif private: + void UpdateMetricsOnFlushingEventQueue(const EventBatchItem& item) { + mOutEventsCnt->Add(item.EventSize()); + mTotalDelayMs->Add( + item.EventSize() + * std::chrono::time_point_cast(std::chrono::system_clock::now()) + .time_since_epoch() + .count() + - item.TotalEnqueTimeMs()); + mBufferedGroupsCnt->Sub(1); + mBufferedEventsCnt->Sub(item.EventSize()); + mBufferedDataSizeByte->Sub(item.DataSize()); + } + + void UpdateMetricsOnFlushingGroupQueue() { + mOutEventsCnt->Add(mGroupQueue->EventSize()); + mTotalDelayMs->Add( + mGroupQueue->EventSize() + * std::chrono::time_point_cast(std::chrono::system_clock::now()) + .time_since_epoch() + .count() + - mGroupQueue->TotalEnqueTimeMs()); + mBufferedGroupsCnt->Sub(mGroupQueue->GroupSize()); + mBufferedEventsCnt->Sub(mGroupQueue->EventSize()); + mBufferedDataSizeByte->Sub(mGroupQueue->DataSize()); + } + std::mutex mMux; std::map> mEventQueueMap; EventFlushStrategy mEventFlushStrategy; @@ -219,6 +289,16 @@ class Batcher { Flusher* mFlusher = nullptr; + mutable MetricsRecordRef mMetricsRecordRef; + CounterPtr mInEventsCnt; + CounterPtr mInGroupDataSizeBytes; + CounterPtr mOutEventsCnt; + CounterPtr mTotalDelayMs; + IntGaugePtr mEventBatchItemsCnt; + IntGaugePtr mBufferedGroupsCnt; + IntGaugePtr mBufferedEventsCnt; + IntGaugePtr mBufferedDataSizeByte; + #ifdef APSARA_UNIT_TEST_MAIN friend class BatcherUnittest; #endif diff --git a/core/pipeline/plugin/instance/FlusherInstance.cpp b/core/pipeline/plugin/instance/FlusherInstance.cpp index 9623a96b75..997f75fe4b 100644 --- a/core/pipeline/plugin/instance/FlusherInstance.cpp +++ b/core/pipeline/plugin/instance/FlusherInstance.cpp @@ -25,14 +25,14 @@ bool FlusherInstance::Init(const Json::Value& config, PipelineContext& context, return false; } - mFlusherInRecordsTotal = mPlugin->GetMetricsRecordRef().CreateCounter(METRIC_FLUSHER_IN_RECORDS_TOTAL); - mFlusherInRecordsSizeBytes = mPlugin->GetMetricsRecordRef().CreateCounter(METRIC_FLUSHER_IN_RECORDS_SIZE_BYTES); + mInEventsCnt = mPlugin->GetMetricsRecordRef().CreateCounter(METRIC_IN_EVENTS_CNT); + mInGroupDataSizeBytes = mPlugin->GetMetricsRecordRef().CreateCounter(METRIC_IN_EVENT_GROUP_SIZE_BYTES); return true; } bool FlusherInstance::Send(PipelineEventGroup&& g) { - mFlusherInRecordsTotal->Add(g.GetEvents().size()); - mFlusherInRecordsSizeBytes->Add(g.DataSize()); + mInEventsCnt->Add(g.GetEvents().size()); + mInGroupDataSizeBytes->Add(g.DataSize()); return mPlugin->Send(std::move(g)); } diff --git a/core/pipeline/plugin/instance/FlusherInstance.h b/core/pipeline/plugin/instance/FlusherInstance.h index 2900a594df..4abec0d766 100644 --- a/core/pipeline/plugin/instance/FlusherInstance.h +++ b/core/pipeline/plugin/instance/FlusherInstance.h @@ -46,8 +46,8 @@ class FlusherInstance : public PluginInstance { private: std::unique_ptr mPlugin; - CounterPtr mFlusherInRecordsTotal; - CounterPtr mFlusherInRecordsSizeBytes; + CounterPtr mInEventsCnt; + CounterPtr mInGroupDataSizeBytes; }; } // namespace logtail diff --git a/core/pipeline/plugin/instance/ProcessorInstance.cpp b/core/pipeline/plugin/instance/ProcessorInstance.cpp index 8790886cd4..70dbc5a172 100644 --- a/core/pipeline/plugin/instance/ProcessorInstance.cpp +++ b/core/pipeline/plugin/instance/ProcessorInstance.cpp @@ -49,11 +49,9 @@ void ProcessorInstance::Process(vector& logGroupList) { mProcInRecordsTotal->Add(logGroup.GetEvents().size()); } - uint64_t startTime = GetCurrentTimeInMilliSeconds(); + auto before = chrono::system_clock::now(); mPlugin->Process(logGroupList); - uint64_t durationTime = GetCurrentTimeInMilliSeconds() - startTime; - - mProcTimeMS->Add(durationTime); + mProcTimeMS->Add(chrono::duration_cast(chrono::system_clock::now() - before).count()); for (const auto& logGroup : logGroupList) { mProcOutRecordsTotal->Add(logGroup.GetEvents().size()); diff --git a/core/pipeline/queue/QueueInterface.h b/core/pipeline/queue/QueueInterface.h index 5c2e3b82cb..1c450013ad 100644 --- a/core/pipeline/queue/QueueInterface.h +++ b/core/pipeline/queue/QueueInterface.h @@ -33,10 +33,10 @@ class QueueInterface { {METRIC_LABEL_CONFIG_NAME, ctx.GetConfigName()}, }); - mInItemsCnt = mMetricsRecordRef.CreateCounter("in_items_cnt"); - mInItemDataSizeBytes = mMetricsRecordRef.CreateCounter("in_item_data_size_bytes"); - mOutItemsCnt = mMetricsRecordRef.CreateCounter("out_items_cnt"); - mTotalDelayMs = mMetricsRecordRef.CreateCounter("total_delay_ms"); + mInItemsCnt = mMetricsRecordRef.CreateCounter(METRIC_IN_ITEMS_CNT); + mInItemDataSizeBytes = mMetricsRecordRef.CreateCounter(METRIC_IN_ITEM_SIZE_BYTES); + mOutItemsCnt = mMetricsRecordRef.CreateCounter(METRIC_OUT_ITEMS_CNT); + mTotalDelayMs = mMetricsRecordRef.CreateCounter(METRIC_TOTAL_DELAY_MS); mQueueSize = mMetricsRecordRef.CreateIntGauge("queue_size"); mQueueDataSizeByte = mMetricsRecordRef.CreateIntGauge("queue_data_size_bytes"); } diff --git a/core/pipeline/route/Router.cpp b/core/pipeline/route/Router.cpp index e9c4a817cd..73de16bcda 100644 --- a/core/pipeline/route/Router.cpp +++ b/core/pipeline/route/Router.cpp @@ -15,6 +15,7 @@ #include "pipeline/route/Router.h" #include "common/ParamExtractor.h" +#include "monitor/MetricConstants.h" #include "pipeline/Pipeline.h" #include "pipeline/plugin/interface/Flusher.h" @@ -33,10 +34,20 @@ bool Router::Init(std::vector> configs, const P mAlwaysMatchedFlusherIdx.push_back(item.first); } } + + WriteMetrics::GetInstance()->PrepareMetricsRecordRef(mMetricsRecordRef, + {{METRIC_LABEL_PROJECT, ctx.GetProjectName()}, + {METRIC_LABEL_CONFIG_NAME, ctx.GetConfigName()}, + {METRIC_LABEL_KEY_COMPONENT_NAME, "router"}}); + mInEventsCnt = mMetricsRecordRef.CreateCounter(METRIC_IN_EVENTS_CNT); + mInGroupDataSizeBytes = mMetricsRecordRef.CreateCounter(METRIC_IN_EVENT_GROUP_SIZE_BYTES); return true; } vector Router::Route(const PipelineEventGroup& g) const { + mInEventsCnt->Add(g.GetEvents().size()); + mInGroupDataSizeBytes->Add(g.DataSize()); + vector res(mAlwaysMatchedFlusherIdx); for (size_t i = 0; i < mConditions.size(); ++i) { if (mConditions[i].second.Check(g)) { diff --git a/core/pipeline/route/Router.h b/core/pipeline/route/Router.h index c46dedca6e..b036256551 100644 --- a/core/pipeline/route/Router.h +++ b/core/pipeline/route/Router.h @@ -22,6 +22,7 @@ #include #include "models/PipelineEventGroup.h" +#include "monitor/LogtailMetric.h" #include "pipeline/route/Condition.h" namespace logtail { @@ -37,6 +38,10 @@ class Router { std::vector> mConditions; std::vector mAlwaysMatchedFlusherIdx; + mutable MetricsRecordRef mMetricsRecordRef; + CounterPtr mInEventsCnt; + CounterPtr mInGroupDataSizeBytes; + #ifdef APSARA_UNIT_TEST_MAIN friend class RouterUnittest; friend class PipelineUnittest; diff --git a/core/pipeline/serializer/SLSSerializer.cpp b/core/pipeline/serializer/SLSSerializer.cpp index 600e709f68..f24030c100 100644 --- a/core/pipeline/serializer/SLSSerializer.cpp +++ b/core/pipeline/serializer/SLSSerializer.cpp @@ -17,14 +17,14 @@ #include "application/Application.h" #include "common/Flags.h" #include "common/TimeUtil.h" -#include "pipeline/compression/CompressType.h" +#include "common/compression/CompressType.h" #include "plugin/flusher/sls/FlusherSLS.h" DEFINE_FLAG_INT32(max_send_log_group_size, "bytes", 10 * 1024 * 1024); const std::string METRIC_RESERVED_KEY_NAME = "__name__"; -const std::string METRIC_RESERVED_KEY_LABELS = "__labels__"; +const std::string METRIC_RESERVED_KEY_LABELS = "__labels__"; const std::string METRIC_RESERVED_KEY_VALUE = "__value__"; const std::string METRIC_RESERVED_KEY_TIME_NANO = "__time_nano__"; @@ -35,7 +35,31 @@ using namespace std; namespace logtail { +template <> +bool Serializer>::DoSerialize(vector&& p, + std::string& output, + std::string& errorMsg) { + auto inputSize = 0; + for (auto& item : p) { + inputSize += item.mData.size(); + } + mInItemsCnt->Add(1); + mInItemSizeBytes->Add(inputSize); + auto before = std::chrono::system_clock::now(); + auto res = Serialize(std::move(p), output, errorMsg); + mTotalDelayMs->Add( + std::chrono::duration_cast(std::chrono::system_clock::now() - before).count()); + + if (res) { + mOutItemsCnt->Add(1); + mOutItemSizeBytes->Add(output.size()); + } else { + mDiscardedItemsCnt->Add(1); + mDiscardedItemSizeBytes->Add(inputSize); + } + return res; +} bool SLSEventGroupSerializer::Serialize(BatchedEvents&& group, string& res, string& errorMsg) { sls_logs::LogGroup logGroup; @@ -56,7 +80,7 @@ bool SLSEventGroupSerializer::Serialize(BatchedEvents&& group, string& res, stri } else if (e.Is()) { const auto& metricEvent = e.Cast(); if (metricEvent.Is()) { - continue; + continue; } auto log = logGroup.add_logs(); std::ostringstream oss; @@ -78,9 +102,10 @@ bool SLSEventGroupSerializer::Serialize(BatchedEvents&& group, string& res, stri logPtr = log->add_contents(); logPtr->set_key(METRIC_RESERVED_KEY_TIME_NANO); if (metricEvent.GetTimestampNanosecond()) { - logPtr->set_value(std::to_string(metricEvent.GetTimestamp()) + NumberToDigitString(metricEvent.GetTimestampNanosecond().value(), 9)); + logPtr->set_value(std::to_string(metricEvent.GetTimestamp()) + + NumberToDigitString(metricEvent.GetTimestampNanosecond().value(), 9)); } else { - logPtr->set_value(std::to_string(metricEvent.GetTimestamp())); + logPtr->set_value(std::to_string(metricEvent.GetTimestamp())); } // set __value__ if (metricEvent.Is()) { @@ -88,7 +113,7 @@ bool SLSEventGroupSerializer::Serialize(BatchedEvents&& group, string& res, stri logPtr = log->add_contents(); logPtr->set_key(METRIC_RESERVED_KEY_VALUE); logPtr->set_value(std::to_string(value)); - } + } // set __name__ logPtr = log->add_contents(); logPtr->set_key(METRIC_RESERVED_KEY_NAME); @@ -122,9 +147,7 @@ bool SLSEventGroupSerializer::Serialize(BatchedEvents&& group, string& res, stri return true; } -bool SLSEventGroupListSerializer::Serialize(vector&& v, - string& res, - string& errorMsg) { +bool SLSEventGroupListSerializer::Serialize(vector&& v, string& res, string& errorMsg) { sls_logs::SlsLogPackageList logPackageList; for (const auto& item : v) { auto package = logPackageList.add_packages(); diff --git a/core/pipeline/serializer/SLSSerializer.h b/core/pipeline/serializer/SLSSerializer.h index f19d0f8e3a..c9bdb56798 100644 --- a/core/pipeline/serializer/SLSSerializer.h +++ b/core/pipeline/serializer/SLSSerializer.h @@ -27,6 +27,7 @@ class SLSEventGroupSerializer : public Serializer { public: SLSEventGroupSerializer(Flusher* f) : Serializer(f) {} +private: bool Serialize(BatchedEvents&& p, std::string& res, std::string& errorMsg) override; }; @@ -37,10 +38,14 @@ struct CompressedLogGroup { CompressedLogGroup(std::string&& data, size_t rawSize) : mData(std::move(data)), mRawSize(rawSize) {} }; +template<> +bool Serializer>::DoSerialize(std::vector&& p, std::string& output, std::string& errorMsg); + class SLSEventGroupListSerializer : public Serializer> { public: SLSEventGroupListSerializer(Flusher* f) : Serializer>(f) {} +private: bool Serialize(std::vector&& v, std::string& res, std::string& errorMsg) override; }; diff --git a/core/pipeline/serializer/Serializer.h b/core/pipeline/serializer/Serializer.h index 968f882a91..b632124b7a 100644 --- a/core/pipeline/serializer/Serializer.h +++ b/core/pipeline/serializer/Serializer.h @@ -16,27 +16,92 @@ #pragma once +#include #include -#include "pipeline/batch/BatchedEvents.h" #include "models/PipelineEventPtr.h" +#include "monitor/MetricConstants.h" +#include "pipeline/batch/BatchedEvents.h" #include "pipeline/plugin/interface/Flusher.h" namespace logtail { +inline size_t GetInputSize(const PipelineEventPtr& p) { + return p->DataSize(); +} + +inline size_t GetInputSize(const BatchedEvents& p) { + return p.mSizeBytes; +} + +inline size_t GetInputSize(const BatchedEventsList& p) { + size_t size = 0; + for (const auto& e : p) { + size += GetInputSize(e); + } + return size; +} + // T: PipelineEventPtr, BatchedEvents, BatchedEventsList template class Serializer { public: - Serializer() = default; - Serializer(Flusher* f) : mFlusher(f) {} + Serializer(Flusher* f) : mFlusher(f) { + WriteMetrics::GetInstance()->PrepareMetricsRecordRef( + mMetricsRecordRef, + {{METRIC_LABEL_PROJECT, f->GetContext().GetProjectName()}, + {METRIC_LABEL_CONFIG_NAME, f->GetContext().GetConfigName()}, + {METRIC_LABEL_KEY_COMPONENT_NAME, "serializer"}, + {METRIC_LABEL_KEY_FLUSHER_NODE_ID, f->GetNodeID()}}); + mInItemsCnt = mMetricsRecordRef.CreateCounter(METRIC_IN_ITEMS_CNT); + mInItemSizeBytes = mMetricsRecordRef.CreateCounter(METRIC_IN_ITEM_SIZE_BYTES); + mOutItemsCnt = mMetricsRecordRef.CreateCounter(METRIC_OUT_ITEMS_CNT); + mOutItemSizeBytes = mMetricsRecordRef.CreateCounter(METRIC_OUT_ITEM_SIZE_BYTES); + mDiscardedItemsCnt = mMetricsRecordRef.CreateCounter("discarded_items_cnt"); + mDiscardedItemSizeBytes = mMetricsRecordRef.CreateCounter("discarded_item_data_size_bytes"); + mTotalDelayMs = mMetricsRecordRef.CreateCounter(METRIC_TOTAL_DELAY_MS); + } virtual ~Serializer() = default; - virtual bool Serialize(T&& p, std::string& res, std::string& errorMsg) = 0; + bool DoSerialize(T&& p, std::string& output, std::string& errorMsg) { + auto inputSize = GetInputSize(p); + mInItemsCnt->Add(1); + mInItemSizeBytes->Add(inputSize); + + auto before = std::chrono::system_clock::now(); + auto res = Serialize(std::move(p), output, errorMsg); + mTotalDelayMs->Add( + std::chrono::duration_cast(std::chrono::system_clock::now() - before).count()); + + if (res) { + mOutItemsCnt->Add(1); + mOutItemSizeBytes->Add(output.size()); + } else { + mDiscardedItemsCnt->Add(1); + mDiscardedItemSizeBytes->Add(inputSize); + } + return res; + } protected: // if serialized output contains output related info, it can be obtained via this member const Flusher* mFlusher = nullptr; + + mutable MetricsRecordRef mMetricsRecordRef; + CounterPtr mInItemsCnt; + CounterPtr mInItemSizeBytes; + CounterPtr mOutItemsCnt; + CounterPtr mOutItemSizeBytes; + CounterPtr mDiscardedItemsCnt; + CounterPtr mDiscardedItemSizeBytes; + CounterPtr mTotalDelayMs; + +private: + virtual bool Serialize(T&& p, std::string& res, std::string& errorMsg) = 0; + +#ifdef APSARA_UNIT_TEST_MAIN + friend class SerializerUnittest; +#endif }; using EventSerializer = Serializer; diff --git a/core/plugin/flusher/sls/FlusherSLS.cpp b/core/plugin/flusher/sls/FlusherSLS.cpp index 5df36854c9..5f965a4535 100644 --- a/core/plugin/flusher/sls/FlusherSLS.cpp +++ b/core/plugin/flusher/sls/FlusherSLS.cpp @@ -23,9 +23,9 @@ #include "common/LogtailCommonFlags.h" #include "common/ParamExtractor.h" #include "common/TimeUtil.h" +#include "common/compression/CompressorFactory.h" #include "pipeline/Pipeline.h" #include "pipeline/batch/FlushStrategy.h" -#include "pipeline/compression/CompressorFactory.h" #include "pipeline/queue/QueueKeyManager.h" #include "pipeline/queue/SLSSenderQueueItem.h" #include "pipeline/queue/SenderQueueManager.h" @@ -445,7 +445,7 @@ bool FlusherSLS::Init(const Json::Value& config, Json::Value& optionalGoPipeline // CompressType if (BOOL_FLAG(sls_client_send_compress)) { - mCompressor = CompressorFactory::GetInstance()->Create(config, *mContext, sName, CompressType::LZ4); + mCompressor = CompressorFactory::GetInstance()->Create(config, *mContext, sName, mNodeID, CompressType::LZ4); } mGroupSerializer = make_unique(this); @@ -809,7 +809,7 @@ bool FlusherSLS::Send(string&& data, const string& shardHashKey, const string& l string compressedData; if (mCompressor) { string errorMsg; - if (!mCompressor->Compress(data, compressedData, errorMsg)) { + if (!mCompressor->DoCompress(data, compressedData, errorMsg)) { LOG_WARNING(mContext->GetLogger(), ("failed to compress data", errorMsg)("action", "discard data")("plugin", sName)("config", mContext->GetConfigName())); @@ -866,7 +866,7 @@ bool FlusherSLS::SerializeAndPush(PipelineEventGroup&& group) { std::move(group.GetExactlyOnceCheckpoint())); AddPackId(g); string errorMsg; - if (!mGroupSerializer->Serialize(std::move(g), serializedData, errorMsg)) { + if (!mGroupSerializer->DoSerialize(std::move(g), serializedData, errorMsg)) { LOG_WARNING(mContext->GetLogger(), ("failed to serialize event group", errorMsg)("action", "discard data")("plugin", sName)("config", mContext->GetConfigName())); @@ -880,7 +880,7 @@ bool FlusherSLS::SerializeAndPush(PipelineEventGroup&& group) { return false; } if (mCompressor) { - if (!mCompressor->Compress(serializedData, compressedData, errorMsg)) { + if (!mCompressor->DoCompress(serializedData, compressedData, errorMsg)) { LOG_WARNING(mContext->GetLogger(), ("failed to compress event group", errorMsg)("action", "discard data")("plugin", sName)("config", mContext->GetConfigName())); @@ -926,7 +926,7 @@ bool FlusherSLS::SerializeAndPush(BatchedEventsList&& groupList) { } AddPackId(group); string errorMsg; - if (!mGroupSerializer->Serialize(std::move(group), serializedData, errorMsg)) { + if (!mGroupSerializer->DoSerialize(std::move(group), serializedData, errorMsg)) { LOG_WARNING(mContext->GetLogger(), ("failed to serialize event group", errorMsg)("action", "discard data")("plugin", sName)("config", mContext->GetConfigName())); @@ -941,7 +941,7 @@ bool FlusherSLS::SerializeAndPush(BatchedEventsList&& groupList) { continue; } if (mCompressor) { - if (!mCompressor->Compress(serializedData, compressedData, errorMsg)) { + if (!mCompressor->DoCompress(serializedData, compressedData, errorMsg)) { LOG_WARNING(mContext->GetLogger(), ("failed to compress event group", errorMsg)("action", "discard data")("plugin", sName)("config", mContext->GetConfigName())); @@ -991,7 +991,7 @@ bool FlusherSLS::SerializeAndPush(BatchedEventsList&& groupList) { } if (enablePackageList) { string errorMsg; - mGroupListSerializer->Serialize(std::move(compressedLogGroups), serializedData, errorMsg); + mGroupListSerializer->DoSerialize(std::move(compressedLogGroups), serializedData, errorMsg); allSucceeded = Flusher::PushToQueue(make_unique( std::move(serializedData), packageSize, this, mQueueKey, mLogstore, RawDataType::EVENT_GROUP_LIST)) diff --git a/core/plugin/flusher/sls/FlusherSLS.h b/core/plugin/flusher/sls/FlusherSLS.h index 2840b94fe4..7e3d4658a1 100644 --- a/core/plugin/flusher/sls/FlusherSLS.h +++ b/core/plugin/flusher/sls/FlusherSLS.h @@ -24,12 +24,12 @@ #include #include +#include "common/compression/Compressor.h" +#include "models/PipelineEventGroup.h" #include "pipeline/batch/BatchStatus.h" #include "pipeline/batch/Batcher.h" -#include "pipeline/compression/Compressor.h" -#include "models/PipelineEventGroup.h" -#include "pipeline/plugin/interface/HttpFlusher.h" #include "pipeline/limiter/ConcurrencyLimiter.h" +#include "pipeline/plugin/interface/HttpFlusher.h" #include "pipeline/serializer/SLSSerializer.h" namespace logtail { diff --git a/core/profile_sender/ProfileSender.cpp b/core/profile_sender/ProfileSender.cpp index 1121e91216..7098f31ce1 100644 --- a/core/profile_sender/ProfileSender.cpp +++ b/core/profile_sender/ProfileSender.cpp @@ -28,7 +28,7 @@ #include "sdk/Exception.h" #include "sls_control/SLSControl.h" // TODO: temporarily used -#include "pipeline/compression/CompressorFactory.h" +#include "common/compression/CompressorFactory.h" using namespace std; @@ -101,8 +101,8 @@ void ProfileSender::SetProfileProjectName(const string& region, const string& pr flusher.mAliuid = STRING_FLAG(logtail_profile_aliuid); // logstore is given at send time // TODO: temporarily used - flusher.mCompressor - = CompressorFactory::GetInstance()->Create(Json::Value(), PipelineContext(), "flusher_sls", CompressType::ZSTD); + flusher.mCompressor = CompressorFactory::GetInstance()->Create( + Json::Value(), PipelineContext(), "flusher_sls", "", CompressType::ZSTD); } FlusherSLS* ProfileSender::GetFlusher(const string& region) { diff --git a/core/unittest/batch/BatchItemUnittest.cpp b/core/unittest/batch/BatchItemUnittest.cpp index 71d2d7fa59..0aac5d30b3 100644 --- a/core/unittest/batch/BatchItemUnittest.cpp +++ b/core/unittest/batch/BatchItemUnittest.cpp @@ -92,6 +92,7 @@ void EventBatchItemUnittest::TestAdd() { APSARA_TEST_EQUAL(1U, mItem.mBatch.mEvents.size()); APSARA_TEST_EQUAL(1U, mItem.GetStatus().GetCnt()); APSARA_TEST_EQUAL(size, mItem.GetStatus().GetSize()); + APSARA_TEST_NOT_EQUAL(0, mItem.mTotalEnqueTimeMs); } void EventBatchItemUnittest::TestFlushEmpty() { @@ -116,10 +117,12 @@ void EventBatchItemUnittest::TestFlushGroupBatchItem() { sEventGroup->AddLogEvent(); PipelineEventPtr& e = sEventGroup->MutableEvents().back(); mItem.Add(std::move(e)); + auto size = mItem.DataSize(); GroupBatchItem res; mItem.Flush(res); APSARA_TEST_EQUAL(1U, res.mGroups.size()); + APSARA_TEST_EQUAL(size, res.mGroups[0].mSizeBytes); APSARA_TEST_TRUE(mItem.IsEmpty()); APSARA_TEST_TRUE(mItem.mBatch.mTags.mInner.empty()); @@ -130,12 +133,14 @@ void EventBatchItemUnittest::TestFlushGroupBatchItem() { APSARA_TEST_EQUAL(0U, mItem.GetStatus().GetCnt()); APSARA_TEST_EQUAL(0U, mItem.GetStatus().GetSize()); APSARA_TEST_EQUAL(0, mItem.GetStatus().GetCreateTime()); + APSARA_TEST_EQUAL(0, mItem.mTotalEnqueTimeMs); } void EventBatchItemUnittest::TestFlushBatchedEvensList() { sEventGroup->AddLogEvent(); PipelineEventPtr& e = sEventGroup->MutableEvents().back(); mItem.Add(std::move(e)); + auto size = mItem.DataSize(); BatchedEventsList res; mItem.Flush(res); @@ -145,6 +150,7 @@ void EventBatchItemUnittest::TestFlushBatchedEvensList() { APSARA_TEST_NOT_EQUAL(nullptr, res[0].mExactlyOnceCheckpoint); APSARA_TEST_STREQ("pack_id", res[0].mPackIdPrefix.data()); APSARA_TEST_EQUAL(1U, res[0].mSourceBuffers.size()); + APSARA_TEST_EQUAL(size, res[0].mSizeBytes); APSARA_TEST_TRUE(mItem.IsEmpty()); APSARA_TEST_TRUE(mItem.mBatch.mTags.mInner.empty()); @@ -155,12 +161,14 @@ void EventBatchItemUnittest::TestFlushBatchedEvensList() { APSARA_TEST_EQUAL(0U, mItem.GetStatus().GetCnt()); APSARA_TEST_EQUAL(0U, mItem.GetStatus().GetSize()); APSARA_TEST_EQUAL(0, mItem.GetStatus().GetCreateTime()); + APSARA_TEST_EQUAL(0, mItem.mTotalEnqueTimeMs); } void EventBatchItemUnittest::TestFlushBatchedEvensLists() { sEventGroup->AddLogEvent(); PipelineEventPtr& e = sEventGroup->MutableEvents().back(); mItem.Add(std::move(e)); + auto size = mItem.DataSize(); vector res; mItem.Flush(res); @@ -171,6 +179,7 @@ void EventBatchItemUnittest::TestFlushBatchedEvensLists() { APSARA_TEST_NOT_EQUAL(nullptr, res[0][0].mExactlyOnceCheckpoint); APSARA_TEST_STREQ("pack_id", res[0][0].mPackIdPrefix.data()); APSARA_TEST_EQUAL(1U, res[0][0].mSourceBuffers.size()); + APSARA_TEST_EQUAL(size, res[0][0].mSizeBytes); APSARA_TEST_TRUE(mItem.IsEmpty()); APSARA_TEST_TRUE(mItem.mBatch.mTags.mInner.empty()); @@ -181,6 +190,7 @@ void EventBatchItemUnittest::TestFlushBatchedEvensLists() { APSARA_TEST_EQUAL(0U, mItem.GetStatus().GetCnt()); APSARA_TEST_EQUAL(0U, mItem.GetStatus().GetSize()); APSARA_TEST_EQUAL(0, mItem.GetStatus().GetCreateTime()); + APSARA_TEST_EQUAL(0, mItem.mTotalEnqueTimeMs); } void EventBatchItemUnittest::TestExactlyOnce() { @@ -223,6 +233,7 @@ class GroupBatchItemUnittest : public ::testing::Test { mBatch.mEvents = std::move(eventGroup.MutableEvents()); mBatch.mSourceBuffers.emplace_back(std::move(eventGroup.GetSourceBuffer())); mBatch.mTags = std::move(eventGroup.GetSizedTags()); + mBatch.mSizeBytes = 100; } void TearDown() override { @@ -236,11 +247,15 @@ class GroupBatchItemUnittest : public ::testing::Test { }; void GroupBatchItemUnittest::TestAdd() { - size_t size = mBatch.DataSize(); - mItem.Add(std::move(mBatch)); + size_t size = mBatch.mSizeBytes; + mItem.Add(std::move(mBatch), 1234567890000); APSARA_TEST_EQUAL(1U, mItem.mGroups.size()); APSARA_TEST_EQUAL(size, mItem.GetStatus().GetSize()); + APSARA_TEST_EQUAL(1234567890000, mItem.TotalEnqueTimeMs()); + APSARA_TEST_EQUAL(1U, mItem.EventSize()); + APSARA_TEST_EQUAL(1U, mItem.GroupSize()); + APSARA_TEST_EQUAL(100U, mItem.DataSize()); } void GroupBatchItemUnittest::TestFlushEmpty() { @@ -257,7 +272,7 @@ void GroupBatchItemUnittest::TestFlushEmpty() { } void GroupBatchItemUnittest::TestFlushBatchedEvensList() { - mItem.Add(std::move(mBatch)); + mItem.Add(std::move(mBatch), 1234567890000); BatchedEventsList res; mItem.Flush(res); @@ -266,10 +281,14 @@ void GroupBatchItemUnittest::TestFlushBatchedEvensList() { APSARA_TEST_TRUE(mItem.IsEmpty()); APSARA_TEST_EQUAL(0U, mItem.GetStatus().GetSize()); APSARA_TEST_EQUAL(0, mItem.GetStatus().GetCreateTime()); + APSARA_TEST_EQUAL(0, mItem.TotalEnqueTimeMs()); + APSARA_TEST_EQUAL(0U, mItem.EventSize()); + APSARA_TEST_EQUAL(0U, mItem.GroupSize()); + APSARA_TEST_EQUAL(0U, mItem.DataSize()); } void GroupBatchItemUnittest::TestFlushBatchedEvensLists() { - mItem.Add(std::move(mBatch)); + mItem.Add(std::move(mBatch), 1234567890000); vector res; mItem.Flush(res); @@ -279,6 +298,10 @@ void GroupBatchItemUnittest::TestFlushBatchedEvensLists() { APSARA_TEST_TRUE(mItem.IsEmpty()); APSARA_TEST_EQUAL(0U, mItem.GetStatus().GetSize()); APSARA_TEST_EQUAL(0, mItem.GetStatus().GetCreateTime()); + APSARA_TEST_EQUAL(0, mItem.TotalEnqueTimeMs()); + APSARA_TEST_EQUAL(0U, mItem.EventSize()); + APSARA_TEST_EQUAL(0U, mItem.GroupSize()); + APSARA_TEST_EQUAL(0U, mItem.DataSize()); } UNIT_TEST_CASE(GroupBatchItemUnittest, TestAdd) diff --git a/core/unittest/batch/BatchStatusUnittest.cpp b/core/unittest/batch/BatchStatusUnittest.cpp index 23932bef46..094333fd3d 100644 --- a/core/unittest/batch/BatchStatusUnittest.cpp +++ b/core/unittest/batch/BatchStatusUnittest.cpp @@ -124,6 +124,7 @@ class GroupBatchStatusUnittest : public ::testing::Test { void TestUpdate(); protected: + static void SetUpTestCase() { sBatch.mSizeBytes = 100; } void SetUp() override { mStatus.Reset(); } private: @@ -142,10 +143,10 @@ void GroupBatchStatusUnittest::TestReset() { void GroupBatchStatusUnittest::TestUpdate() { mStatus.Update(sBatch); time_t createTime = mStatus.GetCreateTime(); - APSARA_TEST_EQUAL(sBatch.DataSize(), mStatus.GetSize()); + APSARA_TEST_EQUAL(sBatch.mSizeBytes, mStatus.GetSize()); mStatus.Update(sBatch); - APSARA_TEST_EQUAL(2 * sBatch.DataSize(), mStatus.GetSize()); + APSARA_TEST_EQUAL(2 * sBatch.mSizeBytes, mStatus.GetSize()); APSARA_TEST_EQUAL(createTime, mStatus.GetCreateTime()); } diff --git a/core/unittest/batch/BatcherUnittest.cpp b/core/unittest/batch/BatcherUnittest.cpp index 8d40721565..799153692d 100644 --- a/core/unittest/batch/BatcherUnittest.cpp +++ b/core/unittest/batch/BatcherUnittest.cpp @@ -12,9 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. - -#include "pipeline/batch/Batcher.h" #include "common/JsonUtil.h" +#include "pipeline/batch/Batcher.h" #include "unittest/Unittest.h" #include "unittest/plugin/PluginMock.h" @@ -34,6 +33,7 @@ class BatcherUnittest : public ::testing::Test { void TestFlushGroupQueue(); void TestFlushAllWithoutGroupBatch(); void TestFlushAllWithGroupBatch(); + void TestMetric(); protected: static void SetUpTestCase() { sFlusher = make_unique(); } @@ -42,6 +42,7 @@ class BatcherUnittest : public ::testing::Test { mCtx.SetConfigName("test_config"); sFlusher->SetContext(mCtx); sFlusher->SetMetricsRecordRef(FlusherMock::sName, "1", "1", "1"); + sFlusher->SetNodeID("1"); } void TearDown() override { TimeoutFlushManager::GetInstance()->mTimeoutRecords.clear(); } @@ -559,6 +560,63 @@ void BatcherUnittest::TestFlushAllWithGroupBatch() { APSARA_TEST_STREQ("pack_id", res[1][0].mPackIdPrefix.data()); } +void BatcherUnittest::TestMetric() { + { + DefaultFlushStrategyOptions strategy; + strategy.mMaxCnt = 2; + strategy.mMaxSizeBytes = 1000; + strategy.mTimeoutSecs = 3; + + Batcher<> batch; + batch.Init(Json::Value(), sFlusher.get(), strategy, false); + + PipelineEventGroup g = CreateEventGroup(3); + auto groupSize = g.DataSize(); + auto batchSize = groupSize - 2 * g.GetEvents()[0]->DataSize(); + + vector res; + batch.Add(std::move(g), res); + APSARA_TEST_EQUAL(5U, batch.mMetricsRecordRef->GetLabels()->size()); + APSARA_TEST_TRUE(batch.mMetricsRecordRef.HasLabel(METRIC_LABEL_PROJECT, "")); + APSARA_TEST_TRUE(batch.mMetricsRecordRef.HasLabel(METRIC_LABEL_CONFIG_NAME, "test_config")); + APSARA_TEST_TRUE(batch.mMetricsRecordRef.HasLabel(METRIC_LABEL_KEY_COMPONENT_NAME, "batcher")); + APSARA_TEST_TRUE(batch.mMetricsRecordRef.HasLabel(METRIC_LABEL_KEY_FLUSHER_NODE_ID, "1")); + APSARA_TEST_TRUE(batch.mMetricsRecordRef.HasLabel("enable_group_batch", "false")); + APSARA_TEST_EQUAL(3U, batch.mInEventsCnt->GetValue()); + APSARA_TEST_EQUAL(groupSize, batch.mInGroupDataSizeBytes->GetValue()); + APSARA_TEST_EQUAL(2U, batch.mOutEventsCnt->GetValue()); + APSARA_TEST_EQUAL(1U, batch.mEventBatchItemsCnt->GetValue()); + APSARA_TEST_EQUAL(1U, batch.mBufferedGroupsCnt->GetValue()); + APSARA_TEST_EQUAL(1U, batch.mBufferedEventsCnt->GetValue()); + APSARA_TEST_EQUAL(batchSize, batch.mBufferedDataSizeByte->GetValue()); + } + { + DefaultFlushStrategyOptions strategy; + strategy.mMaxCnt = 2; + strategy.mMaxSizeBytes = 1000; + strategy.mTimeoutSecs = 3; + + Batcher<> batch; + batch.Init(Json::Value(), sFlusher.get(), strategy, true); + + PipelineEventGroup g = CreateEventGroup(3); + auto groupSize = g.DataSize(); + auto batchSize = groupSize - 2 * g.GetEvents()[0]->DataSize(); + + vector res; + batch.Add(std::move(g), res); + batch.FlushQueue(0, res[0]); + APSARA_TEST_TRUE(batch.mMetricsRecordRef.HasLabel("enable_group_batch", "true")); + APSARA_TEST_EQUAL(3U, batch.mInEventsCnt->GetValue()); + APSARA_TEST_EQUAL(groupSize, batch.mInGroupDataSizeBytes->GetValue()); + APSARA_TEST_EQUAL(2U, batch.mOutEventsCnt->GetValue()); + APSARA_TEST_EQUAL(1U, batch.mEventBatchItemsCnt->GetValue()); + APSARA_TEST_EQUAL(1U, batch.mBufferedGroupsCnt->GetValue()); + APSARA_TEST_EQUAL(1U, batch.mBufferedEventsCnt->GetValue()); + APSARA_TEST_EQUAL(batchSize, batch.mBufferedDataSizeByte->GetValue()); + } +} + PipelineEventGroup BatcherUnittest::CreateEventGroup(size_t cnt) { PipelineEventGroup group(make_shared()); group.SetTag(string("key"), string("val")); @@ -581,6 +639,7 @@ UNIT_TEST_CASE(BatcherUnittest, TestFlushEventQueueWithGroupBatch) UNIT_TEST_CASE(BatcherUnittest, TestFlushGroupQueue) UNIT_TEST_CASE(BatcherUnittest, TestFlushAllWithoutGroupBatch) UNIT_TEST_CASE(BatcherUnittest, TestFlushAllWithGroupBatch) +UNIT_TEST_CASE(BatcherUnittest, TestMetric) } // namespace logtail diff --git a/core/unittest/compression/CMakeLists.txt b/core/unittest/compression/CMakeLists.txt index 442a8db893..60f1655e5d 100644 --- a/core/unittest/compression/CMakeLists.txt +++ b/core/unittest/compression/CMakeLists.txt @@ -18,6 +18,9 @@ project(compression_unittest) add_executable(compressor_factory_unittest CompressorFactoryUnittest.cpp) target_link_libraries(compressor_factory_unittest ${UT_BASE_TARGET}) +add_executable(compressor_unittest CompressorUnittest.cpp) +target_link_libraries(compressor_unittest ${UT_BASE_TARGET}) + add_executable(lz4_compressor_unittest LZ4CompressorUnittest.cpp) target_link_libraries(lz4_compressor_unittest ${UT_BASE_TARGET}) @@ -26,5 +29,6 @@ target_link_libraries(zstd_compressor_unittest ${UT_BASE_TARGET}) include(GoogleTest) gtest_discover_tests(compressor_factory_unittest) +gtest_discover_tests(compressor_unittest) gtest_discover_tests(lz4_compressor_unittest) gtest_discover_tests(zstd_compressor_unittest) diff --git a/core/unittest/compression/CompressorFactoryUnittest.cpp b/core/unittest/compression/CompressorFactoryUnittest.cpp index 7093a37ff3..84e159ee34 100644 --- a/core/unittest/compression/CompressorFactoryUnittest.cpp +++ b/core/unittest/compression/CompressorFactoryUnittest.cpp @@ -12,7 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include "pipeline/compression/CompressorFactory.h" +#include "monitor/MetricConstants.h" +#include "common/compression/CompressorFactory.h" #include "unittest/Unittest.h" using namespace std; @@ -23,56 +24,64 @@ class CompressorFactoryUnittest : public ::testing::Test { public: void TestCreate(); void TestCompressTypeToString(); + void TestMetric(); protected: - void SetUp() { mCtx.SetConfigName("test_config"); } + void SetUp() { + mCtx.SetConfigName("test_config"); + mFlusherId = "1"; + } private: PipelineContext mCtx; + string mFlusherId; }; void CompressorFactoryUnittest::TestCreate() { { // use default - auto compressor - = CompressorFactory::GetInstance()->Create(Json::Value(), mCtx, "test_plugin", CompressType::LZ4); + auto compressor = CompressorFactory::GetInstance()->Create( + Json::Value(), mCtx, "test_plugin", mFlusherId, CompressType::LZ4); APSARA_TEST_EQUAL(CompressType::LZ4, compressor->GetCompressType()); } { // lz4 Json::Value config; config["CompressType"] = "lz4"; - auto compressor = CompressorFactory::GetInstance()->Create(config, mCtx, "test_plugin", CompressType::ZSTD); + auto compressor + = CompressorFactory::GetInstance()->Create(config, mCtx, "test_plugin", mFlusherId, CompressType::ZSTD); APSARA_TEST_EQUAL(CompressType::LZ4, compressor->GetCompressType()); } { // zstd Json::Value config; config["CompressType"] = "zstd"; - auto compressor = CompressorFactory::GetInstance()->Create(config, mCtx, "test_plugin", CompressType::LZ4); + auto compressor + = CompressorFactory::GetInstance()->Create(config, mCtx, "test_plugin", mFlusherId, CompressType::LZ4); APSARA_TEST_EQUAL(CompressType::ZSTD, compressor->GetCompressType()); } { // none Json::Value config; config["CompressType"] = "none"; - auto compressor = CompressorFactory::GetInstance()->Create(config, mCtx, "test_plugin", CompressType::LZ4); + auto compressor + = CompressorFactory::GetInstance()->Create(config, mCtx, "test_plugin", mFlusherId, CompressType::LZ4); APSARA_TEST_EQUAL(nullptr, compressor); } { // unknown Json::Value config; config["CompressType"] = "unknown"; - auto compressor - = CompressorFactory::GetInstance()->Create(Json::Value(), mCtx, "test_plugin", CompressType::LZ4); + auto compressor = CompressorFactory::GetInstance()->Create( + Json::Value(), mCtx, "test_plugin", mFlusherId, CompressType::LZ4); APSARA_TEST_EQUAL(CompressType::LZ4, compressor->GetCompressType()); } { // invalid Json::Value config; config["CompressType"] = 123; - auto compressor - = CompressorFactory::GetInstance()->Create(Json::Value(), mCtx, "test_plugin", CompressType::LZ4); + auto compressor = CompressorFactory::GetInstance()->Create( + Json::Value(), mCtx, "test_plugin", mFlusherId, CompressType::LZ4); APSARA_TEST_EQUAL(CompressType::LZ4, compressor->GetCompressType()); } } @@ -83,8 +92,19 @@ void CompressorFactoryUnittest::TestCompressTypeToString() { APSARA_TEST_STREQ("none", CompressTypeToString(CompressType::NONE).data()); } +void CompressorFactoryUnittest::TestMetric() { + auto compressor + = CompressorFactory::GetInstance()->Create(Json::Value(), mCtx, "test_plugin", mFlusherId, CompressType::LZ4); + APSARA_TEST_EQUAL(4U, compressor->mMetricsRecordRef->GetLabels()->size()); + APSARA_TEST_TRUE(compressor->mMetricsRecordRef.HasLabel(METRIC_LABEL_PROJECT, "")); + APSARA_TEST_TRUE(compressor->mMetricsRecordRef.HasLabel(METRIC_LABEL_CONFIG_NAME, "test_config")); + APSARA_TEST_TRUE(compressor->mMetricsRecordRef.HasLabel(METRIC_LABEL_KEY_COMPONENT_NAME, "compressor")); + APSARA_TEST_TRUE(compressor->mMetricsRecordRef.HasLabel(METRIC_LABEL_KEY_FLUSHER_NODE_ID, mFlusherId)); +} + UNIT_TEST_CASE(CompressorFactoryUnittest, TestCreate) UNIT_TEST_CASE(CompressorFactoryUnittest, TestCompressTypeToString) +UNIT_TEST_CASE(CompressorFactoryUnittest, TestMetric) } // namespace logtail diff --git a/core/unittest/compression/CompressorUnittest.cpp b/core/unittest/compression/CompressorUnittest.cpp new file mode 100644 index 0000000000..54576c63a4 --- /dev/null +++ b/core/unittest/compression/CompressorUnittest.cpp @@ -0,0 +1,79 @@ +// Copyright 2024 iLogtail Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "common/compression/LZ4Compressor.h" +#include "monitor/MetricConstants.h" +#include "unittest/Unittest.h" + +using namespace std; + +namespace logtail { + +class CompressorMock : public Compressor { +public: + CompressorMock(CompressType type) : Compressor(type) {}; + + bool UnCompress(const std::string& input, std::string& output, std::string& errorMsg) override { return true; } + +private: + bool Compress(const std::string& input, std::string& output, std::string& errorMsg) override { + if (input == "failed") { + return false; + } + output = input.substr(0, input.size() / 2); + return true; + } +}; + +class CompressorUnittest : public ::testing::Test { +public: + void TestMetric(); +}; + +void CompressorUnittest::TestMetric() { + { + CompressorMock compressor(CompressType::MOCK); + compressor.SetMetricRecordRef({}); + string input = "hello world"; + string output; + string errorMsg; + compressor.DoCompress(input, output, errorMsg); + APSARA_TEST_EQUAL(1U, compressor.mInItemsCnt->GetValue()); + APSARA_TEST_EQUAL(input.size(), compressor.mInItemSizeBytes->GetValue()); + APSARA_TEST_EQUAL(1U, compressor.mOutItemsCnt->GetValue()); + APSARA_TEST_EQUAL(output.size(), compressor.mOutItemSizeBytes->GetValue()); + APSARA_TEST_EQUAL(0U, compressor.mDiscardedItemsCnt->GetValue()); + APSARA_TEST_EQUAL(0U, compressor.mDiscardedItemSizeBytes->GetValue()); + } + { + CompressorMock compressor(CompressType::MOCK); + compressor.SetMetricRecordRef({}); + string input = "failed"; + string output; + string errorMsg; + compressor.DoCompress(input, output, errorMsg); + APSARA_TEST_EQUAL(1U, compressor.mInItemsCnt->GetValue()); + APSARA_TEST_EQUAL(input.size(), compressor.mInItemSizeBytes->GetValue()); + APSARA_TEST_EQUAL(0U, compressor.mOutItemsCnt->GetValue()); + APSARA_TEST_EQUAL(0U, compressor.mOutItemSizeBytes->GetValue()); + APSARA_TEST_EQUAL(1U, compressor.mDiscardedItemsCnt->GetValue()); + APSARA_TEST_EQUAL(input.size(), compressor.mDiscardedItemSizeBytes->GetValue()); + } +} + +UNIT_TEST_CASE(CompressorUnittest, TestMetric) + +} // namespace logtail + +UNIT_TEST_MAIN diff --git a/core/unittest/compression/LZ4CompressorUnittest.cpp b/core/unittest/compression/LZ4CompressorUnittest.cpp index 7bd62300f7..e852c063cf 100644 --- a/core/unittest/compression/LZ4CompressorUnittest.cpp +++ b/core/unittest/compression/LZ4CompressorUnittest.cpp @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include "pipeline/compression/LZ4Compressor.h" +#include "common/compression/LZ4Compressor.h" #include "unittest/Unittest.h" using namespace std; @@ -29,7 +29,7 @@ void LZ4CompressorUnittest::TestCompress() { string input = "hello world"; string output; string errorMsg; - APSARA_TEST_TRUE(compressor.Compress(input, output, errorMsg)); + APSARA_TEST_TRUE(compressor.DoCompress(input, output, errorMsg)); string decompressed; decompressed.resize(input.size()); APSARA_TEST_TRUE(compressor.UnCompress(output, decompressed, errorMsg)); diff --git a/core/unittest/compression/ZstdCompressorUnittest.cpp b/core/unittest/compression/ZstdCompressorUnittest.cpp index 6362907173..0e165b1987 100644 --- a/core/unittest/compression/ZstdCompressorUnittest.cpp +++ b/core/unittest/compression/ZstdCompressorUnittest.cpp @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include "pipeline/compression/ZstdCompressor.h" +#include "common/compression/ZstdCompressor.h" #include "unittest/Unittest.h" using namespace std; @@ -29,7 +29,7 @@ void ZstdCompressorUnittest::TestCompress() { string input = "hello world"; string output; string errorMsg; - APSARA_TEST_TRUE(compressor.Compress(input, output, errorMsg)); + APSARA_TEST_TRUE(compressor.DoCompress(input, output, errorMsg)); string decompressed; decompressed.resize(input.size()); APSARA_TEST_TRUE(compressor.UnCompress(output, decompressed, errorMsg)); diff --git a/core/unittest/flusher/FlusherSLSUnittest.cpp b/core/unittest/flusher/FlusherSLSUnittest.cpp index 52a3d2f510..71e13cbcc6 100644 --- a/core/unittest/flusher/FlusherSLSUnittest.cpp +++ b/core/unittest/flusher/FlusherSLSUnittest.cpp @@ -22,10 +22,7 @@ #ifdef __ENTERPRISE__ #include "config/provider/EnterpriseConfigProvider.h" #endif -#include "pipeline/compression/CompressorFactory.h" -#include "plugin/flusher/sls/FlusherSLS.h" -#include "plugin/flusher/sls/PackIdManager.h" -#include "plugin/flusher/sls/SLSClientManager.h" +#include "common/compression/CompressorFactory.h" #include "pipeline/Pipeline.h" #include "pipeline/PipelineContext.h" #include "pipeline/queue/ExactlyOnceQueueManager.h" @@ -33,6 +30,9 @@ #include "pipeline/queue/QueueKeyManager.h" #include "pipeline/queue/SLSSenderQueueItem.h" #include "pipeline/queue/SenderQueueManager.h" +#include "plugin/flusher/sls/FlusherSLS.h" +#include "plugin/flusher/sls/PackIdManager.h" +#include "plugin/flusher/sls/SLSClientManager.h" #include "unittest/Unittest.h" DECLARE_FLAG_INT32(batch_send_interval); @@ -617,7 +617,7 @@ void FlusherSLSUnittest::TestSend() { APSARA_TEST_EQUAL(cpt, item->mExactlyOnceCheckpoint); auto compressor - = CompressorFactory::GetInstance()->Create(Json::Value(), ctx, "flusher_sls", CompressType::LZ4); + = CompressorFactory::GetInstance()->Create(Json::Value(), ctx, "flusher_sls", "1", CompressType::LZ4); string output, errorMsg; output.resize(item->mRawSize); APSARA_TEST_TRUE(compressor->UnCompress(item->mData, output, errorMsg)); @@ -671,7 +671,7 @@ void FlusherSLSUnittest::TestSend() { APSARA_TEST_EQUAL(checkpoints[0], item->mExactlyOnceCheckpoint); auto compressor - = CompressorFactory::GetInstance()->Create(Json::Value(), ctx, "flusher_sls", CompressType::LZ4); + = CompressorFactory::GetInstance()->Create(Json::Value(), ctx, "flusher_sls", "1", CompressType::LZ4); string output, errorMsg; output.resize(item->mRawSize); APSARA_TEST_TRUE(compressor->UnCompress(item->mData, output, errorMsg)); @@ -750,7 +750,7 @@ void FlusherSLSUnittest::TestSend() { APSARA_TEST_EQUAL(flusher.mLogstore, item->mLogstore); auto compressor - = CompressorFactory::GetInstance()->Create(Json::Value(), ctx, "flusher_sls", CompressType::LZ4); + = CompressorFactory::GetInstance()->Create(Json::Value(), ctx, "flusher_sls", "1", CompressType::LZ4); string output, errorMsg; output.resize(item->mRawSize); APSARA_TEST_TRUE(compressor->UnCompress(item->mData, output, errorMsg)); @@ -844,7 +844,7 @@ void FlusherSLSUnittest::TestSend() { APSARA_TEST_EQUAL(flusher.mLogstore, item->mLogstore); auto compressor - = CompressorFactory::GetInstance()->Create(Json::Value(), ctx, "flusher_sls", CompressType::LZ4); + = CompressorFactory::GetInstance()->Create(Json::Value(), ctx, "flusher_sls", "1", CompressType::LZ4); sls_logs::SlsLogPackageList packageList; APSARA_TEST_TRUE(packageList.ParseFromString(item->mData)); @@ -1017,7 +1017,7 @@ void FlusherSLSUnittest::OnGoPipelineSend() { APSARA_TEST_EQUAL("other_logstore", item->mLogstore); auto compressor - = CompressorFactory::GetInstance()->Create(Json::Value(), ctx, "flusher_sls", CompressType::LZ4); + = CompressorFactory::GetInstance()->Create(Json::Value(), ctx, "flusher_sls", "1", CompressType::LZ4); string output; output.resize(item->mRawSize); APSARA_TEST_TRUE(compressor->UnCompress(item->mData, output, errorMsg)); @@ -1040,7 +1040,7 @@ void FlusherSLSUnittest::OnGoPipelineSend() { flusher.mProject = "test_project"; flusher.mLogstore = "test_logstore"; flusher.mCompressor = CompressorFactory::GetInstance()->Create( - Json::Value(), PipelineContext(), "flusher_sls", CompressType::LZ4); + Json::Value(), PipelineContext(), "flusher_sls", "1", CompressType::LZ4); APSARA_TEST_TRUE(flusher.Send("content", "")); @@ -1060,7 +1060,7 @@ void FlusherSLSUnittest::OnGoPipelineSend() { APSARA_TEST_EQUAL("test_logstore", item->mLogstore); auto compressor - = CompressorFactory::GetInstance()->Create(Json::Value(), ctx, "flusher_sls", CompressType::LZ4); + = CompressorFactory::GetInstance()->Create(Json::Value(), ctx, "flusher_sls", "1", CompressType::LZ4); string output; output.resize(item->mRawSize); string errorMsg; diff --git a/core/unittest/pipeline/PipelineUnittest.cpp b/core/unittest/pipeline/PipelineUnittest.cpp index 98b41a5ff6..d4d76b5138 100644 --- a/core/unittest/pipeline/PipelineUnittest.cpp +++ b/core/unittest/pipeline/PipelineUnittest.cpp @@ -18,17 +18,17 @@ #include #include "app_config/AppConfig.h" -#include "pipeline/batch/TimeoutFlushManager.h" #include "common/JsonUtil.h" #include "config/PipelineConfig.h" -#include "plugin/input/InputFeedbackInterfaceRegistry.h" #include "pipeline/Pipeline.h" +#include "pipeline/batch/TimeoutFlushManager.h" #include "pipeline/plugin/PluginRegistry.h" -#include "plugin/processor/inner/ProcessorSplitLogStringNative.h" -#include "plugin/processor/inner/ProcessorSplitMultilineLogStringNative.h" #include "pipeline/queue/BoundedProcessQueue.h" #include "pipeline/queue/ProcessQueueManager.h" #include "pipeline/queue/QueueKeyManager.h" +#include "plugin/input/InputFeedbackInterfaceRegistry.h" +#include "plugin/processor/inner/ProcessorSplitLogStringNative.h" +#include "plugin/processor/inner/ProcessorSplitMultilineLogStringNative.h" #include "unittest/Unittest.h" #include "unittest/plugin/PluginMock.h" @@ -112,6 +112,9 @@ void PipelineUnittest::OnSuccessfulInit() const { APSARA_TEST_EQUAL("test_region", pipeline->GetContext().GetRegion()); APSARA_TEST_EQUAL(QueueKeyManager::GetInstance()->GetKey("test_config-flusher_sls-test_project#test_logstore"), pipeline->GetContext().GetLogstoreKey()); + APSARA_TEST_EQUAL(2U, pipeline->mMetricsRecordRef->GetLabels()->size()); + APSARA_TEST_TRUE(pipeline->mMetricsRecordRef.HasLabel(METRIC_LABEL_CONFIG_NAME, configName)); + APSARA_TEST_TRUE(pipeline->mMetricsRecordRef.HasLabel(METRIC_LABEL_PROJECT, "test_project")); // without sls flusher configStr = R"( @@ -2684,12 +2687,24 @@ void PipelineUnittest::TestProcess() const { processor->Init(Json::Value(), ctx); pipeline.mProcessorLine.emplace_back(std::move(processor)); - vector group; - group.emplace_back(make_shared()); - pipeline.Process(group, 0); + WriteMetrics::GetInstance()->PrepareMetricsRecordRef(pipeline.mMetricsRecordRef, {}); + pipeline.mProcessorsInEventsCnt = pipeline.mMetricsRecordRef.CreateCounter("processors_in_events_cnt"); + pipeline.mProcessorsInGroupsCnt = pipeline.mMetricsRecordRef.CreateCounter("processors_in_event_groups_cnt"); + pipeline.mProcessorsInGroupDataSizeBytes + = pipeline.mMetricsRecordRef.CreateCounter("processors_in_event_group_data_size_bytes"); + pipeline.mProcessorsTotalDelayMs = pipeline.mMetricsRecordRef.CreateCounter("processors_total_delay_ms"); + + vector groups; + groups.emplace_back(make_shared()); + groups.back().AddLogEvent(); + auto size = groups.back().DataSize(); + pipeline.Process(groups, 0); APSARA_TEST_EQUAL( 1U, static_cast(pipeline.mInputs[0]->GetInnerProcessors()[0]->mPlugin.get())->mCnt); APSARA_TEST_EQUAL(1U, static_cast(pipeline.mProcessorLine[0]->mPlugin.get())->mCnt); + APSARA_TEST_EQUAL(1U, pipeline.mProcessorsInEventsCnt->GetValue()); + APSARA_TEST_EQUAL(1U, pipeline.mProcessorsInGroupsCnt->GetValue()); + APSARA_TEST_EQUAL(size, pipeline.mProcessorsInGroupDataSizeBytes->GetValue()); } void PipelineUnittest::TestSend() const { diff --git a/core/unittest/route/RouterUnittest.cpp b/core/unittest/route/RouterUnittest.cpp index bee7bb1803..3e42ff722a 100644 --- a/core/unittest/route/RouterUnittest.cpp +++ b/core/unittest/route/RouterUnittest.cpp @@ -13,6 +13,7 @@ // limitations under the License. #include "common/JsonUtil.h" +#include "monitor/MetricConstants.h" #include "pipeline/Pipeline.h" #include "pipeline/route/Router.h" #include "unittest/Unittest.h" @@ -25,6 +26,10 @@ class RouterUnittest : public testing::Test { public: void TestInit(); void TestRoute(); + void TestMetric(); + +protected: + void SetUp() override { ctx.SetConfigName("test_config"); } private: PipelineContext ctx; @@ -124,8 +129,44 @@ void RouterUnittest::TestRoute() { } } +void RouterUnittest::TestMetric() { + Json::Value configJson; + string errorMsg; + string configStr = R"( + [ + { + "Type": "event_type", + "Value": "log" + } + ] + )"; + APSARA_TEST_TRUE(ParseJsonTable(configStr, configJson, errorMsg)); + vector> configs; + for (Json::Value::ArrayIndex i = 0; i < configJson.size(); ++i) { + configs.emplace_back(i, &configJson[i]); + } + configs.emplace_back(configJson.size(), nullptr); + + Router router; + router.Init(configs, ctx); + + APSARA_TEST_EQUAL(3U, router.mMetricsRecordRef->GetLabels()->size()); + APSARA_TEST_TRUE(router.mMetricsRecordRef.HasLabel(METRIC_LABEL_PROJECT, "")); + APSARA_TEST_TRUE(router.mMetricsRecordRef.HasLabel(METRIC_LABEL_CONFIG_NAME, "test_config")); + APSARA_TEST_TRUE(router.mMetricsRecordRef.HasLabel(METRIC_LABEL_KEY_COMPONENT_NAME, "router")); + + PipelineEventGroup g(make_shared()); + g.AddLogEvent(); + auto size = g.DataSize(); + router.Route(g); + + APSARA_TEST_EQUAL(1U, router.mInEventsCnt->GetValue()); + APSARA_TEST_EQUAL(size, router.mInGroupDataSizeBytes->GetValue()); +} + UNIT_TEST_CASE(RouterUnittest, TestInit) UNIT_TEST_CASE(RouterUnittest, TestRoute) +UNIT_TEST_CASE(RouterUnittest, TestMetric) } // namespace logtail diff --git a/core/unittest/serializer/CMakeLists.txt b/core/unittest/serializer/CMakeLists.txt index 11c1436ee5..b471e3b81a 100644 --- a/core/unittest/serializer/CMakeLists.txt +++ b/core/unittest/serializer/CMakeLists.txt @@ -15,8 +15,12 @@ cmake_minimum_required(VERSION 3.22) project(serializer_unittest) +add_executable(serializer_unittest SerializerUnittest.cpp) +target_link_libraries(serializer_unittest ${UT_BASE_TARGET}) + add_executable(sls_serializer_unittest SLSSerializerUnittest.cpp) target_link_libraries(sls_serializer_unittest ${UT_BASE_TARGET}) include(GoogleTest) +gtest_discover_tests(serializer_unittest) gtest_discover_tests(sls_serializer_unittest) diff --git a/core/unittest/serializer/SLSSerializerUnittest.cpp b/core/unittest/serializer/SLSSerializerUnittest.cpp index a3695df4d1..7d57cef290 100644 --- a/core/unittest/serializer/SLSSerializerUnittest.cpp +++ b/core/unittest/serializer/SLSSerializerUnittest.cpp @@ -52,7 +52,7 @@ void SLSSerializerUnittest::TestSerializeEventGroup() { { // nano second disabled, and set string res, errorMsg; - APSARA_TEST_TRUE(serializer.Serialize(CreateBatchedEvents(false), res, errorMsg)); + APSARA_TEST_TRUE(serializer.DoSerialize(CreateBatchedEvents(false), res, errorMsg)); sls_logs::LogGroup logGroup; APSARA_TEST_TRUE(logGroup.ParseFromString(res)); APSARA_TEST_EQUAL(1, logGroup.logs_size()); @@ -72,7 +72,7 @@ void SLSSerializerUnittest::TestSerializeEventGroup() { // nano second enabled, and set const_cast(mCtx.GetGlobalConfig()).mEnableTimestampNanosecond = true; string res, errorMsg; - APSARA_TEST_TRUE(serializer.Serialize(CreateBatchedEvents(true), res, errorMsg)); + APSARA_TEST_TRUE(serializer.DoSerialize(CreateBatchedEvents(true), res, errorMsg)); sls_logs::LogGroup logGroup; APSARA_TEST_TRUE(logGroup.ParseFromString(res)); APSARA_TEST_EQUAL(1234567890U, logGroup.logs(0).time()); @@ -83,7 +83,7 @@ void SLSSerializerUnittest::TestSerializeEventGroup() { // nano second enabled, not set const_cast(mCtx.GetGlobalConfig()).mEnableTimestampNanosecond = true; string res, errorMsg; - APSARA_TEST_TRUE(serializer.Serialize(CreateBatchedEvents(false), res, errorMsg)); + APSARA_TEST_TRUE(serializer.DoSerialize(CreateBatchedEvents(false), res, errorMsg)); sls_logs::LogGroup logGroup; APSARA_TEST_TRUE(logGroup.ParseFromString(res)); APSARA_TEST_EQUAL(1234567890U, logGroup.logs(0).time()); @@ -94,13 +94,13 @@ void SLSSerializerUnittest::TestSerializeEventGroup() { // log group exceed size limit INT32_FLAG(max_send_log_group_size) = 0; string res, errorMsg; - APSARA_TEST_FALSE(serializer.Serialize(CreateBatchedEvents(true), res, errorMsg)); + APSARA_TEST_FALSE(serializer.DoSerialize(CreateBatchedEvents(true), res, errorMsg)); INT32_FLAG(max_send_log_group_size) = 10 * 1024 * 1024; } { // metric event string res, errorMsg; - APSARA_TEST_TRUE(serializer.Serialize(CreateBatchedMetricEvents(false, 0, false), res, errorMsg)); + APSARA_TEST_TRUE(serializer.DoSerialize(CreateBatchedMetricEvents(false, 0, false), res, errorMsg)); sls_logs::LogGroup logGroup; APSARA_TEST_TRUE(logGroup.ParseFromString(res)); @@ -125,7 +125,7 @@ void SLSSerializerUnittest::TestSerializeEventGroup() { const_cast(mCtx.GetGlobalConfig()).mEnableTimestampNanosecond = true; string res, errorMsg; - APSARA_TEST_TRUE(serializer.Serialize(CreateBatchedMetricEvents(true, 1, false), res, errorMsg)); + APSARA_TEST_TRUE(serializer.DoSerialize(CreateBatchedMetricEvents(true, 1, false), res, errorMsg)); sls_logs::LogGroup logGroup; APSARA_TEST_TRUE(logGroup.ParseFromString(res)); @@ -151,7 +151,7 @@ void SLSSerializerUnittest::TestSerializeEventGroup() { const_cast(mCtx.GetGlobalConfig()).mEnableTimestampNanosecond = true; string res, errorMsg; - APSARA_TEST_TRUE(serializer.Serialize(CreateBatchedMetricEvents(true, 1999999999, false), res, errorMsg)); + APSARA_TEST_TRUE(serializer.DoSerialize(CreateBatchedMetricEvents(true, 1999999999, false), res, errorMsg)); sls_logs::LogGroup logGroup; APSARA_TEST_TRUE(logGroup.ParseFromString(res)); @@ -178,7 +178,7 @@ void SLSSerializerUnittest::TestSerializeEventGroup() { const_cast(mCtx.GetGlobalConfig()).mEnableTimestampNanosecond = true; string res, errorMsg; - APSARA_TEST_TRUE(serializer.Serialize(CreateBatchedMetricEvents(false, 0, true), res, errorMsg)); + APSARA_TEST_TRUE(serializer.DoSerialize(CreateBatchedMetricEvents(false, 0, true), res, errorMsg)); sls_logs::LogGroup logGroup; APSARA_TEST_TRUE(logGroup.ParseFromString(res)); @@ -192,7 +192,7 @@ void SLSSerializerUnittest::TestSerializeEventGroupList() { SLSEventGroupListSerializer serializer(sFlusher.get()); string res, errorMsg; - APSARA_TEST_TRUE(serializer.Serialize(std::move(v), res, errorMsg)); + APSARA_TEST_TRUE(serializer.DoSerialize(std::move(v), res, errorMsg)); sls_logs::SlsLogPackageList logPackageList; APSARA_TEST_TRUE(logPackageList.ParseFromString(res)); APSARA_TEST_EQUAL(1, logPackageList.packages_size()); diff --git a/core/unittest/serializer/SerializerUnittest.cpp b/core/unittest/serializer/SerializerUnittest.cpp new file mode 100644 index 0000000000..ed632e02ae --- /dev/null +++ b/core/unittest/serializer/SerializerUnittest.cpp @@ -0,0 +1,114 @@ +// Copyright 2024 iLogtail Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "monitor/MetricConstants.h" +#include "pipeline/plugin/interface/Flusher.h" +#include "pipeline/serializer/Serializer.h" +#include "unittest/Unittest.h" +#include "unittest/plugin/PluginMock.h" + +using namespace std; + +namespace logtail { + +class SerializerMock : public Serializer { +public: + SerializerMock(Flusher* f) : Serializer(f) {}; + +private: + bool Serialize(BatchedEvents&& p, std::string& res, std::string& errorMsg) override { + if (p.mEvents.empty()) { + return false; + } + res = "result"; + return true; + } +}; + +class SerializerUnittest : public ::testing::Test { +public: + void TestMetric(); + +protected: + static void SetUpTestCase() { sFlusher = make_unique(); } + + void SetUp() override { + mCtx.SetConfigName("test_config"); + sFlusher->SetContext(mCtx); + sFlusher->SetMetricsRecordRef(FlusherMock::sName, "1", "1", "1"); + } + +private: + static unique_ptr sFlusher; + + BatchedEvents CreateBatchedMetricEvents(bool withEvents = true); + + PipelineContext mCtx; +}; + +unique_ptr SerializerUnittest::sFlusher; + +void SerializerUnittest::TestMetric() { + { + SerializerMock serializer(sFlusher.get()); + auto input = CreateBatchedMetricEvents(); + auto inputSize = input.mSizeBytes; + string output; + string errorMsg; + serializer.DoSerialize(std::move(input), output, errorMsg); + APSARA_TEST_EQUAL(1U, serializer.mInItemsCnt->GetValue()); + APSARA_TEST_EQUAL(inputSize, serializer.mInItemSizeBytes->GetValue()); + APSARA_TEST_EQUAL(1U, serializer.mOutItemsCnt->GetValue()); + APSARA_TEST_EQUAL(output.size(), serializer.mOutItemSizeBytes->GetValue()); + APSARA_TEST_EQUAL(0U, serializer.mDiscardedItemsCnt->GetValue()); + APSARA_TEST_EQUAL(0U, serializer.mDiscardedItemSizeBytes->GetValue()); + } + { + SerializerMock serializer(sFlusher.get()); + auto input = CreateBatchedMetricEvents(false); + auto inputSize = input.mSizeBytes; + string output; + string errorMsg; + serializer.DoSerialize(std::move(input), output, errorMsg); + APSARA_TEST_EQUAL(1U, serializer.mInItemsCnt->GetValue()); + APSARA_TEST_EQUAL(inputSize, serializer.mInItemSizeBytes->GetValue()); + APSARA_TEST_EQUAL(0U, serializer.mOutItemsCnt->GetValue()); + APSARA_TEST_EQUAL(0U, serializer.mOutItemSizeBytes->GetValue()); + APSARA_TEST_EQUAL(1U, serializer.mDiscardedItemsCnt->GetValue()); + APSARA_TEST_EQUAL(inputSize, serializer.mDiscardedItemSizeBytes->GetValue()); + } +} + +BatchedEvents SerializerUnittest::CreateBatchedMetricEvents(bool withEvents) { + PipelineEventGroup group(make_shared()); + group.SetTag(string("key"), string("value")); + StringBuffer b = group.GetSourceBuffer()->CopyString(string("pack_id")); + group.SetMetadataNoCopy(EventGroupMetaKey::SOURCE_ID, StringView(b.data, b.size)); + group.SetExactlyOnceCheckpoint(RangeCheckpointPtr(new RangeCheckpoint)); + if (withEvents) { + group.AddLogEvent(); + } + BatchedEvents batch(std::move(group.MutableEvents()), + std::move(group.GetSizedTags()), + std::move(group.GetSourceBuffer()), + group.GetMetadata(EventGroupMetaKey::SOURCE_ID), + std::move(group.GetExactlyOnceCheckpoint())); + return batch; +} + +UNIT_TEST_CASE(SerializerUnittest, TestMetric) + +} // namespace logtail + +UNIT_TEST_MAIN