Add spill stats reporting to PeriodicStatsReporter

tanjialiang · May 14, 2024 · 6ca5d25 · 6ca5d25
1 parent b29d933
commit 6ca5d25
Show file tree

Hide file tree

Showing 5 changed files with 35 additions and 0 deletions.
diff --git a/velox/common/base/Counters.cpp b/velox/common/base/Counters.cpp
@@ -421,5 +421,11 @@ void registerVeloxMetrics() {
   // flushed due to memory reclaiming.
   DEFINE_METRIC(
       kMetricFileWriterEarlyFlushedRawBytes, facebook::velox::StatType::SUM);
+
+  // The current spilling memory usage in bytes.
+  DEFINE_METRIC(kMetricSpillMemoryBytes, facebook::velox::StatType::AVG);
+
+  // The peak spilling memory usage in bytes.
+  DEFINE_METRIC(kMetricSpillPeakMemoryBytes, facebook::velox::StatType::AVG);
 }
 } // namespace facebook::velox
diff --git a/velox/common/base/Counters.h b/velox/common/base/Counters.h
@@ -126,6 +126,12 @@ constexpr folly::StringPiece kMetricSpillFlushTimeMs{
 constexpr folly::StringPiece kMetricSpillWriteTimeMs{
     "velox.spill_write_time_ms"};
 
+constexpr folly::StringPiece kMetricSpillMemoryBytes{
+    "velox.spill_memory_bytes"};
+
+constexpr folly::StringPiece kMetricSpillPeakMemoryBytes{
+    "velox.spill_peak_memory_bytes"};
+
 constexpr folly::StringPiece kMetricFileWriterEarlyFlushedRawBytes{
     "velox.file_writer_early_flushed_raw_bytes"};
 

diff --git a/velox/common/base/PeriodicStatsReporter.cpp b/velox/common/base/PeriodicStatsReporter.cpp
@@ -79,6 +79,10 @@ void PeriodicStatsReporter::start() {
       "report_arbitrator_stats",
       [this]() { reportArbitratorStats(); },
       options_.arbitratorStatsIntervalMs);
+  addTask(
+      "report_spill_stats",
+      [this]() { reportSpillStats(); },
+      options_.spillStatsIntervalMs);
 }
 
 void PeriodicStatsReporter::stop() {
@@ -228,4 +232,13 @@ void PeriodicStatsReporter::reportCacheStats() {
   lastCacheStats_ = cacheStats;
 }
 
+void PeriodicStatsReporter::reportSpillStats() {
+  const auto spillMemoryStats = velox::memory::spillMemoryPool()->stats();
+  LOG(INFO) << "Spill memory usage: current["
+            << velox::succinctBytes(spillMemoryStats.currentBytes) << "] peak["
+            << velox::succinctBytes(spillMemoryStats.peakBytes) << "]";
+  RECORD_METRIC_VALUE(kMetricSpillMemoryBytes, spillMemoryStats.currentBytes);
+  RECORD_METRIC_VALUE(kMetricSpillPeakMemoryBytes, spillMemoryStats.peakBytes);
+}
+
 } // namespace facebook::velox
diff --git a/velox/common/base/PeriodicStatsReporter.h b/velox/common/base/PeriodicStatsReporter.h
@@ -17,6 +17,7 @@
 #pragma once
 
 #include <folly/experimental/ThreadedRepeatingFunctionRunner.h>
+#include "velox/common/base/SpillStats.h"
 #include "velox/common/caching/AsyncDataCache.h"
 #include "velox/common/caching/SsdFile.h"
 #include "velox/common/memory/MemoryArbitrator.h"
@@ -50,6 +51,8 @@ class PeriodicStatsReporter {
     const memory::MemoryArbitrator* arbitrator{nullptr};
     uint64_t arbitratorStatsIntervalMs{60'000};
 
+    uint64_t spillStatsIntervalMs{60'000};
+
     std::string toString() const {
       return fmt::format(
           "allocatorStatsIntervalMs:{}, cacheStatsIntervalMs:{}, "
@@ -90,6 +93,7 @@ class PeriodicStatsReporter {
   void reportCacheStats();
   void reportAllocatorStats();
   void reportArbitratorStats();
+  void reportSpillStats();
 
   const velox::memory::MemoryAllocator* const allocator_{nullptr};
   const velox::cache::AsyncDataCache* const cache_{nullptr};

diff --git a/velox/docs/monitoring/metrics.rst b/velox/docs/monitoring/metrics.rst
@@ -414,6 +414,12 @@ Spilling
    * - file_writer_early_flushed_raw_bytes
      - Sum
      - Number of bytes pre-maturely flushed from file writers because of memory reclaiming.
+   * - spill_memory_bytes
+     - Avg
+     - The current spilling memory usage in bytes.
+   * - spill_peak_memory_bytes
+     - Avg
+     - The peak spilling memory usage in bytes.
 
 Hive Connector
 --------------