diff --git a/velox/common/base/Counters.cpp b/velox/common/base/Counters.cpp index cb431b827b3a5..8d93226dd5e40 100644 --- a/velox/common/base/Counters.cpp +++ b/velox/common/base/Counters.cpp @@ -421,5 +421,11 @@ void registerVeloxMetrics() { // flushed due to memory reclaiming. DEFINE_METRIC( kMetricFileWriterEarlyFlushedRawBytes, facebook::velox::StatType::SUM); + + // The current spilling memory usage in bytes. + DEFINE_METRIC(kMetricSpillMemoryBytes, facebook::velox::StatType::AVG); + + // The peak spilling memory usage in bytes. + DEFINE_METRIC(kMetricSpillPeakMemoryBytes, facebook::velox::StatType::AVG); } } // namespace facebook::velox diff --git a/velox/common/base/Counters.h b/velox/common/base/Counters.h index 522bc4f484950..ccd31ec877a64 100644 --- a/velox/common/base/Counters.h +++ b/velox/common/base/Counters.h @@ -126,6 +126,12 @@ constexpr folly::StringPiece kMetricSpillFlushTimeMs{ constexpr folly::StringPiece kMetricSpillWriteTimeMs{ "velox.spill_write_time_ms"}; +constexpr folly::StringPiece kMetricSpillMemoryBytes{ + "velox.spill_memory_bytes"}; + +constexpr folly::StringPiece kMetricSpillPeakMemoryBytes{ + "velox.spill_peak_memory_bytes"}; + constexpr folly::StringPiece kMetricFileWriterEarlyFlushedRawBytes{ "velox.file_writer_early_flushed_raw_bytes"}; diff --git a/velox/common/base/PeriodicStatsReporter.cpp b/velox/common/base/PeriodicStatsReporter.cpp index 23fdfc2309d70..b444f7bcdbcbd 100644 --- a/velox/common/base/PeriodicStatsReporter.cpp +++ b/velox/common/base/PeriodicStatsReporter.cpp @@ -79,6 +79,10 @@ void PeriodicStatsReporter::start() { "report_arbitrator_stats", [this]() { reportArbitratorStats(); }, options_.arbitratorStatsIntervalMs); + addTask( + "report_spill_stats", + [this]() { reportSpillStats(); }, + options_.spillStatsIntervalMs); } void PeriodicStatsReporter::stop() { @@ -228,4 +232,13 @@ void PeriodicStatsReporter::reportCacheStats() { lastCacheStats_ = cacheStats; } +void PeriodicStatsReporter::reportSpillStats() { + const auto spillMemoryStats = velox::memory::spillMemoryPool()->stats(); + LOG(INFO) << "Spill memory usage: current[" + << velox::succinctBytes(spillMemoryStats.currentBytes) << "] peak[" + << velox::succinctBytes(spillMemoryStats.peakBytes) << "]"; + RECORD_METRIC_VALUE(kMetricSpillMemoryBytes, spillMemoryStats.currentBytes); + RECORD_METRIC_VALUE(kMetricSpillPeakMemoryBytes, spillMemoryStats.peakBytes); +} + } // namespace facebook::velox diff --git a/velox/common/base/PeriodicStatsReporter.h b/velox/common/base/PeriodicStatsReporter.h index 2e5a01a15001c..85a9a9d8a16a4 100644 --- a/velox/common/base/PeriodicStatsReporter.h +++ b/velox/common/base/PeriodicStatsReporter.h @@ -17,6 +17,7 @@ #pragma once #include +#include "velox/common/base/SpillStats.h" #include "velox/common/caching/AsyncDataCache.h" #include "velox/common/caching/SsdFile.h" #include "velox/common/memory/MemoryArbitrator.h" @@ -50,6 +51,8 @@ class PeriodicStatsReporter { const memory::MemoryArbitrator* arbitrator{nullptr}; uint64_t arbitratorStatsIntervalMs{60'000}; + uint64_t spillStatsIntervalMs{60'000}; + std::string toString() const { return fmt::format( "allocatorStatsIntervalMs:{}, cacheStatsIntervalMs:{}, " @@ -90,6 +93,7 @@ class PeriodicStatsReporter { void reportCacheStats(); void reportAllocatorStats(); void reportArbitratorStats(); + void reportSpillStats(); const velox::memory::MemoryAllocator* const allocator_{nullptr}; const velox::cache::AsyncDataCache* const cache_{nullptr}; diff --git a/velox/docs/monitoring/metrics.rst b/velox/docs/monitoring/metrics.rst index 273d13246c913..4b63f0346d850 100644 --- a/velox/docs/monitoring/metrics.rst +++ b/velox/docs/monitoring/metrics.rst @@ -414,6 +414,12 @@ Spilling * - file_writer_early_flushed_raw_bytes - Sum - Number of bytes pre-maturely flushed from file writers because of memory reclaiming. + * - spill_memory_bytes + - Avg + - The current spilling memory usage in bytes. + * - spill_peak_memory_bytes + - Avg + - The peak spilling memory usage in bytes. Hive Connector --------------