count tablet meta memory usage

be/src/olap/metadata_adder.h

@@ -0,0 +1,164 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#pragma once
+
+#include <bvar/bvar.h>
+#include <stdint.h>
+
+namespace doris {
+
+inline bvar::Adder<int64_t> g_total_rowset_meta_mem_size("doris_total_rowset_meta_mem_size");
+inline bvar::Adder<int64_t> g_total_rowset_meta_num("doris_total_rowset_meta_num");
+
+inline bvar::Adder<int64_t> g_total_tablet_meta_mem_size("doris_total_tablet_meta_mem_size");
+inline bvar::Adder<int64_t> g_total_tablet_meta_num("doris_total_tablet_meta_num");
+
+inline bvar::Adder<int64_t> g_total_tablet_column_mem_size("doris_total_tablet_column_mem_size");
+inline bvar::Adder<int64_t> g_total_tablet_column_num("doris_total_tablet_column_num");
+
+inline bvar::Adder<int64_t> g_total_tablet_index_mem_size("doris_total_tablet_index_mem_size");
+inline bvar::Adder<int64_t> g_total_tablet_index_num("doris_total_tablet_index_num");
+
+inline bvar::Adder<int64_t> g_total_tablet_schema_mem_size("doris_total_tablet_schema_mem_size");
+inline bvar::Adder<int64_t> g_total_tablet_schema_num("doris_total_tablet_schema_num");
+
+inline bvar::Adder<int64_t> g_total_segment_mem_size("doris_total_segment_mem_size");
+inline bvar::Adder<int64_t> g_total_segment_num("doris_total_segment_num");
+
+inline bvar::Adder<int64_t> g_total_column_reader_mem_size("doris_total_column_reader_mem_size");
+inline bvar::Adder<int64_t> g_total_column_reader_num("doris_total_column_reader_num");
+
+class RowsetMeta;
+class TabletMeta;
+class TabletColumn;
+class TabletIndex;
+class TabletSchema;
+
+namespace segment_v2 {
+class Segment;
+class ColumnReader;
+}; // namespace segment_v2
+
+/*
+    When a derived Class extends MetadataAdder, then the Class's number and fixed length field's memory can be counted automatically.
+    But if the Class has variable length field, then you should overwrite get_metadata_size and call update_metadata_size when the Class's memory changes.
+
+    There are some special situations that need to be noted:
+    1. when the derived Class override copy constructor, you'd better update memory size(call update_metadata_size) if derived class's 
+    memory changed in its copy constructor or you not call MetadataAdder's copy constructor.
+    2. when the derived Class override operator=, you'd better update memory size(call update_metadata_size) if the derived Class has variable length field;
+
+    Anyway, you should update mem size whenever derived Class's memory changes.
+*/
+
+template <typename T>
+class MetadataAdder {
+private:
+    int64_t _current_meta_size {0};
+
+    void add_mem_size(int64_t val);
+
+    void add_num(int64_t val);
+
+protected:
+    virtual ~MetadataAdder();
+
+    virtual int64_t get_metadata_size() { return sizeof(T); }
+
+    MetadataAdder(const MetadataAdder& other);
+
+    void update_metadata_size();
+
+public:
+    MetadataAdder();
+};
+
+template <typename T>
+MetadataAdder<T>::MetadataAdder(const MetadataAdder<T>& other) {
+    this->_current_meta_size = other._current_meta_size;
+    add_num(1);
+    add_mem_size(this->_current_meta_size);
+}
+
+template <typename T>
+MetadataAdder<T>::MetadataAdder() {
+    this->_current_meta_size = sizeof(T);
+    add_mem_size(this->_current_meta_size);
+    add_num(1);
+}
+
+template <typename T>
+MetadataAdder<T>::~MetadataAdder() {
+    add_mem_size(-_current_meta_size);
+    add_num(-1);
+}
+
+template <typename T>
+void MetadataAdder<T>::update_metadata_size() {
+    int64_t old_size = _current_meta_size;
+    _current_meta_size = get_metadata_size();
+    int64_t size_diff = _current_meta_size - old_size;
+
+    add_mem_size(size_diff);
+}
+
+template <typename T>
+void MetadataAdder<T>::add_mem_size(int64_t val) {
+    if (val == 0) {
+        return;
+    }
+    if constexpr (std::is_same_v<T, RowsetMeta>) {
+        g_total_rowset_meta_mem_size << val;
+    } else if constexpr (std::is_same_v<T, TabletMeta>) {
+        g_total_tablet_meta_mem_size << val;
+    } else if constexpr (std::is_same_v<T, TabletColumn>) {
+        g_total_tablet_column_mem_size << val;
+    } else if constexpr (std::is_same_v<T, TabletIndex>) {
+        g_total_tablet_index_mem_size << val;
+    } else if constexpr (std::is_same_v<T, TabletSchema>) {
+        g_total_tablet_schema_mem_size << val;
+    } else if constexpr (std::is_same_v<T, segment_v2::Segment>) {
+        g_total_segment_mem_size << val;
+    } else if constexpr (std::is_same_v<T, segment_v2::ColumnReader>) {
+        g_total_column_reader_mem_size << val;
+    }
+}
+
+template <typename T>
+void MetadataAdder<T>::add_num(int64_t val) {
+    if (val == 0) {
+        return;
+    }
+    if constexpr (std::is_same_v<T, RowsetMeta>) {
+        g_total_rowset_meta_num << val;
+    } else if constexpr (std::is_same_v<T, TabletMeta>) {
+        g_total_tablet_meta_num << val;
+    } else if constexpr (std::is_same_v<T, TabletColumn>) {
+        g_total_tablet_column_num << val;
+    } else if constexpr (std::is_same_v<T, TabletIndex>) {
+        g_total_tablet_index_num << val;
+    } else if constexpr (std::is_same_v<T, TabletSchema>) {
+        g_total_tablet_schema_num << val;
+    } else if constexpr (std::is_same_v<T, segment_v2::Segment>) {
+        g_total_segment_num << val;
+    } else if constexpr (std::is_same_v<T, segment_v2::ColumnReader>) {
+        g_total_column_reader_num << val;
+    }
+}
+
+}; // namespace doris

be/src/olap/rowset/rowset_meta.cpp

@@ -199,6 +199,7 @@ void RowsetMeta::_init() {
     } else {
         _rowset_id.init(_rowset_meta_pb.rowset_id_v2());
     }
+    update_metadata_size();
 }
 
 void RowsetMeta::add_segments_file_size(const std::vector<size_t>& seg_file_size) {
@@ -255,6 +256,12 @@ void RowsetMeta::merge_rowset_meta(const RowsetMeta& other) {
     if (rowset_state() == RowsetStatePB::BEGIN_PARTIAL_UPDATE) {
         set_rowset_state(RowsetStatePB::COMMITTED);
     }
+
+    update_metadata_size();
+}
+
+int64_t RowsetMeta::get_metadata_size() {
+    return sizeof(RowsetMeta) + _rowset_meta_pb.ByteSizeLong();
 }
 
 InvertedIndexFileInfo RowsetMeta::inverted_index_file_info(int seg_id) {

be/src/olap/rowset/rowset_meta.h

@@ -25,6 +25,7 @@
 #include <vector>
 
 #include "io/fs/file_system.h"
+#include "olap/metadata_adder.h"
 #include "olap/olap_common.h"
 #include "olap/rowset/rowset_fwd.h"
 #include "olap/storage_policy.h"
@@ -33,7 +34,7 @@
 
 namespace doris {
 
-class RowsetMeta {
+class RowsetMeta : public MetadataAdder<RowsetMeta> {
 public:
     RowsetMeta() = default;
     ~RowsetMeta();
@@ -367,6 +368,8 @@ class RowsetMeta {
 
     void update_inverted_index_files_info(const std::vector<InvertedIndexFileInfo>& idx_file_info);
 
+    int64_t get_metadata_size() override;
+
     // Because the member field '_handle' is a raw pointer, use member func 'init' to replace copy ctor
     RowsetMeta(const RowsetMeta&) = delete;
     RowsetMeta operator=(const RowsetMeta&) = delete;

be/src/olap/rowset/segment_v2/column_reader.cpp

@@ -86,8 +86,6 @@ inline bool read_as_string(PrimitiveType type) {
            type == PrimitiveType::TYPE_OBJECT;
 }
 
-static bvar::Adder<size_t> g_column_reader_memory_bytes("doris_column_reader_memory_bytes");
-static bvar::Adder<size_t> g_column_reader_num("doris_column_reader_num");
 Status ColumnReader::create_array(const ColumnReaderOptions& opts, const ColumnMetaPB& meta,
                                   const io::FileReaderSPtr& file_reader,
                                   std::unique_ptr<ColumnReader>* reader) {
@@ -276,15 +274,9 @@ ColumnReader::ColumnReader(const ColumnReaderOptions& opts, const ColumnMetaPB&
     _meta_is_nullable = meta.is_nullable();
     _meta_dict_page = meta.dict_page();
     _meta_compression = meta.compression();
-
-    g_column_reader_memory_bytes << sizeof(*this);
-    g_column_reader_num << 1;
 }
 
-ColumnReader::~ColumnReader() {
-    g_column_reader_memory_bytes << -sizeof(*this);
-    g_column_reader_num << -1;
-}
+ColumnReader::~ColumnReader() {}
 
 Status ColumnReader::init(const ColumnMetaPB* meta) {
     _type_info = get_type_info(meta);

be/src/olap/rowset/segment_v2/column_reader.h

@@ -111,7 +111,7 @@ struct ColumnIteratorOptions {
 // we should do our best to reduce resource usage through share
 // same information, such as OrdinalPageIndex and Page data.
 // This will cache data shared by all reader
-class ColumnReader {
+class ColumnReader : public MetadataAdder<ColumnReader> {
 public:
     // Create an initialized ColumnReader in *reader.
     // This should be a lightweight operation without I/O.

be/src/olap/rowset/segment_v2/segment.cpp

@@ -72,7 +72,7 @@
 #include "vec/olap/vgeneric_iterators.h"
 
 namespace doris::segment_v2 {
-static bvar::Adder<size_t> g_total_segment_num("doris_total_segment_num");
+
 class InvertedIndexIterator;
 
 io::UInt128Wrapper file_cache_key_from_path(const std::string& seg_path) {
@@ -139,18 +139,19 @@ Segment::Segment(uint32_t segment_id, RowsetId rowset_id, TabletSchemaSPtr table
           _meta_mem_usage(0),
           _rowset_id(rowset_id),
           _tablet_schema(std::move(tablet_schema)),
-          _idx_file_info(idx_file_info) {
-    g_total_segment_num << 1;
-}
+          _idx_file_info(idx_file_info) {}
 
-Segment::~Segment() {
-    g_total_segment_num << -1;
-}
+Segment::~Segment() {}
 
 io::UInt128Wrapper Segment::file_cache_key(std::string_view rowset_id, uint32_t seg_id) {
     return io::BlockFileCache::hash(fmt::format("{}_{}.dat", rowset_id, seg_id));
 }
 
+int64_t Segment::get_metadata_size() {
+    return sizeof(Segment) + (_footer_pb ? _footer_pb->ByteSizeLong() : 0) +
+           (_pk_index_meta ? _pk_index_meta->ByteSizeLong() : 0);
+}
+
 Status Segment::_open() {
     _footer_pb = std::make_unique<SegmentFooterPB>();
     RETURN_IF_ERROR(_parse_footer(_footer_pb.get()));
@@ -167,6 +168,9 @@ Status Segment::_open() {
     if (_pk_index_meta != nullptr) {
         _meta_mem_usage += _pk_index_meta->ByteSizeLong();
     }
+
+    update_metadata_size();
+
     _meta_mem_usage += sizeof(*this);
     _meta_mem_usage += _tablet_schema->num_columns() * config::estimated_mem_per_column_reader;
 

be/src/olap/rowset/segment_v2/segment.h

@@ -78,7 +78,7 @@ using SegmentSharedPtr = std::shared_ptr<Segment>;
 // NOTE: This segment is used to a specified TabletSchema, when TabletSchema
 // is changed, this segment can not be used any more. For example, after a schema
 // change finished, client should disable all cached Segment for old TabletSchema.
-class Segment : public std::enable_shared_from_this<Segment> {
+class Segment : public std::enable_shared_from_this<Segment>, public MetadataAdder<Segment> {
 public:
     static Status open(io::FileSystemSPtr fs, const std::string& path, uint32_t segment_id,
                        RowsetId rowset_id, TabletSchemaSPtr tablet_schema,
@@ -92,6 +92,8 @@ class Segment : public std::enable_shared_from_this<Segment> {
 
     ~Segment();
 
+    int64_t get_metadata_size() override;
+
     Status new_iterator(SchemaSPtr schema, const StorageReadOptions& read_options,
                         std::unique_ptr<RowwiseIterator>* iter);
 

be/src/olap/tablet_meta.h

@@ -43,6 +43,7 @@
 #include "io/fs/file_system.h"
 #include "olap/binlog_config.h"
 #include "olap/lru_cache.h"
+#include "olap/metadata_adder.h"
 #include "olap/olap_common.h"
 #include "olap/rowset/rowset_meta.h"
 #include "olap/tablet_schema.h"
@@ -90,7 +91,7 @@ class TBinlogConfig;
 
 // Class encapsulates meta of tablet.
 // The concurrency control is handled in Tablet Class, not in this class.
-class TabletMeta {
+class TabletMeta : public MetadataAdder<TabletMeta> {
 public:
     static TabletMetaSharedPtr create(
             const TCreateTabletReq& request, const TabletUid& tablet_uid, uint64_t shard_id,

be/src/olap/tablet_schema.cpp

@@ -53,8 +53,6 @@
 
 namespace doris {
 
-static bvar::Adder<size_t> g_total_tablet_schema_num("doris_total_tablet_schema_num");
-
 FieldType TabletColumn::get_field_type_by_type(PrimitiveType primitiveType) {
     switch (primitiveType) {
     case PrimitiveType::INVALID_TYPE:
@@ -847,12 +845,8 @@ void TabletIndex::to_schema_pb(TabletIndexPB* index) const {
     }
 }
 
-TabletSchema::TabletSchema() {
-    g_total_tablet_schema_num << 1;
-}
-
-TabletSchema::~TabletSchema() {
-    g_total_tablet_schema_num << -1;
+int64_t TabletSchema::get_metadata_size() {
+    return sizeof(TabletSchema) + _vl_field_mem_size;
 }
 
 void TabletSchema::append_column(TabletColumn column, ColumnType col_type) {
@@ -974,7 +968,10 @@ void TabletSchema::init_from_pb(const TabletSchemaPB& schema, bool ignore_extrac
             ++_num_variant_columns;
         }
         _cols.emplace_back(std::make_shared<TabletColumn>(std::move(column)));
+        _vl_field_mem_size +=
+                sizeof(StringRef) + sizeof(char) * _cols.back()->name().size() + sizeof(size_t);
         _field_name_to_index.emplace(StringRef(_cols.back()->name()), _num_columns);
+        _vl_field_mem_size += sizeof(int32_t) * 2;
         _field_id_to_index[_cols.back()->unique_id()] = _num_columns;
         _num_columns++;
     }
@@ -1017,6 +1014,8 @@ void TabletSchema::init_from_pb(const TabletSchemaPB& schema, bool ignore_extrac
     _row_store_column_unique_ids.assign(schema.row_store_column_unique_ids().begin(),
                                         schema.row_store_column_unique_ids().end());
     _variant_enable_flatten_nested = schema.variant_enable_flatten_nested();
+    _vl_field_mem_size += _row_store_column_unique_ids.capacity() * sizeof(int32_t);
+    update_metadata_size();
 }
 
 void TabletSchema::copy_from(const TabletSchema& tablet_schema) {