[Feature](inverted index) add inverted index reader memory size into …

…searcher cache (#35149)
apache · May 22, 2024 · f77c53d · f77c53d
1 parent 01fa2e6
commit f77c53d
Show file tree

Hide file tree

Showing 4 changed files with 8 additions and 1 deletion.
diff --git a/be/src/clucene b/be/src/clucene
diff --git a/be/src/olap/rowset/segment_v2/inverted_index_reader.cpp b/be/src/olap/rowset/segment_v2/inverted_index_reader.cpp
@@ -280,6 +280,8 @@ Status InvertedIndexReader::create_index_searcher(lucene::store::Directory* dir,
     if (std::string(dir->getObjectName()) == "DorisCompoundReader") {
         static_cast<DorisCompoundReader*>(dir)->getDorisIndexInput()->setIdxFileCache(false);
     }
+    // NOTE: before mem_tracker hook becomes active, we caculate reader memory size by hand.
+    mem_tracker->consume(index_searcher_builder->get_reader_size());
     return Status::OK();
 };
 

diff --git a/be/src/olap/rowset/segment_v2/inverted_index_searcher.cpp b/be/src/olap/rowset/segment_v2/inverted_index_searcher.cpp
@@ -44,6 +44,7 @@ Status FulltextIndexSearcherBuilder::build(lucene::store::Directory* directory,
         return Status::Error<ErrorCode::INVERTED_INDEX_CLUCENE_ERROR>(
                 "FulltextIndexSearcherBuilder build index_searcher error.");
     }
+    reader_size = reader->getTermInfosRAMUsed();
     // NOTE: need to cl_refcount-- here, so that directory will be deleted when
     // index_searcher is destroyed
     _CLDECDELETE(directory)

diff --git a/be/src/olap/rowset/segment_v2/inverted_index_searcher.h b/be/src/olap/rowset/segment_v2/inverted_index_searcher.h
@@ -61,6 +61,10 @@ class IndexSearcherBuilder {
     virtual Result<IndexSearcherPtr> get_index_searcher(lucene::store::Directory* directory);
     static Result<std::unique_ptr<IndexSearcherBuilder>> create_index_searcher_builder(
             InvertedIndexReaderType reader_type);
+    int64_t get_reader_size() const { return reader_size; }
+
+protected:
+    int64_t reader_size = 0;
 };
 
 class FulltextIndexSearcherBuilder : public IndexSearcherBuilder {