From f77c53dcc248259465197ea4e3022d1286ae6626 Mon Sep 17 00:00:00 2001
From: airborne12 <airborne08@gmail.com>
Date: Wed, 22 May 2024 10:34:00 +0800
Subject: [PATCH] [Feature](inverted index) add inverted index reader memory
 size into searcher cache (#35149)

---
 be/src/clucene                                            | 2 +-
 be/src/olap/rowset/segment_v2/inverted_index_reader.cpp   | 2 ++
 be/src/olap/rowset/segment_v2/inverted_index_searcher.cpp | 1 +
 be/src/olap/rowset/segment_v2/inverted_index_searcher.h   | 4 ++++
 4 files changed, 8 insertions(+), 1 deletion(-)

diff --git a/be/src/clucene b/be/src/clucene
index 315b25ba70adc6..1936207ae4345e 160000
--- a/be/src/clucene
+++ b/be/src/clucene
@@ -1 +1 @@
-Subproject commit 315b25ba70adc6f7f303183897317cc88a860a3a
+Subproject commit 1936207ae4345e45fcb2b4e6b1ed1a318f8c1b1b
diff --git a/be/src/olap/rowset/segment_v2/inverted_index_reader.cpp b/be/src/olap/rowset/segment_v2/inverted_index_reader.cpp
index dc4aa0870b608d..2d9a23dfc9a28a 100644
--- a/be/src/olap/rowset/segment_v2/inverted_index_reader.cpp
+++ b/be/src/olap/rowset/segment_v2/inverted_index_reader.cpp
@@ -280,6 +280,8 @@ Status InvertedIndexReader::create_index_searcher(lucene::store::Directory* dir,
     if (std::string(dir->getObjectName()) == "DorisCompoundReader") {
         static_cast<DorisCompoundReader*>(dir)->getDorisIndexInput()->setIdxFileCache(false);
     }
+    // NOTE: before mem_tracker hook becomes active, we caculate reader memory size by hand.
+    mem_tracker->consume(index_searcher_builder->get_reader_size());
     return Status::OK();
 };
 
diff --git a/be/src/olap/rowset/segment_v2/inverted_index_searcher.cpp b/be/src/olap/rowset/segment_v2/inverted_index_searcher.cpp
index aff9c6ae0e0043..90d455e5754bfd 100644
--- a/be/src/olap/rowset/segment_v2/inverted_index_searcher.cpp
+++ b/be/src/olap/rowset/segment_v2/inverted_index_searcher.cpp
@@ -44,6 +44,7 @@ Status FulltextIndexSearcherBuilder::build(lucene::store::Directory* directory,
         return Status::Error<ErrorCode::INVERTED_INDEX_CLUCENE_ERROR>(
                 "FulltextIndexSearcherBuilder build index_searcher error.");
     }
+    reader_size = reader->getTermInfosRAMUsed();
     // NOTE: need to cl_refcount-- here, so that directory will be deleted when
     // index_searcher is destroyed
     _CLDECDELETE(directory)
diff --git a/be/src/olap/rowset/segment_v2/inverted_index_searcher.h b/be/src/olap/rowset/segment_v2/inverted_index_searcher.h
index b0466ab2df21f9..46297fbf6e25bd 100644
--- a/be/src/olap/rowset/segment_v2/inverted_index_searcher.h
+++ b/be/src/olap/rowset/segment_v2/inverted_index_searcher.h
@@ -61,6 +61,10 @@ class IndexSearcherBuilder {
     virtual Result<IndexSearcherPtr> get_index_searcher(lucene::store::Directory* directory);
     static Result<std::unique_ptr<IndexSearcherBuilder>> create_index_searcher_builder(
             InvertedIndexReaderType reader_type);
+    int64_t get_reader_size() const { return reader_size; }
+
+protected:
+    int64_t reader_size = 0;
 };
 
 class FulltextIndexSearcherBuilder : public IndexSearcherBuilder {