[Refator](point query) change LookupConnectionCache policy to number (#…

…33950)
apache · Apr 22, 2024 · 4c4c272 · 4c4c272
1 parent 3280443
commit 4c4c272
Show file tree

Hide file tree

Showing 5 changed files with 9 additions and 18 deletions.
diff --git a/be/src/common/config.cpp b/be/src/common/config.cpp
@@ -1070,8 +1070,8 @@ DEFINE_mInt64(auto_inc_low_water_level_mark_size_ratio, "3");
 
 // number of threads that fetch auto-inc ranges from FE
 DEFINE_mInt64(auto_inc_fetch_thread_num, "3");
-// default 4GB
-DEFINE_mInt64(lookup_connection_cache_bytes_limit, "4294967296");
+// default max to 2048 connections
+DEFINE_mInt64(lookup_connection_cache_capacity, "2048");
 
 // level of compression when using LZ4_HC, whose defalut value is LZ4HC_CLEVEL_DEFAULT
 DEFINE_mInt64(LZ4_HC_compression_level, "9");

diff --git a/be/src/common/config.h b/be/src/common/config.h
@@ -1128,7 +1128,7 @@ DECLARE_mInt64(auto_inc_low_water_level_mark_size_ratio);
 // number of threads that fetch auto-inc ranges from FE
 DECLARE_mInt64(auto_inc_fetch_thread_num);
 // Max connection cache num for point lookup queries
-DECLARE_mInt64(lookup_connection_cache_bytes_limit);
+DECLARE_mInt64(lookup_connection_cache_capacity);
 
 // level of compression when using LZ4_HC, whose defalut value is LZ4HC_CLEVEL_DEFAULT
 DECLARE_mInt64(LZ4_HC_compression_level);

diff --git a/be/src/runtime/exec_env_init.cpp b/be/src/runtime/exec_env_init.cpp
@@ -487,8 +487,8 @@ Status ExecEnv::_init_mem_env() {
 
     _file_meta_cache = new FileMetaCache(config::max_external_file_meta_cache_num);
 
-    _lookup_connection_cache = LookupConnectionCache::create_global_instance(
-            config::lookup_connection_cache_bytes_limit);
+    _lookup_connection_cache =
+            LookupConnectionCache::create_global_instance(config::lookup_connection_cache_capacity);
 
     // use memory limit
     int64_t inverted_index_cache_limit =

diff --git a/be/src/service/point_query_executor.cpp b/be/src/service/point_query_executor.cpp
@@ -57,7 +57,6 @@ Reusable::~Reusable() {}
 constexpr static int s_preallocted_blocks_num = 32;
 Status Reusable::init(const TDescriptorTable& t_desc_tbl, const std::vector<TExpr>& output_exprs,
                       const TQueryOptions& query_options, size_t block_size) {
-    SCOPED_MEM_COUNT_BY_HOOK(&_mem_size);
     _runtime_state = RuntimeState::create_unique();
     _runtime_state->set_query_options(query_options);
     RETURN_IF_ERROR(DescriptorTbl::create(_runtime_state->obj_pool(), t_desc_tbl, &_desc_tbl));
@@ -111,10 +110,6 @@ void Reusable::return_block(std::unique_ptr<vectorized::Block>& block) {
     }
 }
 
-int64_t Reusable::mem_size() const {
-    return _mem_size;
-}
-
 LookupConnectionCache* LookupConnectionCache::create_global_instance(size_t capacity) {
     DCHECK(ExecEnv::GetInstance()->get_lookup_connection_cache() == nullptr);
     auto* res = new LookupConnectionCache(capacity);

diff --git a/be/src/service/point_query_executor.h b/be/src/service/point_query_executor.h
@@ -91,8 +91,6 @@ class Reusable {
 
     const vectorized::VExprContextSPtrs& output_exprs() { return _output_exprs_ctxs; }
 
-    int64_t mem_size() const;
-
 private:
     // caching TupleDescriptor, output_expr, etc...
     std::unique_ptr<RuntimeState> _runtime_state;
@@ -105,7 +103,6 @@ class Reusable {
     vectorized::DataTypeSerDeSPtrs _data_type_serdes;
     std::unordered_map<uint32_t, uint32_t> _col_uid_to_idx;
     std::vector<std::string> _col_default_values;
-    int64_t _mem_size = 0;
 };
 
 // RowCache is a LRU cache for row store
@@ -214,8 +211,8 @@ class LookupConnectionCache : public LRUCachePolicy {
     friend class PointQueryExecutor;
     LookupConnectionCache(size_t capacity)
             : LRUCachePolicy(CachePolicy::CacheType::LOOKUP_CONNECTION_CACHE, capacity,
-                             LRUCacheType::SIZE, config::tablet_lookup_cache_stale_sweep_time_sec) {
-    }
+                             LRUCacheType::NUMBER,
+                             config::tablet_lookup_cache_stale_sweep_time_sec) {}
 
     static std::string encode_key(__int128_t cache_id) {
         fmt::memory_buffer buffer;
@@ -227,11 +224,10 @@ class LookupConnectionCache : public LRUCachePolicy {
         std::string key = encode_key(cache_id);
         auto* value = new CacheValue;
         value->item = item;
-        LOG(INFO) << "Add item mem size " << item->mem_size()
+        LOG(INFO) << "Add item mem"
                   << ", cache_capacity: " << get_total_capacity()
                   << ", cache_usage: " << get_usage() << ", mem_consum: " << mem_consumption();
-        auto* lru_handle =
-                insert(key, value, item->mem_size(), item->mem_size(), CachePriority::NORMAL);
+        auto* lru_handle = insert(key, value, 1, sizeof(Reusable), CachePriority::NORMAL);
         release(lru_handle);
     }