From 4c4c272808a8e424cc8708ed30c4124846a29ea8 Mon Sep 17 00:00:00 2001 From: lihangyu <15605149486@163.com> Date: Mon, 22 Apr 2024 15:54:48 +0800 Subject: [PATCH] [Refator](point query) change LookupConnectionCache policy to number (#33950) --- be/src/common/config.cpp | 4 ++-- be/src/common/config.h | 2 +- be/src/runtime/exec_env_init.cpp | 4 ++-- be/src/service/point_query_executor.cpp | 5 ----- be/src/service/point_query_executor.h | 12 ++++-------- 5 files changed, 9 insertions(+), 18 deletions(-) diff --git a/be/src/common/config.cpp b/be/src/common/config.cpp index a6972828209ef0..05cfbecb9126c5 100644 --- a/be/src/common/config.cpp +++ b/be/src/common/config.cpp @@ -1070,8 +1070,8 @@ DEFINE_mInt64(auto_inc_low_water_level_mark_size_ratio, "3"); // number of threads that fetch auto-inc ranges from FE DEFINE_mInt64(auto_inc_fetch_thread_num, "3"); -// default 4GB -DEFINE_mInt64(lookup_connection_cache_bytes_limit, "4294967296"); +// default max to 2048 connections +DEFINE_mInt64(lookup_connection_cache_capacity, "2048"); // level of compression when using LZ4_HC, whose defalut value is LZ4HC_CLEVEL_DEFAULT DEFINE_mInt64(LZ4_HC_compression_level, "9"); diff --git a/be/src/common/config.h b/be/src/common/config.h index 2b0ea1cc4dc7ef..2eaf21b04cdd62 100644 --- a/be/src/common/config.h +++ b/be/src/common/config.h @@ -1128,7 +1128,7 @@ DECLARE_mInt64(auto_inc_low_water_level_mark_size_ratio); // number of threads that fetch auto-inc ranges from FE DECLARE_mInt64(auto_inc_fetch_thread_num); // Max connection cache num for point lookup queries -DECLARE_mInt64(lookup_connection_cache_bytes_limit); +DECLARE_mInt64(lookup_connection_cache_capacity); // level of compression when using LZ4_HC, whose defalut value is LZ4HC_CLEVEL_DEFAULT DECLARE_mInt64(LZ4_HC_compression_level); diff --git a/be/src/runtime/exec_env_init.cpp b/be/src/runtime/exec_env_init.cpp index 2c63b9cf3ca71c..490d3ebd83f976 100644 --- a/be/src/runtime/exec_env_init.cpp +++ b/be/src/runtime/exec_env_init.cpp @@ -487,8 +487,8 @@ Status ExecEnv::_init_mem_env() { _file_meta_cache = new FileMetaCache(config::max_external_file_meta_cache_num); - _lookup_connection_cache = LookupConnectionCache::create_global_instance( - config::lookup_connection_cache_bytes_limit); + _lookup_connection_cache = + LookupConnectionCache::create_global_instance(config::lookup_connection_cache_capacity); // use memory limit int64_t inverted_index_cache_limit = diff --git a/be/src/service/point_query_executor.cpp b/be/src/service/point_query_executor.cpp index 2d6c3d9665665d..207cbd42108db3 100644 --- a/be/src/service/point_query_executor.cpp +++ b/be/src/service/point_query_executor.cpp @@ -57,7 +57,6 @@ Reusable::~Reusable() {} constexpr static int s_preallocted_blocks_num = 32; Status Reusable::init(const TDescriptorTable& t_desc_tbl, const std::vector& output_exprs, const TQueryOptions& query_options, size_t block_size) { - SCOPED_MEM_COUNT_BY_HOOK(&_mem_size); _runtime_state = RuntimeState::create_unique(); _runtime_state->set_query_options(query_options); RETURN_IF_ERROR(DescriptorTbl::create(_runtime_state->obj_pool(), t_desc_tbl, &_desc_tbl)); @@ -111,10 +110,6 @@ void Reusable::return_block(std::unique_ptr& block) { } } -int64_t Reusable::mem_size() const { - return _mem_size; -} - LookupConnectionCache* LookupConnectionCache::create_global_instance(size_t capacity) { DCHECK(ExecEnv::GetInstance()->get_lookup_connection_cache() == nullptr); auto* res = new LookupConnectionCache(capacity); diff --git a/be/src/service/point_query_executor.h b/be/src/service/point_query_executor.h index e761a32a8f6323..37345d80f68f62 100644 --- a/be/src/service/point_query_executor.h +++ b/be/src/service/point_query_executor.h @@ -91,8 +91,6 @@ class Reusable { const vectorized::VExprContextSPtrs& output_exprs() { return _output_exprs_ctxs; } - int64_t mem_size() const; - private: // caching TupleDescriptor, output_expr, etc... std::unique_ptr _runtime_state; @@ -105,7 +103,6 @@ class Reusable { vectorized::DataTypeSerDeSPtrs _data_type_serdes; std::unordered_map _col_uid_to_idx; std::vector _col_default_values; - int64_t _mem_size = 0; }; // RowCache is a LRU cache for row store @@ -214,8 +211,8 @@ class LookupConnectionCache : public LRUCachePolicy { friend class PointQueryExecutor; LookupConnectionCache(size_t capacity) : LRUCachePolicy(CachePolicy::CacheType::LOOKUP_CONNECTION_CACHE, capacity, - LRUCacheType::SIZE, config::tablet_lookup_cache_stale_sweep_time_sec) { - } + LRUCacheType::NUMBER, + config::tablet_lookup_cache_stale_sweep_time_sec) {} static std::string encode_key(__int128_t cache_id) { fmt::memory_buffer buffer; @@ -227,11 +224,10 @@ class LookupConnectionCache : public LRUCachePolicy { std::string key = encode_key(cache_id); auto* value = new CacheValue; value->item = item; - LOG(INFO) << "Add item mem size " << item->mem_size() + LOG(INFO) << "Add item mem" << ", cache_capacity: " << get_total_capacity() << ", cache_usage: " << get_usage() << ", mem_consum: " << mem_consumption(); - auto* lru_handle = - insert(key, value, item->mem_size(), item->mem_size(), CachePriority::NORMAL); + auto* lru_handle = insert(key, value, 1, sizeof(Reusable), CachePriority::NORMAL); release(lru_handle); }