diff --git a/be/src/common/config.cpp b/be/src/common/config.cpp index 458c124c3042fc..1b11f3dc667fa2 100644 --- a/be/src/common/config.cpp +++ b/be/src/common/config.cpp @@ -125,8 +125,8 @@ DEFINE_Int64(max_sys_mem_available_low_water_mark_bytes, "6871947673"); DEFINE_Int64(memtable_limiter_reserved_memory_bytes, "838860800"); // The size of the memory that gc wants to release each time, as a percentage of the mem limit. -DEFINE_mString(process_minor_gc_size, "10%"); -DEFINE_mString(process_full_gc_size, "20%"); +DEFINE_mString(process_minor_gc_size, "5%"); +DEFINE_mString(process_full_gc_size, "10%"); // If true, when the process does not exceed the soft mem limit, the query memory will not be limited; // when the process memory exceeds the soft mem limit, the query with the largest ratio between the currently @@ -140,6 +140,8 @@ DEFINE_mBool(enable_stacktrace, "true"); DEFINE_mInt64(stacktrace_in_alloc_large_memory_bytes, "2147483648"); +DEFINE_mInt64(crash_in_alloc_large_memory_bytes, "-1"); + DEFINE_mBool(enable_memory_orphan_check, "false"); // The maximum time a thread waits for full GC. Currently only query will wait for full gc. @@ -588,7 +590,7 @@ DEFINE_mInt32(memory_maintenance_sleep_time_ms, "100"); // After full gc, no longer full gc and minor gc during sleep. // After minor gc, no minor gc during sleep, but full gc is possible. -DEFINE_mInt32(memory_gc_sleep_time_ms, "1000"); +DEFINE_mInt32(memory_gc_sleep_time_ms, "500"); // Sleep time in milliseconds between memtbale flush mgr refresh iterations DEFINE_mInt64(memtable_mem_tracker_refresh_interval_ms, "5"); diff --git a/be/src/common/config.h b/be/src/common/config.h index 2449826936f13c..8fb7acd36d8988 100644 --- a/be/src/common/config.h +++ b/be/src/common/config.h @@ -191,6 +191,10 @@ DECLARE_mBool(enable_stacktrace); // if alloc failed using Doris Allocator, will print stacktrace in error log. // if is -1, disable print stacktrace when alloc large memory. DECLARE_mInt64(stacktrace_in_alloc_large_memory_bytes); +// when alloc memory larger than crash_in_alloc_large_memory_bytes will crash, default -1 means disabled. +// if you need a core dump to analyze large memory allocation, +// modify this parameter to crash when large memory allocation occur will help +DECLARE_mInt64(crash_in_alloc_large_memory_bytes); // default is true. if any memory tracking in Orphan mem tracker will report error. DECLARE_mBool(enable_memory_orphan_check); diff --git a/be/src/runtime/memory/thread_mem_tracker_mgr.h b/be/src/runtime/memory/thread_mem_tracker_mgr.h index d9c4e093a4acfb..73cdd3243da1dc 100644 --- a/be/src/runtime/memory/thread_mem_tracker_mgr.h +++ b/be/src/runtime/memory/thread_mem_tracker_mgr.h @@ -242,17 +242,29 @@ inline void ThreadMemTrackerMgr::consume(int64_t size, int skip_large_memory_che flush_untracked_mem(); } - if (skip_large_memory_check == 0 && doris::config::stacktrace_in_alloc_large_memory_bytes > 0 && - size > doris::config::stacktrace_in_alloc_large_memory_bytes) { - _stop_consume = true; - LOG(WARNING) << fmt::format( - "malloc or new large memory: {}, {}, this is just a warning, not prevent memory " - "alloc, stacktrace:\n{}", - size, - is_attach_query() ? "in query or load: " + print_id(_query_id) - : "not in query or load", - get_stack_trace()); - _stop_consume = false; + if (skip_large_memory_check == 0) { + if (doris::config::stacktrace_in_alloc_large_memory_bytes > 0 && + size > doris::config::stacktrace_in_alloc_large_memory_bytes) { + _stop_consume = true; + LOG(WARNING) << fmt::format( + "alloc large memory: {}, {}, this is just a warning, not prevent memory alloc, " + "stacktrace:\n{}", + size, + is_attach_query() ? "in query or load: " + print_id(_query_id) + : "not in query or load", + get_stack_trace()); + _stop_consume = false; + } + if (doris::config::crash_in_alloc_large_memory_bytes > 0 && + size > doris::config::crash_in_alloc_large_memory_bytes) { + LOG(FATAL) << fmt::format( + "alloc large memory: {}, {}, crash generate core dumpsto help analyze, " + "stacktrace:\n{}", + size, + is_attach_query() ? "in query or load: " + print_id(_query_id) + : "not in query or load", + get_stack_trace()); + } } }