[opt](spill) handle oom exception in spill tasks (#35025) #35171
Conversation
|
Thank you for your contribution to Apache Doris. Since 2024-03-18, the Document has been moved to doris-website. |
|
run buildall |
![github-actions[bot]](https://avatars.githubusercontent.com/in/15368?s=60&v=4){kind=small}
| _spill_write_wait_io_timer); | ||
| } | ||
|
|
||
| Status PartitionedHashJoinProbeLocalState::spill_probe_blocks(RuntimeState* state) { |
There was a problem hiding this comment.
warning: function 'spill_probe_blocks' has cognitive complexity of 59 (threshold 50) [readability-function-cognitive-complexity]
Status PartitionedHashJoinProbeLocalState::spill_probe_blocks(RuntimeState* state) {
^Additional context
be/src/pipeline/exec/partitioned_hash_join_probe_operator.cpp:169: nesting level increased to 1
auto spill_func = [query_id, state, submit_timer, this] {
^be/src/pipeline/exec/partitioned_hash_join_probe_operator.cpp:174: +2, including nesting penalty of 1, nesting level increased to 2
for (uint32_t partition_index = 0; partition_index != p._partition_count;
^be/src/pipeline/exec/partitioned_hash_join_probe_operator.cpp:178: +3, including nesting penalty of 2, nesting level increased to 3
if (partitioned_block && partitioned_block->allocated_bytes() >=
^be/src/pipeline/exec/partitioned_hash_join_probe_operator.cpp:178: +1
if (partitioned_block && partitioned_block->allocated_bytes() >=
^be/src/pipeline/exec/partitioned_hash_join_probe_operator.cpp:185: +3, including nesting penalty of 2, nesting level increased to 3
if (!spilling_stream) {
^be/src/pipeline/exec/partitioned_hash_join_probe_operator.cpp:186: +4, including nesting penalty of 3, nesting level increased to 4
RETURN_IF_ERROR(ExecEnv::GetInstance()->spill_stream_mgr()->register_spill_stream(
^be/src/common/status.h:614: expanded from macro 'RETURN_IF_ERROR'
do { \
^be/src/pipeline/exec/partitioned_hash_join_probe_operator.cpp:186: +5, including nesting penalty of 4, nesting level increased to 5
RETURN_IF_ERROR(ExecEnv::GetInstance()->spill_stream_mgr()->register_spill_stream(
^be/src/common/status.h:616: expanded from macro 'RETURN_IF_ERROR'
if (UNLIKELY(!_status_.ok())) { \
^be/src/pipeline/exec/partitioned_hash_join_probe_operator.cpp:190: +4, including nesting penalty of 3, nesting level increased to 4
RETURN_IF_ERROR(spilling_stream->prepare_spill());
^be/src/common/status.h:614: expanded from macro 'RETURN_IF_ERROR'
do { \
^be/src/pipeline/exec/partitioned_hash_join_probe_operator.cpp:190: +5, including nesting penalty of 4, nesting level increased to 5
RETURN_IF_ERROR(spilling_stream->prepare_spill());
^be/src/common/status.h:616: expanded from macro 'RETURN_IF_ERROR'
if (UNLIKELY(!_status_.ok())) { \
^be/src/pipeline/exec/partitioned_hash_join_probe_operator.cpp:197: +3, including nesting penalty of 2, nesting level increased to 3
while (!blocks.empty() && !state->is_cancelled()) {
^be/src/pipeline/exec/partitioned_hash_join_probe_operator.cpp:197: +1
while (!blocks.empty() && !state->is_cancelled()) {
^be/src/pipeline/exec/partitioned_hash_join_probe_operator.cpp:200: +4, including nesting penalty of 3, nesting level increased to 4
RETURN_IF_ERROR(spilling_stream->spill_block(state, block, false));
^be/src/common/status.h:614: expanded from macro 'RETURN_IF_ERROR'
do { \
^be/src/pipeline/exec/partitioned_hash_join_probe_operator.cpp:200: +5, including nesting penalty of 4, nesting level increased to 5
RETURN_IF_ERROR(spilling_stream->spill_block(state, block, false));
^be/src/common/status.h:616: expanded from macro 'RETURN_IF_ERROR'
if (UNLIKELY(!_status_.ok())) { \
^be/src/pipeline/exec/partitioned_hash_join_probe_operator.cpp:211: nesting level increased to 1
auto exception_catch_func = [query_id, mem_tracker, shared_state_holder, execution_context,
^be/src/pipeline/exec/partitioned_hash_join_probe_operator.cpp:216: +2, including nesting penalty of 1, nesting level increased to 2
if (shared_state_sptr) {
^be/src/pipeline/exec/partitioned_hash_join_probe_operator.cpp:219: +2, including nesting penalty of 1, nesting level increased to 2
if (!shared_state_sptr || !execution_context_lock) {
^be/src/pipeline/exec/partitioned_hash_join_probe_operator.cpp:219: +1
if (!shared_state_sptr || !execution_context_lock) {
^be/src/pipeline/exec/partitioned_hash_join_probe_operator.cpp:225: nesting level increased to 2
auto status = [&]() { RETURN_IF_CATCH_EXCEPTION({ return spill_func(); }); }();
^be/src/pipeline/exec/partitioned_hash_join_probe_operator.cpp:225: +3, including nesting penalty of 2, nesting level increased to 3
auto status = [&]() { RETURN_IF_CATCH_EXCEPTION({ return spill_func(); }); }();
^be/src/common/exception.h:89: expanded from macro 'RETURN_IF_CATCH_EXCEPTION'
do { \
^be/src/pipeline/exec/partitioned_hash_join_probe_operator.cpp:225: +4, including nesting penalty of 3, nesting level increased to 4
auto status = [&]() { RETURN_IF_CATCH_EXCEPTION({ return spill_func(); }); }();
^be/src/common/exception.h:94: expanded from macro 'RETURN_IF_CATCH_EXCEPTION'
} catch (const doris::Exception& e) { \
^be/src/pipeline/exec/partitioned_hash_join_probe_operator.cpp:225: +5, including nesting penalty of 4, nesting level increased to 5
auto status = [&]() { RETURN_IF_CATCH_EXCEPTION({ return spill_func(); }); }();
^be/src/common/exception.h:95: expanded from macro 'RETURN_IF_CATCH_EXCEPTION'
if (e.code() == doris::ErrorCode::MEM_ALLOC_FAILED) { \
^be/src/pipeline/exec/partitioned_hash_join_probe_operator.cpp:227: +2, including nesting penalty of 1, nesting level increased to 2
if (!status.ok()) {
^| _spill_write_wait_io_timer); | ||
| } | ||
|
|
||
| Status PartitionedHashJoinProbeLocalState::spill_probe_blocks(RuntimeState* state) { |
There was a problem hiding this comment.
warning: function 'spill_probe_blocks' exceeds recommended size/complexity thresholds [readability-function-size]
Status PartitionedHashJoinProbeLocalState::spill_probe_blocks(RuntimeState* state) {
^Additional context
be/src/pipeline/exec/partitioned_hash_join_probe_operator.cpp:155: 81 lines including whitespace and comments (threshold 80)
Status PartitionedHashJoinProbeLocalState::spill_probe_blocks(RuntimeState* state) {
^
|
run buildall |
2 similar comments
|
run buildall |
|
run buildall |
|
run buildall |
| _spill_write_wait_io_timer); | ||
| } | ||
|
|
||
| Status PartitionedHashJoinProbeLocalState::spill_probe_blocks(RuntimeState* state) { |
There was a problem hiding this comment.
warning: function 'spill_probe_blocks' has cognitive complexity of 59 (threshold 50) [readability-function-cognitive-complexity]
Status PartitionedHashJoinProbeLocalState::spill_probe_blocks(RuntimeState* state) {
^Additional context
be/src/pipeline/exec/partitioned_hash_join_probe_operator.cpp:169: nesting level increased to 1
auto spill_func = [query_id, state, submit_timer, this] {
^be/src/pipeline/exec/partitioned_hash_join_probe_operator.cpp:174: +2, including nesting penalty of 1, nesting level increased to 2
for (uint32_t partition_index = 0; partition_index != p._partition_count;
^be/src/pipeline/exec/partitioned_hash_join_probe_operator.cpp:178: +3, including nesting penalty of 2, nesting level increased to 3
if (partitioned_block && partitioned_block->allocated_bytes() >=
^be/src/pipeline/exec/partitioned_hash_join_probe_operator.cpp:178: +1
if (partitioned_block && partitioned_block->allocated_bytes() >=
^be/src/pipeline/exec/partitioned_hash_join_probe_operator.cpp:185: +3, including nesting penalty of 2, nesting level increased to 3
if (!spilling_stream) {
^be/src/pipeline/exec/partitioned_hash_join_probe_operator.cpp:186: +4, including nesting penalty of 3, nesting level increased to 4
RETURN_IF_ERROR(ExecEnv::GetInstance()->spill_stream_mgr()->register_spill_stream(
^be/src/common/status.h:612: expanded from macro 'RETURN_IF_ERROR'
do { \
^be/src/pipeline/exec/partitioned_hash_join_probe_operator.cpp:186: +5, including nesting penalty of 4, nesting level increased to 5
RETURN_IF_ERROR(ExecEnv::GetInstance()->spill_stream_mgr()->register_spill_stream(
^be/src/common/status.h:614: expanded from macro 'RETURN_IF_ERROR'
if (UNLIKELY(!_status_.ok())) { \
^be/src/pipeline/exec/partitioned_hash_join_probe_operator.cpp:190: +4, including nesting penalty of 3, nesting level increased to 4
RETURN_IF_ERROR(spilling_stream->prepare_spill());
^be/src/common/status.h:612: expanded from macro 'RETURN_IF_ERROR'
do { \
^be/src/pipeline/exec/partitioned_hash_join_probe_operator.cpp:190: +5, including nesting penalty of 4, nesting level increased to 5
RETURN_IF_ERROR(spilling_stream->prepare_spill());
^be/src/common/status.h:614: expanded from macro 'RETURN_IF_ERROR'
if (UNLIKELY(!_status_.ok())) { \
^be/src/pipeline/exec/partitioned_hash_join_probe_operator.cpp:197: +3, including nesting penalty of 2, nesting level increased to 3
while (!blocks.empty() && !state->is_cancelled()) {
^be/src/pipeline/exec/partitioned_hash_join_probe_operator.cpp:197: +1
while (!blocks.empty() && !state->is_cancelled()) {
^be/src/pipeline/exec/partitioned_hash_join_probe_operator.cpp:200: +4, including nesting penalty of 3, nesting level increased to 4
RETURN_IF_ERROR(spilling_stream->spill_block(state, block, false));
^be/src/common/status.h:612: expanded from macro 'RETURN_IF_ERROR'
do { \
^be/src/pipeline/exec/partitioned_hash_join_probe_operator.cpp:200: +5, including nesting penalty of 4, nesting level increased to 5
RETURN_IF_ERROR(spilling_stream->spill_block(state, block, false));
^be/src/common/status.h:614: expanded from macro 'RETURN_IF_ERROR'
if (UNLIKELY(!_status_.ok())) { \
^be/src/pipeline/exec/partitioned_hash_join_probe_operator.cpp:211: nesting level increased to 1
auto exception_catch_func = [query_id, mem_tracker, shared_state_holder, execution_context,
^be/src/pipeline/exec/partitioned_hash_join_probe_operator.cpp:216: +2, including nesting penalty of 1, nesting level increased to 2
if (shared_state_sptr) {
^be/src/pipeline/exec/partitioned_hash_join_probe_operator.cpp:219: +2, including nesting penalty of 1, nesting level increased to 2
if (!shared_state_sptr || !execution_context_lock) {
^be/src/pipeline/exec/partitioned_hash_join_probe_operator.cpp:219: +1
if (!shared_state_sptr || !execution_context_lock) {
^be/src/pipeline/exec/partitioned_hash_join_probe_operator.cpp:225: nesting level increased to 2
auto status = [&]() { RETURN_IF_CATCH_EXCEPTION({ return spill_func(); }); }();
^be/src/pipeline/exec/partitioned_hash_join_probe_operator.cpp:225: +3, including nesting penalty of 2, nesting level increased to 3
auto status = [&]() { RETURN_IF_CATCH_EXCEPTION({ return spill_func(); }); }();
^be/src/common/exception.h:89: expanded from macro 'RETURN_IF_CATCH_EXCEPTION'
do { \
^be/src/pipeline/exec/partitioned_hash_join_probe_operator.cpp:225: +4, including nesting penalty of 3, nesting level increased to 4
auto status = [&]() { RETURN_IF_CATCH_EXCEPTION({ return spill_func(); }); }();
^be/src/common/exception.h:94: expanded from macro 'RETURN_IF_CATCH_EXCEPTION'
} catch (const doris::Exception& e) { \
^be/src/pipeline/exec/partitioned_hash_join_probe_operator.cpp:225: +5, including nesting penalty of 4, nesting level increased to 5
auto status = [&]() { RETURN_IF_CATCH_EXCEPTION({ return spill_func(); }); }();
^be/src/common/exception.h:95: expanded from macro 'RETURN_IF_CATCH_EXCEPTION'
if (e.code() == doris::ErrorCode::MEM_ALLOC_FAILED) { \
^be/src/pipeline/exec/partitioned_hash_join_probe_operator.cpp:227: +2, including nesting penalty of 1, nesting level increased to 2
if (!status.ok()) {
^
|
run buildall |
|
run buildall |
|
run buildall |
|
TeamCity be ut coverage result: |
Proposed changes
pick #35025
Further comments
If this is a relatively large or complex change, kick off the discussion at [email protected] by explaining why you chose the solution you did and what alternatives you considered, etc...