diff --git a/be/src/pipeline/exec/hashjoin_build_sink.cpp b/be/src/pipeline/exec/hashjoin_build_sink.cpp
index bd618d245984a2f..98fa2b341ac2935 100644
--- a/be/src/pipeline/exec/hashjoin_build_sink.cpp
+++ b/be/src/pipeline/exec/hashjoin_build_sink.cpp
@@ -113,7 +113,7 @@ Status HashJoinBuildSinkLocalState::open(RuntimeState* state) {
 Status HashJoinBuildSinkLocalState::close(RuntimeState* state, Status exec_status) {
     auto p = _parent->cast<HashJoinBuildSinkOperatorX>();
     Defer defer {[&]() {
-        if (_should_build_hash_table) {
+        if (_should_build_hash_table && !p._shared_hashtable_controller) {
             // The build side hash key column maybe no need output, but we need to keep the column in block
             // because it is used to compare with probe side hash key column
             if (p._should_keep_hash_key_column && _build_col_ids.size() == 1) {
@@ -123,7 +123,7 @@ Status HashJoinBuildSinkLocalState::close(RuntimeState* state, Status exec_statu
             if (_shared_state->build_block) {
                 // release the memory of unused column in probe stage
                 _shared_state->build_block->clear_column_mem_not_keep(
-                        p._should_keep_column_flags, p._shared_hash_table_context.get() != nullptr);
+                        p._should_keep_column_flags, true);
             }
         }