diff --git a/fe/fe-core/src/main/java/org/apache/doris/cloud/catalog/CloudTabletRebalancer.java b/fe/fe-core/src/main/java/org/apache/doris/cloud/catalog/CloudTabletRebalancer.java
index dccce2c3dcb5f8..78947afdb11e39 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/cloud/catalog/CloudTabletRebalancer.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/cloud/catalog/CloudTabletRebalancer.java
@@ -727,6 +727,7 @@ private void sendPreHeatingRpc(Tablet pickedTablet, long srcBe, long destBe) thr
         TNetworkAddress address = null;
         Backend srcBackend = cloudSystemInfoService.getBackend(srcBe);
         Backend destBackend = cloudSystemInfoService.getBackend(destBe);
+        boolean ok = true;
         try {
             address = new TNetworkAddress(destBackend.getHost(), destBackend.getBePort());
             client = ClientPool.backendPool.borrowObject(address);
@@ -743,10 +744,14 @@ private void sendPreHeatingRpc(Tablet pickedTablet, long srcBe, long destBe) thr
             }
         } catch (Exception e) {
             LOG.warn("send pre heating rpc error. backend[{}]", destBackend.getId(), e);
-            ClientPool.backendPool.invalidateObject(address, client);
+            ok = false;
             throw e;
         } finally {
-            ClientPool.backendPool.returnObject(address, client);
+            if (ok) {
+                ClientPool.backendPool.returnObject(address, client);
+            } else {
+                ClientPool.backendPool.invalidateObject(address, client);
+            }
         }
     }
 
@@ -754,6 +759,7 @@ private Map<Long, Boolean> sendCheckWarmUpCacheAsyncRpc(List<Long> tabletIds, lo
         BackendService.Client client = null;
         TNetworkAddress address = null;
         Backend destBackend = cloudSystemInfoService.getBackend(be);
+        boolean ok = true;
         try {
             address = new TNetworkAddress(destBackend.getHost(), destBackend.getBePort());
             client = ClientPool.backendPool.borrowObject(address);
@@ -770,9 +776,13 @@ private Map<Long, Boolean> sendCheckWarmUpCacheAsyncRpc(List<Long> tabletIds, lo
             return result.getTaskDone();
         } catch (Exception e) {
             LOG.warn("send check pre cache rpc error. backend[{}]", destBackend.getId(), e);
-            ClientPool.backendPool.invalidateObject(address, client);
+            ok = false;
         } finally {
-            ClientPool.backendPool.returnObject(address, client);
+            if (ok) {
+                ClientPool.backendPool.returnObject(address, client);
+            } else {
+                ClientPool.backendPool.invalidateObject(address, client);
+            }
         }
         return null;
     }
diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/util/ProfileManager.java b/fe/fe-core/src/main/java/org/apache/doris/common/util/ProfileManager.java
index c6fc3307fe0b29..dfd54e473f8a4d 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/common/util/ProfileManager.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/common/util/ProfileManager.java
@@ -257,9 +257,10 @@ private static TGetRealtimeExecStatusResponse getRealtimeQueryProfile(
             client = ClientPool.backendPool.borrowObject(targetBackend);
         } catch (Exception e) {
             LOG.warn("Fetch a agent client failed, address: {}", targetBackend.toString());
+            ClientPool.backendPool.invalidateObject(targetBackend, client);
             return resp;
         }
-
+        boolean ok = true;
         try {
             TGetRealtimeExecStatusRequest req = new TGetRealtimeExecStatusRequest();
             req.setId(queryID);
@@ -267,9 +268,13 @@ private static TGetRealtimeExecStatusResponse getRealtimeQueryProfile(
         } catch (TException e) {
             LOG.warn("Got exception when getRealtimeExecStatus, query {} backend {}",
                     DebugUtil.printId(queryID), targetBackend.toString(), e);
-            ClientPool.backendPool.invalidateObject(targetBackend, client);
+            ok = false;
         } finally {
-            ClientPool.backendPool.returnObject(targetBackend, client);
+            if (ok) {
+                ClientPool.backendPool.returnObject(targetBackend, client);
+            } else {
+                ClientPool.backendPool.invalidateObject(targetBackend, client);
+            }
         }
 
         if (!resp.isSetStatus()) {