diff --git a/be/src/olap/olap_server.cpp b/be/src/olap/olap_server.cpp
index 0e47cf224e0463..166e86ce1b43a6 100644
--- a/be/src/olap/olap_server.cpp
+++ b/be/src/olap/olap_server.cpp
@@ -249,9 +249,8 @@ Status StorageEngine::start_bg_threads() {
 }
 
 void StorageEngine::_fd_cache_clean_callback() {
-    int32_t interval = 600;
+    int32_t interval = config::cache_clean_interval;
     while (!_stop_background_threads_latch.wait_for(std::chrono::seconds(interval))) {
-        interval = config::cache_clean_interval;
         if (interval <= 0) {
             LOG(WARNING) << "config of file descriptor clean interval is illegal: [" << interval
                          << "], force set to 3600 ";
diff --git a/be/src/olap/segment_loader.cpp b/be/src/olap/segment_loader.cpp
index aad8ac517a319b..94b0f9bca5b68c 100644
--- a/be/src/olap/segment_loader.cpp
+++ b/be/src/olap/segment_loader.cpp
@@ -87,7 +87,7 @@ Status SegmentLoader::load_segments(const BetaRowsetSharedPtr& rowset,
     return Status::OK();
 }
 
-void SegmentLoader::erase_segment(const SegmentCache::CacheKey& key) {
+void SegmentLoader::erase_segments(const SegmentCache::CacheKey& key) {
     _segment_cache->erase(key);
 }
 
diff --git a/be/src/olap/segment_loader.h b/be/src/olap/segment_loader.h
index 600692750e1ffa..784dc8e9473b32 100644
--- a/be/src/olap/segment_loader.h
+++ b/be/src/olap/segment_loader.h
@@ -111,7 +111,7 @@ class SegmentLoader {
     Status load_segments(const BetaRowsetSharedPtr& rowset, SegmentCacheHandle* cache_handle,
                          bool use_cache = false);
 
-    void erase_segment(const SegmentCache::CacheKey& key);
+    void erase_segments(const SegmentCache::CacheKey& key);
 
 private:
     SegmentLoader();
diff --git a/be/src/olap/tablet.cpp b/be/src/olap/tablet.cpp
index 13a887c429b9f2..98283a0b0a229a 100644
--- a/be/src/olap/tablet.cpp
+++ b/be/src/olap/tablet.cpp
@@ -665,6 +665,8 @@ void Tablet::_delete_stale_rowset_by_version(const Version& version) {
         return;
     }
     _tablet_meta->delete_stale_rs_meta_by_version(version);
+    // If the stale rowset was deleted, it need to remove the fds directly
+    SegmentLoader::instance()->erase_segments(SegmentCache::CacheKey(rowset_meta->rowset_id()));
     VLOG_NOTICE << "delete stale rowset. tablet=" << full_name() << ", version=" << version;
 }
 
diff --git a/be/src/olap/txn_manager.cpp b/be/src/olap/txn_manager.cpp
index e3dc722b0bac40..7b69033b1019bb 100644
--- a/be/src/olap/txn_manager.cpp
+++ b/be/src/olap/txn_manager.cpp
@@ -378,7 +378,7 @@ Status TxnManager::publish_txn(OlapMeta* meta, TPartitionId partition_id,
             rowset->merge_rowset_meta(transient_rowset->rowset_meta());
 
             // erase segment cache cause we will add a segment to rowset
-            SegmentLoader::instance()->erase_segment(rowset->rowset_id());
+            SegmentLoader::instance()->erase_segments(rowset->rowset_id());
         }
         stats->partial_update_write_segment_us = MonotonicMicros() - t3;
         int64_t t4 = MonotonicMicros();
diff --git a/be/src/vec/olap/vertical_block_reader.cpp b/be/src/vec/olap/vertical_block_reader.cpp
index 75d9127de2d26a..7d6f9bc5022dea 100644
--- a/be/src/vec/olap/vertical_block_reader.cpp
+++ b/be/src/vec/olap/vertical_block_reader.cpp
@@ -69,8 +69,9 @@ Status VerticalBlockReader::_get_segment_iterators(const ReaderParams& read_para
         // segment iterator will be inited here
         // In vertical compaction, every group will load segment so we should cache
         // segment to avoid tot many s3 head request
+        bool use_cache = !rs_split.rs_reader->rowset()->is_local();
         RETURN_IF_ERROR(rs_split.rs_reader->get_segment_iterators(&_reader_context, segment_iters,
-                                                                  {}, true));
+                                                                  {}, use_cache));
         // if segments overlapping, all segment iterator should be inited in
         // heap merge iterator. If segments are none overlapping, only first segment of this
         // rowset will be inited and push to heap, other segment will be inited later when current