diff --git a/llmfoundry/data/packing.py b/llmfoundry/data/packing.py
index f78fef3b55..6a93783c67 100644
--- a/llmfoundry/data/packing.py
+++ b/llmfoundry/data/packing.py
@@ -305,6 +305,8 @@ def auto_packing_ratio(dataloader_cfg: DictConfig,
             break
         packing_ratio = packing_ratio_candidate
 
+    clean_stale_shared_memory()
+
     # Select the minimum packing ratio across all ranks.
     if dist.is_available() and dist.is_initialized():
         device = get_device(None)
@@ -408,8 +410,6 @@ def profile(raw_batch_size: int) -> Tuple[float, float]:
         padding, waste = profile(raw_batch_size)
         yield (packing_ratio, padding, waste)
 
-    clean_stale_shared_memory()
-
 
 if __name__ == '__main__':