move to auto only

mosaicml · Dec 7, 2023 · 3339924 · 3339924
1 parent 3f92f9c
commit 3339924
Showing 1 changed file with 2 additions and 2 deletions.
diff --git a/llmfoundry/data/packing.py b/llmfoundry/data/packing.py
@@ -305,6 +305,8 @@ def auto_packing_ratio(dataloader_cfg: DictConfig,
             break
         packing_ratio = packing_ratio_candidate
 
+    clean_stale_shared_memory()
+
     # Select the minimum packing ratio across all ranks.
     if dist.is_available() and dist.is_initialized():
         device = get_device(None)
@@ -408,8 +410,6 @@ def profile(raw_batch_size: int) -> Tuple[float, float]:
         padding, waste = profile(raw_batch_size)
         yield (packing_ratio, padding, waste)
 
-    clean_stale_shared_memory()
-
 
 if __name__ == '__main__':