From d4a981770baab18b41f566010d535bb6be573f03 Mon Sep 17 00:00:00 2001 From: Fabian Isensee Date: Tue, 9 Jan 2024 11:53:58 +0100 Subject: [PATCH] remove barrier prints --- nnunetv2/training/nnUNetTrainer/nnUNetTrainer.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/nnunetv2/training/nnUNetTrainer/nnUNetTrainer.py b/nnunetv2/training/nnUNetTrainer/nnUNetTrainer.py index 145554ce2..d355fd5ea 100644 --- a/nnunetv2/training/nnUNetTrainer/nnUNetTrainer.py +++ b/nnunetv2/training/nnUNetTrainer/nnUNetTrainer.py @@ -1151,7 +1151,6 @@ def perform_actual_validation(self, save_probabilities: bool = False): _, val_keys = self.do_split() if self.is_ddp: last_barrier_at_idx = len(val_keys) // dist.get_world_size() - 1 - print(f'last barrier at idx {last_barrier_at_idx}') val_keys = val_keys[self.local_rank:: dist.get_world_size()] # we cannot just have barriers all over the place because the number of keys each GPU receives can be @@ -1240,7 +1239,6 @@ def perform_actual_validation(self, save_probabilities: bool = False): )) # if we don't barrier from time to time we will get nccl timeouts for large datsets. Yuck. if self.is_ddp and i < last_barrier_at_idx and (i + 1) % 20 == 0: - print(f'index {i}. Barrier rank {self.local_rank}') dist.barrier() _ = [r.get() for r in results]