Merge branch 'rwkv-x-eagle-notebooks' of https://github.com/RWKV/RWKV…

…-infctx-trainer into rwkv-x-eagle-notebooks
RWKV · Feb 7, 2024 · 6e792a2 · 6e792a2
2 parents 5ffe8cb + bea9014
commit 6e792a2
Showing 1 changed file with 4 additions and 0 deletions.
diff --git a/RWKV-v5/src/data.py b/RWKV-v5/src/data.py
@@ -1603,6 +1603,8 @@ def train_dataloader(self):
             shuffle=self.dataloader_shuffle_training and not self.sort_by_length,
             num_replicas=self.trainer.world_size,
             rank=self.trainer.global_rank,
+            ## This is required due to multi node alignment errors
+            drop_last=True
         )
 
         microbatch_size = 1
@@ -1638,6 +1640,8 @@ def val_dataloader(self):
             shuffle=False, 
             num_replicas=self.trainer.world_size,
             rank=self.trainer.global_rank,
+            ## This is required due to multi node alignment errors
+            drop_last=True
         )
 
         microbatch_size = 1