Add more logging

mosaicml · Apr 24, 2024 · 67cfe9b · 67cfe9b
1 parent 5fb3db0
commit 67cfe9b
Showing 1 changed file with 2 additions and 0 deletions.
diff --git a/llmfoundry/data/finetuning/tasks.py b/llmfoundry/data/finetuning/tasks.py
@@ -834,6 +834,8 @@ def dataset_mapper(example: Dict):
                 desc='Tokenizing dataset',
             )
 
+            log.info('Finished tokenizing dataset.')
+
             # Use multiprocessing to introduce a custom timeout for this operation.
             # A temporary workaround to avoid indefinite hangs observed occasionally due to
             # filter_dataset not properly exiting and the dist.barrier() below not timing out.