typecast shuffle_block_size because of issue (#581)

mosaicml · Sep 6, 2023 · 9558a40 · 9558a40
1 parent e891c30
commit 9558a40
Showing 1 changed file with 4 additions and 0 deletions.
diff --git a/llmfoundry/data/text_data.py b/llmfoundry/data/text_data.py
@@ -112,6 +112,10 @@ def __init__(self,
                         f'local directory {local} does not contain split {split}'
                     )
 
+        # TODO: discover where yamls are being converted incorrect, but temporary workaround
+        if isinstance(shuffle_block_size, float):
+            shuffle_block_size = int(shuffle_block_size)
+
         # Build Dataset
         super().__init__(
             streams=streams,