Update random

stanford-crfm · Aug 29, 2023 · 5b15805 · 5b15805
1 parent 78d0824
commit 5b15805
Showing 1 changed file with 1 addition and 1 deletion.
diff --git a/scripts/data_overlap/load_documents.py b/scripts/data_overlap/load_documents.py
@@ -115,7 +115,7 @@ def get_the_pile_document_iterator(file_path: str) -> Iterator[str]:
     """
     with open(file_path, "r") as f:
         import random
-        if random.random() < 1 / 1000:
+        if random.random() < 1 / 10:
             for line in f:
                 yield json.loads(line)["text"]