From 126f71b691c9b1e15583306b33617c6ca47f8d0e Mon Sep 17 00:00:00 2001 From: "@picocreator (Eugene Cheah)" Date: Sun, 20 Aug 2023 03:25:37 +0000 Subject: [PATCH] disabling source_data_dir --- RWKV-v4neo/config-example.yaml | 4 ++-- RWKV-v4neo/config-minimum-example.yaml | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/RWKV-v4neo/config-example.yaml b/RWKV-v4neo/config-example.yaml index e6009075..d369d70e 100644 --- a/RWKV-v4neo/config-example.yaml +++ b/RWKV-v4neo/config-example.yaml @@ -343,9 +343,9 @@ data: # source_dataset_params: # language: en - # Use data_dir, if you are using source=text/json/etc + # Use source_data_dir, if you are using source=text/json/etc # If using relative path, this should be relative to the trainer script path - source_data_dir: ../dataset-text/ + # source_data_dir: ../dataset-text/ # After loading the dataset, split out test data used for validation, # This process is skipped if the dataset includes a test split diff --git a/RWKV-v4neo/config-minimum-example.yaml b/RWKV-v4neo/config-minimum-example.yaml index 17fd48ae..d551bab9 100644 --- a/RWKV-v4neo/config-minimum-example.yaml +++ b/RWKV-v4neo/config-minimum-example.yaml @@ -160,9 +160,9 @@ data: # source: "teven/enwiki_00k" # Hugging face dataset # source: text # Text mode, used with source_data_dir - # Use data_dir, if you are using source=text/json/etc + # Use source_data_dir, if you are using source=text/json/etc # If using relative path, this should be relative to the trainer script path - source_data_dir: ../dataset-json-dir/ + # source_data_dir: ../dataset-json-dir/ # Tokenizer to use, use either the inbuilt 'neox', or 'world' tokenizer # If using a custom tokenizer, provide the HF tokenizer name/path