fix a dataset_map bug of custom_dataset

InternLM · Jun 17, 2024 · f0e2729 · f0e2729
1 parent 6bbc274
commit f0e2729
Show file tree

Hide file tree

Showing 3 changed files with 6 additions and 6 deletions.
diff --git a/xtuner/configs/custom_dataset/sft/internlm/internlm2_chat_1_8b_qlora_custom_sft_e1.py b/xtuner/configs/custom_dataset/sft/internlm/internlm2_chat_1_8b_qlora_custom_sft_e1.py
@@ -26,7 +26,7 @@
 
 from xtuner.dataset import process_hf_dataset
 from xtuner.dataset.collate_fns import default_collate_fn
-from xtuner.dataset.map_fns import template_map_fn_factory
+from xtuner.dataset.map_fns import openai_map_fn, template_map_fn_factory
 from xtuner.engine.hooks import (DatasetInfoHook, EvaluateChatHook,
                                  VarlenAttnArgsToMessageHubHook)
 from xtuner.engine.runner import TrainLoop
@@ -111,7 +111,7 @@
     dataset=dict(type=load_dataset, path='json', data_files=data_files),
     tokenizer=tokenizer,
     max_length=max_length,
-    dataset_map_fn=None,
+    dataset_map_fn=openai_map_fn,
     template_map_fn=dict(
         type=template_map_fn_factory, template=prompt_template),
     remove_unused_columns=True,

diff --git a/xtuner/configs/custom_dataset/sft/internlm/internlm2_chat_20b_qlora_custom_sft_e1.py b/xtuner/configs/custom_dataset/sft/internlm/internlm2_chat_20b_qlora_custom_sft_e1.py
@@ -26,7 +26,7 @@
 
 from xtuner.dataset import process_hf_dataset
 from xtuner.dataset.collate_fns import default_collate_fn
-from xtuner.dataset.map_fns import template_map_fn_factory
+from xtuner.dataset.map_fns import openai_map_fn, template_map_fn_factory
 from xtuner.engine.hooks import (DatasetInfoHook, EvaluateChatHook,
                                  VarlenAttnArgsToMessageHubHook)
 from xtuner.engine.runner import TrainLoop
@@ -111,7 +111,7 @@
     dataset=dict(type=load_dataset, path='json', data_files=data_files),
     tokenizer=tokenizer,
     max_length=max_length,
-    dataset_map_fn=None,
+    dataset_map_fn=openai_map_fn,
     template_map_fn=dict(
         type=template_map_fn_factory, template=prompt_template),
     remove_unused_columns=True,

diff --git a/xtuner/configs/custom_dataset/sft/internlm/internlm2_chat_7b_qlora_custom_sft_e1.py b/xtuner/configs/custom_dataset/sft/internlm/internlm2_chat_7b_qlora_custom_sft_e1.py
@@ -26,7 +26,7 @@
 
 from xtuner.dataset import process_hf_dataset
 from xtuner.dataset.collate_fns import default_collate_fn
-from xtuner.dataset.map_fns import template_map_fn_factory
+from xtuner.dataset.map_fns import openai_map_fn, template_map_fn_factory
 from xtuner.engine.hooks import (DatasetInfoHook, EvaluateChatHook,
                                  VarlenAttnArgsToMessageHubHook)
 from xtuner.engine.runner import TrainLoop
@@ -111,7 +111,7 @@
     dataset=dict(type=load_dataset, path='json', data_files=data_files),
     tokenizer=tokenizer,
     max_length=max_length,
-    dataset_map_fn=None,
+    dataset_map_fn=openai_map_fn,
     template_map_fn=dict(
         type=template_map_fn_factory, template=prompt_template),
     remove_unused_columns=True,