[Usability] Add padding_side to dataset fingerprint

OptimalScale · Jun 14, 2024 · f9d99e1 · f9d99e1
1 parent 4fb2e0a
commit f9d99e1
Show file tree

Hide file tree

Showing 2 changed files with 2 additions and 0 deletions.
diff --git a/src/lmflow/models/hf_decoder_model.py b/src/lmflow/models/hf_decoder_model.py
@@ -246,6 +246,7 @@ def tokenize(self, dataset, add_special_tokens=True, *args, **kwargs):
                 (
                     raw_datasets.get_fingerprint()
                     + str(self.tokenizer)
+                    + f'###padding_side={self.tokenizer.padding_side}'
                     + ('###conversation_template=' + str(conversation_template) if "conversation" in dataset_type else "")
                     + f'###disable_group_texts={data_args.disable_group_texts}'
                     + f'###block_size={data_args.block_size}'

diff --git a/src/lmflow/models/hf_text_regression_model.py b/src/lmflow/models/hf_text_regression_model.py
@@ -205,6 +205,7 @@ def tokenize(
                 (
                     raw_datasets.get_fingerprint()
                     + str(self.tokenizer)
+                    + f'###padding_side={self.tokenizer.padding_side}'
                     + ('###conversation_template=' + str(conversation_template) if "conversation" in dataset_type else "")
                     + f'###disable_group_texts={data_args.disable_group_texts}'
                     + f'###block_size={data_args.block_size}'