From af88782007be67fb69902b8a0d0f20786cb85a14 Mon Sep 17 00:00:00 2001 From: Vitaliy Chiley <6439018+vchiley@users.noreply.github.com> Date: Mon, 26 Jun 2023 17:15:16 -0700 Subject: [PATCH] lint --- llmfoundry/data/finetuning/tasks.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/llmfoundry/data/finetuning/tasks.py b/llmfoundry/data/finetuning/tasks.py index 2e10ac0ed4..8c3eb94227 100644 --- a/llmfoundry/data/finetuning/tasks.py +++ b/llmfoundry/data/finetuning/tasks.py @@ -255,8 +255,7 @@ def dataset_mapper(example: Dict): remove_columns=columns_to_remove, ) prompt_length_filtered_dataset = tokenized_dataset.filter( - lambda example: len(example['input_ids']) < max_seq_len, - ) + lambda example: len(example['input_ids']) < max_seq_len) examples_removed = len(tokenized_dataset) - len( prompt_length_filtered_dataset)