From e6cdce5297baa14d5f4b620a1c843b6821c7b5ad Mon Sep 17 00:00:00 2001 From: Tim Miller Date: Fri, 1 Sep 2023 20:05:14 -0400 Subject: [PATCH] Fix processor to assume all tasks if no task is specified (matching json) --- src/cnlpt/cnlp_processors.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/cnlpt/cnlp_processors.py b/src/cnlpt/cnlp_processors.py index f446898e..1918b46e 100644 --- a/src/cnlpt/cnlp_processors.py +++ b/src/cnlpt/cnlp_processors.py @@ -185,8 +185,11 @@ def __init__(self, data_dir: str, tasks: Set[str] = None, max_train_items=-1): dataset_tasks = first_split.features.keys() - set( ["text", "text_a", "text_b"] ) - active_tasks = set(tasks).intersection(dataset_tasks) - active_tasks = list(active_tasks) + if tasks is None: + active_tasks = list(dataset_tasks) + else: + active_tasks = set(tasks).intersection(dataset_tasks) + active_tasks = list(active_tasks) active_tasks.sort() self.dataset.task_output_modes = {} elif ext_check_file.endswith("json"):