diff --git a/configs/datasets/mmlu_pro/mmlu_pro_0shot_cot_gen_08c1de.py b/configs/datasets/mmlu_pro/mmlu_pro_0shot_cot_gen_08c1de.py index df9b8bc37..341e13cf5 100644 --- a/configs/datasets/mmlu_pro/mmlu_pro_0shot_cot_gen_08c1de.py +++ b/configs/datasets/mmlu_pro/mmlu_pro_0shot_cot_gen_08c1de.py @@ -56,6 +56,7 @@ dict( abbr=f'mmlu_pro_{category.replace(" ", "_")}', type=MMLUProDataset, + path='opencompass/mmlu_pro', category=category, reader_cfg=mmlu_pro_reader_cfg, infer_cfg=mmlu_pro_infer_cfg, diff --git a/configs/datasets/mmlu_pro/mmlu_pro_gen_cdbebf.py b/configs/datasets/mmlu_pro/mmlu_pro_gen_cdbebf.py index 7dd143b1e..42c30131f 100644 --- a/configs/datasets/mmlu_pro/mmlu_pro_gen_cdbebf.py +++ b/configs/datasets/mmlu_pro/mmlu_pro_gen_cdbebf.py @@ -51,6 +51,7 @@ dict( abbr=f'mmlu_pro_{category.replace(" ", "_")}', type=MMLUProDataset, + path='opencompass/mmlu_pro', category=category, reader_cfg=mmlu_pro_reader_cfg, infer_cfg=mmlu_pro_infer_cfg, diff --git a/opencompass/configs/datasets/mmlu_pro/mmlu_pro_0shot_cot_gen_08c1de.py b/opencompass/configs/datasets/mmlu_pro/mmlu_pro_0shot_cot_gen_08c1de.py index df9b8bc37..341e13cf5 100644 --- a/opencompass/configs/datasets/mmlu_pro/mmlu_pro_0shot_cot_gen_08c1de.py +++ b/opencompass/configs/datasets/mmlu_pro/mmlu_pro_0shot_cot_gen_08c1de.py @@ -56,6 +56,7 @@ dict( abbr=f'mmlu_pro_{category.replace(" ", "_")}', type=MMLUProDataset, + path='opencompass/mmlu_pro', category=category, reader_cfg=mmlu_pro_reader_cfg, infer_cfg=mmlu_pro_infer_cfg, diff --git a/opencompass/configs/datasets/mmlu_pro/mmlu_pro_gen_cdbebf.py b/opencompass/configs/datasets/mmlu_pro/mmlu_pro_gen_cdbebf.py index 7dd143b1e..42c30131f 100644 --- a/opencompass/configs/datasets/mmlu_pro/mmlu_pro_gen_cdbebf.py +++ b/opencompass/configs/datasets/mmlu_pro/mmlu_pro_gen_cdbebf.py @@ -51,6 +51,7 @@ dict( abbr=f'mmlu_pro_{category.replace(" ", "_")}', type=MMLUProDataset, + path='opencompass/mmlu_pro', category=category, reader_cfg=mmlu_pro_reader_cfg, infer_cfg=mmlu_pro_infer_cfg, diff --git a/opencompass/datasets/mmlu_pro.py b/opencompass/datasets/mmlu_pro.py index b7debbba7..d5e2a4e4a 100644 --- a/opencompass/datasets/mmlu_pro.py +++ b/opencompass/datasets/mmlu_pro.py @@ -4,6 +4,7 @@ from datasets import load_dataset from opencompass.registry import LOAD_DATASET +from opencompass.utils import get_data_path from .base import BaseDataset @@ -24,8 +25,9 @@ def _parse(item): class MMLUProDataset(BaseDataset): @staticmethod - def load(category: str): - mmlu_pro = load_dataset('TIGER-Lab/MMLU-Pro') + def load(path: str, category: str): + path = get_data_path(path) + mmlu_pro = load_dataset(path) mmlu_pro = mmlu_pro.filter(lambda x: x['category'] == category) mmlu_pro = mmlu_pro.map(_parse) return mmlu_pro diff --git a/opencompass/utils/datasets_info.py b/opencompass/utils/datasets_info.py index af14251c6..617c796c0 100644 --- a/opencompass/utils/datasets_info.py +++ b/opencompass/utils/datasets_info.py @@ -175,6 +175,12 @@ "hf_id": "opencompass/mmlu", "local": "./data/mmlu/", }, + # MMLU_PRO + "opencompass/mmlu_pro": { + "ms_id": "", + "hf_id": "", + "local": "./data/mmlu_pro", + }, # NQ "opencompass/natural_question": { "ms_id": "opencompass/natural_question", @@ -386,4 +392,8 @@ "url": "http://opencompass.oss-cn-shanghai.aliyuncs.com/datasets/data/SQuAD2.0.zip", "md5": "1321cbf9349e1102a57d31d1b2bfdd7e", }, + "mmlu_pro": { + "url": "http://opencompass.oss-cn-shanghai.aliyuncs.com/datasets/data/mmlu_pro.zip", + "md5": "e3200c7380f4cea5f13c768f2815fabb", + }, }