[Feature] Mmlu-pro auto-download (#1464)

* update * update * update * update * update
open-compass · Aug 30, 2024 · 9693be4 · 9693be4
1 parent f342097
commit 9693be4
Show file tree

Hide file tree

Showing 6 changed files with 18 additions and 2 deletions.
diff --git a/configs/datasets/mmlu_pro/mmlu_pro_0shot_cot_gen_08c1de.py b/configs/datasets/mmlu_pro/mmlu_pro_0shot_cot_gen_08c1de.py
@@ -56,6 +56,7 @@
         dict(
             abbr=f'mmlu_pro_{category.replace(" ", "_")}',
             type=MMLUProDataset,
+            path='opencompass/mmlu_pro',
             category=category,
             reader_cfg=mmlu_pro_reader_cfg,
             infer_cfg=mmlu_pro_infer_cfg,

diff --git a/configs/datasets/mmlu_pro/mmlu_pro_gen_cdbebf.py b/configs/datasets/mmlu_pro/mmlu_pro_gen_cdbebf.py
@@ -51,6 +51,7 @@
         dict(
             abbr=f'mmlu_pro_{category.replace(" ", "_")}',
             type=MMLUProDataset,
+            path='opencompass/mmlu_pro',
             category=category,
             reader_cfg=mmlu_pro_reader_cfg,
             infer_cfg=mmlu_pro_infer_cfg,

diff --git a/opencompass/configs/datasets/mmlu_pro/mmlu_pro_0shot_cot_gen_08c1de.py b/opencompass/configs/datasets/mmlu_pro/mmlu_pro_0shot_cot_gen_08c1de.py
@@ -56,6 +56,7 @@
         dict(
             abbr=f'mmlu_pro_{category.replace(" ", "_")}',
             type=MMLUProDataset,
+            path='opencompass/mmlu_pro',
             category=category,
             reader_cfg=mmlu_pro_reader_cfg,
             infer_cfg=mmlu_pro_infer_cfg,

diff --git a/opencompass/configs/datasets/mmlu_pro/mmlu_pro_gen_cdbebf.py b/opencompass/configs/datasets/mmlu_pro/mmlu_pro_gen_cdbebf.py
@@ -51,6 +51,7 @@
         dict(
             abbr=f'mmlu_pro_{category.replace(" ", "_")}',
             type=MMLUProDataset,
+            path='opencompass/mmlu_pro',
             category=category,
             reader_cfg=mmlu_pro_reader_cfg,
             infer_cfg=mmlu_pro_infer_cfg,

diff --git a/opencompass/datasets/mmlu_pro.py b/opencompass/datasets/mmlu_pro.py
@@ -4,6 +4,7 @@
 from datasets import load_dataset
 
 from opencompass.registry import LOAD_DATASET
+from opencompass.utils import get_data_path
 
 from .base import BaseDataset
 
@@ -24,8 +25,9 @@ def _parse(item):
 class MMLUProDataset(BaseDataset):
 
     @staticmethod
-    def load(category: str):
-        mmlu_pro = load_dataset('TIGER-Lab/MMLU-Pro')
+    def load(path: str, category: str):
+        path = get_data_path(path)
+        mmlu_pro = load_dataset(path)
         mmlu_pro = mmlu_pro.filter(lambda x: x['category'] == category)
         mmlu_pro = mmlu_pro.map(_parse)
         return mmlu_pro
diff --git a/opencompass/utils/datasets_info.py b/opencompass/utils/datasets_info.py
@@ -175,6 +175,12 @@
         "hf_id": "opencompass/mmlu",
         "local": "./data/mmlu/",
     },
+    # MMLU_PRO
+    "opencompass/mmlu_pro": {
+        "ms_id": "",
+        "hf_id": "",
+        "local": "./data/mmlu_pro",
+    },
     # NQ
     "opencompass/natural_question": {
         "ms_id": "opencompass/natural_question",
@@ -386,4 +392,8 @@
         "url": "http://opencompass.oss-cn-shanghai.aliyuncs.com/datasets/data/SQuAD2.0.zip",
         "md5": "1321cbf9349e1102a57d31d1b2bfdd7e",
     },
+    "mmlu_pro": {
+        "url": "http://opencompass.oss-cn-shanghai.aliyuncs.com/datasets/data/mmlu_pro.zip",
+        "md5": "e3200c7380f4cea5f13c768f2815fabb",
+    },
 }