Add SVAMP dataset

configs/datasets/SVAMP/svamp_gen.py

@@ -1,4 +1,4 @@
 from mmengine.config import read_base
 
 with read_base():
-    from .svamp_gen_5edbdb import svamp_datasets  # noqa: F401, F403
+    from .svamp_gen_fb25e4 import svamp_datasets  # noqa: F401, F403

configs/datasets/SVAMP/svamp_gen_fb25e4.py

@@ -0,0 +1,36 @@
+from opencompass.openicl.icl_prompt_template import PromptTemplate
+from opencompass.openicl.icl_retriever import ZeroRetriever
+from opencompass.openicl.icl_inferencer import GenInferencer
+from opencompass.datasets import SVAMPDataset, gsm8k_postprocess, Gsm8kEvaluator
+
+svamp_infer_cfg = dict(
+    prompt_template=dict(
+        type=PromptTemplate,
+        template=dict(
+            round=[
+                dict(role='HUMAN', prompt="Question: There are 87 oranges and 290 bananas in Philip's collection. If the bananas are organized into 2 groups and oranges are organized into 93 groups How big is each group of bananas?\nLet's think step by step\nAnswer:"),
+                dict(role='BOT', prompt="To find the size of each group of bananas, we divide the total number of bananas (290) by the number of groups (2): 290 / 2 = 145. Therefore, each group of bananas contains 145 bananas. The answer is 145.\n"),
+                dict(role='HUMAN', prompt="Question: Marco and his dad went strawberry picking. Marco's dad's strawberries weighed 11 pounds. If together their strawberries weighed 30 pounds. How much did Marco's strawberries weigh?\nLet's think step by step\nAnswer:"),
+                dict(role='BOT', prompt="To find Marco's strawberries' weight, we subtract his dad's strawberries' weight (11 pounds) from the total weight of their strawberries (30 pounds): 30 - 11 = 19. Therefore, Marco's strawberries weighed 19 pounds. The answer is 19.\n"),
+                dict(role='HUMAN', prompt="Question: Edward spent $ 6 to buy 2 books each book costing him the same amount of money. Now he has $ 12. How much did each book cost?\nLet's think step by step\nAnswer:"),
+                dict(role='BOT', prompt="To find the cost of each book, we subtract the initial amount of money Edward had ($6) from the current amount of money he has ($12) and divide it by the number of books (2): (12 - 6) / 2 = 6 / 2 = 3 Therefore, each book cost $3. The answer is 3.\n"),
+                dict(role='HUMAN', prompt="Question: Frank was reading through his favorite book. The book had 3 chapters, each with the same number of pages. It has a total of 594 pages. It took Frank 607 days to finish the book. How many pages are in each chapter?\nLet's think step by step\nAnswer:"),
+                dict(role='BOT', prompt="To find the number of pages in each chapter, we divide the total number of pages in the book (594) by the number of chapters (3): 594 / 3 = 198. Therefore, each chapter has 198 pages. The answer is 198.\n"),
+                dict(role='HUMAN', prompt="Question: {question}\nLet's think step by step\nAnswer:"),
+            ],
+        )),
+    retriever=dict(type=ZeroRetriever),
+    inferencer=dict(type=GenInferencer, max_out_len=512))
+
+svamp_eval_cfg = dict(evaluator=dict(type=Gsm8kEvaluator),
+                      pred_postprocessor=dict(type=gsm8k_postprocess))
+
+svamp_datasets = [
+    dict(
+        abbr='svamp',
+        type=SVAMPDataset,
+        path='./data/svamp/test.jsonl',
+        reader_cfg=dict(input_columns=['question'], output_column='answer'),
+        infer_cfg=svamp_infer_cfg,
+        eval_cfg=svamp_eval_cfg)
+]

opencompass/datasets/__init__.py

@@ -21,10 +21,13 @@
 from .cmnli import *  # noqa: F401, F403
 from .cmrc import *  # noqa: F401, F403
 from .commonsenseqa import *  # noqa: F401, F403
+from .commonsenseqa_cn import *  # noqa: F401, F403
 from .copa import *  # noqa: F401, F403
 from .crowspairs import *  # noqa: F401, F403
+from .crowspairs_cn import *  # noqa: F401, F403
 from .csl import *  # noqa: F401, F403
 from .cvalues import *  # noqa: F401, F403
+from .douknow import *  # noqa: F401, F403
 from .drcd import *  # noqa: F401, F403
 from .drop import *  # noqa: F401, F403
 from .ds1000 import *  # noqa: F401, F403
@@ -55,6 +58,7 @@
 from .multirc import *  # noqa: F401, F403
 from .narrativeqa import *  # noqa: F401, F403
 from .natural_question import *  # noqa: F401, F403
+from .natural_question_cn import *  # noqa: F401, F403
 from .obqa import *  # noqa: F401, F403
 from .piqa import *  # noqa: F401, F403
 from .py150 import *  # noqa: F401, F403
@@ -73,7 +77,6 @@
 from .summedits import *  # noqa: F401, F403
 from .summscreen import *  # noqa: F401, F403
 from .svamp import *  # noqa: F401, F403
-from .tabmwp import *  # noqa: F401, F403
 from .TheoremQA import *  # noqa: F401, F403
 from .tnews import *  # noqa: F401, F403
 from .triviaqa import *  # noqa: F401, F403

opencompass/datasets/svamp.py

@@ -1,6 +1,6 @@
 import json
 
-from datasets import Dataset, DatasetDict
+from datasets import Dataset
 
 from opencompass.registry import LOAD_DATASET
 
@@ -16,8 +16,8 @@ def load(path):
         with open(path, 'r', encoding='utf-8') as f:
             for line in f:
                 line = json.loads(line.strip())
-                question = line['Question']
+                question = line['Body'] + ' ' + line['Question']
                 answer = str(int(line['Answer']))
                 dataset.append({'question': question, 'answer': answer})
         dataset = Dataset.from_list(dataset)
-        return DatasetDict(dataset)
+        return dataset