From dbacd36379ee8e066f2ccdf1d3ec93b78157883d Mon Sep 17 00:00:00 2001
From: liushz <qq1791167085@163.com>
Date: Mon, 20 Nov 2023 19:40:41 +0800
Subject: [PATCH] Add aritch to mathbench (#607)

---
 .../MathBench/mathbench_gen_ccd638.py         | 58 +++++++++++++++++++
 opencompass/datasets/mathbench.py             |  5 +-
 2 files changed, 62 insertions(+), 1 deletion(-)
 create mode 100644 configs/datasets/MathBench/mathbench_gen_ccd638.py

diff --git a/configs/datasets/MathBench/mathbench_gen_ccd638.py b/configs/datasets/MathBench/mathbench_gen_ccd638.py
new file mode 100644
index 000000000..bc1c96893
--- /dev/null
+++ b/configs/datasets/MathBench/mathbench_gen_ccd638.py
@@ -0,0 +1,58 @@
+from opencompass.openicl.icl_prompt_template import PromptTemplate
+from opencompass.openicl.icl_retriever import ZeroRetriever
+from opencompass.openicl.icl_inferencer import GenInferencer
+from opencompass.openicl.icl_evaluator import AccEvaluator
+from opencompass.datasets import MathBenchDataset, mathbench_postprocess
+
+cloze_prompts ={
+    "cloze_arith_en": [
+                dict(role='HUMAN', prompt='Q: Calculate (341/11)/(9/(-6)*(-2)/3).'),
+                dict(role='BOT', prompt='A: First, (9/(-6)*(-2)/3) can be simplified by : 9/(-6) = -1.5, -1.5 * (-2) = 3, 3 / 3 = 1. So, (9/(-6)*(-2)/3) is equal to 1. Now, we have `(341/11)/1` equals `341/11`. Finally, calculate `341/11 = 31`. The answer is 31.\n'),
+                dict(role='HUMAN', prompt='Q: In base 14, what is 5 - 638d8d?'),
+                dict(role='BOT', prompt='A: 5 - 638d8d = -638d88. The answer is -638d88.\n'),
+                dict(role='HUMAN', prompt='Q: What is -491354 times -0.34?'),
+                dict(role='BOT', prompt='A: The product of -491354 and -0.34 is 167060.36. The answer is 167060.36.\n'),
+                dict(role='HUMAN', prompt='Q: What is the value of (-55)/(6930/(-382)) + (0 - 3)?.'), 
+                dict(role='BOT', prompt='A: First, (-55)/(6930/(-382)) = (-55)/(-(6930/382)) = 55*382/6930 = 21010/6930 = 2101/693. Then, 2101/693 + (0 - 3) = 2101/693 - 3 = 2101/693 - 3*693/693 = (2101-2079)/693 = 22/693 = 2/63. The answer is 2/63.\n'),    
+                dict(role='HUMAN', prompt='Q: {question}'),
+                dict(role='BOT', prompt='A: {answer}\n'),
+]
+}
+
+mathbench_sets = {
+    'arithmetic': ['cloze_arith_en'],
+}
+
+mathbench_datasets = []
+
+for _split in list(mathbench_sets.keys()):
+    for _name in mathbench_sets[_split]:
+        mathbench_infer_cfg = dict(
+            prompt_template=dict(
+                type=PromptTemplate,
+                template=dict(
+                    round=cloze_prompts[_name],
+                    ),
+            ),
+            retriever=dict(type=ZeroRetriever),
+            inferencer=dict(type=GenInferencer, max_out_len=512),
+        )
+
+        mathbench_eval_cfg = dict(
+            evaluator=dict(type=AccEvaluator),
+            pred_postprocessor=dict(type=mathbench_postprocess, name=_name))
+
+        mathbench_datasets.append(
+            dict(
+                type=MathBenchDataset,
+                path=f"./data/mathbench/{_split}",
+                name=_name,
+                with_circular=False,
+                abbr="mathbench-arithmetic" + _split + '-' + _name,
+                reader_cfg=dict(
+                    input_columns=["question"],
+                    output_column="answer"
+                    ),
+                infer_cfg=mathbench_infer_cfg,
+                eval_cfg=mathbench_eval_cfg,
+            ))
diff --git a/opencompass/datasets/mathbench.py b/opencompass/datasets/mathbench.py
index 0a7ebb157..995a758e7 100644
--- a/opencompass/datasets/mathbench.py
+++ b/opencompass/datasets/mathbench.py
@@ -87,6 +87,7 @@ def load(path: str, name: str, with_circular: bool = True):
 
 @TEXT_POSTPROCESSORS.register_module()
 def mathbench_postprocess(text: str, name: str) -> str:
+    split = False
     ans = text
     if '_cn' in name:
         ans_line = ans.split('答案是')
@@ -94,10 +95,12 @@ def mathbench_postprocess(text: str, name: str) -> str:
         ans_line = ans.split('The answer is')
     if len(ans_line) != 1:
         ans = ans_line[1].strip()
+        split = True
 
     output = re.sub(r'(\d),(\d)', r'\1\2', ans)
     numbers = re.findall(r'-?\d*\.?/?\d+|\d+', output)
+
     if numbers:
-        return numbers[-1]
+        return numbers[0] if split else numbers[-1]
 
     return ans