diff --git a/configs/datasets/MathBench/mathbench_gen_ccd638.py b/configs/datasets/MathBench/mathbench_gen_ccd638.py new file mode 100644 index 000000000..bc1c96893 --- /dev/null +++ b/configs/datasets/MathBench/mathbench_gen_ccd638.py @@ -0,0 +1,58 @@ +from opencompass.openicl.icl_prompt_template import PromptTemplate +from opencompass.openicl.icl_retriever import ZeroRetriever +from opencompass.openicl.icl_inferencer import GenInferencer +from opencompass.openicl.icl_evaluator import AccEvaluator +from opencompass.datasets import MathBenchDataset, mathbench_postprocess + +cloze_prompts ={ + "cloze_arith_en": [ + dict(role='HUMAN', prompt='Q: Calculate (341/11)/(9/(-6)*(-2)/3).'), + dict(role='BOT', prompt='A: First, (9/(-6)*(-2)/3) can be simplified by : 9/(-6) = -1.5, -1.5 * (-2) = 3, 3 / 3 = 1. So, (9/(-6)*(-2)/3) is equal to 1. Now, we have `(341/11)/1` equals `341/11`. Finally, calculate `341/11 = 31`. The answer is 31.\n'), + dict(role='HUMAN', prompt='Q: In base 14, what is 5 - 638d8d?'), + dict(role='BOT', prompt='A: 5 - 638d8d = -638d88. The answer is -638d88.\n'), + dict(role='HUMAN', prompt='Q: What is -491354 times -0.34?'), + dict(role='BOT', prompt='A: The product of -491354 and -0.34 is 167060.36. The answer is 167060.36.\n'), + dict(role='HUMAN', prompt='Q: What is the value of (-55)/(6930/(-382)) + (0 - 3)?.'), + dict(role='BOT', prompt='A: First, (-55)/(6930/(-382)) = (-55)/(-(6930/382)) = 55*382/6930 = 21010/6930 = 2101/693. Then, 2101/693 + (0 - 3) = 2101/693 - 3 = 2101/693 - 3*693/693 = (2101-2079)/693 = 22/693 = 2/63. The answer is 2/63.\n'), + dict(role='HUMAN', prompt='Q: {question}'), + dict(role='BOT', prompt='A: {answer}\n'), +] +} + +mathbench_sets = { + 'arithmetic': ['cloze_arith_en'], +} + +mathbench_datasets = [] + +for _split in list(mathbench_sets.keys()): + for _name in mathbench_sets[_split]: + mathbench_infer_cfg = dict( + prompt_template=dict( + type=PromptTemplate, + template=dict( + round=cloze_prompts[_name], + ), + ), + retriever=dict(type=ZeroRetriever), + inferencer=dict(type=GenInferencer, max_out_len=512), + ) + + mathbench_eval_cfg = dict( + evaluator=dict(type=AccEvaluator), + pred_postprocessor=dict(type=mathbench_postprocess, name=_name)) + + mathbench_datasets.append( + dict( + type=MathBenchDataset, + path=f"./data/mathbench/{_split}", + name=_name, + with_circular=False, + abbr="mathbench-arithmetic" + _split + '-' + _name, + reader_cfg=dict( + input_columns=["question"], + output_column="answer" + ), + infer_cfg=mathbench_infer_cfg, + eval_cfg=mathbench_eval_cfg, + )) diff --git a/opencompass/datasets/mathbench.py b/opencompass/datasets/mathbench.py index 0a7ebb157..995a758e7 100644 --- a/opencompass/datasets/mathbench.py +++ b/opencompass/datasets/mathbench.py @@ -87,6 +87,7 @@ def load(path: str, name: str, with_circular: bool = True): @TEXT_POSTPROCESSORS.register_module() def mathbench_postprocess(text: str, name: str) -> str: + split = False ans = text if '_cn' in name: ans_line = ans.split('答案是') @@ -94,10 +95,12 @@ def mathbench_postprocess(text: str, name: str) -> str: ans_line = ans.split('The answer is') if len(ans_line) != 1: ans = ans_line[1].strip() + split = True output = re.sub(r'(\d),(\d)', r'\1\2', ans) numbers = re.findall(r'-?\d*\.?/?\d+|\d+', output) + if numbers: - return numbers[-1] + return numbers[0] if split else numbers[-1] return ans