From a48ef112725bd6a41566a99b0c6140c2830a0e35 Mon Sep 17 00:00:00 2001
From: liushz <liuhongwei@pjlab.rog.cn>
Date: Wed, 18 Sep 2024 14:01:22 +0800
Subject: [PATCH] Update MathBench & WikiBench for FullBench

---
 .../wikibench/wikibench_gen_f96ece.py         | 56 +++++++++++++++++++
 .../wikibench/wikibench_gen_f96ece.py         | 56 +++++++++++++++++++
 2 files changed, 112 insertions(+)
 create mode 100644 configs/datasets/wikibench/wikibench_gen_f96ece.py
 create mode 100644 opencompass/configs/datasets/wikibench/wikibench_gen_f96ece.py

diff --git a/configs/datasets/wikibench/wikibench_gen_f96ece.py b/configs/datasets/wikibench/wikibench_gen_f96ece.py
new file mode 100644
index 000000000..5bf9d34ed
--- /dev/null
+++ b/configs/datasets/wikibench/wikibench_gen_f96ece.py
@@ -0,0 +1,56 @@
+from opencompass.openicl.icl_prompt_template import PromptTemplate
+from opencompass.openicl.icl_retriever import ZeroRetriever
+from opencompass.openicl.icl_inferencer import GenInferencer
+from opencompass.openicl.icl_evaluator import CircularEvaluator, AccEvaluator
+from opencompass.datasets import WikiBenchDataset
+from opencompass.utils.text_postprocessors import first_option_postprocess
+
+
+single_choice_prompts = {
+    'single_choice_cn': '以下是一道单项选择题，请你根据你了解的知识给出正确的答案选项。\n下面是你要回答的题目：\n{question}\n答案选项：',
+}
+
+wikibench_sets = {
+    'wiki': ['single_choice_cn'],
+}
+
+do_circular = True
+
+wikibench_datasets = []
+
+for _split in list(wikibench_sets.keys()):
+    for _name in wikibench_sets[_split]:
+        wikibench_infer_cfg = dict(
+            ice_template=dict(
+                type=PromptTemplate,
+                template=dict(
+                    begin='</E>',
+                    round=[
+                        dict(role='HUMAN', prompt=single_choice_prompts[_name]),
+                        dict(role='BOT', prompt='{answer}'),
+                    ],
+                ),
+                ice_token='</E>',
+            ),
+            retriever=dict(type=ZeroRetriever),
+            inferencer=dict(type=GenInferencer),
+        )
+        wikibench_eval_cfg = dict(
+            evaluator=dict(type=CircularEvaluator if do_circular else AccEvaluator),
+            pred_postprocessor=dict(type=first_option_postprocess, options='ABCD'),
+        )
+
+        wikibench_datasets.append(
+            dict(
+                type=WikiBenchDataset,
+                path=f'./data/WikiBench/{_name}.jsonl',
+                name='circular_' + _name if do_circular else _name,
+                abbr='wikibench-' + _split + '-' + _name + 'circular' if do_circular else '',
+                reader_cfg=dict(
+                    input_columns=['question'],
+                    output_column='answer',
+                ),
+                infer_cfg=wikibench_infer_cfg,
+                eval_cfg=wikibench_eval_cfg,
+            )
+        )
diff --git a/opencompass/configs/datasets/wikibench/wikibench_gen_f96ece.py b/opencompass/configs/datasets/wikibench/wikibench_gen_f96ece.py
new file mode 100644
index 000000000..5bf9d34ed
--- /dev/null
+++ b/opencompass/configs/datasets/wikibench/wikibench_gen_f96ece.py
@@ -0,0 +1,56 @@
+from opencompass.openicl.icl_prompt_template import PromptTemplate
+from opencompass.openicl.icl_retriever import ZeroRetriever
+from opencompass.openicl.icl_inferencer import GenInferencer
+from opencompass.openicl.icl_evaluator import CircularEvaluator, AccEvaluator
+from opencompass.datasets import WikiBenchDataset
+from opencompass.utils.text_postprocessors import first_option_postprocess
+
+
+single_choice_prompts = {
+    'single_choice_cn': '以下是一道单项选择题，请你根据你了解的知识给出正确的答案选项。\n下面是你要回答的题目：\n{question}\n答案选项：',
+}
+
+wikibench_sets = {
+    'wiki': ['single_choice_cn'],
+}
+
+do_circular = True
+
+wikibench_datasets = []
+
+for _split in list(wikibench_sets.keys()):
+    for _name in wikibench_sets[_split]:
+        wikibench_infer_cfg = dict(
+            ice_template=dict(
+                type=PromptTemplate,
+                template=dict(
+                    begin='</E>',
+                    round=[
+                        dict(role='HUMAN', prompt=single_choice_prompts[_name]),
+                        dict(role='BOT', prompt='{answer}'),
+                    ],
+                ),
+                ice_token='</E>',
+            ),
+            retriever=dict(type=ZeroRetriever),
+            inferencer=dict(type=GenInferencer),
+        )
+        wikibench_eval_cfg = dict(
+            evaluator=dict(type=CircularEvaluator if do_circular else AccEvaluator),
+            pred_postprocessor=dict(type=first_option_postprocess, options='ABCD'),
+        )
+
+        wikibench_datasets.append(
+            dict(
+                type=WikiBenchDataset,
+                path=f'./data/WikiBench/{_name}.jsonl',
+                name='circular_' + _name if do_circular else _name,
+                abbr='wikibench-' + _split + '-' + _name + 'circular' if do_circular else '',
+                reader_cfg=dict(
+                    input_columns=['question'],
+                    output_column='answer',
+                ),
+                infer_cfg=wikibench_infer_cfg,
+                eval_cfg=wikibench_eval_cfg,
+            )
+        )