From 757b24e18851f285b85390eec569829e73f277de Mon Sep 17 00:00:00 2001 From: liushz Date: Tue, 12 Sep 2023 06:44:22 +0000 Subject: [PATCH] Update kaoshi --- configs/datasets/Kaoshi/Kaoshi_gen_6666.py | 2 +- opencompass/datasets/Kaoshi.py | 22 +++++++++------------- 2 files changed, 10 insertions(+), 14 deletions(-) diff --git a/configs/datasets/Kaoshi/Kaoshi_gen_6666.py b/configs/datasets/Kaoshi/Kaoshi_gen_6666.py index e3b061ed2..f751c2799 100644 --- a/configs/datasets/Kaoshi/Kaoshi_gen_6666.py +++ b/configs/datasets/Kaoshi/Kaoshi_gen_6666.py @@ -6,7 +6,7 @@ prompts = { "单选题" : "请你做一道单项选择题\n请你一步一步思考并将思考过程写在【解析】和之间。你将从A，B，C，D中选出正确的答案，并写在【答案】和之间，答案应只包含最终结果，不要添加额外词语。\n例如：【答案】: A \n完整的题目回答的格式如下：\n【解析】 ... \n【答案】 ... \n请你严格按照上述格式作答。\n题目如下：", "多选题" : "请你做一道多项选择题\n请你一步一步思考并将思考过程写在【解析】和之间。你将从多个选项中选出正确的答案，答案可能是一个到多个选项，奇怪将其写在【答案】和之间，答案应只包含最终结果，不要添加额外词语。\n例如：【答案】: A D \n完整的题目回答的格式如下：\n【解析】 ... \n【答案】 ... \n请你严格按照上述格式作答。\n题目如下：", - "填空题" : "请解答下面的填空题\n仔细阅读题目，解答其中的问题，请你一步步思考并将思考过程写在【解析】和之间。请把你的答案写在【答案】和之间，答案应只包含最终结果，不要添加额外词语。\n完整的题目回答格式如下：\n【解析】 ...\n【答案】...\n请你严格按照上述格式作答。\n题目如下:", + "填空题" : "请解答下面的填空题\n仔细阅读题目，解答其中的问题，请你一步步思考并将思考过程写在【解析】和之间。请把你的答案写在【答案】和之间，答案应只包含最终结果，不要添加额外词语。\n完整的题目回答格式如下：\n【解析】 ... \n【答案】... \n请你严格按照上述格式作答。\n题目如下:", "完形填空" : "请你做一道英语完形填空题,其中包含二十个小题。\n请你一步一步思考。每一题你将从A，B，C，D中选出正确的答案，并写在【答案】和之间。\n例如：（1）【答案】 A \n（2）【答案】 B \n请你严格按照上述格式作答。\n", "七选五": "请回答下面的问题，将符合题意的五个选项的字母写在【答案】和之间，例如：【答案】 A B C D E \n请严格按照上述格式作答。题目如下：\n", "判断题" : "请回答下面的判断题，将你的判断结果写在【答案】和之间，若给定表述正确时回答：\n【答案】正确 \n 表述错误时回答：\n【答案】错误 \n请严格按照上述格式作答。题目如下：\n", diff --git a/opencompass/datasets/Kaoshi.py b/opencompass/datasets/Kaoshi.py index 9acaac218..ecb4df869 100644 --- a/opencompass/datasets/Kaoshi.py +++ b/opencompass/datasets/Kaoshi.py @@ -34,9 +34,8 @@ def load(path: str, name: str): data_list.append(data) return Dataset.from_list(data_list) - -valid_kaoshi__question_types = [ - 'single_choice', 'multi_choice', 'multi_question_choice' +valid_kaoshi_question_types = [ + 'single_choice', 'multi_choice', 'multi_question_choice', 'five_out_of_seven', 'cloze', 'judgment' ] @@ -45,7 +44,7 @@ class KaoshiEvaluator(BaseEvaluator): def __init__(self, question_type) -> None: super().__init__() - assert question_type in valid_kaoshi__question_types + assert question_type in valid_kaoshi_question_types self.question_type = question_type def do_predictions_postprocess(self, model_output, answer_lenth=None): @@ -93,7 +92,7 @@ def do_predictions_postprocess(self, model_output, answer_lenth=None): for k in range(min(5, len(temp))): model_answer.append(temp[k]) - elif self.question_type == 'judgment': + elif self.question_type in ['cloze', 'judgment']: model_answer = [] temp = re.findall(r'【答案】(.*?) ', model_output) if len(temp) > 0: @@ -107,10 +106,7 @@ def ensure_same_length(self, pred, refr): return ['Z'] * len(refr) def score(self, predictions, references): - if self.question_type not in [ - 'single_choice', 'multi_choice', 'multi_question_choice', - 'five_out_of_seven' - ]: + if self.question_type not in valid_kaoshi_question_types: return {'score': 0} elif self.question_type == 'multi_choice': correct_score, total_score = 0, 0 @@ -135,7 +131,7 @@ def score(self, predictions, references): pred = self.do_predictions_postprocess(pred, len(refr)) else: pred = self.do_predictions_postprocess(pred) - if self.question_type == 'judgment': + if self.question_type in ['cloze', 'judgment']: refr = [refr] pred = self.ensure_same_length(pred, refr) for p, r in zip(pred, refr): @@ -145,12 +141,12 @@ def score(self, predictions, references): return {'score': correct_score / total_score * 100} -for question_type in valid_kaoshi__question_types: +for question_type in valid_kaoshi_question_types: # fix classic closure problem def _kaoshi_register(question_type): ICL_EVALUATORS.register_module( - name='KaoshiEvaluator' + '_' + question_type, + name='KaoshiEvaluator' + "_" + question_type, module=lambda *args, **kwargs: KaoshiEvaluator( question_type=question_type, *args, **kwargs)) - _kaoshi_register(question_type) + _kaoshi_register(question_type) \ No newline at end of file