Update Kaoshi

configs/datasets/Kaoshi/Kaoshi_gen.py

@@ -1,4 +1,4 @@
 from mmengine.config import read_base
 
 with read_base():
-    from .Kaoshi_gen_86aca2 import Kaoshi_datasets  # noqa: F401, F403
+    from .Kaoshi_gen_86aca2 import kaoshi_datasets  # noqa: F401, F403

configs/datasets/Kaoshi/Kaoshi_gen_86aca2.py

@@ -21,7 +21,7 @@
 
 zh2en = {'单选题': 'single_choice', '多选题': 'multi_choice', '完形填空': 'multi_question_choice', '判断题': 'judgment', '填空题': 'cloze', '七选五': 'five_out_of_seven'}
 
-Kaoshi_datasets = []
+kaoshi_datasets = []
 
 for _type in list(splits_with_type.keys()):
     for _split in splits_with_type[_type]:
@@ -52,7 +52,8 @@
         }
         _eval_cfg = {
             "evaluator": {
-                "type": "KaoshiEvaluator" + "_" + zh2en[_type],
+                "type": "KaoshiEvaluator",
+                  "question_type": zh2en[_type],
             },
             "pred_role": "BOT",
         }
@@ -67,7 +68,7 @@
             "eval_cfg": _eval_cfg,
         }
 
-        Kaoshi_datasets.append(_dataset)
+        kaoshi_datasets.append(_dataset)
 
 _temporary_variables = [k for k in globals() if k.startswith('_')]
 for _t in _temporary_variables:

opencompass/datasets/Kaoshi.py

@@ -4,26 +4,23 @@
 from datasets import Dataset
 
 from opencompass.openicl.icl_evaluator import BaseEvaluator
-from opencompass.registry import ICL_EVALUATORS, LOAD_DATASET
 
 from .base import BaseDataset
 
 
 def get_number(options):
 
     result_string = ''
-    for i, percentage in enumerate(options, start=65):
-        result_string += f'{chr(i)}. {percentage}\n'
+    for i, option in enumerate(options, start=65):
+        result_string += f'{chr(i)}. {option}\n'
     return result_string
 
 
-@LOAD_DATASET.register_module()
 class KaoshiDataset(BaseDataset):
 
     @staticmethod
     def load(path: str, name: str):
         data_list = []
-        _type = path.split('/')[-1].replace('.jsonl', '')
         with open(path, encoding='utf-8') as f:
             for line in f:
                 data = json.loads(line)
@@ -139,14 +136,3 @@ def score(self, predictions, references):
                         correct_score += 1
                     total_score += 1
             return {'score': correct_score / total_score * 100}
-
-
-for question_type in valid_kaoshi_question_types:
-    # fix classic closure problem
-    def _kaoshi_register(question_type):
-        ICL_EVALUATORS.register_module(
-            name='KaoshiEvaluator' + '_' + question_type,
-            module=lambda *args, **kwargs: KaoshiEvaluator(
-                question_type=question_type, *args, **kwargs))
-
-    _kaoshi_register(question_type)