[Feature] Add dingo test

configs/eval_dingo.py

@@ -0,0 +1,41 @@
+from mmengine.config import read_base
+
+from opencompass.openicl.icl_prompt_template import PromptTemplate
+from opencompass.openicl.icl_retriever import ZeroRetriever
+from opencompass.openicl.icl_inferencer import GenInferencer
+from opencompass.datasets import dingoDataset, dingoEvaluator
+
+
+with read_base():
+    from .models.hf_internlm.hf_internlm_7b import models
+
+dingo_paths = [
+    './data/dingo/en_192.csv',
+    './data/dingo/zh_170.csv',
+]
+
+dingo_datasets = []
+for path in dingo_paths:
+    dingo_reader_cfg = dict(input_columns='input', output_column=None)
+    dingo_infer_cfg = dict(
+        prompt_template=dict(
+            type=PromptTemplate,
+            template=dict(round=[dict(role='HUMAN', prompt='{input}')])),
+        retriever=dict(type=ZeroRetriever),
+        inferencer=dict(type=GenInferencer),
+    )
+    dingo_eval_cfg = dict(evaluator=dict(type=dingoEvaluator), pred_role='BOT')
+
+    dingo_datasets.append(
+        dict(
+            abbr='dingo_' + path.split('/')[-1].split('.csv')[0],
+            type=dingoDataset,
+            path=path,
+            reader_cfg=dingo_reader_cfg,
+            infer_cfg=dingo_infer_cfg,
+            eval_cfg=dingo_eval_cfg,
+        ))
+
+datasets = dingo_datasets
+
+work_dir = './outputs/eval_dingo'

opencompass/datasets/__init__.py

@@ -89,6 +89,7 @@
 from .OpenFinData import *  # noqa: F401, F403
 from .piqa import *  # noqa: F401, F403
 from .py150 import *  # noqa: F401, F403
+from .dingo import * # noqa: F401, F403
 from .qasper import *  # noqa: F401, F403
 from .qaspercut import *  # noqa: F401, F403
 from .QuALITY import *  # noqa: F401, F403

opencompass/datasets/dingo.py

@@ -0,0 +1,78 @@
+# flake8: nodingo
+# yapf: disable
+import os
+import csv
+import json
+import time
+from typing import List
+from datasets import Dataset
+
+from opencompass.openicl.icl_evaluator import BaseEvaluator
+from opencompass.registry import ICL_EVALUATORS, LOAD_DATASET
+
+from .base import BaseDataset
+
+try:
+    from dingo.model.model import Model
+    from dingo.io import InputArgs
+    from dingo.exec import Executor
+except Exception:
+    raise ModuleNotFoundError('=========== dingo register fail. please try: pip install dingo-python. ===========')
+
+@LOAD_DATASET.register_module()
+class dingoDataset(BaseDataset):
+
+    @staticmethod
+    def load(path: str):
+        raw_data = []
+        with open(path, encoding='utf-8') as f:
+            reader = csv.reader(f, delimiter=';')
+            for row in reader:
+                if len(row) < 1:
+                    row = ['']
+                raw_data.append({'input': row[0]})
+        return Dataset.from_list(raw_data)
+
+
+@LOAD_DATASET.register_module()
+class dingoLongDataset(BaseDataset):
+
+    @staticmethod
+    def load(path: str):
+        raw_data = []
+        with open(path, 'r', encoding='utf-8') as f:
+            for line in f:
+                raw_data.append({'input': json.loads(line).get('input')})
+        return Dataset.from_list(raw_data)
+
+
+@ICL_EVALUATORS.register_module()
+class dingoEvaluator(BaseEvaluator):
+
+    def score(self, origin_prompt: List, predictions: List) -> dict:
+        current_time = time.strftime('%Y%m%d_%H%M%S', time.localtime())
+        file_data = [{'prompt':pmt, 'prediction':prd} for pmt, prd in zip(origin_prompt, predictions)]
+        file_name = 'dingo_file_' + current_time + '.jsonl'
+        with open(file_name, 'a', encoding='utf-8') as f:
+            for d in file_data:
+                json.dump(d, f, ensure_ascii=False)
+                f.write('\n')
+
+        input_data = {
+            "eval_models": ["llm_base"],
+            "input_path": file_name,
+            "output_path": "./outputs/dingo/",
+            "dataset": "local",
+            "datasource": "local",  # If not fill in this item, it will be the same as "dataset"
+            "data_format": "jsonl",
+            "column_prompt": ["prompt"],
+            "column_content": ["prediction"],
+        }
+        # Model.apply_config(input_data['custom_config_path'])
+        input_args = InputArgs(**input_data)
+        executor = Executor.exec_map["local"](input_args)
+        result = executor.execute()
+        summary = result[0].to_dict()
+
+        os.remove(file_name)
+        return summary