diff --git a/configs/datasets/rolebench/instruction_generalization_eng.py b/configs/datasets/rolebench/instruction_generalization_eng.py new file mode 100644 index 000000000..09c2fcae1 --- /dev/null +++ b/configs/datasets/rolebench/instruction_generalization_eng.py @@ -0,0 +1,41 @@ +from opencompass.openicl.icl_prompt_template import PromptTemplate +from opencompass.openicl.icl_retriever import ZeroRetriever +from opencompass.openicl.icl_inferencer import GenInferencer +from opencompass.openicl.icl_evaluator import RougeEvaluator +from opencompass.datasets.rolebench import InstructionGeneralizationEnglishDataset + +instruction_generalization_eng_reader_cfg = dict( + input_columns=['role', 'desc', 'question'], + output_column='answer', + train_split='train', + test_split='test' +) + +instruction_generalization_eng_infer_cfg = dict( + prompt_template=dict( + type=PromptTemplate, + template=dict( + begin=[ + dict(role='SYSTEM', fallback_role='HUMAN', prompt='You are {role}, your description is: {desc}. Now please answer some questions to accurately show your personality traits! Your speaking style should fully imitate the personality role assigned to you! Please do not expose that you are an artificial intelligence model or a language model, you must always remember that you are only assigned one personality role. Don’t be verbose or too formal or polite when speaking.'), + ], + round=[ + dict(role='HUMAN', prompt='{question}'), + dict(role='BOT', prompt=''), + ], )), + retriever=dict(type=ZeroRetriever), + inferencer=dict(type=GenInferencer, max_out_len=512) +) + +instruction_generalization_eng_eval_cfg = dict( + evaluator=dict(type=RougeEvaluator), + pred_role='BOT' +) + +instruction_generalization_eng_datasets = [ + dict( + type=InstructionGeneralizationEnglishDataset, + path='ZenMoore/RoleBench', + reader_cfg=instruction_generalization_eng_reader_cfg, + infer_cfg=instruction_generalization_eng_infer_cfg, + eval_cfg=instruction_generalization_eng_eval_cfg) +] diff --git a/configs/datasets/rolebench/instruction_generalization_zh.py b/configs/datasets/rolebench/instruction_generalization_zh.py new file mode 100644 index 000000000..231c5e771 --- /dev/null +++ b/configs/datasets/rolebench/instruction_generalization_zh.py @@ -0,0 +1,41 @@ +from opencompass.openicl.icl_prompt_template import PromptTemplate +from opencompass.openicl.icl_retriever import ZeroRetriever +from opencompass.openicl.icl_inferencer import GenInferencer +from opencompass.openicl.icl_evaluator import RougeEvaluator +from opencompass.datasets.rolebench import InstructionGeneralizationChineseDataset + +instruction_generalization_zh_reader_cfg = dict( + input_columns=['role', 'desc', 'question'], + output_column='answer', + train_split='train', + test_split='test' +) + +instruction_generalization_zh_infer_cfg = dict( + prompt_template=dict( + type=PromptTemplate, + template=dict( + begin=[ + dict(role='SYSTEM', fallback_role='HUMAN', prompt='你是{role}，你的特征描述是：{desc}。现在请你回答我的一些问题，以准确展现你的人格特征！你的说话风格要全面模仿被赋予的人格角色！请不要暴露你是人工智能模型或者语言模型，你要时刻记住你只被赋予的一个人格角色。说话不要嗦，也不要太过于正式或礼貌。'), + ], + round=[ + dict(role='HUMAN', prompt='{question}'), + dict(role='BOT', prompt=''), + ], )), + retriever=dict(type=ZeroRetriever), + inferencer=dict(type=GenInferencer, max_out_len=512) +) + +instruction_generalization_zh_eval_cfg = dict( + evaluator=dict(type=RougeEvaluator), + pred_role='BOT' +) + +instruction_generalization_zh_datasets = [ + dict( + type=InstructionGeneralizationChineseDataset, + path='ZenMoore/RoleBench', + reader_cfg=instruction_generalization_zh_reader_cfg, + infer_cfg=instruction_generalization_zh_infer_cfg, + eval_cfg=instruction_generalization_zh_eval_cfg) +] diff --git a/configs/datasets/rolebench/role_generalization_eng.py b/configs/datasets/rolebench/role_generalization_eng.py new file mode 100644 index 000000000..168909a28 --- /dev/null +++ b/configs/datasets/rolebench/role_generalization_eng.py @@ -0,0 +1,41 @@ +from opencompass.openicl.icl_prompt_template import PromptTemplate +from opencompass.openicl.icl_retriever import ZeroRetriever +from opencompass.openicl.icl_inferencer import GenInferencer +from opencompass.openicl.icl_evaluator import RougeEvaluator +from opencompass.datasets.rolebench import RoleGeneralizationEnglishDataset + +role_generalization_eng_reader_cfg = dict( + input_columns=['role', 'desc', 'question'], + output_column='answer', + train_split='train', + test_split='test' +) + +role_generalization_eng_infer_cfg = dict( + prompt_template=dict( + type=PromptTemplate, + template=dict( + begin=[ + dict(role='SYSTEM', fallback_role='HUMAN', prompt='You are {role}, your description is: {desc}. Now please answer some questions to accurately show your personality traits! Your speaking style should fully imitate the personality role assigned to you! Please do not expose that you are an artificial intelligence model or a language model, you must always remember that you are only assigned one personality role. Don’t be verbose or too formal or polite when speaking.'), + ], + round=[ + dict(role='HUMAN', prompt='{question}'), + dict(role='BOT', prompt=''), + ], )), + retriever=dict(type=ZeroRetriever), + inferencer=dict(type=GenInferencer, max_out_len=512) +) + +role_generalization_eng_eval_cfg = dict( + evaluator=dict(type=RougeEvaluator), + pred_role='BOT' +) + +role_generalization_eng_datasets = [ + dict( + type=RoleGeneralizationEnglishDataset, + path='ZenMoore/RoleBench', + reader_cfg=role_generalization_eng_reader_cfg, + infer_cfg=role_generalization_eng_infer_cfg, + eval_cfg=role_generalization_eng_eval_cfg) +] diff --git a/opencompass/datasets/rolebench.py b/opencompass/datasets/rolebench.py new file mode 100644 index 000000000..22e772206 --- /dev/null +++ b/opencompass/datasets/rolebench.py @@ -0,0 +1,84 @@ +import json +import os + +from datasets import Dataset, DatasetDict + +from opencompass.registry import LOAD_DATASET + +from .base import BaseDataset + + +@LOAD_DATASET.register_module() +class RoleBenchBaseDataset(BaseDataset): + + @staticmethod + def load_single(source_file, desc_list): + with open(source_file, 'r', encoding='utf-8') as f: + source_data = [json.loads(line) for line in f.readlines()] + dataset = [{ + 'role': item['role'], + 'desc': desc_list[item['role']], + 'question': item['question'], + 'answer': item['generated'][0] + } for item in source_data] + return dataset + + @staticmethod + def load_desc(path): + with open(path, 'r', encoding='utf-8') as f: + desc_list = json.load(f) + return desc_list + + @staticmethod + def load_dataset(path, desc_list): + train_data_list = RoleBenchBaseDataset.load_single( + os.path.join(path, 'general/train.jsonl'), desc_list) + train_data_list.extend( + RoleBenchBaseDataset.load_single( + os.path.join(path, 'role_specific/train.jsonl'), desc_list)) + test_dataset = RoleBenchBaseDataset.load_single( + os.path.join(path, 'general/test.jsonl'), desc_list) + test_dataset.extend( + RoleBenchBaseDataset.load_single( + os.path.join(path, 'role_specific/test.jsonl'), desc_list)) + return Dataset.from_list(train_data_list).shuffle( + seed=42), Dataset.from_list(test_dataset).shuffle(seed=42) + + +@LOAD_DATASET.register_module() +class InstructionGeneralizationEnglishDataset(RoleBenchBaseDataset): + + @staticmethod + def load(path): + desc_list = RoleBenchBaseDataset.load_desc( + os.path.join(path, 'profiles-eng/desc.json')) + path = os.path.join(path, 'rolebench-eng/instruction-generalization') + train_dataset, test_dataset = RoleBenchBaseDataset.load_dataset( + path, desc_list) + return DatasetDict({'train': train_dataset, 'test': test_dataset}) + + +@LOAD_DATASET.register_module() +class RoleGeneralizationEnglishDataset(RoleBenchBaseDataset): + + @staticmethod + def load(path): + desc_list = RoleBenchBaseDataset.load_desc( + os.path.join(path, 'profiles-eng/desc.json')) + path = os.path.join(path, 'rolebench-eng/role-generalization') + train_dataset, test_dataset = RoleBenchBaseDataset.load_dataset( + path, desc_list) + return DatasetDict({'train': train_dataset, 'test': test_dataset}) + + +@LOAD_DATASET.register_module() +class InstructionGeneralizationChineseDataset(RoleBenchBaseDataset): + + @staticmethod + def load(path): + desc_list = RoleBenchBaseDataset.load_desc( + os.path.join(path, 'profiles-zh/desc.json')) + path = os.path.join(path, 'rolebench-zh') + train_dataset, test_dataset = RoleBenchBaseDataset.load_dataset( + path, desc_list) + return DatasetDict({'train': train_dataset, 'test': test_dataset})