-
Notifications
You must be signed in to change notification settings - Fork 405
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
* added rolebench * 修改了不合理的变量名 * 修改了评论中的变量名
- Loading branch information
Showing
4 changed files
with
207 additions
and
0 deletions.
There are no files selected for viewing
41 changes: 41 additions & 0 deletions
41
configs/datasets/rolebench/instruction_generalization_eng.py
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,41 @@ | ||
from opencompass.openicl.icl_prompt_template import PromptTemplate | ||
from opencompass.openicl.icl_retriever import ZeroRetriever | ||
from opencompass.openicl.icl_inferencer import GenInferencer | ||
from opencompass.openicl.icl_evaluator import RougeEvaluator | ||
from opencompass.datasets.rolebench import InstructionGeneralizationEnglishDataset | ||
|
||
instruction_generalization_eng_reader_cfg = dict( | ||
input_columns=['role', 'desc', 'question'], | ||
output_column='answer', | ||
train_split='train', | ||
test_split='test' | ||
) | ||
|
||
instruction_generalization_eng_infer_cfg = dict( | ||
prompt_template=dict( | ||
type=PromptTemplate, | ||
template=dict( | ||
begin=[ | ||
dict(role='SYSTEM', fallback_role='HUMAN', prompt='You are {role}, your description is: {desc}. Now please answer some questions to accurately show your personality traits! Your speaking style should fully imitate the personality role assigned to you! Please do not expose that you are an artificial intelligence model or a language model, you must always remember that you are only assigned one personality role. Don’t be verbose or too formal or polite when speaking.'), | ||
], | ||
round=[ | ||
dict(role='HUMAN', prompt='{question}'), | ||
dict(role='BOT', prompt=''), | ||
], )), | ||
retriever=dict(type=ZeroRetriever), | ||
inferencer=dict(type=GenInferencer, max_out_len=512) | ||
) | ||
|
||
instruction_generalization_eng_eval_cfg = dict( | ||
evaluator=dict(type=RougeEvaluator), | ||
pred_role='BOT' | ||
) | ||
|
||
instruction_generalization_eng_datasets = [ | ||
dict( | ||
type=InstructionGeneralizationEnglishDataset, | ||
path='ZenMoore/RoleBench', | ||
reader_cfg=instruction_generalization_eng_reader_cfg, | ||
infer_cfg=instruction_generalization_eng_infer_cfg, | ||
eval_cfg=instruction_generalization_eng_eval_cfg) | ||
] |
41 changes: 41 additions & 0 deletions
41
configs/datasets/rolebench/instruction_generalization_zh.py
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,41 @@ | ||
from opencompass.openicl.icl_prompt_template import PromptTemplate | ||
from opencompass.openicl.icl_retriever import ZeroRetriever | ||
from opencompass.openicl.icl_inferencer import GenInferencer | ||
from opencompass.openicl.icl_evaluator import RougeEvaluator | ||
from opencompass.datasets.rolebench import InstructionGeneralizationChineseDataset | ||
|
||
instruction_generalization_zh_reader_cfg = dict( | ||
input_columns=['role', 'desc', 'question'], | ||
output_column='answer', | ||
train_split='train', | ||
test_split='test' | ||
) | ||
|
||
instruction_generalization_zh_infer_cfg = dict( | ||
prompt_template=dict( | ||
type=PromptTemplate, | ||
template=dict( | ||
begin=[ | ||
dict(role='SYSTEM', fallback_role='HUMAN', prompt='你是{role},你的特征描述是:{desc}。现在请你回答我的一些问题,以准确展现你的人格特征!你的说话风格要全面模仿被赋予的人格角色!请不要暴露你是人工智能模型或者语言模型,你要时刻记住你只被赋予的一个人格角色。说话不要嗦,也不要太过于正式或礼貌。'), | ||
], | ||
round=[ | ||
dict(role='HUMAN', prompt='{question}'), | ||
dict(role='BOT', prompt=''), | ||
], )), | ||
retriever=dict(type=ZeroRetriever), | ||
inferencer=dict(type=GenInferencer, max_out_len=512) | ||
) | ||
|
||
instruction_generalization_zh_eval_cfg = dict( | ||
evaluator=dict(type=RougeEvaluator), | ||
pred_role='BOT' | ||
) | ||
|
||
instruction_generalization_zh_datasets = [ | ||
dict( | ||
type=InstructionGeneralizationChineseDataset, | ||
path='ZenMoore/RoleBench', | ||
reader_cfg=instruction_generalization_zh_reader_cfg, | ||
infer_cfg=instruction_generalization_zh_infer_cfg, | ||
eval_cfg=instruction_generalization_zh_eval_cfg) | ||
] |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,41 @@ | ||
from opencompass.openicl.icl_prompt_template import PromptTemplate | ||
from opencompass.openicl.icl_retriever import ZeroRetriever | ||
from opencompass.openicl.icl_inferencer import GenInferencer | ||
from opencompass.openicl.icl_evaluator import RougeEvaluator | ||
from opencompass.datasets.rolebench import RoleGeneralizationEnglishDataset | ||
|
||
role_generalization_eng_reader_cfg = dict( | ||
input_columns=['role', 'desc', 'question'], | ||
output_column='answer', | ||
train_split='train', | ||
test_split='test' | ||
) | ||
|
||
role_generalization_eng_infer_cfg = dict( | ||
prompt_template=dict( | ||
type=PromptTemplate, | ||
template=dict( | ||
begin=[ | ||
dict(role='SYSTEM', fallback_role='HUMAN', prompt='You are {role}, your description is: {desc}. Now please answer some questions to accurately show your personality traits! Your speaking style should fully imitate the personality role assigned to you! Please do not expose that you are an artificial intelligence model or a language model, you must always remember that you are only assigned one personality role. Don’t be verbose or too formal or polite when speaking.'), | ||
], | ||
round=[ | ||
dict(role='HUMAN', prompt='{question}'), | ||
dict(role='BOT', prompt=''), | ||
], )), | ||
retriever=dict(type=ZeroRetriever), | ||
inferencer=dict(type=GenInferencer, max_out_len=512) | ||
) | ||
|
||
role_generalization_eng_eval_cfg = dict( | ||
evaluator=dict(type=RougeEvaluator), | ||
pred_role='BOT' | ||
) | ||
|
||
role_generalization_eng_datasets = [ | ||
dict( | ||
type=RoleGeneralizationEnglishDataset, | ||
path='ZenMoore/RoleBench', | ||
reader_cfg=role_generalization_eng_reader_cfg, | ||
infer_cfg=role_generalization_eng_infer_cfg, | ||
eval_cfg=role_generalization_eng_eval_cfg) | ||
] |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,84 @@ | ||
import json | ||
import os | ||
|
||
from datasets import Dataset, DatasetDict | ||
|
||
from opencompass.registry import LOAD_DATASET | ||
|
||
from .base import BaseDataset | ||
|
||
|
||
@LOAD_DATASET.register_module() | ||
class RoleBenchBaseDataset(BaseDataset): | ||
|
||
@staticmethod | ||
def load_single(source_file, desc_list): | ||
with open(source_file, 'r', encoding='utf-8') as f: | ||
source_data = [json.loads(line) for line in f.readlines()] | ||
dataset = [{ | ||
'role': item['role'], | ||
'desc': desc_list[item['role']], | ||
'question': item['question'], | ||
'answer': item['generated'][0] | ||
} for item in source_data] | ||
return dataset | ||
|
||
@staticmethod | ||
def load_desc(path): | ||
with open(path, 'r', encoding='utf-8') as f: | ||
desc_list = json.load(f) | ||
return desc_list | ||
|
||
@staticmethod | ||
def load_dataset(path, desc_list): | ||
train_data_list = RoleBenchBaseDataset.load_single( | ||
os.path.join(path, 'general/train.jsonl'), desc_list) | ||
train_data_list.extend( | ||
RoleBenchBaseDataset.load_single( | ||
os.path.join(path, 'role_specific/train.jsonl'), desc_list)) | ||
test_dataset = RoleBenchBaseDataset.load_single( | ||
os.path.join(path, 'general/test.jsonl'), desc_list) | ||
test_dataset.extend( | ||
RoleBenchBaseDataset.load_single( | ||
os.path.join(path, 'role_specific/test.jsonl'), desc_list)) | ||
return Dataset.from_list(train_data_list).shuffle( | ||
seed=42), Dataset.from_list(test_dataset).shuffle(seed=42) | ||
|
||
|
||
@LOAD_DATASET.register_module() | ||
class InstructionGeneralizationEnglishDataset(RoleBenchBaseDataset): | ||
|
||
@staticmethod | ||
def load(path): | ||
desc_list = RoleBenchBaseDataset.load_desc( | ||
os.path.join(path, 'profiles-eng/desc.json')) | ||
path = os.path.join(path, 'rolebench-eng/instruction-generalization') | ||
train_dataset, test_dataset = RoleBenchBaseDataset.load_dataset( | ||
path, desc_list) | ||
return DatasetDict({'train': train_dataset, 'test': test_dataset}) | ||
|
||
|
||
@LOAD_DATASET.register_module() | ||
class RoleGeneralizationEnglishDataset(RoleBenchBaseDataset): | ||
|
||
@staticmethod | ||
def load(path): | ||
desc_list = RoleBenchBaseDataset.load_desc( | ||
os.path.join(path, 'profiles-eng/desc.json')) | ||
path = os.path.join(path, 'rolebench-eng/role-generalization') | ||
train_dataset, test_dataset = RoleBenchBaseDataset.load_dataset( | ||
path, desc_list) | ||
return DatasetDict({'train': train_dataset, 'test': test_dataset}) | ||
|
||
|
||
@LOAD_DATASET.register_module() | ||
class InstructionGeneralizationChineseDataset(RoleBenchBaseDataset): | ||
|
||
@staticmethod | ||
def load(path): | ||
desc_list = RoleBenchBaseDataset.load_desc( | ||
os.path.join(path, 'profiles-zh/desc.json')) | ||
path = os.path.join(path, 'rolebench-zh') | ||
train_dataset, test_dataset = RoleBenchBaseDataset.load_dataset( | ||
path, desc_list) | ||
return DatasetDict({'train': train_dataset, 'test': test_dataset}) |