-
Notifications
You must be signed in to change notification settings - Fork 435
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
[Feature] Support chat style inferencer. (#643)
* [Feature] Support chat style inferencer. * [Fix] use new prompt * [Fix] use new prompt --------- Co-authored-by: yingfhu <[email protected]>
- Loading branch information
Showing
12 changed files
with
782 additions
and
468 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,55 @@ | ||
from opencompass.openicl.icl_prompt_template import PromptTemplate | ||
from opencompass.openicl.icl_retriever import ZeroRetriever | ||
from opencompass.openicl.icl_inferencer import AgentInferencer | ||
from opencompass.datasets import ( | ||
GSM8KDataset, | ||
gsm8k_postprocess, | ||
gsm8k_dataset_postprocess, | ||
Gsm8kAgentEvaluator, | ||
) | ||
|
||
gsm8k_reader_cfg = dict(input_columns=["question"], output_column="answer") | ||
|
||
gsm8k_infer_cfg = dict( | ||
prompt_template=dict( | ||
type=PromptTemplate, | ||
template=dict( | ||
round=[ | ||
# # ################################### NEW SHOT ################################### | ||
dict(role='HUMAN', prompt='Mark\'s basketball team scores 25 2 pointers, 8 3 pointers and 10 free throws. Their opponents score double the 2 pointers but half the 3 pointers and free throws. What\'s the total number of points scored by both teams added together?'), | ||
dict(role='BOT', prompt='Tool:PythonInterpreter\nTool Input:def solution():\n mark_pointers_2 = 25 * 2\n mark_pointers_3 = 8 * 3\n mark_free_throws = 10 * 1\n mark_points_scored = mark_pointers_2 + mark_pointers_3 + mark_free_throws\n opponents_pointers_2 = mark_pointers_2 * 2\n opponents_pointers_3 = mark_pointers_3 / 2\n opponents_free_throws = mark_free_throws / 2\n opponents_points_scored = opponents_pointers_2 + opponents_pointers_3 + opponents_free_throws\n total_points_scored = mark_points_scored + opponents_points_scored\n result = total_points_scored\n return result'), | ||
dict(role='SYSTEM', prompt='Response:210'), | ||
dict(role='BOT', prompt='Thought: According to the response, I got the answer\nFinalAnswer: 210'), | ||
|
||
dict(role='HUMAN', prompt='Bella has two times as many marbles as frisbees. She also has 20 more frisbees than deck cards. If she buys 2/5 times more of each item, what would be the total number of the items she will have if she currently has 60 marbles?'), | ||
dict(role='BOT', prompt='Tool:PythonInterpreter\nTool Input:def solution():\n marbles = 60\n num_increased_marbles = marbles * 2 / 5\n num_total_marbles = marbles + num_increased_marbles\n frisbees = marbles / 2\n num_increased_frisbees = frisbees * 2 / 5\n num_total_frisbees = frisbees + num_increased_frisbees\n deck_cards = frisbees - 20\n num_increased_deck_cards = deck_cards * 2 / 5\n num_total_deck_cards = deck_cards + num_increased_deck_cards\n num_total = num_total_marbles + num_total_frisbees + num_total_deck_cards\n result = num_total\n return result'), | ||
dict(role='SYSTEM', prompt='Response:140'), | ||
dict(role='BOT', prompt='Thought: According to the response, I got the answer\nFinalAnswer: 140'), | ||
|
||
dict(role='HUMAN', prompt='A group of 4 fruit baskets contains 9 apples, 15 oranges, and 14 bananas in the first three baskets and 2 less of each fruit in the fourth basket. How many fruits are there?'), | ||
dict(role='BOT', prompt="""Tool:PythonInterpreter\nTool Input:def solution():\n num_fruits_per_first_three_basket = 9 + 15 + 14\n num_fruits_first_three_basket = num_fruits_per_first_three_basket * 3\n num_apple_fourth_basket = 9 - 2\n num_orange_fourth_basket = 15 - 2\n num_banana_fourth_basket = 14 - 2\n num_fruits_fourth_basket = num_apple_fourth_basket + num_orange_fourth_basket + num_banana_fourth_basket\n num_fruits_total = num_fruits_first_three_basket + num_fruits_fourth_basket\n result = num_fruits_total\n return result"""), | ||
dict(role='SYSTEM', prompt='Response:146'), | ||
dict(role='BOT', prompt='Thought: According to the response, I got the answer\nFinalAnswer: 146'), | ||
|
||
dict(role='HUMAN', prompt='{question}'), | ||
])), | ||
retriever=dict(type=ZeroRetriever), | ||
inferencer=dict(type=AgentInferencer), | ||
) | ||
|
||
gsm8k_eval_cfg = dict( | ||
evaluator=dict(type=Gsm8kAgentEvaluator), | ||
pred_postprocessor=dict(type=gsm8k_postprocess), | ||
dataset_postprocessor=dict(type=gsm8k_dataset_postprocess), | ||
) | ||
|
||
gsm8k_datasets = [ | ||
dict( | ||
abbr='gsm8k', | ||
type=GSM8KDataset, | ||
path='./data/gsm8k', | ||
reader_cfg=gsm8k_reader_cfg, | ||
infer_cfg=gsm8k_infer_cfg, | ||
eval_cfg=gsm8k_eval_cfg, | ||
) | ||
] |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,55 @@ | ||
from mmengine.config import read_base | ||
from opencompass.models.openai_api import OpenAI | ||
from opencompass.partitioners import SizePartitioner | ||
from opencompass.runners import LocalRunner | ||
from opencompass.tasks import OpenICLInferTask | ||
from opencompass.models.lagent import LagentAgent | ||
from lagent import PythonInterpreter, ReAct | ||
from lagent.agents.react import ReActProtocol | ||
|
||
with read_base(): | ||
from .datasets.gsm8k.gsm8k_agent_gen_3ac57d import gsm8k_datasets as datasets | ||
|
||
system_prompt = """You are a helpful assistant which use tools to solve mathematical reasoning questions. The code must be a function, and the function name must be 'solution'. For mathematics, please use code tool to calculate. The example format is as follows: | ||
``` | ||
def solution(): | ||
variable_names_with_real_meaning = func(variable) | ||
return variable_names_with_real_meaning | ||
```""" | ||
|
||
protocol = dict( | ||
type=ReActProtocol, | ||
action=dict(role="ACTION", begin="Tool:", end="\n"), | ||
action_input=dict(role="ARGS", begin="Tool Input:", end="\n"), | ||
finish=dict(role="FINISH", begin="FinalAnswer:", end="\n"), | ||
call_protocol=system_prompt, | ||
) | ||
|
||
models = [ | ||
dict( | ||
abbr='gpt-3.5-react', | ||
type=LagentAgent, | ||
agent_type=ReAct, | ||
max_turn=3, | ||
llm=dict( | ||
type=OpenAI, | ||
path='gpt-3.5-turbo', | ||
key='ENV', | ||
query_per_second=1, | ||
max_seq_len=4096, | ||
), | ||
actions=[ | ||
dict(type=PythonInterpreter), | ||
], | ||
protocol=protocol, | ||
batch_size=1, | ||
), | ||
] | ||
|
||
infer = dict( | ||
partitioner=dict(type=SizePartitioner, max_task_size=1000), | ||
runner=dict( | ||
type=LocalRunner, | ||
max_num_workers=16, | ||
task=dict(type=OpenICLInferTask)), | ||
) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,82 @@ | ||
from lagent.agents.react import ReActProtocol | ||
from mmengine.config import read_base | ||
|
||
from opencompass.lagent.actions.ipython_interpreter import IPythonInterpreter | ||
from opencompass.lagent.agents.react import CIReAct | ||
from opencompass.models.lagent import CodeAgent | ||
from opencompass.models.openai_api import OpenAI | ||
from opencompass.partitioners import SizePartitioner | ||
from opencompass.runners import LocalRunner | ||
from opencompass.tasks import OpenICLInferTask | ||
|
||
with read_base(): | ||
from .datasets.CIBench.CIBench_gen_eb42f9 import \ | ||
cibench_datasets as datasets | ||
|
||
FORCE_STOP_PROMPT_EN = """You should directly give results based on history information.""" | ||
|
||
FEWSHOT_INSTRUCTION = """\ | ||
You are an assistant who can utilize external tools. | ||
{tool_description} | ||
To use a tool, please response with the following format: | ||
``` | ||
{thought} Think what you need to solve, do you need to use tools? | ||
{action} The tool name, should be one of [{action_names}]. | ||
{action_input} The input to the tool that you want to use. | ||
``` | ||
The tool will give you response after your response using the following format: | ||
``` | ||
{response} the results after call the tool. | ||
``` | ||
Therefore DO NOT generate tool response by yourself. | ||
Also please follow the guidelines: | ||
1. Always use code interpreter to solve the problem. | ||
2. The generated codes should always in a markdown code block format. | ||
3. The generated codes will be executed in an ipython manner and the results will be cached. | ||
4. Your responded code should always be simple and only solves the problem in current step. | ||
Begin! | ||
""" | ||
|
||
IPYTHON_INTERPRETER_DESCRIPTION = '''\ | ||
It can run Python code in a manner as jupyter notebook. The code must be a valid code that contains only python method.''' | ||
|
||
models = [ | ||
dict( | ||
abbr='gpt-3.5-code', | ||
type=CodeAgent, | ||
agent_type=CIReAct, | ||
max_turn=3, | ||
llm=dict( | ||
type=OpenAI, | ||
path='gpt-3.5-turbo', | ||
key='ENV', | ||
query_per_second=1, | ||
max_seq_len=4096, | ||
), | ||
actions=[ | ||
dict(type=IPythonInterpreter, | ||
description=IPYTHON_INTERPRETER_DESCRIPTION) | ||
], | ||
protocol=dict( | ||
type=ReActProtocol, | ||
call_protocol=FEWSHOT_INSTRUCTION, | ||
force_stop=FORCE_STOP_PROMPT_EN, | ||
finish=dict(role='FINISH', begin='Final Answer:', end='\n'), | ||
), | ||
batch_size=1, | ||
), | ||
] | ||
|
||
for dataset in datasets: | ||
# Evaluate on every assistant response | ||
dataset['infer_cfg']['inferencer']['infer_mode'] = 'every' | ||
|
||
infer = dict( | ||
partitioner=dict(type=SizePartitioner, max_task_size=1000), | ||
runner=dict( | ||
type=LocalRunner, | ||
max_num_workers=16, | ||
task=dict(type=OpenICLInferTask)), | ||
) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,35 @@ | ||
from mmengine.config import read_base | ||
|
||
from opencompass.models.openai_api import OpenAI | ||
from opencompass.openicl import ChatInferencer | ||
from opencompass.partitioners import SizePartitioner | ||
from opencompass.runners import LocalRunner | ||
from opencompass.tasks import OpenICLInferTask | ||
|
||
with read_base(): | ||
from .datasets.gsm8k.gsm8k_gen_1d7fe4 import gsm8k_datasets as datasets | ||
|
||
models = [ | ||
dict( | ||
abbr='gpt-3.5', | ||
type=OpenAI, | ||
path='gpt-3.5-turbo', | ||
key='ENV', | ||
max_out_len=100, | ||
max_seq_len=2048, | ||
batch_size=16, | ||
run_cfg=dict(num_gpus=1, num_procs=1), | ||
) | ||
] | ||
|
||
for dataset in datasets: | ||
# Use ChatInferencer instead of GenInferencer | ||
dataset['infer_cfg']['inferencer'] = dict(type=ChatInferencer) | ||
|
||
infer = dict( | ||
partitioner=dict(type=SizePartitioner, max_task_size=1000), | ||
runner=dict( | ||
type=LocalRunner, | ||
max_num_workers=16, | ||
task=dict(type=OpenICLInferTask)), | ||
) |
This file was deleted.
Oops, something went wrong.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.