From 9740bbabd3f1bd506777d086387e93c83aef7850 Mon Sep 17 00:00:00 2001 From: yingfhu Date: Wed, 15 Nov 2023 21:29:02 +0800 Subject: [PATCH] [Feat] support pass@k --- configs/eval_code_passk.py | 27 +++++---------------- configs/eval_code_passk_repeat_dataset.py | 29 ++++++----------------- 2 files changed, 13 insertions(+), 43 deletions(-) diff --git a/configs/eval_code_passk.py b/configs/eval_code_passk.py index a4a4d7266..41fd20465 100644 --- a/configs/eval_code_passk.py +++ b/configs/eval_code_passk.py @@ -3,9 +3,9 @@ from mmengine.config import read_base from opencompass.partitioners import SizePartitioner from opencompass.models import HuggingFaceCausalLM -from opencompass.runners import SlurmSequentialRunner -from opencompass.partitioners import SizePartitioner, NaivePartitioner -from opencompass.tasks import OpenICLInferTask, OpenICLEvalTask +from opencompass.runners import LocalRunner +from opencompass.partitioners import SizePartitioner +from opencompass.tasks import OpenICLInferTask from opencompass.datasets import MBPP_V2Dataset, MBPPPassKEvaluator with read_base(): @@ -49,21 +49,6 @@ infer = dict( partitioner=dict(type=SizePartitioner, max_task_size=300), runner=dict( - type=SlurmSequentialRunner, - max_num_workers=256, - partition="llmit", - quotatype="reserved", - task=dict(type=OpenICLInferTask), - ), -) - -eval = dict( - partitioner=dict(type=NaivePartitioner, n=10), - runner=dict( - type=SlurmSequentialRunner, - max_num_workers=256, - partition="llmit", - quotatype="reserved", - task=dict(type=OpenICLEvalTask), - ), -) + type=LocalRunner, max_num_workers=16, + task=dict(type=OpenICLInferTask)), +) \ No newline at end of file diff --git a/configs/eval_code_passk_repeat_dataset.py b/configs/eval_code_passk_repeat_dataset.py index c424f39e7..0950186da 100644 --- a/configs/eval_code_passk_repeat_dataset.py +++ b/configs/eval_code_passk_repeat_dataset.py @@ -3,9 +3,9 @@ from mmengine.config import read_base from opencompass.partitioners import SizePartitioner from opencompass.models import HuggingFaceCausalLM -from opencompass.runners import SlurmSequentialRunner -from opencompass.partitioners import SizePartitioner, NaivePartitioner -from opencompass.tasks import OpenICLInferTask, OpenICLEvalTask +from opencompass.runners import LocalRunner +from opencompass.partitioners import SizePartitioner +from opencompass.tasks import OpenICLInferTask from opencompass.datasets import MBPP_V2Dataset, MBPPPassKEvaluator with read_base(): @@ -58,23 +58,8 @@ infer = dict( - partitioner=dict(type=SizePartitioner, max_task_size=6000), + partitioner=dict(type=SizePartitioner, max_task_size=600), runner=dict( - type=SlurmSequentialRunner, - max_num_workers=256, - partition="llmit", - quotatype="reserved", - task=dict(type=OpenICLInferTask), - ), -) - -eval = dict( - partitioner=dict(type=NaivePartitioner, n=1), - runner=dict( - type=SlurmSequentialRunner, - max_num_workers=256, - partition="llmit", - quotatype="reserved", - task=dict(type=OpenICLEvalTask), - ), -) + type=LocalRunner, max_num_workers=16, + task=dict(type=OpenICLInferTask)), +) \ No newline at end of file