[Feat] support pass@k

open-compass · Nov 15, 2023 · 9740bba · 9740bba
1 parent cc09d1a
commit 9740bba
Show file tree

Hide file tree

Showing 2 changed files with 13 additions and 43 deletions.
diff --git a/configs/eval_code_passk.py b/configs/eval_code_passk.py
@@ -3,9 +3,9 @@
 from mmengine.config import read_base
 from opencompass.partitioners import SizePartitioner
 from opencompass.models import HuggingFaceCausalLM
-from opencompass.runners import SlurmSequentialRunner
-from opencompass.partitioners import SizePartitioner, NaivePartitioner
-from opencompass.tasks import OpenICLInferTask, OpenICLEvalTask
+from opencompass.runners import LocalRunner
+from opencompass.partitioners import SizePartitioner
+from opencompass.tasks import OpenICLInferTask
 from opencompass.datasets import MBPP_V2Dataset, MBPPPassKEvaluator
 
 with read_base():
@@ -49,21 +49,6 @@
 infer = dict(
     partitioner=dict(type=SizePartitioner, max_task_size=300),
     runner=dict(
-        type=SlurmSequentialRunner,
-        max_num_workers=256,
-        partition="llmit",
-        quotatype="reserved",
-        task=dict(type=OpenICLInferTask),
-    ),
-)
-
-eval = dict(
-    partitioner=dict(type=NaivePartitioner, n=10),
-    runner=dict(
-        type=SlurmSequentialRunner,
-        max_num_workers=256,
-        partition="llmit",
-        quotatype="reserved",
-        task=dict(type=OpenICLEvalTask),
-    ),
-)
+        type=LocalRunner, max_num_workers=16,
+        task=dict(type=OpenICLInferTask)),
+)
diff --git a/configs/eval_code_passk_repeat_dataset.py b/configs/eval_code_passk_repeat_dataset.py
@@ -3,9 +3,9 @@
 from mmengine.config import read_base
 from opencompass.partitioners import SizePartitioner
 from opencompass.models import HuggingFaceCausalLM
-from opencompass.runners import SlurmSequentialRunner
-from opencompass.partitioners import SizePartitioner, NaivePartitioner
-from opencompass.tasks import OpenICLInferTask, OpenICLEvalTask
+from opencompass.runners import LocalRunner
+from opencompass.partitioners import SizePartitioner
+from opencompass.tasks import OpenICLInferTask
 from opencompass.datasets import MBPP_V2Dataset, MBPPPassKEvaluator
 
 with read_base():
@@ -58,23 +58,8 @@
 
 
 infer = dict(
-    partitioner=dict(type=SizePartitioner, max_task_size=6000),
+    partitioner=dict(type=SizePartitioner, max_task_size=600),
     runner=dict(
-        type=SlurmSequentialRunner,
-        max_num_workers=256,
-        partition="llmit",
-        quotatype="reserved",
-        task=dict(type=OpenICLInferTask),
-    ),
-)
-
-eval = dict(
-    partitioner=dict(type=NaivePartitioner, n=1),
-    runner=dict(
-        type=SlurmSequentialRunner,
-        max_num_workers=256,
-        partition="llmit",
-        quotatype="reserved",
-        task=dict(type=OpenICLEvalTask),
-    ),
-)
+        type=LocalRunner, max_num_workers=16,
+        task=dict(type=OpenICLInferTask)),
+)