[Fix] Quick fix (#995)

open-compass · Mar 22, 2024 · 0665bb9 · 0665bb9
1 parent 1d31985
commit 0665bb9
Show file tree

Hide file tree

Showing 6 changed files with 7 additions and 178 deletions.
diff --git a/configs/eval_subjective_alignbench.py b/configs/eval_subjective_alignbench.py
@@ -3,7 +3,7 @@
 with read_base():
     from .datasets.subjective.alignbench.alignbench_judgeby_critiquellm import subjective_datasets
 
-from opencompass.models import HuggingFaceCausalLM, HuggingFace, HuggingFaceChatGLM3
+from opencompass.models import HuggingFaceCausalLM, HuggingFace, HuggingFaceChatGLM3, OpenAI
 from opencompass.models.openai_api import OpenAIAllesAPIN
 from opencompass.partitioners import NaivePartitioner, SizePartitioner
 from opencompass.partitioners.sub_naive import SubjectiveNaivePartitioner
@@ -51,26 +51,14 @@
 
 datasets = [*subjective_datasets]
 
-infer = dict(
-    partitioner=dict(type=NaivePartitioner),
-    runner=dict(
-        type=SlurmSequentialRunner,
-        partition='llmeval',
-        quotatype='auto',
-        max_num_workers=256,
-        task=dict(type=OpenICLInferTask),
-    ),
-)
-
 # -------------Evalation Stage ----------------------------------------
 
 ## ------------- JudgeLLM Configuration
 judge_model = dict(
     abbr='GPT4-Turbo',
-    type=OpenAIAllesAPIN,
+    type=OpenAI,
     path='gpt-4-1106-preview',
     key='xxxx',  # The key will be obtained from $OPENAI_API_KEY, but you can write down your key here as well
-    url='xxxx',
     meta_template=api_meta_template,
     query_per_second=16,
     max_out_len=2048,

diff --git a/configs/eval_subjective_alpacaeval.py b/configs/eval_subjective_alpacaeval.py
@@ -68,16 +68,7 @@
     temperature=1,
 )  # Re-inference gpt4's predictions or you can choose to use the pre-commited gpt4's predictions
 
-infer = dict(
-    partitioner=dict(type=NaivePartitioner),
-    runner=dict(
-        type=SlurmSequentialRunner,
-        partition='llmeval',
-        quotatype='auto',
-        max_num_workers=256,
-        task=dict(type=OpenICLInferTask),
-    ),
-)
+
 
 # -------------Evalation Stage ----------------------------------------
 

diff --git a/configs/eval_subjective_compassarena.py b/configs/eval_subjective_compassarena.py
@@ -69,17 +69,6 @@
     temperature=1,
 )  # Re-inference gpt4's predictions or you can choose to use the pre-commited gpt4's predictions
 
-infer = dict(
-    partitioner=dict(type=SizePartitioner, strategy='split', max_task_size=10000),
-    runner=dict(
-        type=SlurmSequentialRunner,
-        partition='llm_dev2',
-        quotatype='auto',
-        max_num_workers=256,
-        task=dict(type=OpenICLInferTask),
-    ),
-)
-
 # -------------Evalation Stage ----------------------------------------
 
 ## ------------- JudgeLLM Configuration

diff --git a/configs/eval_subjective_corev2.py b/configs/eval_subjective_corev2.py
diff --git a/configs/eval_subjective_creationbench.py b/configs/eval_subjective_creationbench.py
@@ -3,7 +3,7 @@
 with read_base():
     from .datasets.subjective.creationbench.creationbench_judgeby_gpt4_withref import subjective_datasets
 
-from opencompass.models import HuggingFaceCausalLM, HuggingFace, HuggingFaceChatGLM3
+from opencompass.models import HuggingFaceCausalLM, HuggingFace, HuggingFaceChatGLM3, OpenAI
 from opencompass.models.openai_api import OpenAIAllesAPIN
 from opencompass.partitioners import NaivePartitioner, SizePartitioner
 from opencompass.partitioners.sub_naive import SubjectiveNaivePartitioner
@@ -51,26 +51,14 @@
 
 datasets = [*subjective_datasets]
 
-infer = dict(
-    partitioner=dict(type=NaivePartitioner),
-    runner=dict(
-        type=SlurmSequentialRunner,
-        partition='llmeval',
-        quotatype='auto',
-        max_num_workers=256,
-        task=dict(type=OpenICLInferTask),
-    ),
-)
-
 # -------------Evalation Stage ----------------------------------------
 
 ## ------------- JudgeLLM Configuration
 judge_model = dict(
     abbr='GPT4-Turbo',
-    type=OpenAIAllesAPIN,
+    type=OpenAI,
     path='gpt-4-1106-preview',
     key='xxxx',  # The key will be obtained from $OPENAI_API_KEY, but you can write down your key here as well
-    url='xxxx',
     meta_template=api_meta_template,
     query_per_second=16,
     max_out_len=2048,

diff --git a/configs/eval_subjective_mtbench.py b/configs/eval_subjective_mtbench.py
@@ -4,7 +4,7 @@
     from .datasets.subjective.multiround.mtbench_single_judge_diff_temp import subjective_datasets
     # from .datasets.subjective.multiround.mtbench_pair_judge import subjective_datasets
 
-from opencompass.models import HuggingFaceCausalLM, HuggingFace, HuggingFaceChatGLM3
+from opencompass.models import HuggingFaceCausalLM, HuggingFace, HuggingFaceChatGLM3, OpenAI
 from opencompass.models.openai_api import OpenAIAllesAPIN
 from opencompass.partitioners import NaivePartitioner, SizePartitioner
 from opencompass.partitioners.sub_naive import SubjectiveNaivePartitioner
@@ -59,26 +59,14 @@
 
 datasets = [*subjective_datasets]
 
-infer = dict(
-    partitioner=dict(type=SizePartitioner, strategy='split', max_task_size=10000),
-    runner=dict(
-        type=SlurmSequentialRunner,
-        partition='llm_dev2',
-        quotatype='auto',
-        max_num_workers=256,
-        task=dict(type=OpenICLInferTask),
-    ),
-)
-
 # -------------Evalation Stage ----------------------------------------
 
 ## ------------- JudgeLLM Configuration
 judge_model = dict(
     abbr='GPT4-Turbo',
-    type=OpenAIAllesAPIN,
+    type=OpenAI,
     path='gpt-4-0613', # To compare with the official leaderboard, please use gpt4-0613
     key='xxxx',  # The key will be obtained from $OPENAI_API_KEY, but you can write down your key here as well
-    url='xxxx',
     meta_template=api_meta_template,
     query_per_second=16,
     max_out_len=2048,