[Fix] Fix ruler_16k_gen (#1643)

open-compass · Oct 29, 2024 · 46affab · 46affab
1 parent 8172af4
commit 46affab
Show file tree

Hide file tree

Showing 3 changed files with 18 additions and 17 deletions.
diff --git a/configs/datasets/ruler/ruler_16k_gen.py b/configs/datasets/ruler/ruler_16k_gen.py
@@ -21,7 +21,7 @@
 
 # Different seq length
 for max_seq_len, abbr_suffix in zip(max_seq_lens, abbr_suffixs):
-    for dataset in import_datasets:
+    for dataset in import_ds:
         tmp_dataset = dataset.deepcopy()
         tmp_dataset['abbr'] = tmp_dataset['abbr'] + '_' + abbr_suffix
         tmp_dataset['num_samples'] = NUM_SAMPLES

diff --git a/configs/eval_ruler.py b/configs/eval_ruler.py
@@ -1,29 +1,32 @@
+from mmengine.config import read_base
+
 from opencompass.partitioners import (
     NaivePartitioner,
     NumWorkerPartitioner,
 )
-from mmengine.config import read_base
 from opencompass.runners import LocalRunner
-from opencompass.tasks import OpenICLInferTask, OpenICLEvalTask
+from opencompass.tasks import OpenICLEvalTask, OpenICLInferTask
 
 with read_base():
-    from opencompass.configs.models.qwen.lmdeploy_qwen2_7b_instruct import (
-        models as qwen2_7b_instruct_model,
+    from opencompass.configs.datasets.ruler.ruler_cwe_gen import cwe_datasets  # CWE
+    from opencompass.configs.datasets.ruler.ruler_fwe_gen import fwe_datasets  # FWE
+    from opencompass.configs.datasets.ruler.ruler_niah_gen import niah_datasets  # Niah
+    from opencompass.configs.datasets.ruler.ruler_qa_gen import qa_datasets  # QA
+    from opencompass.configs.datasets.ruler.ruler_vt_gen import vt_datasets  # VT
+    from opencompass.configs.models.hf_internlm.lmdeploy_internlm2_5_7b_chat_1m import (
+        models as internlm2_5_7b_chat_1m,
     )
     from opencompass.configs.models.hf_llama.lmdeploy_llama3_8b_instruct import (
         models as llama3_8b_instruct_model,
     )
-    from opencompass.configs.models.hf_internlm.lmdeploy_internlm2_5_7b_chat_1m import (
-        models as internlm2_5_7b_chat_1m,
+    from opencompass.configs.models.qwen.lmdeploy_qwen2_7b_instruct import (
+        models as qwen2_7b_instruct_model,
     )
-    from opencompass.configs.datasets.ruler.ruler_niah_gen import niah_datasets  # Niah
-    from opencompass.configs.datasets.ruler.ruler_vt_gen import vt_datasets  # VT
-    from opencompass.configs.datasets.ruler.ruler_fwe_gen import fwe_datasets  # FWE
-    from opencompass.configs.datasets.ruler.ruler_cwe_gen import cwe_datasets  # CWE
-    from opencompass.configs.datasets.ruler.ruler_qa_gen import qa_datasets  # QA
     from opencompass.configs.summarizers.groups.ruler import ruler_summary_groups
 
-import_datasets = sum((v for k, v in locals().items() if k.endswith('_datasets')), [])
+import_datasets = sum(
+    [niah_datasets, vt_datasets, fwe_datasets, cwe_datasets, qa_datasets], []
+)
 
 # Evaluation config
 NUM_SAMPLES = 500
@@ -84,9 +87,7 @@
 
 summarizer = dict(
     dataset_abbrs=abbr_suffixs,
-    summary_groups=sum(
-        [v for k, v in locals().items() if k.endswith('_summary_groups')], []
-    ),
+    summary_groups=sum([ruler_summary_groups], []),
 )
 
 

diff --git a/opencompass/configs/datasets/ruler/ruler_16k_gen.py b/opencompass/configs/datasets/ruler/ruler_16k_gen.py
@@ -21,7 +21,7 @@
 
 # Different seq length
 for max_seq_len, abbr_suffix in zip(max_seq_lens, abbr_suffixs):
-    for dataset in import_datasets:
+    for dataset in import_ds:
         tmp_dataset = dataset.deepcopy()
         tmp_dataset['abbr'] = tmp_dataset['abbr'] + '_' + abbr_suffix
         tmp_dataset['num_samples'] = NUM_SAMPLES