From 46affab882b383b77ced5471690ca1f237a1f09d Mon Sep 17 00:00:00 2001
From: Chang Lan <c.lan@apple.com>
Date: Tue, 29 Oct 2024 02:58:43 -0700
Subject: [PATCH] [Fix] Fix ruler_16k_gen (#1643)

---
 configs/datasets/ruler/ruler_16k_gen.py       |  2 +-
 configs/eval_ruler.py                         | 31 ++++++++++---------
 .../configs/datasets/ruler/ruler_16k_gen.py   |  2 +-
 3 files changed, 18 insertions(+), 17 deletions(-)

diff --git a/configs/datasets/ruler/ruler_16k_gen.py b/configs/datasets/ruler/ruler_16k_gen.py
index 5a8a9c47f..faab3ccab 100644
--- a/configs/datasets/ruler/ruler_16k_gen.py
+++ b/configs/datasets/ruler/ruler_16k_gen.py
@@ -21,7 +21,7 @@
 
 # Different seq length
 for max_seq_len, abbr_suffix in zip(max_seq_lens, abbr_suffixs):
-    for dataset in import_datasets:
+    for dataset in import_ds:
         tmp_dataset = dataset.deepcopy()
         tmp_dataset['abbr'] = tmp_dataset['abbr'] + '_' + abbr_suffix
         tmp_dataset['num_samples'] = NUM_SAMPLES
diff --git a/configs/eval_ruler.py b/configs/eval_ruler.py
index c9b4efb1f..0a32d6efb 100644
--- a/configs/eval_ruler.py
+++ b/configs/eval_ruler.py
@@ -1,29 +1,32 @@
+from mmengine.config import read_base
+
 from opencompass.partitioners import (
     NaivePartitioner,
     NumWorkerPartitioner,
 )
-from mmengine.config import read_base
 from opencompass.runners import LocalRunner
-from opencompass.tasks import OpenICLInferTask, OpenICLEvalTask
+from opencompass.tasks import OpenICLEvalTask, OpenICLInferTask
 
 with read_base():
-    from opencompass.configs.models.qwen.lmdeploy_qwen2_7b_instruct import (
-        models as qwen2_7b_instruct_model,
+    from opencompass.configs.datasets.ruler.ruler_cwe_gen import cwe_datasets  # CWE
+    from opencompass.configs.datasets.ruler.ruler_fwe_gen import fwe_datasets  # FWE
+    from opencompass.configs.datasets.ruler.ruler_niah_gen import niah_datasets  # Niah
+    from opencompass.configs.datasets.ruler.ruler_qa_gen import qa_datasets  # QA
+    from opencompass.configs.datasets.ruler.ruler_vt_gen import vt_datasets  # VT
+    from opencompass.configs.models.hf_internlm.lmdeploy_internlm2_5_7b_chat_1m import (
+        models as internlm2_5_7b_chat_1m,
     )
     from opencompass.configs.models.hf_llama.lmdeploy_llama3_8b_instruct import (
         models as llama3_8b_instruct_model,
     )
-    from opencompass.configs.models.hf_internlm.lmdeploy_internlm2_5_7b_chat_1m import (
-        models as internlm2_5_7b_chat_1m,
+    from opencompass.configs.models.qwen.lmdeploy_qwen2_7b_instruct import (
+        models as qwen2_7b_instruct_model,
     )
-    from opencompass.configs.datasets.ruler.ruler_niah_gen import niah_datasets  # Niah
-    from opencompass.configs.datasets.ruler.ruler_vt_gen import vt_datasets  # VT
-    from opencompass.configs.datasets.ruler.ruler_fwe_gen import fwe_datasets  # FWE
-    from opencompass.configs.datasets.ruler.ruler_cwe_gen import cwe_datasets  # CWE
-    from opencompass.configs.datasets.ruler.ruler_qa_gen import qa_datasets  # QA
     from opencompass.configs.summarizers.groups.ruler import ruler_summary_groups
 
-import_datasets = sum((v for k, v in locals().items() if k.endswith('_datasets')), [])
+import_datasets = sum(
+    [niah_datasets, vt_datasets, fwe_datasets, cwe_datasets, qa_datasets], []
+)
 
 # Evaluation config
 NUM_SAMPLES = 500
@@ -84,9 +87,7 @@
 
 summarizer = dict(
     dataset_abbrs=abbr_suffixs,
-    summary_groups=sum(
-        [v for k, v in locals().items() if k.endswith('_summary_groups')], []
-    ),
+    summary_groups=sum([ruler_summary_groups], []),
 )
 
 
diff --git a/opencompass/configs/datasets/ruler/ruler_16k_gen.py b/opencompass/configs/datasets/ruler/ruler_16k_gen.py
index 5a8a9c47f..faab3ccab 100644
--- a/opencompass/configs/datasets/ruler/ruler_16k_gen.py
+++ b/opencompass/configs/datasets/ruler/ruler_16k_gen.py
@@ -21,7 +21,7 @@
 
 # Different seq length
 for max_seq_len, abbr_suffix in zip(max_seq_lens, abbr_suffixs):
-    for dataset in import_datasets:
+    for dataset in import_ds:
         tmp_dataset = dataset.deepcopy()
         tmp_dataset['abbr'] = tmp_dataset['abbr'] + '_' + abbr_suffix
         tmp_dataset['num_samples'] = NUM_SAMPLES