From 4c87e777d855636b9eda7ec87bcbbf12b62caed3 Mon Sep 17 00:00:00 2001 From: RunningLeon Date: Thu, 1 Feb 2024 22:31:14 +0800 Subject: [PATCH] [Feature] Add end_str for turbomind (#859) * fix * update * fix internlm1 * fix docs * remove sys --- configs/eval_internlm_chat_turbomind.py | 31 ++++++++++++++++++ .../advanced_guides/evaluation_turbomind.md | 1 + .../advanced_guides/evaluation_turbomind.md | 1 + opencompass/models/turbomind.py | 32 ++++++++++++++----- 4 files changed, 57 insertions(+), 8 deletions(-) diff --git a/configs/eval_internlm_chat_turbomind.py b/configs/eval_internlm_chat_turbomind.py index 0118ebc23..7430c6797 100644 --- a/configs/eval_internlm_chat_turbomind.py +++ b/configs/eval_internlm_chat_turbomind.py @@ -23,6 +23,14 @@ ], eos_token_id=103028) +internlm2_meta_template = dict( + round=[ + dict(role='HUMAN', begin='<|im_start|>user\n', end='<|im_end|>\n'), + dict(role='BOT', begin='<|im_start|>assistant\n', end='<|im_end|>\n', generate=True), + ], + eos_token_id=92542 +) + # config for internlm-chat-7b internlm_chat_7b = dict( type=TurboMindModel, @@ -41,6 +49,28 @@ concurrency=32, meta_template=internlm_meta_template, run_cfg=dict(num_gpus=1, num_procs=1), + end_str='', +) + +# config for internlm-chat-7b +internlm2_chat_7b = dict( + type=TurboMindModel, + abbr='internlm2-chat-7b-turbomind', + path='internlm/internlm2-chat-7b', + engine_config=dict(session_len=2048, + max_batch_size=32, + rope_scaling_factor=1.0), + gen_config=dict(top_k=1, + top_p=0.8, + temperature=1.0, + max_new_tokens=100), + max_out_len=100, + max_seq_len=2048, + batch_size=32, + concurrency=32, + meta_template=internlm2_meta_template, + run_cfg=dict(num_gpus=1, num_procs=1), + end_str='<|im_end|>' ) # config for internlm-chat-20b @@ -61,6 +91,7 @@ concurrency=8, meta_template=internlm_meta_template, run_cfg=dict(num_gpus=1, num_procs=1), + end_str='', ) models = [internlm_chat_20b] diff --git a/docs/en/advanced_guides/evaluation_turbomind.md b/docs/en/advanced_guides/evaluation_turbomind.md index 6e9a5b1a8..9ace22f5e 100644 --- a/docs/en/advanced_guides/evaluation_turbomind.md +++ b/docs/en/advanced_guides/evaluation_turbomind.md @@ -56,6 +56,7 @@ internlm_20b = dict( batch_size=8, concurrency=8, run_cfg=dict(num_gpus=1, num_procs=1), + end_str='' ) models = [internlm_20b] diff --git a/docs/zh_cn/advanced_guides/evaluation_turbomind.md b/docs/zh_cn/advanced_guides/evaluation_turbomind.md index be86e1729..f52142e1d 100644 --- a/docs/zh_cn/advanced_guides/evaluation_turbomind.md +++ b/docs/zh_cn/advanced_guides/evaluation_turbomind.md @@ -56,6 +56,7 @@ internlm_20b = dict( batch_size=8, concurrency=8, run_cfg=dict(num_gpus=1, num_procs=1), + end_str='' ) models = [internlm_20b] diff --git a/opencompass/models/turbomind.py b/opencompass/models/turbomind.py index 1a17a7ba8..c5606d93f 100644 --- a/opencompass/models/turbomind.py +++ b/opencompass/models/turbomind.py @@ -34,6 +34,9 @@ class TurboMindModel(BaseModel): arguments like session_len, max_batch_size for TurboMind. gen_config (Dict, optional): Generation config to set arguments like top_k, top_p, temperature. + end_str (str, optional): Whether to trim generated strings with end_str + if the model has special ending strings that are not handled well. + Defaults to None. """ def __init__(self, @@ -42,7 +45,8 @@ def __init__(self, max_seq_len: int = 2048, meta_template: Optional[Dict] = None, engine_config: Optional[Dict] = None, - gen_config: Optional[Dict] = None): + gen_config: Optional[Dict] = None, + end_str: Optional[str] = None): super().__init__(path=path, max_seq_len=max_seq_len, meta_template=meta_template) @@ -62,6 +66,7 @@ def __init__(self, ] self.generator_ids = [i + 1 for i in range(concurrency)] self.gen_config = gen_config + self.end_str = end_str def generate( self, @@ -90,11 +95,15 @@ def generate( for batch_input in batch_inputs: with ThreadPoolExecutor() as executor: _results = list( - executor.map(self._generate, - self.generators[:len(batch_input)], - self.generator_ids[:len(batch_input)], - batch_input, [max_out_len] * len(batch_input), - [self.gen_config] * len(batch_input))) + executor.map( + self._generate, + self.generators[:len(batch_input)], + self.generator_ids[:len(batch_input)], + batch_input, + [max_out_len] * len(batch_input), + [self.gen_config] * len(batch_input), + [self.end_str] * len(batch_input), + )) results += _results return results @@ -114,7 +123,8 @@ def _generate(self, session_id, prompt: str or PromptList, max_out_len: int, - gen_config=None) -> str: + gen_config=None, + end_str: Optional[str] = None) -> str: """Generate results given a list of inputs. Args: @@ -124,7 +134,10 @@ def _generate(self, max_out_len (int): The maximum length of the output. gen_config (EngineGenerationConfig, optional): Generation config to set arguments like top_k, top_p, temperature. - + end_str (str, optional): Whether to trim generated strings + with end_str if the model has special ending strings + that are not handled well. + Defaults to None. Returns: str: The generated string. """ @@ -144,4 +157,7 @@ def _generate(self, _, output_ids, _ = outputs response = self.tokenizer.decode(output_ids) response = valid_str(response) + # used to trim + if end_str: + response = response.split(end_str)[0] return response