Add model postprocess function

configs/datasets/gsm8k/gsm8k_xfinder_gen_a58960.py

@@ -23,9 +23,13 @@
 gsm8k_eval_cfg = dict(
     evaluator=dict(type=MATHEvaluator, version='v2'),
     pred_postprocessor=dict(type=math_postprocess_v2),
-    model_postprocessor=dict(type=xfinder_postprocess, question_type='math', xfinder_model_name='xFinder-qwen1505', xfiner_api_url='http://0.0.0.0:23333/v1,http://0.0.0.0:23334/v1'),
     dataset_postprocessor=dict(type=gsm8k_dataset_postprocess),
-)
+    model_postprocessor=dict(
+        type=xfinder_postprocess,
+        question_type='math',
+        xfinder_model_name='xFinder-qwen1505',
+        xfiner_api_url='http://0.0.0.0:23333/v1,http://0.0.0.0:23334/v1')
+    )
 
 gsm8k_datasets = [
     dict(

configs/datasets/mmlu/mmlu_xfinder_gen_4d595a.py

@@ -108,8 +108,13 @@
 
     mmlu_eval_cfg = dict(
         evaluator=dict(type=AccwithDetailsEvaluator),
-        model_postprocessor=dict(type=xfinder_postprocess, question_type='alphabet_option', xfinder_model_name='xFinder-qwen1505', xfiner_api_url='http://0.0.0.0:23333/v1,http://0.0.0.0:23334/v1'),
-        pred_postprocessor=dict(type=first_option_postprocess, options='ABCD'))
+        pred_postprocessor=dict(type=first_option_postprocess, options='ABCD'),
+        model_postprocessor=dict(
+            type=xfinder_postprocess,
+            question_type='alphabet_option',
+            xfinder_model_name='xFinder-qwen1505',
+            xfiner_api_url='http://0.0.0.0:23333/v1,http://0.0.0.0:23334/v1')
+        )
 
     mmlu_datasets.append(
         dict(

configs/datasets/nq/nq_xfinder_gen_3dcea1.py

@@ -17,9 +17,14 @@
     retriever=dict(type=ZeroRetriever),
     inferencer=dict(type=GenInferencer))
 
-nq_eval_cfg = dict(evaluator=dict(type=NQEvaluator), pred_role='BOT',
-                model_postprocessor=dict(type=xfinder_postprocess, question_type='short_text', xfinder_model_name='xFinder-qwen1505', xfiner_api_url='http://0.0.0.0:23333/v1,http://0.0.0.0:23334/v1'),
-)
+nq_eval_cfg = dict(
+    evaluator=dict(type=NQEvaluator), pred_role='BOT',
+    model_postprocessor=dict(
+        type=xfinder_postprocess,
+        question_type='short_text',
+        xfinder_model_name='xFinder-qwen1505',
+        xfiner_api_url='http://0.0.0.0:23333/v1,http://0.0.0.0:23334/v1')
+    )
 
 nq_datasets = [
     dict(

opencompass/configs/datasets/gsm8k/gsm8k_xfinder_gen_a58960.py

@@ -23,9 +23,13 @@
 gsm8k_eval_cfg = dict(
     evaluator=dict(type=MATHEvaluator, version='v2'),
     pred_postprocessor=dict(type=math_postprocess_v2),
-    model_postprocessor=dict(type=xfinder_postprocess, question_type='math', xfinder_model_name='xFinder-qwen1505', xfiner_api_url='http://0.0.0.0:23333/v1,http://0.0.0.0:23334/v1'),
     dataset_postprocessor=dict(type=gsm8k_dataset_postprocess),
-)
+    model_postprocessor=dict(
+        type=xfinder_postprocess,
+        question_type='math',
+        xfinder_model_name='xFinder-qwen1505',
+        xfiner_api_url='http://0.0.0.0:23333/v1,http://0.0.0.0:23334/v1')
+    )
 
 gsm8k_datasets = [
     dict(

opencompass/configs/datasets/mmlu/mmlu_xfinder_gen_4d595a.py

@@ -108,8 +108,13 @@
 
     mmlu_eval_cfg = dict(
         evaluator=dict(type=AccwithDetailsEvaluator),
-        model_postprocessor=dict(type=xfinder_postprocess, question_type='alphabet_option', xfinder_model_name='xFinder-qwen1505', xfiner_api_url='http://0.0.0.0:23333/v1,http://0.0.0.0:23334/v1'),
-        pred_postprocessor=dict(type=first_option_postprocess, options='ABCD'))
+        pred_postprocessor=dict(type=first_option_postprocess, options='ABCD'),
+        model_postprocessor=dict(
+            type=xfinder_postprocess,
+            question_type='alphabet_option',
+            xfinder_model_name='xFinder-qwen1505',
+            xfiner_api_url='http://0.0.0.0:23333/v1,http://0.0.0.0:23334/v1')
+        )
 
     mmlu_datasets.append(
         dict(

opencompass/configs/datasets/nq/nq_xfinder_gen_3dcea1.py

@@ -17,9 +17,14 @@
     retriever=dict(type=ZeroRetriever),
     inferencer=dict(type=GenInferencer))
 
-nq_eval_cfg = dict(evaluator=dict(type=NQEvaluator), pred_role='BOT',
-                model_postprocessor=dict(type=xfinder_postprocess, question_type='short_text', xfinder_model_name='xFinder-qwen1505', xfiner_api_url='http://0.0.0.0:23333/v1,http://0.0.0.0:23334/v1'),
-)
+nq_eval_cfg = dict(
+    evaluator=dict(type=NQEvaluator), pred_role='BOT',
+    model_postprocessor=dict(
+        type=xfinder_postprocess,
+        question_type='short_text',
+        xfinder_model_name='xFinder-qwen1505',
+        xfiner_api_url='http://0.0.0.0:23333/v1,http://0.0.0.0:23334/v1')
+    )
 
 nq_datasets = [
     dict(

opencompass/utils/model_postprocessors.py

@@ -6,8 +6,9 @@
 
 from opencompass.registry import TEXT_POSTPROCESSORS
 
-from .xfinder.Extractor import Extractor
-from .xfinder.xfinder_utils import DataProcessor, convert_to_xfinder_format
+from .postprocessors.xfinder.extractor import Extractor
+from .postprocessors.xfinder.xfinder_utils import (DataProcessor,
+                                                   convert_to_xfinder_format)
 
 
 def gen_output(ori_data, extractor):

opencompass/utils/xfinder/README.md → opencompass/utils/postprocessors/xfinder/README.md

@@ -69,17 +69,9 @@ Here are some examples of the question type supported from the official xFinder
 
 ## How to Use Model Postprocess in OpenCompass
 
-We make the postprocess as a common postprocess function in OpenCompass, so you can use it by setting the `postprocess` parameter in the `predict` function of OpenCompass. It can be used with the default postprocess regularization extract function at the same time. The only thing you need to do is to deploy the postprocess model server and set the `model_postprocessor` to the original `eval_cfg` in the dataset configuration, like the following example:
-
-```python
-from opencompass.utils.model_postprocessors import xfinder_postprocess
+### Step 1: Deploy the Postprocess Model Server
 
-...
-
-    model_postprocessor=dict(type=xfinder_postprocess, question_type='math', xfinder_model_name='xFinder-qwen1505', xfiner_api_url='http://0.0.0.0:23333/v1,http://0.0.0.0:23334/v1'),
-```
-
-The `question_type` is the type of the question, which can be one of the four types mentioned above. The `xfinder_model_name` is the name of the model you deploying the model server, and the `xfiner_api_url` is the URL of the model server. For now, there are two xFinder model can use, you can download them from Huggingface model hub:
+For now, there are two xFinder models can use, you can download them from Huggingface model hub:
 
 1. **IAAR-Shanghai/xFinder-qwen1505**
 2. **IAAR-Shanghai/xFinder-llama38it**
@@ -90,6 +82,28 @@ You can use LMDeploy or vLLM to deploy the xFinder model server, for example, yo
 lmdeploy serve api_server IAAR-Shanghai/xFinder-qwen1505  --model-name xFinder-qwen1505  --server-port 23333 --backend turbomind --tp 1
 ```
 
+### Step 2: Set the Postprocess Model Config in the Dataset Configuration
+
+We make the postprocess as a common postprocess function in OpenCompass, so you can use it by setting the `postprocess` parameter in the `predict` function of OpenCompass. It can be used with the default postprocess regularization extract function at the same time. The only thing you need to do is to deploy the postprocess model server and set the `model_postprocessor` to the original `eval_cfg` in the dataset configuration, like the following example:
+
+```python
+from opencompass.utils.model_postprocessors import xfinder_postprocess
+
+...
+
+    model_postprocessor=dict(
+        type=xfinder_postprocess,
+        question_type='math',
+        xfinder_model_name='xFinder-qwen1505',
+        xfiner_api_url='http://0.0.0.0:23333/v1,http://0.0.0.0:23334/v1')
+```
+
+Explanation of the parameters:
+
+- `question_type`: the type of the question, which can be one of the three types mentioned above.
+- `xfinder_model_name`: the name of the model you deploying the model server.
+- `xfiner_api_url`: the URL of the model server, you can set multiple URLs with `,` to use multiple model servers, which can accelerate the postprocess speed.
+
 📢：**Please attention following points**:
 
 1. Now only support extract questions with Zero-shot setting.
@@ -123,9 +137,13 @@ gsm8k_infer_cfg = dict(
 gsm8k_eval_cfg = dict(
     evaluator=dict(type=MATHEvaluator, version='v2'),
     pred_postprocessor=dict(type=math_postprocess_v2),
-    model_postprocessor=dict(type=xfinder_postprocess, question_type='math', xfinder_model_name='xFinder-qwen1505', xfiner_api_url='http://0.0.0.0:23333/v1,http://0.0.0.0:23334/v1'), # Add model postprocess
     dataset_postprocessor=dict(type=gsm8k_dataset_postprocess),
-)
+    model_postprocessor=dict(
+        type=xfinder_postprocess,
+        question_type='math',
+        xfinder_model_name='xFinder-qwen1505',
+        xfiner_api_url='http://0.0.0.0:23333/v1,http://0.0.0.0:23334/v1')
+    )
 
 gsm8k_datasets = [
     dict(
@@ -145,10 +163,10 @@ You can also use the `--dump-eval-details` command to dump the detailed evaluati
 
 ## Results Comparison with Different Question Types
 
-We have tested the model postprocess method with XFinder model on the GSM8K, MMLU, Natural Questions (NQ) datasets with above settings, and the results are as follows:
+We have tested the model postprocess method with XFinder model on the GSM8K, MMLU, Natural Questions (NQ) datasets for `Meta-Llama-3-8B-Instruct` with above settings, and the results are as follows:
 
-| Dataset | Type            | Regex Postprocess Score | Model Postprocess Score |
-| ------- | --------------- | ----------------------- | ----------------------- |
-| gsm8k   | math            | 73.46                   | 78.09                   |
-| nq      | short_text      | 22.33                   | 37.53                   |
-| mmlu    | alphabet_option | 67.89                   | 67.93                   |
+| Dataset | Type            | Config Name              | Regex Postprocess Score | Model Postprocess Score |
+| ------- | --------------- | ------------------------ | ----------------------- | ----------------------- |
+| gsm8k   | math            | gsm8k_xfinder_gen_a58960 | 73.46                   | 78.09                   |
+| nq      | short_text      | nq_xfinder_gen_3dcea1    | 22.33                   | 37.53                   |
+| mmlu    | alphabet_option | mmlu_xfinder_gen_4d595a  | 67.89                   | 67.93                   |