From 71a1e6cc1859d8692cea134946bdc249ad0e5400 Mon Sep 17 00:00:00 2001 From: RunningLeon Date: Thu, 14 Sep 2023 14:13:17 +0800 Subject: [PATCH] fix torch2onnx for mmdet3d (#2425) * fix mmdet3d * fix * resolve comments * fix rtmdet ncnn * update docs --- .github/md-link-config.json | 3 ++ .github/workflows/regression-test.yml | 4 +- .../single-stage_ncnn_static-640x640.py | 4 ++ docs/en/04-supported-codebases/mmdet3d.md | 45 +++++-------------- docs/zh_cn/04-supported-codebases/mmdet3d.md | 45 +++++-------------- .../codebase/mmdet/deploy/object_detection.py | 4 +- .../codebase/mmdet3d/deploy/mono_detection.py | 3 +- mmdeploy/codebase/mmdet3d/models/base.py | 10 +++-- .../codebase/mmdet3d/models/mvx_two_stage.py | 14 +++--- .../mmdet3d/models/single_stage_mono3d.py | 11 +++-- tests/regression/mmdet.yml | 5 ++- .../test_mmdet3d/test_mmdet3d_models.py | 2 +- 12 files changed, 64 insertions(+), 86 deletions(-) create mode 100644 configs/mmdet/detection/single-stage_ncnn_static-640x640.py diff --git a/.github/md-link-config.json b/.github/md-link-config.json index 76986cbd01..e389a1b5ac 100644 --- a/.github/md-link-config.json +++ b/.github/md-link-config.json @@ -15,6 +15,9 @@ }, { "pattern": "^http://localhost" + }, + { + "pattern": "^https://openmmlab.com" } ], "httpHeaders": [ diff --git a/.github/workflows/regression-test.yml b/.github/workflows/regression-test.yml index 69a68525f4..81803aa7af 100644 --- a/.github/workflows/regression-test.yml +++ b/.github/workflows/regression-test.yml @@ -17,7 +17,7 @@ on: required: true description: 'Whether to start regression test on Linux x86_64' type: boolean - default: false + default: true test_windows: required: true description: 'Whether to start regression test on Windows' @@ -52,7 +52,7 @@ on: required: false description: 'Do not change it unless you know what you are doing!' type: string - default: 'https://e2e1-14-136-99-158.ngrok-free.app' + default: 'https://4ec1-14-136-99-158.ngrok-free.app' concurrency: group: ${{ github.workflow }}-${{ github.ref }} diff --git a/configs/mmdet/detection/single-stage_ncnn_static-640x640.py b/configs/mmdet/detection/single-stage_ncnn_static-640x640.py new file mode 100644 index 0000000000..630ef69acb --- /dev/null +++ b/configs/mmdet/detection/single-stage_ncnn_static-640x640.py @@ -0,0 +1,4 @@ +_base_ = ['../_base_/base_static.py', '../../_base_/backends/ncnn.py'] + +codebase_config = dict(model_type='ncnn_end2end') +onnx_config = dict(output_names=['detection_output'], input_shape=[640, 640]) diff --git a/docs/en/04-supported-codebases/mmdet3d.md b/docs/en/04-supported-codebases/mmdet3d.md index 5ad374f9e8..253270fadc 100644 --- a/docs/en/04-supported-codebases/mmdet3d.md +++ b/docs/en/04-supported-codebases/mmdet3d.md @@ -12,37 +12,12 @@ ______________________________________________________________________ ## Install mmdet3d -These branches are required for mmdet3d deployment - -| codebase | commit | -| :------: | :-------: | -| mmdet3d | v1.1.0rc1 | -| mmcv | v2.0.0rc1 | -| mmdet | v3.0.0rc1 | -| mmseg | v1.0.0rc0 | - -First checkout and install mmcv/mmdet/mmseg/mmdet3d - -```bash -python3 -m pip install openmim --user -python3 -m mim install mmcv==2.0.0rc1 mmdet==3.0.0rc1 mmseg==1.0.0rc0 --user - -git clone https://github.com/open-mmlab/mmdetection3d --branch v1.1.0rc1 -cd mmdetection3d -python3 -m pip install . -cd - -``` - -After installation, `tools/check_env.py` should display mmdet3d version normally +We could install mmdet3d through [mim](https://github.com/open-mmlab/mim). +For other ways of installation, please refer to [here](https://mmdetection3d.readthedocs.io/en/latest/get_started.html#installation) ```bash -python3 tools/check_env.py -.. -11/11 13:56:19 - mmengine - INFO - **********Codebase information********** -11/11 13:56:19 - mmengine - INFO - mmdet: 3.0.0rc1 -11/11 13:56:19 - mmengine - INFO - mmseg: 1.0.0rc0 -.. -11/11 13:56:19 - mmengine - INFO - mmdet3d: 1.1.0rc1 +python3 -m pip install -U openmim +python3 -m mim install "mmdet3d>=1.1.0" ``` ## Convert model @@ -50,11 +25,15 @@ python3 tools/check_env.py For example, use `tools/deploy.py` to convert centerpoint to onnxruntime format ```bash -export MODEL_CONFIG=/path/to/mmdetection3d/configs/centerpoint/centerpoint_pillar02_second_secfpn_head-circlenms_8xb4-cyclic-20e_nus-3d.py +# cd to mmdeploy root directory +# download config and model +mim download mmdet3d --config centerpoint_pillar02_second_secfpn_head-circlenms_8xb4-cyclic-20e_nus-3d --dest . + +export MODEL_CONFIG=centerpoint_pillar02_second_secfpn_head-circlenms_8xb4-cyclic-20e_nus-3d.py -export MODEL_PATH=https://download.openmmlab.com/mmdetection3d/v1.0.0_models/centerpoint/centerpoint_02pillar_second_secfpn_circlenms_4x8_cyclic_20e_nus/centerpoint_02pillar_second_secfpn_circlenms_4x8_cyclic_20e_nus_20210816_064624-0f3299c0.pth +export MODEL_PATH=centerpoint_02pillar_second_secfpn_circlenms_4x8_cyclic_20e_nus_20220811_031844-191a3822.pth -export TEST_DATA=/path/to/mmdetection3d/tests/data/nuscenes/sweeps/LIDAR_TOP/n008-2018-09-18-12-07-26-0400__LIDAR_TOP__1537287083900561.pcd.bin +export TEST_DATA=tests/data/n008-2018-08-01-15-16-36-0400__LIDAR_TOP__1533151612397179.pcd.bin python3 tools/deploy.py configs/mmdet3d/voxel-detection/voxel-detection_onnxruntime_dynamic.py $MODEL_CONFIG $MODEL_PATH $TEST_DATA --work-dir centerpoint ``` @@ -82,4 +61,4 @@ The caller needs to refer to the corresponding [python implementation](../../../ | [pointpillars](https://github.com/open-mmlab/mmdetection3d/blob/main/configs/pointpillars/pointpillars_hv_secfpn_8xb6-160e_kitti-3d-3class.py) | voxel detection | KITTI | ✔️ | ✔️ | ✔️ | | [smoke](https://github.com/open-mmlab/mmdetection3d/blob/main/configs/smoke/smoke_dla34_dlaneck_gn-all_4xb8-6x_kitti-mono3d.py) | monocular detection | KITTI | ✔️ | x | ✔️ | -- Make sure trt >= 8.4 for some bug fixed, such as ScatterND, dynamic shape crash and so on. +- Make sure trt >= 8.6 for some bug fixed, such as ScatterND, dynamic shape crash and so on. diff --git a/docs/zh_cn/04-supported-codebases/mmdet3d.md b/docs/zh_cn/04-supported-codebases/mmdet3d.md index 5461d95436..e0b0119bb8 100644 --- a/docs/zh_cn/04-supported-codebases/mmdet3d.md +++ b/docs/zh_cn/04-supported-codebases/mmdet3d.md @@ -12,37 +12,12 @@ ______________________________________________________________________ ## 安装 mmdet3d -因为依赖的 codebase 不在 master 分支，所以要切到相应分支： - -| codebase | commit | -| :------: | :-------: | -| mmdet3d | v1.1.0rc1 | -| mmcv | v2.0.0rc1 | -| mmdet | v3.0.0rc1 | -| mmseg | v1.0.0rc0 | - -先安装前置依赖 mmcv/mmdet/mmseg，再安装 mmdet3d - -```bash -python3 -m pip install openmim --user -python3 -m mim install mmcv==2.0.0rc1 mmdet==3.0.0rc1 mmseg==1.0.0rc0 --user - -git clone https://github.com/open-mmlab/mmdetection3d --branch v1.1.0rc1 -cd mmdetection3d -python3 -m pip install . -cd - -``` - -成功后 `tools/check_env.py` 应能正常显示 mmdet3d 版本号。 +我们可以通过 [mim](https://github.com/open-mmlab/mim) 来安装 mmdet3d. +更多安装方式可参考该[文档](https://mmdetection3d.readthedocs.io/en/latest/get_started.html#installation) ```bash -python3 tools/check_env.py -.. -11/11 13:56:19 - mmengine - INFO - **********Codebase information********** -11/11 13:56:19 - mmengine - INFO - mmdet: 3.0.0rc1 -11/11 13:56:19 - mmengine - INFO - mmseg: 1.0.0rc0 -.. -11/11 13:56:19 - mmengine - INFO - mmdet3d: 1.1.0rc1 +python3 -m pip install -U openmim +python3 -m mim install "mmdet3d>=1.1.0" ``` ## 模型转换 @@ -50,11 +25,15 @@ python3 tools/check_env.py 使用 `tools/deploy.py` 把 mmdet3d 转到相应后端，以 centerpoint onnxruntime 为例： ```bash -export MODEL_CONFIG=/path/to/mmdetection3d/configs/centerpoint/centerpoint_pillar02_second_secfpn_head-circlenms_8xb4-cyclic-20e_nus-3d.py +# 切换到 mmdeploy 根目录 +# 通过mim下载centerpoint模型 +mim download mmdet3d --config centerpoint_pillar02_second_secfpn_head-circlenms_8xb4-cyclic-20e_nus-3d --dest . + +export MODEL_CONFIG=centerpoint_pillar02_second_secfpn_head-circlenms_8xb4-cyclic-20e_nus-3d.py -export MODEL_PATH=https://download.openmmlab.com/mmdetection3d/v1.0.0_models/centerpoint/centerpoint_02pillar_second_secfpn_circlenms_4x8_cyclic_20e_nus/centerpoint_02pillar_second_secfpn_circlenms_4x8_cyclic_20e_nus_20210816_064624-0f3299c0.pth +export MODEL_PATH=centerpoint_02pillar_second_secfpn_circlenms_4x8_cyclic_20e_nus_20220811_031844-191a3822.pth -export TEST_DATA=/path/to/mmdetection3d/tests/data/nuscenes/sweeps/LIDAR_TOP/n008-2018-09-18-12-07-26-0400__LIDAR_TOP__1537287083900561.pcd.bin +export TEST_DATA=tests/data/n008-2018-08-01-15-16-36-0400__LIDAR_TOP__1533151612397179.pcd.bin python3 tools/deploy.py configs/mmdet3d/voxel-detection/voxel-detection_onnxruntime_dynamic.py $MODEL_CONFIG $MODEL_PATH $TEST_DATA --work-dir centerpoint ``` @@ -80,4 +59,4 @@ ls -lah centerpoint | [pointpillars](https://github.com/open-mmlab/mmdetection3d/blob/main/configs/pointpillars/pointpillars_hv_secfpn_8xb6-160e_kitti-3d-3class.py) | voxel detection | KITTI | ✔️ | ✔️ | ✔️ | | [smoke](https://github.com/open-mmlab/mmdetection3d/blob/main/configs/smoke/smoke_dla34_dlaneck_gn-all_4xb8-6x_kitti-mono3d.py) | monocular detection | KITTI | ✔️ | x | ✔️ | -- 考虑到 ScatterND、动态 shape 等已知问题，请确保 trt >= 8.4 +- 考虑到 ScatterND、动态 shape 等已知问题，请确保 trt >= 8.6 diff --git a/mmdeploy/codebase/mmdet/deploy/object_detection.py b/mmdeploy/codebase/mmdet/deploy/object_detection.py index 92152d73cf..40c5a5b0cf 100644 --- a/mmdeploy/codebase/mmdet/deploy/object_detection.py +++ b/mmdeploy/codebase/mmdet/deploy/object_detection.py @@ -70,8 +70,10 @@ def process_model_config(model_cfg: Config, if transform.type == 'Resize': pipeline[i].keep_ratio = False pipeline[i].scale = tuple(input_shape) - if transform.type in ('YOLOv5KeepRatioResize', 'LetterResize'): + elif transform.type in ('YOLOv5KeepRatioResize', 'LetterResize'): pipeline[i].scale = tuple(input_shape) + elif transform.type == 'Pad' and 'size' in transform: + pipeline[i].size = tuple(input_shape) pipeline = [ transform for transform in pipeline diff --git a/mmdeploy/codebase/mmdet3d/deploy/mono_detection.py b/mmdeploy/codebase/mmdet3d/deploy/mono_detection.py index 4066445fde..aac4840ea3 100644 --- a/mmdeploy/codebase/mmdet3d/deploy/mono_detection.py +++ b/mmdeploy/codebase/mmdet3d/deploy/mono_detection.py @@ -124,7 +124,8 @@ def create_input( if data_preprocessor is not None: collate_data = data_preprocessor(collate_data, False) - inputs = collate_data['inputs'] + assert 'inputs' in collate_data + inputs = collate_data['inputs']['imgs'] else: inputs = collate_data['inputs'] return collate_data, inputs diff --git a/mmdeploy/codebase/mmdet3d/models/base.py b/mmdeploy/codebase/mmdet3d/models/base.py index 4410e77e2e..d736dbacb9 100644 --- a/mmdeploy/codebase/mmdet3d/models/base.py +++ b/mmdeploy/codebase/mmdet3d/models/base.py @@ -10,16 +10,18 @@ 'mmdet3d.models.detectors.Base3DDetector.forward' # noqa: E501 ) def basedetector__forward(self, - inputs: list, + voxels: torch.Tensor, + num_points: torch.Tensor, + coors: torch.Tensor, data_samples=None, **kwargs) -> Tuple[List[torch.Tensor]]: """Extract features of images.""" batch_inputs_dict = { 'voxels': { - 'voxels': inputs[0], - 'num_points': inputs[1], - 'coors': inputs[2] + 'voxels': voxels, + 'num_points': num_points, + 'coors': coors } } return self._forward(batch_inputs_dict, data_samples, **kwargs) diff --git a/mmdeploy/codebase/mmdet3d/models/mvx_two_stage.py b/mmdeploy/codebase/mmdet3d/models/mvx_two_stage.py index b9d62e684f..17ad711339 100644 --- a/mmdeploy/codebase/mmdet3d/models/mvx_two_stage.py +++ b/mmdeploy/codebase/mmdet3d/models/mvx_two_stage.py @@ -46,11 +46,15 @@ def mvxtwostagedetector__extract_feat(self, batch_inputs_dict: dict) -> tuple: @FUNCTION_REWRITER.register_rewriter( 'mmdet3d.models.detectors.mvx_two_stage.MVXTwoStageDetector.forward') -def mvxtwostagedetector__forward(self, inputs: list, **kwargs): +def mvxtwostagedetector__forward(self, voxels: torch.Tensor, + num_points: torch.Tensor, coors: torch.Tensor, + **kwargs): """Rewrite this func to remove voxelize op. Args: - inputs (list): input list comprises voxels, num_points and coors + voxels (Tensor): input voxels + num_points (Tensor): input num_points + coors (Tensor): input coors Returns: tuple: A tuple of classification scores, bbox and direction @@ -70,9 +74,9 @@ def mvxtwostagedetector__forward(self, inputs: list, **kwargs): deploy_cfg = ctx.cfg batch_inputs_dict = { 'voxels': { - 'voxels': inputs[0], - 'num_points': inputs[1], - 'coors': inputs[2] + 'voxels': voxels, + 'num_points': num_points, + 'coors': coors } } diff --git a/mmdeploy/codebase/mmdet3d/models/single_stage_mono3d.py b/mmdeploy/codebase/mmdet3d/models/single_stage_mono3d.py index 635c686cb8..6c10cf4079 100755 --- a/mmdeploy/codebase/mmdet3d/models/single_stage_mono3d.py +++ b/mmdeploy/codebase/mmdet3d/models/single_stage_mono3d.py @@ -1,19 +1,22 @@ # Copyright (c) OpenMMLab. All rights reserved. +from torch import Tensor + from mmdeploy.core import FUNCTION_REWRITER @FUNCTION_REWRITER.register_rewriter( 'mmdet3d.models.detectors.single_stage_mono3d.' 'SingleStageMono3DDetector.forward') -def singlestagemono3ddetector__forward(self, inputs: list, **kwargs): - """Rewrite this func to r. +def singlestagemono3ddetector__forward(self, inputs: Tensor, **kwargs): + """Rewrite to support feed inputs of Tensor type. Args: - inputs (dict): Input dict comprises `imgs` + inputs (Tensor): Input image Returns: list: two torch.Tensor """ - x = self.extract_feat(inputs) + + x = self.extract_feat({'imgs': inputs}) results = self.bbox_head.forward(x) return results[0], results[1] diff --git a/tests/regression/mmdet.yml b/tests/regression/mmdet.yml index f0e813ce8e..68fd53ad45 100644 --- a/tests/regression/mmdet.yml +++ b/tests/regression/mmdet.yml @@ -348,10 +348,11 @@ models: - configs/rtmdet/rtmdet_s_8xb32-300e_coco.py pipelines: - *pipeline_ort_dynamic_fp32 - - deploy_config: configs/mmdet/detection/detection_tensorrt_dynamic-64x64-800x800.py + - deploy_config: configs/mmdet/detection/detection_tensorrt_static-640x640.py convert_image: *convert_image backend_test: *default_backend_test - sdk_config: *sdk_dynamic + - deploy_config: configs/mmdet/detection/single-stage_ncnn_static-640x640.py + convert_image: *convert_image - name: SOLO metafile: configs/solo/metafile.yml diff --git a/tests/test_codebase/test_mmdet3d/test_mmdet3d_models.py b/tests/test_codebase/test_mmdet3d/test_mmdet3d_models.py index 22e925ccd0..aa24e9f780 100644 --- a/tests/test_codebase/test_mmdet3d/test_mmdet3d_models.py +++ b/tests/test_codebase/test_mmdet3d/test_mmdet3d_models.py @@ -157,7 +157,7 @@ def test_pointpillars(backend_type: Backend): cfg=deploy_cfg, backend=deploy_cfg.backend_config.type, opset=deploy_cfg.onnx_config.opset_version): - outputs = model.forward(data) + outputs = model.forward(*data) assert len(outputs) == 3