OpenPCDet LiDar Object Detection Node

.github/templates/docker_context/docker_context.sh

@@ -39,6 +39,7 @@ while read -r module; do
         # Temporarily skip perception services that have too large image size
         if  [[ "$service_out" == "lane_detection" ]] || \
             [[ "$service_out" == "camera_object_detection" ]] || \
+            [[ "$service_out" == "lidar_object_detection" ]] || \
             [[ "$service_out" == "semantic_segmentation" ]]; then
             continue
         fi

docker/perception/lidar_object_detection/lidar_object_detection.Dockerfile

@@ -1,55 +1,92 @@
-ARG BASE_IMAGE=ghcr.io/watonomous/wato_monorepo/base:humble-ubuntu22.04
-
+ARG BASE_IMAGE=ghcr.io/watonomous/wato_monorepo/base:cuda11.7-humble-ubuntu22.04-devel
 ################################ Source ################################
 FROM ${BASE_IMAGE} as source
-
 WORKDIR ${AMENT_WS}/src
-
 # Copy in source code 
 COPY src/perception/lidar_object_detection lidar_object_detection
-COPY src/wato_msgs/sample_msgs sample_msgs
-
 # Scan for rosdeps
 RUN apt-get -qq update && rosdep update && \
     rosdep install --from-paths . --ignore-src -r -s \
-        | grep 'apt-get install' \
-        | awk '{print $3}' \
-        | sort  > /tmp/colcon_install_list
-
+    | grep 'apt-get install' \
+    | awk '{print $3}' \
+    | sort  > /tmp/colcon_install_list
+WORKDIR /home/bolty
+RUN git clone https://github.com/WATonomous/OpenPCDet.git && \
+    cd OpenPCDet \ 
+    && git checkout 06fd4f862329625ff9ed850464330816e54531f8
+################################ INSTALL OpenCV with CUDA Support ##############
+WORKDIR /opt
+RUN git clone -b 4.x https://github.com/opencv/opencv.git && \
+    cd opencv && git checkout 4.5.5
+RUN git clone -b 4.x https://github.com/opencv/opencv_contrib.git && \
+    cd opencv_contrib && git checkout 4.5.5
+WORKDIR /opt/opencv/build
+RUN cmake -D CMAKE_BUILD_TYPE=RELEASE \
+    -D CMAKE_INSTALL_PREFIX=/usr/local \
+    -D OPENCV_GENERATE_PKGCONFIG=ON \
+    -D BUILD_EXAMPLES=OFF \
+    -D INSTALL_PYTHON_EXAMPLES=OFF \
+    -D INSTALL_C_EXAMPLES=OFF \
+    -D PYTHON_EXECUTABLE=$(which python3) \
+    -D PYTHON3_INCLUDE_DIR=$(python3 -c "from distutils.sysconfig import get_python_inc; print(get_python_inc())") \
+    -D PYTHON3_PACKAGES_PATH=$(python3 -c "from distutils.sysconfig import get_python_lib; print(get_python_lib())") \
+    -D BUILD_opencv_python3=ON \
+    -D OPENCV_EXTRA_MODULES_PATH=../../opencv_contrib/modules/ \
+    -D WITH_GSTREAMER=ON \
+    -D WITH_CUDA=ON \
+    -D ENABLE_PRECOMPILED_HEADERS=OFF \
+    .. && make -j$(nproc) && make install && ldconfig
+RUN rm -rf /opt/opencv /opt/opencv_contrib
 ################################# Dependencies ################################
 FROM ${BASE_IMAGE} as dependencies
-
+RUN apt-get update && apt-get install -y \
+    git zip unzip libssl-dev libcairo2-dev lsb-release libgoogle-glog-dev libgflags-dev libatlas-base-dev libeigen3-dev software-properties-common \
+    build-essential cmake pkg-config libapr1-dev autoconf automake libtool curl libc6 libboost-all-dev debconf libomp5 libstdc++6 \
+    libqt5core5a libqt5xml5 libqt5gui5 libqt5widgets5 libqt5concurrent5 libqt5opengl5 libcap2 libusb-1.0-0 libatk-adaptor neovim \
+    python3-pip python3-setuptools \
+    && apt-get -qq autoremove -y && apt-get -qq autoclean && apt-get -qq clean && \
+    rm -rf /var/lib/apt/lists/* /root/* /root/.ros /tmp/* /usr/share/doc
+# Set environment variables
+ENV CUDA_HOME /usr/local/cuda
+ENV LD_LIBRARY_PATH /usr/local/cuda/lib64:${LD_LIBRARY_PATH}
+ENV PATH /usr/local/cuda/bin:${PATH}
+ENV OpenCV_DIR=/usr/share/OpenCV
+# Install Python dependencies
+RUN pip3 install torch==1.13.1+cu116 torchvision==0.14.1+cu116 torchaudio==0.13.1 --extra-index-url https://download.pytorch.org/whl/cu116
+RUN pip3 install torch-scatter -f https://data.pyg.org/whl/torch-1.13.1+cu116.html
+ENV TORCH_CUDA_ARCH_LIST="3.5;5.0;6.0;6.1;7.0;7.5;8.0;8.6+PTX"
+RUN pip3 install spconv-cu116 pyquaternion numpy==1.23 pillow==8.4 mayavi open3d av2
 # Install Rosdep requirements
 COPY --from=source /tmp/colcon_install_list /tmp/colcon_install_list
+RUN apt-get update && apt-fast update
 RUN apt-fast install -qq -y --no-install-recommends $(cat /tmp/colcon_install_list)
-
 # Copy in source code from source stage
 WORKDIR ${AMENT_WS}
 COPY --from=source ${AMENT_WS}/src src
-
 # Dependency Cleanup
 WORKDIR /
 RUN apt-get -qq autoremove -y && apt-get -qq autoclean && apt-get -qq clean && \
     rm -rf /root/* /root/.ros /tmp/* /var/lib/apt/lists/* /usr/share/doc/*
-
-################################ Build ################################
+################################ Build #######################################
 FROM dependencies as build
-
+COPY --from=source /home/bolty/OpenPCDet /home/bolty/OpenPCDet
+WORKDIR /home/bolty/OpenPCDet
+RUN pip3 install -r requirements.txt
+RUN pip3 install kornia==0.6.8
+RUN pip3 install nuscenes-devkit==1.0.5
+WORKDIR /home/bolty/OpenPCDet
+RUN python3 setup.py develop
 # Build ROS2 packages
 WORKDIR ${AMENT_WS}
 RUN . /opt/ros/$ROS_DISTRO/setup.sh && \
     colcon build \
-        --cmake-args -DCMAKE_BUILD_TYPE=Release
-
+    --cmake-args -DCMAKE_BUILD_TYPE=Release
 # Entrypoint will run before any CMD on launch. Sources ~/opt/<ROS_DISTRO>/setup.bash and ~/ament_ws/install/setup.bash
 COPY docker/wato_ros_entrypoint.sh ${AMENT_WS}/wato_ros_entrypoint.sh
 ENTRYPOINT ["./wato_ros_entrypoint.sh"]
-
-################################ Prod ################################
+################################ Prod #########################################
 FROM build as deploy
-
 # Source Cleanup and Security Setup
 RUN chown -R $USER:$USER ${AMENT_WS}
 RUN rm -rf src/*
-
 USER ${USER}

modules/docker-compose.perception.yaml

@@ -34,7 +34,6 @@ services:
       - /mnt/wato-drive2/perception_models/yolov8m.pt:/perception_models/yolov8m.pt
       - /mnt/wato-drive2/perception_models/traffic_light.pt:/perception_models/traffic_light.pt
       - /mnt/wato-drive2/perception_models/traffic_signs_v0.pt:/perception_models/traffic_signs_v1.pt
-
   lidar_object_detection:
     build:
       context: ..
@@ -44,8 +43,9 @@ services:
         - "${PERCEPTION_LIDAR_OBJECT_DETECTION_IMAGE}:build_main"
       target: deploy
     image: "${PERCEPTION_LIDAR_OBJECT_DETECTION_IMAGE}:${TAG}"
-    command: /bin/bash -c "ros2 launch lidar_object_detection lidar_object_detection.launch.py"
-
+    command: /bin/bash -c "ros2 launch lidar_object_detection nuscenes_launch.py"
+    volumes:
+      - /mnt/wato-drive2/perception_models/transfusion_trained_model.pth:/home/bolty/OpenPCDet/models/transfusion_trained_model.pth
   semantic_segmentation:
     build:
       context: ..
@@ -97,4 +97,4 @@ services:
         - "${PERCEPTION_DEPTH_ESTIMATION_IMAGE}:build_main"
       target: deploy
     image: "${PERCEPTION_DEPTH_ESTIMATION_IMAGE}:${TAG}"
-    command: /bin/bash -c "ros2 launch depth_estimation eve.launch.py"
+    command: /bin/bash -c "ros2 launch depth_estimation eve.launch.py"

src/perception/lidar_object_detection/lidar_object_detection/config/eve_config.yaml

@@ -0,0 +1,5 @@
+lidar_object_detection_node:
+  ros__parameters:
+    lidar_topic: /velodyne_points
+    model_config_path: /home/bolty/OpenPCDet/tools/cfgs/nuscenes_models/transfusion_lidar.yaml
+    model_path: /home/bolty/OpenPCDet/models/transfusion_trained_model.pth

src/perception/lidar_object_detection/lidar_object_detection/config/nuscenes_config.yaml

@@ -0,0 +1,6 @@
+lidar_object_detection_node:
+  ros__parameters:
+    lidar_topic: /LIDAR_TOP
+    model_config_path: /home/bolty/OpenPCDet/tools/cfgs/nuscenes_models/transfusion_lidar.yaml
+    model_path: /home/bolty/OpenPCDet/models/cbgs_transfusion_lidar.pth
+    enable_detection: true

src/perception/lidar_object_detection/lidar_object_detection/launch/eve_launch.py

@@ -0,0 +1,27 @@
+from launch import LaunchDescription
+from launch_ros.actions import Node
+from ament_index_python.packages import get_package_share_directory
+import os
+
+
+def generate_launch_description():
+    ld = LaunchDescription()
+    config = os.path.join(
+        get_package_share_directory('lidar_object_detection'),
+        'config',
+        'eve_config.yaml'
+    )
+
+    # nodes
+    lidar_object_detection = Node(
+        package='lidar_object_detection',
+        executable='lidar_object_detection_node',
+        name='lidar_object_detection_node',
+        parameters=[config],
+        arguments=['--ros-args', '--log-level', 'info']
+    )
+
+    # finalize
+    ld.add_action(lidar_object_detection)
+
+    return ld

src/perception/lidar_object_detection/lidar_object_detection/launch/nuscenes_launch.py

@@ -0,0 +1,27 @@
+from launch import LaunchDescription
+from launch_ros.actions import Node
+from ament_index_python.packages import get_package_share_directory
+import os
+
+
+def generate_launch_description():
+    ld = LaunchDescription()
+    config = os.path.join(
+        get_package_share_directory('lidar_object_detection'),
+        'config',
+        'nuscenes_config.yaml'
+    )
+
+    # nodes
+    lidar_object_detection = Node(
+        package='lidar_object_detection',
+        executable='lidar_object_detection_node',
+        name='lidar_object_detection_node',
+        parameters=[config],
+        arguments=['--ros-args', '--log-level', 'info']
+    )
+
+    # finalize
+    ld.add_action(lidar_object_detection)
+
+    return ld

src/perception/lidar_object_detection/lidar_object_detection/lidar_object_detection/label_server.py

@@ -0,0 +1,37 @@
+#!/usr/bin/env python
+
+import rclpy
+from rclpy.node import Node
+from vision_msgs.msg import VisionInfo
+
+
+class LabelServer(Node):
+    def __init__(self):
+        super().__init__('label_server')
+        self.publisher_ = self.create_publisher(VisionInfo, 'vision_info', 10)
+        self.label_mapping = {
+            1: "car",
+            2: "pedestrian",
+            3: "cyclist",
+        }
+        self.publish_labels()
+
+    def publish_labels(self):
+        vision_info_msg = VisionInfo()
+        vision_info_msg.database_location = "memory"
+        vision_info_msg.database_version = "1.0"
+        for label_id, class_name in self.label_mapping.items():
+            vision_info_msg.class_map[label_id] = class_name
+        self.publisher_.publish(vision_info_msg)
+
+
+def main(args=None):
+    rclpy.init(args=args)
+    label_server = LabelServer()
+    rclpy.spin(label_server)
+    label_server.destroy_node()
+    rclpy.shutdown()
+
+
+if __name__ == '__main__':
+    main()