Real time semantic segmentation node

docker/perception/semantic_segmentation/semantic_segmentation.Dockerfile

@@ -1,13 +1,22 @@
-ARG BASE_IMAGE=ghcr.io/watonomous/wato_monorepo/base:humble-ubuntu22.04
+ARG BASE_BUILD_IMAGE=ghcr.io/watonomous/wato_monorepo/base:cuda12.0-humble-ubuntu22.04-devel
+ARG BASE_PROD_IMAGE=ghcr.io/watonomous/wato_monorepo/base:cuda12.0-humble-ubuntu22.04
+ARG PADDLE_INFERENCE_BUILD_URL=ghcr.io/watonomous/perception_paddlepaddle_inference_build_cuda-12.0
+################################ Build library ################################
+FROM ${PADDLE_INFERENCE_BUILD_URL} as PADDLE_INFERENCE_BUILD
 
 ################################ Source ################################
-FROM ${BASE_IMAGE} as source
+FROM ${BASE_BUILD_IMAGE} as source
 
 WORKDIR ${AMENT_WS}/src
 
+# Copy in the paddle inference library
+RUN mkdir -p semantic_segmentation/src
+COPY --from=PADDLE_INFERENCE_BUILD /paddle/paddle_inference_cuda120_build.tar /paddle/paddle_inference_cuda120_build.tar
+RUN tar -xvf /paddle/paddle_inference_cuda120_build.tar -C /paddle/
+RUN rm /paddle/paddle_inference_cuda120_build.tar
+
 # Copy in source code 
 COPY src/perception/semantic_segmentation semantic_segmentation
-COPY src/wato_msgs/sample_msgs sample_msgs
 
 # Scan for rosdeps
 RUN apt-get -qq update && rosdep update && \
@@ -17,7 +26,9 @@ RUN apt-get -qq update && rosdep update && \
         | sort  > /tmp/colcon_install_list
 
 ################################# Dependencies ################################
-FROM ${BASE_IMAGE} as dependencies
+FROM ${BASE_BUILD_IMAGE} as dependencies
+
+RUN apt update && apt install -y tensorrt ros-humble-cv-bridge libopencv-dev
 
 # Install Rosdep requirements
 COPY --from=source /tmp/colcon_install_list /tmp/colcon_install_list
@@ -26,6 +37,7 @@ RUN apt-fast install -qq -y --no-install-recommends $(cat /tmp/colcon_install_li
 # Copy in source code from source stage
 WORKDIR ${AMENT_WS}
 COPY --from=source ${AMENT_WS}/src src
+COPY --from=source /paddle /paddle
 
 # Dependency Cleanup
 WORKDIR /
@@ -43,10 +55,44 @@ RUN . /opt/ros/$ROS_DISTRO/setup.sh && \
 
 # Entrypoint will run before any CMD on launch. Sources ~/opt/<ROS_DISTRO>/setup.bash and ~/ament_ws/install/setup.bash
 COPY docker/wato_ros_entrypoint.sh ${AMENT_WS}/wato_ros_entrypoint.sh
+
+# Add runtime libraries to path
+ENV LD_LIBRARY_PATH=$LD_LIBRARY_PATH:${AMENT_WS}/install/semantic_segmentation/lib/
+ENV LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/lib
+
 ENTRYPOINT ["./wato_ros_entrypoint.sh"]
 
-################################ Prod ################################
-FROM build as deploy
+# ################################ Prod ################################
+FROM ${BASE_PROD_IMAGE} as deploy
+
+
+# Install runtime libs
+RUN apt-get update && apt-get install -y \
+    ros-humble-cv-bridge \
+    tensorrt
+
+# Copy the compiled binary to the runtime image
+COPY --from=build ${AMENT_WS} ${AMENT_WS}
+
+WORKDIR ${AMENT_WS}
+
+# Copy in the paddle inference library
+RUN mkdir -p install/semantic_segmentation/lib/
+COPY --from=PADDLE_INFERENCE_BUILD /paddle/paddle_inference_cuda120_build.tar install/semantic_segmentation/lib/paddle_inference_cuda120_build.tar
+RUN tar -xvf install/semantic_segmentation/lib/paddle_inference_cuda120_build.tar -C install/semantic_segmentation/lib
+RUN rm install/semantic_segmentation/lib/paddle_inference_cuda120_build.tar
+
+# Add runtime libraries to path
+ENV LD_LIBRARY_PATH=$LD_LIBRARY_PATH:${AMENT_WS}/install/semantic_segmentation/lib/
+ENV LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/lib
+ENV LD_LIBRARY_PATH=$LD_LIBRARY_PATH:${AMENT_WS}/install/semantic_segmentation/lib/paddle_inference_cuda120_build/paddle_inference_install_dir/paddle/lib/
+
+ENV LD_LIBRARY_PATH=$LD_LIBRARY_PATH:${AMENT_WS}/install/semantic_segmentation/lib/paddle_inference_cuda120_build/paddle_inference_install_dir/third_party/install/cryptopp/lib/
+ENV LD_LIBRARY_PATH=$LD_LIBRARY_PATH:${AMENT_WS}/install/semantic_segmentation/lib/paddle_inference_cuda120_build/paddle_inference_install_dir/third_party/install/mkldnn/lib/
+ENV LD_LIBRARY_PATH=$LD_LIBRARY_PATH:${AMENT_WS}/install/semantic_segmentation/lib/paddle_inference_cuda120_build/paddle_inference_install_dir/third_party/install/mklml/lib/
+
+COPY docker/wato_ros_entrypoint.sh ${AMENT_WS}/wato_ros_entrypoint.sh
+ENTRYPOINT ["./wato_ros_entrypoint.sh"]
 
 # Source Cleanup and Security Setup
 RUN chown -R $USER:$USER ${AMENT_WS}

modules/docker-compose.perception.yaml

@@ -55,7 +55,16 @@ services:
         - "${PERCEPTION_SEMANTIC_SEGMENTATION_IMAGE}:build_main"
       target: deploy
     image: "${PERCEPTION_SEMANTIC_SEGMENTATION_IMAGE}:${TAG}"
-    command: /bin/bash -c "ros2 launch semantic_segmentation semantic_segmentation.launch.py"
+    command: /bin/bash -c "ros2 launch semantic_segmentation eve.launch.py"
+    deploy:
+      resources:
+        reservations:
+          devices:
+            - driver: nvidia
+              count: 1
+              capabilities: [ gpu ]
+    volumes: 
+      - /mnt/wato-drive2/perception-weights/semantic_segmentation:/perception_models/semantic_segmentation
 
   lane_detection:
     build:

src/perception/semantic_segmentation/CMakeLists.txt

@@ -1,14 +1,96 @@
 cmake_minimum_required(VERSION 3.8)
-project(semantic_segmentation)
+project(semantic_segmentation LANGUAGES CXX CUDA)
 
+set(PADDLE_LIB "/paddle/paddle_inference_cuda120_build/paddle_inference_install_dir") # Set this path as per your setup
+
+option(WITH_MKL "Compile demo with MKL/OpenBlas support, default use MKL." ON)
+option(WITH_GPU "Compile demo with GPU/CPU, default use CPU." ON)
+option(WITH_STATIC_LIB "Compile demo with static/shared library, default use static." ON)
+option(USE_TENSORRT "Compile demo with TensorRT." ON)
+option(WITH_ROCM "Compile demo with rocm." OFF)
+
+# Compiler options
 if(CMAKE_COMPILER_IS_GNUCXX OR CMAKE_CXX_COMPILER_ID MATCHES "Clang")
   add_compile_options(-Wall -Wextra -Wpedantic)
 endif()
 
-# find dependencies
+# Default to C++14
+if(NOT CMAKE_CXX_STANDARD)
+  set(CMAKE_CXX_STANDARD 14)
+endif()
+
+# Platform-independent settings
+include_directories("${PADDLE_LIB}/")
+set(PADDLE_LIB_THIRD_PARTY_PATH "${PADDLE_LIB}/third_party/install/")
+include_directories(${PADDLE_LIB_THIRD_PARTY_PATH}protobuf/include ${PADDLE_LIB_THIRD_PARTY_PATH}glog/include ${PADDLE_LIB_THIRD_PARTY_PATH}gflags/include ${PADDLE_LIB_THIRD_PARTY_PATH}xxhash/include ${PADDLE_LIB_THIRD_PARTY_PATH}cryptopp/include ${PADDLE_LIB_THIRD_PARTY_PATH}onnxruntime/include ${PADDLE_LIB_THIRD_PARTY_PATH}paddle2onnx/include)
+
+link_directories(${PADDLE_LIB_THIRD_PARTY_PATH}protobuf/lib ${PADDLE_LIB_THIRD_PARTY_PATH}glog/lib ${PADDLE_LIB_THIRD_PARTY_PATH}gflags/lib ${PADDLE_LIB_THIRD_PARTY_PATH}xxhash/lib ${PADDLE_LIB_THIRD_PARTY_PATH}cryptopp/lib ${PADDLE_LIB_THIRD_PARTY_PATH}onnxruntime/lib ${PADDLE_LIB_THIRD_PARTY_PATH}paddle2onnx/lib ${PADDLE_LIB}/paddle/lib)
+
+# Find packages
+find_package(rclcpp REQUIRED)
+find_package(std_msgs REQUIRED)
+find_package(sensor_msgs REQUIRED)
+find_package(geometry_msgs REQUIRED)
+find_package(image_transport REQUIRED)
+find_package(OpenCV REQUIRED)
+find_package(cv_bridge REQUIRED)
+find_package(CUDA REQUIRED)
 find_package(ament_cmake REQUIRED)
-# uncomment the following section in order to fill in
-# further dependencies manually.
-# find_package(<dependency> REQUIRED)
+find_package(yaml-cpp REQUIRED)
+
+# Include directories
+include_directories(include 
+  ${rclcpp_INCLUDE_DIRS}
+  ${std_msgs_INCLUDE_DIRS}
+  ${sensor_msgs_INCLUDE_DIRS}
+  ${geometry_msgs_INCLUDE_DIRS}
+  ${image_transport_INCLUDE_DIRS}
+  ${CUDA_INCLUDE_DIRS}
+  ${YAML_CPP_INCLUDE_DIRS}
+  /usr/local/include
+  ${PADDLE_LIB}/paddle/include
+  ${PADDLE_LIB}/third_party/install/mkldnn/include
+  )
+
+
+# Link directories
+link_directories(/usr/local/lib ${YAML_CPP_LIBRARIES})
+
+
+# Set additional dependencies
+set(EXTRA_LIBS "-lrt -ldl -lpthread")
+set(DEPS ${rclcpp_LIBRARIES} ${std_msgs_LIBRARIES} ${sensor_msgs_LIBRARIES} ${geometry_msgs_LIBRARIES} ${image_transport_LIBRARIES} ${CUDA_LIBRARIES} ${YAML_CPP_LIBRARIES} ${EXTRA_LIBS}
+${PADDLE_LIB}/paddle/lib/libpaddle_inference.so
+${PADDLE_LIB}/paddle/lib/libphi.so
+${PADDLE_LIB}/paddle/lib/libcommon.so
+${PADDLE_LIB}/third_party/install/mkldnn/lib/libdnnl.so.3
+${PADDLE_LIB}/third_party/install/mklml/lib/libiomp5.so
+${PADDLE_LIB}/third_party/install/mklml/lib/libmklml_intel.so
+)
+
+# Add your executable
+add_executable(semantic_segmentation src/semantic_segmentation_node.cpp)
+target_link_libraries(semantic_segmentation ${DEPS})
+
+# Additional installation commands
+install(DIRECTORY launch config resource DESTINATION share/${PROJECT_NAME}/)
+install(TARGETS semantic_segmentation DESTINATION lib/${PROJECT_NAME})
+# Install shared libraries
+install(FILES ${PADDLE_LIB}/paddle/lib/libpaddle_inference.so
+              ${PADDLE_LIB}/paddle/lib/libphi.so
+              ${PADDLE_LIB}/paddle/lib/libcommon.so
+              ${PADDLE_LIB}/third_party/install/mkldnn/lib/libdnnl.so.3
+              ${PADDLE_LIB}/third_party/install/mklml/lib/libiomp5.so
+              ${PADDLE_LIB}/third_party/install/mklml/lib/libmklml_intel.so
+        DESTINATION /usr/local/lib)
 
+ament_target_dependencies(semantic_segmentation
+  rclcpp
+  std_msgs
+  sensor_msgs
+  cv_bridge
+  geometry_msgs
+  image_transport
+  OpenCV
+)
 ament_package()

src/perception/semantic_segmentation/config/eve_config.yaml

@@ -0,0 +1,8 @@
+semantic_segmentation_node:
+  ros__parameters:
+    input_topic: /camera/left/image_color
+    publish_topic: /camera/left/segmentations
+    publish_vis_topic: /camera/left/segmentations_viz
+    model_path: /perception_models/semantic_segmentation/pp_liteseg_infer_model/
+    save_dir: /segmented
+    save_images: false

src/perception/semantic_segmentation/launch/eve.launch.py

@@ -0,0 +1,21 @@
+from launch import LaunchDescription
+from launch_ros.actions import Node
+from ament_index_python.packages import get_package_share_directory
+import os
+
+
+def generate_launch_description():
+    config = os.path.join(
+        get_package_share_directory('semantic_segmentation'),
+        'config',
+        'eve_config.yaml'
+    )
+
+    return LaunchDescription([
+        Node(
+            package='semantic_segmentation',
+            executable='semantic_segmentation',
+            name='semantic_segmentation_node',
+            parameters=[config]
+        )
+    ])

src/perception/semantic_segmentation/package.xml

@@ -3,13 +3,25 @@
 <package format="3">
   <name>semantic_segmentation</name>
   <version>0.0.0</version>
-  <description>TODO: Package description</description>
-  <maintainer email="[email protected]">bolty</maintainer>
-  <license>TODO: License declaration</license>
+  <description>Package for semantic segmentation</description>
+  <maintainer email="[email protected]">Justin Leung</maintainer>
+  <maintainer email="[email protected]">Steven Gong</maintainer>
+  <license>Apache-2.0</license>
 
   <buildtool_depend>ament_cmake</buildtool_depend>
 
+  <test_depend>ament_lint_auto</test_depend>
+  <test_depend>ament_lint_common</test_depend>
+  <depend>sensor_msgs</depend>
+  <depend>geometry_msgs</depend>
+  <depend>std_msgs</depend>
+  <depend>OpenCV</depend>
+  <depend>cv_bridge</depend>
+  <depend>image_transport</depend>
 
+  <build_depend>rosidl_default_generators</build_depend>
+  <exec_depend>rosidl_default_runtime</exec_depend>
+  <member_of_group>rosidl_interface_packages</member_of_group>
   <export>
     <build_type>ament_cmake</build_type>
   </export>

src/perception/semantic_segmentation/resource/deploy.yaml

@@ -0,0 +1,12 @@
+Deploy:
+  input_shape:
+  - -1
+  - 3
+  - -1
+  - -1
+  model: model.pdmodel
+  output_dtype: int32
+  output_op: argmax
+  params: model.pdiparams
+  transforms:
+  - type: Normalize