Real time semantic segmentation node

docker/perception/semantic_segmentation/semantic_segmentation.Dockerfile

@@ -1,13 +1,23 @@
-ARG BASE_IMAGE=ghcr.io/watonomous/wato_monorepo/base:humble-ubuntu22.04
+ARG BASE_IMAGE=ghcr.io/watonomous/wato_monorepo/base:cuda12.0-humble-ubuntu22.04-devel
+ARG RUNTIME_IMAGE=ghcr.io/watonomous/wato_monorepo/base:cuda12.0-humble-ubuntu22.04
+ARG PADDLE_INFERENCE_BUILD_URL=ghcr.io/watonomous/perception_paddlepaddle_inference_build_cuda-12.0
+################################ Build library ################################
+FROM ${PADDLE_INFERENCE_BUILD_URL} as PADDLE_INFERENCE_BUILD
 
 ################################ Source ################################
 FROM ${BASE_IMAGE} as source
 
 WORKDIR ${AMENT_WS}/src
 
+# Copy in the paddle inference library
+RUN mkdir -p semantic_segmentation/src
+COPY --from=PADDLE_INFERENCE_BUILD /paddle/paddle_inference_cuda120_build.tar /paddle/paddle_inference_cuda120_build.tar
+RUN tar -xvf /paddle/paddle_inference_cuda120_build.tar -C /paddle/
+RUN rm /paddle/paddle_inference_cuda120_build.tar
+
 # Copy in source code 
 COPY src/perception/semantic_segmentation semantic_segmentation
-COPY src/wato_msgs/sample_msgs sample_msgs
+COPY src/perception/perception_utils perception_utils
 
 # Scan for rosdeps
 RUN apt-get -qq update && rosdep update && \
@@ -19,12 +29,15 @@ RUN apt-get -qq update && rosdep update && \
 ################################# Dependencies ################################
 FROM ${BASE_IMAGE} as dependencies
 
+RUN apt update && apt install -y tensorrt ros-humble-cv-bridge libopencv-dev
+
 # Install Rosdep requirements
 COPY --from=source /tmp/colcon_install_list /tmp/colcon_install_list
 RUN apt-fast install -qq -y --no-install-recommends $(cat /tmp/colcon_install_list)
 
 # Copy in source code from source stage
 WORKDIR ${AMENT_WS}
+COPY --from=source /paddle /paddle
 COPY --from=source ${AMENT_WS}/src src
 
 # Dependency Cleanup
@@ -43,10 +56,44 @@ RUN . /opt/ros/$ROS_DISTRO/setup.sh && \
 
 # Entrypoint will run before any CMD on launch. Sources ~/opt/<ROS_DISTRO>/setup.bash and ~/ament_ws/install/setup.bash
 COPY docker/wato_ros_entrypoint.sh ${AMENT_WS}/wato_ros_entrypoint.sh
+
+# Add runtime libraries to path
+ENV LD_LIBRARY_PATH=$LD_LIBRARY_PATH:${AMENT_WS}/install/semantic_segmentation/lib/
+ENV LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/lib
+
 ENTRYPOINT ["./wato_ros_entrypoint.sh"]
 
-################################ Prod ################################
-FROM build as deploy
+# ################################ Prod ################################
+FROM ${RUNTIME_IMAGE} as deploy
+
+
+# Install runtime libs
+RUN apt-get update && apt-get install -y \
+    ros-humble-cv-bridge \
+    tensorrt
+
+# Copy the compiled binary to the runtime image
+COPY --from=build ${AMENT_WS} ${AMENT_WS}
+
+WORKDIR ${AMENT_WS}
+
+# Copy in the paddle inference library
+RUN mkdir -p install/semantic_segmentation/lib/
+COPY --from=PADDLE_INFERENCE_BUILD /paddle/paddle_inference_cuda120_build.tar install/semantic_segmentation/lib/paddle_inference_cuda120_build.tar
+RUN tar -xvf install/semantic_segmentation/lib/paddle_inference_cuda120_build.tar -C install/semantic_segmentation/lib
+RUN rm install/semantic_segmentation/lib/paddle_inference_cuda120_build.tar
+
+# Add runtime libraries to path
+ENV LD_LIBRARY_PATH=$LD_LIBRARY_PATH:${AMENT_WS}/install/semantic_segmentation/lib/
+ENV LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/lib
+ENV LD_LIBRARY_PATH=$LD_LIBRARY_PATH:${AMENT_WS}/install/semantic_segmentation/lib/paddle_inference_cuda120_build/paddle_inference_install_dir/paddle/lib/
+
+ENV LD_LIBRARY_PATH=$LD_LIBRARY_PATH:${AMENT_WS}/install/semantic_segmentation/lib/paddle_inference_cuda120_build/paddle_inference_install_dir/third_party/install/cryptopp/lib/
+ENV LD_LIBRARY_PATH=$LD_LIBRARY_PATH:${AMENT_WS}/install/semantic_segmentation/lib/paddle_inference_cuda120_build/paddle_inference_install_dir/third_party/install/mkldnn/lib/
+ENV LD_LIBRARY_PATH=$LD_LIBRARY_PATH:${AMENT_WS}/install/semantic_segmentation/lib/paddle_inference_cuda120_build/paddle_inference_install_dir/third_party/install/mklml/lib/
+
+COPY docker/wato_ros_entrypoint.sh ${AMENT_WS}/wato_ros_entrypoint.sh
+ENTRYPOINT ["./wato_ros_entrypoint.sh"]
 
 # Source Cleanup and Security Setup
 RUN chown -R $USER:$USER ${AMENT_WS}

modules/docker-compose.perception.yaml

@@ -55,7 +55,16 @@ services:
         - "${PERCEPTION_SEMANTIC_SEGMENTATION_IMAGE}:build_main"
       target: deploy
     image: "${PERCEPTION_SEMANTIC_SEGMENTATION_IMAGE}:${TAG}"
-    command: /bin/bash -c "ros2 launch semantic_segmentation semantic_segmentation.launch.py"
+    command: /bin/bash -c "ros2 launch semantic_segmentation eve.launch.py"
+    deploy:
+      resources:
+        reservations:
+          devices:
+            - driver: nvidia
+              count: 1
+              capabilities: [ gpu ]
+    volumes: 
+      - /mnt/wato-drive2/perception-weights/semantic_segmentation/pp_liteseg_infer_model:/perception_models/semantic_segmentation/pp_liteseg_infer_model
 
   lane_detection:
     build:

src/perception/perception_utils/CMakeLists.txt

@@ -0,0 +1,49 @@
+cmake_minimum_required(VERSION 3.5)
+project(perception_utils)
+
+# Default to C++14
+if(NOT CMAKE_CXX_STANDARD)
+  set(CMAKE_CXX_STANDARD 14)
+endif()
+
+if(CMAKE_COMPILER_IS_GNUCXX OR CMAKE_CXX_COMPILER_ID MATCHES "Clang")
+  add_compile_options(-Wall -Wextra -Wpedantic)
+endif()
+
+# Find dependencies
+find_package(ament_cmake REQUIRED)
+find_package(rclcpp REQUIRED)
+find_package(OpenCV REQUIRED)
+
+# Include header files
+include_directories(include
+  ${OpenCV_INCLUDE_DIRS}
+)
+
+# Declare a C++ library
+add_library(perception_utils SHARED
+  src/camera_utils.cpp
+)
+
+# Specify libraries to link a library or executable target against
+ament_target_dependencies(perception_utils
+  rclcpp
+  OpenCV)
+
+install(TARGETS perception_utils
+  EXPORT export_perception_utils
+  ARCHIVE DESTINATION lib
+  LIBRARY DESTINATION lib
+  RUNTIME DESTINATION bin
+)
+
+ament_export_targets(export_perception_utils HAS_LIBRARY_TARGET)
+ament_export_include_directories(include)
+ament_export_dependencies(rclcpp)
+
+# Install header files
+install(DIRECTORY include/
+  DESTINATION include
+)
+
+ament_package()

src/perception/perception_utils/include/perception_utils/camera_utils.hpp

@@ -0,0 +1,14 @@
+#ifndef CAMERA_UTILS_HPP_
+#define CAMERA_UTILS_HPP_
+
+#include <opencv2/opencv.hpp>
+
+namespace CameraUtils {
+// Swaps the channels of an image from HWC to CHW format
+void hwc_img_2_chw_data(const cv::Mat& hwc_img, float* data);
+cv::Mat resize_image_aspect_ratio(const cv::Mat& original_image, int max_width, int max_height);
+cv::Mat resize_with_padding(const cv::Mat& original_image, int target_width, int target_height);
+cv::Mat resize_from_center(const cv::Mat& original_image, int target_width, int target_height);
+};  // namespace CameraUtils
+
+#endif  // CAMERA_UTILS_HPP_

src/perception/perception_utils/package.xml

@@ -0,0 +1,18 @@
+<?xml version="1.0"?>
+<package format="3">
+  <name>perception_utils</name>
+  <version>0.0.1</version>
+  <description>A utility library for perception tasks in ROS 2.</description>
+
+  <maintainer email="[email protected]">Justin Leung</maintainer>
+  <license>Apache-2.0</license>
+
+  <buildtool_depend>ament_cmake</buildtool_depend>
+  <depend>rclcpp</depend>
+  <depend>opencv</depend>
+
+  <export>
+    <build_type>ament_cmake</build_type>
+  </export>
+
+</package>

src/perception/perception_utils/src/camera_utils.cpp

@@ -0,0 +1,97 @@
+#include "perception_utils/camera_utils.hpp"
+#include <opencv2/opencv.hpp>
+
+// Swaps the channels of an image from HWC to CHW format
+void CameraUtils::hwc_img_2_chw_data(const cv::Mat& hwc_img, float* data) {
+  int rows = hwc_img.rows;
+  int cols = hwc_img.cols;
+  int chs = hwc_img.channels();
+  for (int i = 0; i < chs; ++i) {
+    cv::extractChannel(hwc_img, cv::Mat(rows, cols, CV_32FC1, data + i * rows * cols), i);
+  }
+}
+
+cv::Mat CameraUtils::resize_image_aspect_ratio(const cv::Mat& original_image, int max_width,
+                                               int max_height) {
+  int original_width = original_image.cols;
+  int original_height = original_image.rows;
+  double original_aspect_ratio = (double)original_width / original_height;
+
+  int new_width, new_height;
+  double max_aspect_ratio = (double)max_width / max_height;
+
+  if (original_aspect_ratio > max_aspect_ratio) {
+    // Width is the limiting factor
+    new_width = max_width;
+    new_height = static_cast<int>(max_width / original_aspect_ratio);
+  } else {
+    // Height is the limiting factor
+    new_height = max_height;
+    new_width = static_cast<int>(max_height * original_aspect_ratio);
+  }
+
+  cv::Mat resized_image;
+  cv::resize(original_image, resized_image, cv::Size(new_width, new_height));
+
+  return resized_image;
+}
+
+cv::Mat CameraUtils::resize_with_padding(const cv::Mat& original_image, int target_width,
+                                         int target_height) {
+  int original_width = original_image.cols;
+  int original_height = original_image.rows;
+
+  double target_ratio = (double)target_width / target_height;
+  double original_ratio = (double)original_width / original_height;
+
+  int new_width, new_height;
+
+  if (original_ratio > target_ratio) {
+    // Original is wider. Fit to width and pad height.
+    new_width = target_width;
+    new_height =
+        static_cast<int>(original_height * (static_cast<double>(target_width) / original_width));
+  } else {
+    // Original is taller. Fit to height and pad width.
+    new_height = target_height;
+    new_width =
+        static_cast<int>(original_width * (static_cast<double>(target_height) / original_height));
+  }
+
+  cv::Mat resized_image;
+  cv::resize(original_image, resized_image, cv::Size(new_width, new_height));
+
+  int top = (target_height - new_height) / 2;
+  int bottom = target_height - new_height - top;
+  int left = (target_width - new_width) / 2;
+  int right = target_width - new_width - left;
+
+  cv::Mat padded_image;
+  cv::copyMakeBorder(resized_image, padded_image, top, bottom, left, right, cv::BORDER_CONSTANT,
+                     cv::Scalar(0, 0, 0));
+
+  return padded_image;
+}
+
+cv::Mat CameraUtils::resize_from_center(const cv::Mat& original_image, int target_width,
+                                        int target_height) {
+  int original_width = original_image.cols;
+  int original_height = original_image.rows;
+
+  // Calculate the new height maintaining the aspect ratio
+  double target_ratio = (double)target_width / target_height;
+  int new_height = static_cast<int>(original_width / target_ratio);
+
+  // Calculate the cropping area
+  int startY = (original_height - new_height) / 2;
+
+  // Crop the image from the center
+  cv::Rect roi(0, startY, original_width, new_height);
+  cv::Mat cropped_image = original_image(roi);
+
+  // Resize the cropped image
+  cv::Mat resized_image;
+  cv::resize(cropped_image, resized_image, cv::Size(target_width, target_height));
+
+  return resized_image;
+}