Generate messages in downstream builds (#339)

Expose message generation functionality to downstream packages and remove message generation from the msgs package. This new approach will now distribute the proto files, but will no longer distribute a binary library that downstream users will link against. Instead, downstream users can generate the messages that they need themselves or add new message definitions in their own packages. A summary of the changes required to make this happen: * Expose the mechanisms we use for message generation * This requires installing the protoc generator as well as our python script. * Add cmake functionality to let downstream users discover available messages and retrieve the installation path of the generator and python script (handled via cmake extras) * Create a core library that has the gz-msgs functionality minus the generated code. * Create a compiled library that maintains the CLI functionality. * Make conversions a header-only operation, so that the core library doesn't need to depend on the generated messages. Signed-off-by: Michael Carroll <[email protected]> Co-authored-by: Ian Chen <[email protected]> Co-authored-by: Addisu Z. Taddese <[email protected]>
gazebosim · Jun 23, 2023 · 0c78b96 · 0c78b96
1 parent 3c62ef7
commit 0c78b96
Show file tree

Hide file tree

Showing 72 changed files with 4,131 additions and 3,120 deletions.
diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -17,13 +17,14 @@ find_package(gz-cmake3 REQUIRED)
 set(CMAKE_CXX_STANDARD 17)
 set(CMAKE_CXX_STANDARD_REQUIRED ON)
 
-gz_configure_project(VERSION_SUFFIX)
+gz_configure_project(VERSION_SUFFIX
+  CONFIG_EXTRAS "gz-msgs-extras.cmake.in")
 
-if (UNIX AND NOT APPLE)
-  set (EXTRA_TEST_LIB_DEPS stdc++fs)
-else()
-  set (EXTRA_TEST_LIB_DEPS)
-endif()
+# Install cmake support files
+install(
+  DIRECTORY cmake/
+  DESTINATION "lib/cmake/${PROJECT_NAME}"
+)
 
 #============================================================================
 # Set project-specific options
@@ -71,6 +72,11 @@ gz_find_package(GzProtobuf
                  COMPONENTS all
                  PRETTY Protobuf)
 
+#--------------------------------------
+# Find gz-utils
+gz_find_package(gz-utils2 REQUIRED)
+set(GZ_UTILS_VER ${gz-utils2_VERSION_MAJOR})
+
 #--------------------------------------
 # Find gz-math
 gz_find_package(gz-math7 REQUIRED)
@@ -90,7 +96,7 @@ gz_find_package(TINYXML2 REQUIRED PRIVATE PRETTY tinyxml2)
 #============================================================================
 # Configure the build
 #============================================================================
-gz_configure_build(QUIT_IF_BUILD_ERRORS)
+gz_configure_build(QUIT_IF_BUILD_ERRORS COMPONENTS compiled)
 
 #============================================================================
 # gz command line support
@@ -126,7 +132,6 @@ configure_file(${CMAKE_SOURCE_DIR}/tutorials.md.in ${CMAKE_BINARY_DIR}/tutorials
 gz_create_docs(
   API_MAINPAGE_MD "${CMAKE_BINARY_DIR}/api.md"
   TUTORIALS_MAINPAGE_MD "${CMAKE_BINARY_DIR}/tutorials.md"
-  AUTOGENERATED_DOC "${CMAKE_BINARY_DIR}/include/gz/msgs/details"
   TAGFILES
     "${GZ-MATH_DOXYGEN_TAGFILE} = ${GZ-MATH_API_URL}"
 )

diff --git a/Migration.md b/Migration.md
@@ -5,6 +5,16 @@ Deprecated code produces compile-time warnings. These warning serve as
 notification to users that their code should be upgraded. The next major
 release will remove the deprecated code.
 
+## Gazebo Msgs 9.X to 10.X
+
+### Breaking changes
+
+1. The way that messages are included by downstream projects has been changed.  
+  The messages package will now only install `.proto` files, and it is the responsibility of downstream
+  users of the msgs library to generate corresponding headers and source files via cmake macros.
+  * For more information, consult the `using_gz_msgs` example.
+  * Note that there will no longer be Ruby generated messages, this support will be restored as-needed. 
+
 ## Gazebo Msgs 8.X to 9.X
 
 1. **SuppressWarnings.hh** is deprecated and isn't part of `msgs.hh` anymore.

diff --git a/cmake/gz_msgs_factory.cmake b/cmake/gz_msgs_factory.cmake
@@ -0,0 +1,71 @@
+##################################################
+# A function that generates factory methods for generated gz-msgs
+# The output of this is are a header and source file that can be used as part of your library
+# for the gz::msgs::Factory
+# Options:
+# One value arguments:
+#   FACTORY_GEN_SCRIPT  - Location of the factory generator script
+#   PROTO_PACKAGE       - Protobuf package the file belongs to (e.g. "gz.msgs")
+#   PROTOC_EXEC         - Path to protoc
+#   OUTPUT_CPP_DIR      - Path where C++ files are saved
+#   OUTPUT_CPP_HH_VAR   - A CMake variable name containing a list that the C++ headers should be appended to
+#   OUTPUT_CPP_CC_VAR   - A Cmake variable name containing a list that the C++ sources should be appended to
+# Multi value arguments
+#   INPUT_PROTOS        - List of input proto files
+#   PROTO_PATH          - Base directory of the proto files
+function(gz_msgs_factory)
+  set(options "")
+  set(oneValueArgs
+    FACTORY_GEN_SCRIPT
+    PROTO_PACKAGE
+    OUTPUT_CPP_DIR
+    OUTPUT_CPP_HH_VAR
+    OUTPUT_CPP_CC_VAR)
+  set(multiValueArgs INPUT_PROTOS PROTO_PATH)
+
+  cmake_parse_arguments(gz_msgs_factory "${options}" "${oneValueArgs}" "${multiValueArgs}" ${ARGN})
+
+  _gz_msgs_proto_pkg_to_path(${gz_msgs_factory_PROTO_PACKAGE} proto_package_dir)
+
+  set(output_header "${gz_msgs_factory_OUTPUT_CPP_DIR}/${proto_package_dir}/MessageTypes.hh")
+  set(output_source "${gz_msgs_factory_OUTPUT_CPP_DIR}/${proto_package_dir}/register.cc")
+
+  list(APPEND ${gz_msgs_factory_OUTPUT_CPP_HH_VAR} ${output_header})
+  list(APPEND ${gz_msgs_factory_OUTPUT_CPP_CC_VAR} ${output_source})
+
+  list(APPEND output_files ${output_header})
+  list(APPEND output_files ${output_source})
+
+  set(${gz_msgs_factory_OUTPUT_CPP_HH_VAR} ${${gz_msgs_factory_OUTPUT_CPP_HH_VAR}} PARENT_SCOPE)
+  set(${gz_msgs_factory_OUTPUT_CPP_CC_VAR} ${${gz_msgs_factory_OUTPUT_CPP_CC_VAR}} PARENT_SCOPE)
+
+  set(depends_index)
+  # Full path to an index file, which contains all defined message types for that proto file
+  foreach(proto_file ${gz_msgs_factory_INPUT_PROTOS})
+    # Get a unique path (gz.msgs.foo -> gz_msgs_foo) for naming the index
+    _gz_msgs_proto_to_unique(${proto_file} ${gz_msgs_factory_PROTO_PACKAGE} UNIQUE_NAME)
+    set(input_index "${gz_msgs_factory_OUTPUT_CPP_DIR}/${UNIQUE_NAME}.pb_index")
+    list(APPEND depends_index ${input_index})
+  endforeach()
+
+  set(GENERATE_ARGS
+    --output-cpp-path "${gz_msgs_factory_OUTPUT_CPP_DIR}"
+    --proto-package "${gz_msgs_factory_PROTO_PACKAGE}"
+    --proto-path "${gz_msgs_factory_PROTO_PATH}"
+    --protos "${gz_msgs_factory_INPUT_PROTOS}"
+    )
+
+  add_custom_command(
+    OUTPUT ${output_files}
+    COMMAND Python3::Interpreter
+    ARGS ${gz_msgs_factory_FACTORY_GEN_SCRIPT} ${GENERATE_ARGS}
+    DEPENDS
+      ${depends_index}
+    # While the script is executed in the source directory, it does not write
+    # to the source tree.  All outputs are stored in the build directory.
+    WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}
+    COMMENT "Running factory generator"
+    VERBATIM
+  )
+
+endfunction()
diff --git a/cmake/gz_msgs_generate.cmake b/cmake/gz_msgs_generate.cmake
@@ -0,0 +1,127 @@
+##################################################
+# The implementation of gz_msgs_generate_messages
+# Options:
+# One value arguments:
+#   MSGS_GEN_SCRIPT     - Location of the messge generator script
+#   FACTORY_GEN_SCRIPT  - Location of the factory generator script
+#   GZ_PROTOC_PLUGIN    - Location of the gazebo generator plugin
+#   PROTO_PATH          - Base directory of the proto files
+#   PROTO_PACKAGE       - Protobuf package the file belongs to (e.g. "gz.msgs")
+#   MSGS_LIB            - gz-msgs library to link to
+#   TARGET              - Target (static library) to create
+# Multi value arguments
+#   INPUT_PROTOS        - List of input proto files
+#   DEPENDENCIES        - List of generated messages targets that these messages depend on
+#                         Primarily used when generating new custom messages downstream
+#                         that depend on gz-msgs
+function(gz_msgs_generate_messages_impl)
+  set(options "")
+  set(oneValueArgs TARGET PROTO_PACKAGE MSGS_GEN_SCRIPT GZ_PROTOC_PLUGIN FACTORY_GEN_SCRIPT MSGS_LIB PROTO_PATH)
+  set(multiValueArgs INPUT_PROTOS DEPENDENCIES)
+
+  cmake_parse_arguments(generate_messages "${options}" "${oneValueArgs}" "${multiValueArgs}" ${ARGN})
+  _gz_msgs_proto_pkg_to_string(${generate_messages_PROTO_PACKAGE} gen_dir)
+
+  # Extract dependency information from targets
+  set(depends_proto_paths)
+  set(depends_includes)
+
+  foreach(dep ${generate_messages_DEPENDENCIES})
+    get_target_property(dep_proto_path ${dep} PROTO_DIR)
+    get_target_property(dep_proto_include_path ${dep} PROTO_INCLUDE_DIR)
+
+    list(APPEND depends_proto_paths ${dep_proto_path})
+    list(APPEND depends_includes ${dep_proto_include_path})
+  endforeach()
+
+  foreach(proto_file ${generate_messages_INPUT_PROTOS})
+    gz_msgs_protoc(
+      MSGS_GEN_SCRIPT
+        ${generate_messages_MSGS_GEN_SCRIPT}
+      PROTO_PACKAGE
+        ${generate_messages_PROTO_PACKAGE}
+      GENERATE_CPP
+      INPUT_PROTO
+        ${proto_file}
+      PROTOC_EXEC
+        protobuf::protoc
+      GZ_PROTOC_PLUGIN
+        ${generate_messages_GZ_PROTOC_PLUGIN}
+      OUTPUT_CPP_DIR
+        "${PROJECT_BINARY_DIR}/${gen_dir}_gen"
+      OUTPUT_INCLUDES
+        gen_includes
+      OUTPUT_CPP_HH_VAR
+        gen_headers
+      OUTPUT_DETAIL_CPP_HH_VAR
+        gen_detail_headers
+      OUTPUT_CPP_CC_VAR
+        gen_sources
+      PROTO_PATH
+        ${generate_messages_PROTO_PATH}
+      DEPENDENCY_PROTO_PATHS
+        ${depends_proto_paths}
+    )
+  endforeach()
+
+  gz_msgs_factory(
+    FACTORY_GEN_SCRIPT
+      ${generate_messages_FACTORY_GEN_SCRIPT}
+    PROTO_PACKAGE
+      ${generate_messages_PROTO_PACKAGE}
+    INPUT_PROTOS
+      ${generate_messages_INPUT_PROTOS}
+    OUTPUT_CPP_DIR
+      "${PROJECT_BINARY_DIR}/${gen_dir}_gen"
+    OUTPUT_CPP_HH_VAR
+      gen_factory_headers
+    OUTPUT_CPP_CC_VAR
+      gen_factory_sources
+    PROTO_PATH
+      ${generate_messages_PROTO_PATH}
+  )
+
+  set_source_files_properties(
+    ${gen_headers}
+    ${gen_detail_headers}
+    ${gen_sources}
+    ${gen_factory_headers}
+    ${gen_factory_sources}
+    PROPERTIES GENERATED TRUE)
+
+  if(WIN32)
+    set_source_files_properties(${gen_sources}
+      COMPILE_FLAGS "/wd4100 /wd4512 /wd4127 /wd4068 /wd4244 /wd4267 /wd4251 /wd4146")
+  endif()
+
+  if(NOT MSVC)
+    # -Wno-switch-default flags is required for suppressing a warning in some of
+    # the generated protobuf files.
+    set_source_files_properties(${gen_sources} COMPILE_FLAGS "-Wno-switch-default -Wno-float-equal")
+  endif()
+
+  add_library(${generate_messages_TARGET} STATIC ${gen_sources} ${gen_factory_sources})
+
+  # Use position indepedent code (-fPIC), because this library may be linked
+  # into a shared library by the consumer
+  set_property(TARGET ${generate_messages_TARGET} PROPERTY POSITION_INDEPENDENT_CODE ON)
+
+  # Export the messages path and dependency messages paths for potential dependent message libs
+  set(PROTO_DIR)
+  list(APPEND PROTO_DIR ${generate_messages_PROTO_PATH})
+  list(APPEND PROTO_DIR ${depends_proto_paths})
+
+  set(PROTO_INCLUDE_DIR)
+  list(APPEND PROTO_INCLUDE_DIR ${PROJECT_BINARY_DIR}/${gen_dir}_gen)
+  list(APPEND PROTO_INCLUDE_DIR ${depends_includes})
+
+  set_target_properties(${generate_messages_TARGET} PROPERTIES PROTO_DIR "${PROTO_DIR}")
+  set_target_properties(${generate_messages_TARGET} PROPERTIES PROTO_INCLUDE_DIR "${PROTO_INCLUDE_DIR}")
+
+  foreach(dep ${generate_messages_DEPENDENCIES})
+    add_dependencies(${generate_messages_TARGET} ${dep})
+  endforeach()
+
+  target_link_libraries(${generate_messages_TARGET} PUBLIC protobuf::libprotobuf ${generate_messages_MSGS_LIB})
+  target_include_directories(${generate_messages_TARGET} PUBLIC ${PROJECT_BINARY_DIR}/${gen_dir}_gen ${depends_includes})
+endfunction()
diff --git a/cmake/gz_msgs_protoc.cmake b/cmake/gz_msgs_protoc.cmake
@@ -0,0 +1,108 @@
+##################################################
+# A function that calls protoc on a protobuf file
+# Options:
+#   GENERATE_CPP        - generates c++ code for the message if specified
+# One value arguments:
+#   MSGS_GEN_SCRIPT     - Path to the message generation python script
+#   PROTO_PACKAGE       - Protobuf package the file belongs to (e.g. "gz.msgs")
+#   PROTOC_EXEC         - Path to protoc
+#   GZ_PROTOC_PLUGIN    - Path to the gazebo-specific protobuf compiler executable
+#   INPUT_PROTO         - Path to the input .proto file
+#   OUTPUT_CPP_DIR      - Path where C++ files are saved
+#   OUTPUT_INCLUDES     - A CMake variable name containing a list that the C++ header path should be appended to
+#   OUTPUT_CPP_HH_VAR   - A CMake variable name containing a list generated headers should be appended to
+#   OUTPUT_DETAIL_CPP_HH_VAR - A CMake variable name containing a list that the C++ detail headers should be appended to
+#   OUTPUT_CPP_CC_VAR   - A Cmake variable name containing a list that the C++ source files should be appended to
+# Multi value arguments
+#   PROTO_PATH          - Passed to protoc --proto_path
+#   DEPENDENCY_PROTO_PATHS - Passed to protoc --proto_path
+function(gz_msgs_protoc)
+  set(options GENERATE_CPP)
+  set(oneValueArgs
+    MSGS_GEN_SCRIPT
+    PROTO_PACKAGE
+    PROTOC_EXEC
+    GZ_PROTOC_PLUGIN
+    INPUT_PROTO
+    OUTPUT_CPP_DIR
+    OUTPUT_INCLUDES
+    OUTPUT_CPP_HH_VAR
+    OUTPUT_DETAIL_CPP_HH_VAR
+    OUTPUT_CPP_CC_VAR)
+  set(multiValueArgs PROTO_PATH DEPENDENCY_PROTO_PATHS)
+
+  cmake_parse_arguments(gz_msgs_protoc "${options}" "${oneValueArgs}" "${multiValueArgs}" ${ARGN})
+
+  get_filename_component(ABS_FIL ${gz_msgs_protoc_INPUT_PROTO} ABSOLUTE)
+  get_filename_component(FIL_WE ${gz_msgs_protoc_INPUT_PROTO} NAME_WE)
+
+  set(protoc_args)
+  set(output_files)
+
+  _gz_msgs_proto_pkg_to_path(${gz_msgs_protoc_PROTO_PACKAGE} proto_package_dir)
+
+  if(gz_msgs_protoc_GENERATE_CPP)
+    # Full path to gazeob-specific header (${PROJECT_BINARY_DIR}/include/gz/msgs/foo.pb.h)
+    set(output_header "${gz_msgs_protoc_OUTPUT_CPP_DIR}/${proto_package_dir}/${FIL_WE}.pb.h")
+    # Full path to generated protobuf header (${PROJECT_BINARY_DIR}/include/gz/msgs/details/foo.pb.h)
+    set(output_detail_header "${gz_msgs_protoc_OUTPUT_CPP_DIR}/${proto_package_dir}/details/${FIL_WE}.pb.h")
+    # Full path to generated protobuf source (${PROJECT_BINARY_DIR}/include/foo.pb.cc)
+    set(output_source "${gz_msgs_protoc_OUTPUT_CPP_DIR}/${proto_package_dir}/${FIL_WE}.pb.cc")
+
+    _gz_msgs_proto_to_unique(${gz_msgs_protoc_INPUT_PROTO} ${gz_msgs_protoc_PROTO_PACKAGE} UNIQUE_NAME)
+
+    # Full path to an index file, which contains all defined message types for that proto file
+    set(output_index "${gz_msgs_protoc_OUTPUT_CPP_DIR}/${UNIQUE_NAME}.pb_index")
+
+    # Generate a clean relative path (gz/msgs/foo.pb.h)
+    string(REPLACE "${PROJECT_BINARY_DIR}/include/" "" output_include ${output_header})
+    list(APPEND ${gz_msgs_protoc_OUTPUT_INCLUDES} "${output_include}")
+
+    list(APPEND ${gz_msgs_protoc_OUTPUT_CPP_HH_VAR} ${output_header})
+    list(APPEND ${gz_msgs_protoc_OUTPUT_CPP_CC_VAR} ${output_source})
+    list(APPEND ${gz_msgs_protoc_OUTPUT_DETAIL_CPP_HH_VAR} ${output_detail_header})
+
+    list(APPEND output_files ${output_header})
+    list(APPEND output_files ${output_detail_header})
+    list(APPEND output_files ${output_source})
+    list(APPEND output_files ${output_index})
+
+    set(${gz_msgs_protoc_OUTPUT_INCLUDES} ${${gz_msgs_protoc_OUTPUT_INCLUDES}} PARENT_SCOPE)
+    set(${gz_msgs_protoc_OUTPUT_DETAIL_CPP_HH_VAR} ${${gz_msgs_protoc_OUTPUT_DETAIL_CPP_HH_VAR}} PARENT_SCOPE)
+    set(${gz_msgs_protoc_OUTPUT_CPP_HH_VAR} ${${gz_msgs_protoc_OUTPUT_CPP_HH_VAR}} PARENT_SCOPE)
+    set(${gz_msgs_protoc_OUTPUT_CPP_CC_VAR} ${${gz_msgs_protoc_OUTPUT_CPP_CC_VAR}} PARENT_SCOPE)
+  endif()
+
+  set(GENERATE_ARGS
+      --protoc-exec "$<TARGET_FILE:${gz_msgs_protoc_PROTOC_EXEC}>"
+      --gz-generator-bin "${gz_msgs_protoc_GZ_PROTOC_PLUGIN}"
+      --proto-path "${gz_msgs_protoc_PROTO_PATH}"
+      --input-path "${ABS_FIL}"
+  )
+
+  if(gz_msgs_protoc_DEPENDENCY_PROTO_PATHS)
+    list(APPEND GENERATE_ARGS
+      --dependency-proto-paths "${gz_msgs_protoc_DEPENDENCY_PROTO_PATHS}"
+    )
+  endif()
+
+  if(${gz_msgs_protoc_GENERATE_CPP})
+    list(APPEND GENERATE_ARGS
+      --generate-cpp
+      --output-cpp-path "${gz_msgs_protoc_OUTPUT_CPP_DIR}")
+  endif()
+
+  add_custom_command(
+    OUTPUT ${output_files}
+    COMMAND Python3::Interpreter
+    ARGS ${gz_msgs_protoc_MSGS_GEN_SCRIPT} ${GENERATE_ARGS}
+    DEPENDS
+      ${ABS_FIL}
+    # While the script is executed in the source directory, it does not write
+    # to the source tree.  All outputs are stored in the build directory.
+    WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}
+    COMMENT "Running protoc on ${gz_msgs_protoc_INPUT_PROTO}"
+    VERBATIM
+  )
+
+endfunction()