From 9e73fb9da2e14bd0fdf7b258d51cc6e12b92a1de Mon Sep 17 00:00:00 2001 From: Filippo Brizzi Date: Fri, 2 Feb 2024 08:53:58 +0000 Subject: [PATCH] Add new action to create docker image (#16) # Description Add new action to create docker image ## Type of change - New feature (non-breaking change which adds functionality) ## Checklist before requesting a review - [x] I have performed a self-review of my code. - [x] If it is a core feature, I have added thorough tests. - [x] If this is a new component I have added examples. - [x] I updated the README and related documentation. --- .dockerignore | 37 +++++++++ .github/workflows/build.yml | 83 +++++++++++++++++++++ .github/workflows/cmake-single-platform.yml | 41 ---------- CMakePresets.json | 29 ++++++- docker/Dockerfile | 1 - docker/Dockerfile_deps | 17 +++++ docker/build.sh | 28 ++++++- docker/build_deps.sh | 32 ++++++++ docker/version.sh | 5 +- external/CMakeLists.txt | 67 ++++++++++------- modules/utils/src/version_impl.h | 6 +- 11 files changed, 268 insertions(+), 78 deletions(-) create mode 100644 .dockerignore create mode 100644 .github/workflows/build.yml delete mode 100644 .github/workflows/cmake-single-platform.yml create mode 100644 docker/Dockerfile_deps create mode 100755 docker/build_deps.sh diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 00000000..e8e57c95 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,37 @@ +.cache/ +.git/ +.vscode/ +build/ + +# Prerequisites +*.d + +# Compiled Object files +*.slo +*.lo +*.o +*.obj + +# Precompiled Headers +*.gch +*.pch + +# Compiled Dynamic libraries +*.so +*.dylib +*.dll + +# Fortran module files +*.mod +*.smod + +# Compiled Static libraries +*.lai +*.la +*.a +*.lib + +# Executables +*.exe +*.out +*.app diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml new file mode 100644 index 00000000..f6c3a779 --- /dev/null +++ b/.github/workflows/build.yml @@ -0,0 +1,83 @@ +# This starter workflow is for a CMake project running on a single platform. There is a different starter workflow if you need cross-platform coverage. +# See: https://github.com/actions/starter-workflows/blob/main/ci/cmake-multi-platform.yml +name: Build and deploy pipeline + +on: + push: + branches: [ "main" ] + pull_request: + branches: [ "main" ] + +env: + # Customize the CMake build type here (Release, Debug, RelWithDebInfo, etc.) + PRESET: preferred + +jobs: + docker_build_and_publish: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4.1.1 + - uses: dorny/paths-filter@v3.0.0 + id: filter + with: + filters: | + docker: + - 'docker/**' + - name: Docker login + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - name: Build and push the image + if: steps.filter.outputs.docker == 'true' + run: | + cd ${{github.workspace}}/docker + ./build.sh + docker_deps_build_and_publish: + needs: docker_build_and_publish + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4.1.1 + - uses: dorny/paths-filter@v3.0.0 + id: filter + with: + filters: | + external: + - 'external/**' + - name: Docker login + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - name: Build and push the image + if: steps.filter.outputs.external == 'true' + run: | + cd ${{github.workspace}}/docker + ./build_deps.sh + build: + needs: docker_deps_build_and_publish + # TODO: use the docker image published at the step over + runs-on: ubuntu-latest + container: + image: ghcr.io/filippobrizzi/x86_64/eolo-dev_deps:latest + credentials: + username: ${{ github.actor}} + password: ${{secrets.GITHUB_TOKEN}} + steps: + - uses: actions/checkout@v3 + - name: Configure CMake + # Configure CMake in a 'build' subdirectory. `CMAKE_BUILD_TYPE` is only required if you are using a single-configuration generator such as make. + # See https://cmake.org/cmake/help/latest/variable/CMAKE_BUILD_TYPE.html?highlight=cmake_build_type + run: cmake -B ${{github.workspace}}/build --preset CI + # TODO: add a step to run the formatter + - name: Build + # Build your program with the given configuration + run: cmake --build ${{github.workspace}}/build -- all examples + + - name: Test + working-directory: ${{github.workspace}}/build + # Execute tests defined by the CMake configuration. + # See https://cmake.org/cmake/help/latest/manual/ctest.1.html for more detail + run: ninja checks diff --git a/.github/workflows/cmake-single-platform.yml b/.github/workflows/cmake-single-platform.yml deleted file mode 100644 index ab23dc8f..00000000 --- a/.github/workflows/cmake-single-platform.yml +++ /dev/null @@ -1,41 +0,0 @@ -# This starter workflow is for a CMake project running on a single platform. There is a different starter workflow if you need cross-platform coverage. -# See: https://github.com/actions/starter-workflows/blob/main/ci/cmake-multi-platform.yml -name: CMake on a single platform - -on: - push: - branches: [ "main" ] - pull_request: - branches: [ "main" ] - -env: - # Customize the CMake build type here (Release, Debug, RelWithDebInfo, etc.) - PRESET: preferred - -jobs: - build: - # The CMake configure and build commands are platform agnostic and should work equally well on Windows or Mac. - # You can convert this to a matrix build if you need cross-platform coverage. - # See: https://docs.github.com/en/free-pro-team@latest/actions/learn-github-actions/managing-complex-workflows#using-a-build-matrix - runs-on: ubuntu-latest - - steps: - - uses: actions/checkout@v3 - - - name: Install dependencies - run: sudo apt update && sudo apt install -y ninja-build - - - name: Configure CMake - # Configure CMake in a 'build' subdirectory. `CMAKE_BUILD_TYPE` is only required if you are using a single-configuration generator such as make. - # See https://cmake.org/cmake/help/latest/variable/CMAKE_BUILD_TYPE.html?highlight=cmake_build_type - run: cmake -B ${{github.workspace}}/build -DBUILD_MODULES=all -DCMAKE_BUILD_TYPE=Release - - - name: Build - # Build your program with the given configuration - run: cmake --build ${{github.workspace}}/build -- all examples - - - name: Test - working-directory: ${{github.workspace}}/build - # Execute tests defined by the CMake configuration. - # See https://cmake.org/cmake/help/latest/manual/ctest.1.html for more detail - run: make check diff --git a/CMakePresets.json b/CMakePresets.json index 94c325e2..238f4750 100644 --- a/CMakePresets.json +++ b/CMakePresets.json @@ -11,7 +11,34 @@ "cacheVariables": { "BUILD_MODULES": "all" } - } + }, + { + "name": "deps-docker", + "description": "Configuration setting for all modules with preferred toolchain", + "generator": "Ninja", + "binaryDir": "${sourceDir}/build", + "installDir": "/install", + "toolchainFile": "${sourceDir}/toolchains/toolchain_clang.cmake", + "cacheVariables": { + "BUILD_MODULES": "all", + "ENABLE_LINTER": "OFF", + "CMAKE_BUILD_TYPE": "Release" + } + }, + { + "name": "CI", + "description": "Configuration setting for all modules with preferred toolchain", + "generator": "Ninja", + "binaryDir": "${sourceDir}/build", + "installDir": "/install", + "toolchainFile": "${sourceDir}/toolchains/toolchain_clang.cmake", + "cacheVariables": { + "BUILD_MODULES": "all", + "ENABLE_LINTER": "ON", + "CMAKE_BUILD_TYPE": "Release", + "CMAKE_PREFIX_PATH": "/install" + } + } ], "buildPresets": [ { diff --git a/docker/Dockerfile b/docker/Dockerfile index dbe1bd4f..3789c280 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -1,6 +1,5 @@ ARG BASE_IMAGE -# FROM arm64v8/ubuntu:22.04 FROM ${BASE_IMAGE} ENV DEBIAN_FRONTEND=noninteractive diff --git a/docker/Dockerfile_deps b/docker/Dockerfile_deps new file mode 100644 index 00000000..ee11a596 --- /dev/null +++ b/docker/Dockerfile_deps @@ -0,0 +1,17 @@ +ARG BASE_IMAGE +FROM ${BASE_IMAGE} + +SHELL ["/bin/bash", "-c"] + +COPY . /tmp/eolo/ +RUN ls /tmp/eolo/ + +RUN mkdir /tmp/eolo/build +WORKDIR /tmp/eolo/build +RUN . "$HOME/.cargo/env" && cmake ../ --preset deps-docker +RUN ninja all && \ + ninja install + +RUN cd / && rm -rf /tmp/eolo + +WORKDIR / diff --git a/docker/build.sh b/docker/build.sh index c7d747c5..a62ec90d 100755 --- a/docker/build.sh +++ b/docker/build.sh @@ -2,16 +2,36 @@ source ./version.sh - -ARCH=$(uname -m) +HOST_ARCH=$(uname -m) +ARCH=${1:-${HOST_ARCH}} +if [ "${ARCH}" == "aarch64" ]; then + ARCH = "arm64" +fi if [ "${ARCH}" == "x86_64" ]; then # see https://hub.docker.com/r/nvidia/opengl/tags BASE_IMAGE="nvidia/opengl:1.0-glvnd-runtime-ubuntu22.04" -else +elif [ "${ARCH}" == "arm64" ]; then BASE_IMAGE="arm64v8/ubuntu:22.04" +else + echo "Supported arch value: arm64, x86_64" + exit 1 fi ncores=$(cat /proc/cpuinfo | grep processor | wc -l) -docker build -t ${IMAGE}:${VERSION} -f Dockerfile --cpuset-cpus "0-$ncores" --build-arg BASE_IMAGE=${BASE_IMAGE} . +IMAGE_NAME="ghcr.io/filippobrizzi/${ARCH}/${IMAGE}" + +function docker_tag_exists() { + docker manifest inspect ${IMAGE_NAME}:${VERSION} > /dev/null +} + +if docker_tag_exists; then + echo "Image already exists, you can pull it with:" + echo "$ docker pull ${IMAGE_NAME}:${VERSION}" +else + echo "Building image: ${IMAGE_NAME}:${VERSION}" + docker build -t ${IMAGE_NAME}:${VERSION} -f Dockerfile --cpuset-cpus "0-$ncores" --build-arg BASE_IMAGE=${BASE_IMAGE} . --tag ${IMAGE_NAME}:latest + docker push ${IMAGE_NAME}:${VERSION} + docker push ${IMAGE_NAME}:latest +fi diff --git a/docker/build_deps.sh b/docker/build_deps.sh new file mode 100755 index 00000000..699f8926 --- /dev/null +++ b/docker/build_deps.sh @@ -0,0 +1,32 @@ +#!/bin/bash + +source ./version.sh + +HOST_ARCH=$(uname -m) +ARCH=${1:-${HOST_ARCH}} +if [ "${ARCH}" == "aarch64" ]; then + ARCH = "arm64" +fi + +BASE_IMAGE=ghcr.io/filippobrizzi/${ARCH}/${IMAGE}:${VERSION} +docker pull ${BASE_IMAGE} + +SUFFIX=deps +IMAGE_NAME="ghcr.io/filippobrizzi/${ARCH}/${IMAGE}_${SUFFIX}" + +function docker_tag_exists() { + docker manifest inspect ${IMAGE_NAME}:${VERSION} > /dev/null +} + +if docker_tag_exists; then + echo "Image already exists, you can pull it with:" + echo "$ docker pull ${IMAGE_NAME}:${DEPS_VERSION}" +else + echo "Building image: ${IMAGE_NAME}:${DEPS_VERSION}" + pushd ../ + docker build . -t ${IMAGE_NAME}:${DEPS_VERSION} -f docker/Dockerfile_deps --cpuset-cpus "0-$ncores" --build-arg BASE_IMAGE=${BASE_IMAGE} --tag ${IMAGE_NAME}:latest + popd + + docker push ${IMAGE_NAME}:${DEPS_VERSION} + docker push ${IMAGE_NAME}:latest +fi diff --git a/docker/version.sh b/docker/version.sh index 7f3b0702..653e668a 100644 --- a/docker/version.sh +++ b/docker/version.sh @@ -1,3 +1,6 @@ -VERSION=1.0.1 +VERSION=1.0.2 IMAGE=eolo-dev + +# This is the version of the dep image. Increase this number everytime you change `external/CMakeLists.txt` +DEPS_VERSION=1.0.2 diff --git a/external/CMakeLists.txt b/external/CMakeLists.txt index 822cb4c8..e9f726ec 100644 --- a/external/CMakeLists.txt +++ b/external/CMakeLists.txt @@ -34,6 +34,10 @@ set(EP_CMAKE_EXTRA_ARGS -DCMAKE_CXX_STANDARD=${CMAKE_CXX_STANDARD}) +make_directory(${CMAKE_INSTALL_PREFIX}/bin) +make_directory(${CMAKE_INSTALL_PREFIX}/lib) +make_directory(${CMAKE_INSTALL_PREFIX}/include) + # helper macro useful for dependency handling macro(add_dummy_target) if(NOT TARGET ${ARGV0}) @@ -57,23 +61,29 @@ endif() # -------------------------------------------------------------------------------------------------- # zenohcxx if(zenohcxx IN_LIST EXTERNAL_PROJECTS_LIST) - set(ZENOH_PLUGIN_LIB "target/release/libzenoh_plugin_rest.so") - if (CMAKE_SYSTEM_NAME STREQUAL "Darwin") - set(ZENOH_PLUGIN_LIB "target/release/libzenoh_plugin_rest.dylib") + set(zenohc_VERION_REQUIRED 0.10.1-rc) + if (EXISTS "${CMAKE_PREFIX_PATH}/bin/zenohd") + message(STATUS "zenohd: Using from ${CMAKE_PREFIX_PATH}") + add_dummy_target(zenohd) + else() + message(STATUS "zenohd: Building ${zenohc_VERION_REQUIRED} from source") + set(ZENOH_PLUGIN_LIB "target/release/libzenoh_plugin_rest.so") + if (CMAKE_SYSTEM_NAME STREQUAL "Darwin") + set(ZENOH_PLUGIN_LIB "target/release/libzenoh_plugin_rest.dylib") + endif() + ExternalProject_Add( + zenohd + GIT_REPOSITORY "https://github.com/eclipse-zenoh/zenoh.git" + GIT_TAG ${zenohc_VERION_REQUIRED} + GIT_SHALLOW true + CONFIGURE_COMMAND "" + BUILD_COMMAND COMMAND cargo build --release --package zenoh-plugin-rest --package zenohd + BUILD_IN_SOURCE true + INSTALL_COMMAND cp target/release/zenohd ${CMAKE_INSTALL_PREFIX}/bin/ && cp ${ZENOH_PLUGIN_LIB} ${CMAKE_INSTALL_PREFIX}/bin/ + LOG_BUILD ON) endif() - ExternalProject_Add( - zenohd - GIT_REPOSITORY "https://github.com/eclipse-zenoh/zenoh.git" - GIT_TAG 0.10.1-rc # v${zenohc_VERION_REQUIRED} - GIT_SHALLOW true - CONFIGURE_COMMAND "" - BUILD_COMMAND COMMAND cargo build --release --package zenoh-plugin-rest --package zenohd - BUILD_IN_SOURCE true - INSTALL_COMMAND cp target/release/zenohd ${CMAKE_INSTALL_PREFIX}/bin/ && cp ${ZENOH_PLUGIN_LIB} ${CMAKE_INSTALL_PREFIX}/bin/ - LOG_BUILD ON) # install zenoh-c first - # set(zenohc_VERION_REQUIRED 0.11.0.0) find_package(zenohc QUIET) # ${zenohc_VERION_REQUIRED} QUIET) if(zenohc_FOUND) message(STATUS "zenohc: Using version ${zenohc_VERSION} from ${zenohc_DIR}") @@ -82,9 +92,9 @@ if(zenohcxx IN_LIST EXTERNAL_PROJECTS_LIST) message(STATUS "zenohc: Building ${zenohc_VERION_REQUIRED} from source") ExternalProject_Add( zenohc - DEPENDS zenohd + DEPENDS zenohd GIT_REPOSITORY "https://github.com/eclipse-zenoh/zenoh-c.git" - GIT_TAG 408a6eb2f305c965101f19f6925acc0e0a93f764 # v${zenohc_VERION_REQUIRED} + GIT_TAG d1ec83dee421054bed034e0e60abc98a526bae0a # v${zenohc_VERION_REQUIRED} GIT_SHALLOW true CMAKE_ARGS ${EP_CMAKE_EXTRA_ARGS} -DCMAKE_VERBOSE_MAKEFILE=ON) endif() @@ -101,7 +111,7 @@ if(zenohcxx IN_LIST EXTERNAL_PROJECTS_LIST) zenohcxx DEPENDS zenohc GIT_REPOSITORY "https://github.com/eclipse-zenoh/zenoh-cpp.git" - GIT_TAG main # v${zenohcxx_VERION_REQUIRED} + GIT_TAG 10ced293dd72ed5ec55c36b57672e0cc21932761 # v${zenohcxx_VERION_REQUIRED} GIT_SHALLOW true SOURCE_SUBDIR install # configure the 'install' sub-project to avoid # building zenoh-c again @@ -114,7 +124,9 @@ endif() #-------------------------------------------------------------------------------------------------- # Abseil C++ if (absl IN_LIST EXTERNAL_PROJECTS_LIST) - set(ABSEIL_VERSION_REQUIRED 20230802.1) + + set(ABSEIL_VERSION_REQUIRED 20230802) + set(ABSEIL_TAG ${ABSEIL_VERSION_REQUIRED}.1) find_package(absl ${ABSEIL_VERSION_REQUIRED} QUIET) if (absl_FOUND) message(STATUS "Abseil: Using version ${absl_VERSION} from ${absl_DIR}") @@ -129,7 +141,7 @@ if (absl IN_LIST EXTERNAL_PROJECTS_LIST) ExternalProject_Add( absl GIT_REPOSITORY https://github.com/abseil/abseil-cpp - GIT_TAG ${ABSEIL_VERSION_REQUIRED} + GIT_TAG ${ABSEIL_TAG} LOG_BUILD true CMAKE_ARGS ${EP_CMAKE_EXTRA_ARGS} ${ABSEIL_CMAKE_ARGS}) endif() @@ -138,7 +150,8 @@ endif() #-------------------------------------------------------------------------------------------------- # protobuf if (Protobuf IN_LIST EXTERNAL_PROJECTS_LIST) - set(PROTOBUF_VERSION_REQUIRED 25.2) + set(PROTOBUF_TAG 25.2) + set(PROTOBUF_VERSION_REQUIRED 4.${PROTOBUF_TAG}) find_package(Protobuf ${PROTOBUF_VERSION_REQUIRED} QUIET) if(Protobuf_FOUND) message(STATUS "Protobuf: Using version ${Protobuf_VERSION} (${Protobuf_LIBRARIES}, ${Protobuf_PROTOC_LIBRARIES})") @@ -154,7 +167,7 @@ if (Protobuf IN_LIST EXTERNAL_PROJECTS_LIST) Protobuf DEPENDS ${PROTOBUF_DEPENDS} GIT_REPOSITORY "https://github.com/protocolbuffers/protobuf.git" - GIT_TAG "v${PROTOBUF_VERSION_REQUIRED}" + GIT_TAG "v${PROTOBUF_TAG}" GIT_SHALLOW true GIT_SUBMODULES_RECURSE false CMAKE_ARGS ${EP_CMAKE_EXTRA_ARGS} ${PROTOBUF_CMAKE_ARGS} @@ -262,8 +275,8 @@ endif() if(range-v3 IN_LIST EXTERNAL_PROJECTS_LIST) set(RANGE_V3_VERSION_REQUIRED 0.12.0) find_package(range-v3 ${RANGE_V3_VERSION_REQUIRED} QUIET) - if(RANGE_V3_FOUND) - message(STATUS "range-v3: Using ${RANGE_V3_DIR}") + if(range-v3_FOUND) + message(STATUS "range-v3: Using version ${RANGE_V3_VERSION_REQUIRED} from ${range-v3_DIR}") else() message(STATUS "range_v3: Building ${RANGE_V3_VERSION_REQUIRED} from source") ExternalProject_Add( @@ -271,10 +284,10 @@ if(range-v3 IN_LIST EXTERNAL_PROJECTS_LIST) GIT_REPOSITORY "https://github.com/ericniebler/range-v3.git" GIT_TAG ${RANGE_V3_VERSION_REQUIRED} GIT_SHALLOW true - CMAKE_ARGS ${EP_CMAKE_EXTRA_ARGS} - -DRANGE_V3_TESTS=OFF - -DRANGE_V3_EXAMPLES=OFF - -DRANGE_V3_DOCS=OFF + CMAKE_ARGS ${EP_CMAKE_EXTRA_ARGS} + -DRANGE_V3_TESTS=OFF + -DRANGE_V3_EXAMPLES=OFF + -DRANGE_V3_DOCS=OFF ) endif() endif() diff --git a/modules/utils/src/version_impl.h b/modules/utils/src/version_impl.h index 34d62899..30c5fe58 100644 --- a/modules/utils/src/version_impl.h +++ b/modules/utils/src/version_impl.h @@ -17,8 +17,8 @@ static constexpr std::uint8_t VERSION_MAJOR = 0; static constexpr std::uint8_t VERSION_MINOR = 0; static constexpr std::uint16_t VERSION_PATCH = 0; -static constexpr std::string_view REPO_BRANCH = "fix/github_action"; -static constexpr std::string_view BUILD_PROFILE = "RelWithDebInfo"; -static constexpr std::string_view REPO_HASH = "e9649d3"; +static constexpr std::string_view REPO_BRANCH = "github_actions"; +static constexpr std::string_view BUILD_PROFILE = "Release"; +static constexpr std::string_view REPO_HASH = "54ca190"; } // namespace eolo::utils