From 165fd9152cdee7bbdcd875b262cbcad7148ec0d2 Mon Sep 17 00:00:00 2001 From: Andrew Shkrob Date: Sat, 28 Oct 2023 01:41:00 +0200 Subject: [PATCH] [ci] Add test coverage check Signed-off-by: Andrew Shkrob --- .github/workflows/coverage-check.yaml | 185 ++++++++++++++++++++++++++ .github/workflows/linux-check.yaml | 24 ++-- .github/workflows/macos-check.yaml | 2 +- CMakeLists.txt | 7 +- cmake/OmimCoverage.cmake | 41 ++++++ cmake/OmimTesting.cmake | 5 +- gcovr.cfg | 22 +++ 7 files changed, 272 insertions(+), 14 deletions(-) create mode 100644 .github/workflows/coverage-check.yaml create mode 100644 cmake/OmimCoverage.cmake create mode 100644 gcovr.cfg diff --git a/.github/workflows/coverage-check.yaml b/.github/workflows/coverage-check.yaml new file mode 100644 index 0000000000000..9565b2493be77 --- /dev/null +++ b/.github/workflows/coverage-check.yaml @@ -0,0 +1,185 @@ +name: Coverage Report +on: + workflow_dispatch: # Manual trigger + pull_request: + types: + - opened + - synchronize + - labeled + - unlabeled + +# Cancels previous jobs if the same branch or PR was updated again. +concurrency: + group: ${{ github.workflow }}-coverage-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +jobs: + should-run-check: + name: Should run coverage + runs-on: ubuntu-22.04 + outputs: + run-from-pr: ${{ steps.run-from-pr.outputs.run-from-pr }} + manually-triggered: ${{ steps.manually-triggered.outputs.manually-triggered }} + steps: + - name: Check if PR has 'Coverage' label + id: run-from-pr + if: github.event_name == 'pull_request' + env: + PR_NUMBER: ${{ github.event.pull_request.number }} + GH_TOKEN: ${{ github.token }} + run: | + LABEL_NAME="Coverage" + LABELS=$(gh pr view https://github.com/$GITHUB_REPOSITORY/pull/$PR_NUMBER --json labels) + if echo "$LABELS" | jq -e '.labels[].name' | grep -q "$LABEL_NAME"; then + echo "run-from-pr=true" >> $GITHUB_OUTPUT + echo "'Coverage' label found in PR." + fi + - name: Check if manually triggered + id: manually-triggered + if: github.event_name == 'workflow_dispatch' + run: echo "manually-triggered=true" >> $GITHUB_OUTPUT + + coverage: + needs: should-run-check + name: Run coverage + runs-on: ubuntu-22.04 + if: ${{ needs.should-run-check.outputs.run-from-pr == 'true' || needs.should-run-check.outputs.manually-triggered == 'true'}} + outputs: + coverage-summary: ${{ steps.export.outputs.coverage-summary }} + steps: + - name: Free disk space by removing .NET, Android and Haskell + shell: bash + run: | + sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc + + - name: Checkout sources + uses: actions/checkout@v4 + with: + fetch-depth: 100 # enough to get all commits for the current day + + - name: Parallel submodules checkout + shell: bash + run: git submodule update --depth 1 --init --recursive --jobs=$(($(nproc) * 20)) + + - name: Install build tools and dependencies + shell: bash + run: | + sudo apt update -y + sudo apt install -y \ + ninja-build \ + libgl1-mesa-dev \ + libglvnd-dev \ + qt6-base-dev \ + libqt6svg6-dev \ + qt6-positioning-dev \ + libqt6positioning6-plugins \ + libqt6positioning6 \ + llvm + pip install gcovr + + - name: Configure + shell: bash + run: ./configure.sh + + - name: Configure ccache + uses: hendrikmuhs/ccache-action@v1.2 + with: + key: ${{ github.workflow }}-coverage + + - name: CMake + shell: bash + env: + CC: clang-14 + CXX: clang++-14 + CMAKE_C_COMPILER_LAUNCHER: ccache + CMAKE_CXX_COMPILER_LAUNCHER: ccache + # -g1 should slightly reduce build time. + run: | + cmake . -B build -G Ninja -DCMAKE_BUILD_TYPE=Debug \ + -DCMAKE_CXX_FLAGS=-g1 -DCOVERAGE_REPORT=ON + + - name: Compile + shell: bash + working-directory: build + run: ninja + + - name: Generate locales + shell: bash + run: | + sudo locale-gen en_US + sudo locale-gen en_US.UTF-8 + sudo locale-gen es_ES + sudo locale-gen es_ES.UTF-8 + sudo locale-gen fr_FR + sudo locale-gen fr_FR.UTF-8 + sudo locale-gen ru_RU + sudo locale-gen ru_RU.UTF-8 + sudo update-locale + + - name: Tests + shell: bash + working-directory: build + env: + # drape_tests - requires X Window + # generator_integration_tests - https://github.com/organicmaps/organicmaps/issues/225 + # opening_hours_integration_tests - https://github.com/organicmaps/organicmaps/issues/219 + # opening_hours_supported_features_tests - https://github.com/organicmaps/organicmaps/issues/219 + # routing_integration_tests - https://github.com/organicmaps/organicmaps/issues/221 + # shaders_tests - https://github.com/organicmaps/organicmaps/issues/223 + # world_feed_integration_tests - https://github.com/organicmaps/organicmaps/issues/215 + CTEST_EXCLUDE_REGEX: "drape_tests|generator_integration_tests|opening_hours_integration_tests|opening_hours_supported_features_tests|routing_benchmarks|routing_integration_tests|routing_quality_tests|search_quality_tests|storage_integration_tests|shaders_tests|world_feed_integration_tests" + run: | + ln -s ../data data + ctest -L "omim-test" -E "$CTEST_EXCLUDE_REGEX" --output-on-failure + + - name: Run coverage report generation + shell: bash + working-directory: build + run: | + cmake --build . --target omim_coverage + cat coverage_report/summary.txt + + - name: Archive the coverage report + working-directory: build/coverage_report + run: zip -r coverage_report.zip html/ + + - name: Upload artifact + uses: actions/upload-artifact@v2 + with: + name: coverage-report + path: build/coverage_report/coverage_report.zip + + - name: Export coverage summary + id: export + working-directory: build/coverage_report + run: | + echo coverage-summary=$(echo 'Hello, World!' | base64) >> $GITHUB_OUTPUT + + post-coverage-comment: + needs: [should-run-check] + name: Post comment with coverage info + if: ${{ needs.should-run-check.outputs.run-from-pr == 'true' }} + runs-on: ubuntu-22.04 + steps: + - name: Post comment with coverage info + shell: bash + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | +# SUMMARY=$(echo "${{ needs.coverage.outputs.coverage-summary }}" | base64 -d) + SUMMARY='Hello, World!' + ACTION_RUN_URL="https://github.com/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" + PR_NUMBER="${{ github.event.pull_request.number }}" + COMMENT_ID=$(gh api --method GET repos/:owner/:repo/issues/${PR_NUMBER}/comments | jq -r '.[] | select(.user.login == "github-actions" and .body | contains("Coverage Summary:")) | .id') + + COMMENT_BODY="Coverage Summary:\n \\ + \`\`\`\n \\ + ${SUMMARY}\n \\ + \`\`\`\n \\ + Coverage report can be found [here](${ACTION_RUN_URL})." + + if [ -n "$COMMENT_ID" ]; then + gh api --method PATCH repos/:owner/:repo/issues/comments/${COMMENT_ID} --field body="$COMMENT_BODY" + else + gh api --method POST repos/:owner/:repo/issues/${PR_NUMBER}/comments --field body="$COMMENT_BODY" + fi diff --git a/.github/workflows/linux-check.yaml b/.github/workflows/linux-check.yaml index da29e28fbb304..9e9a34bb0bff6 100644 --- a/.github/workflows/linux-check.yaml +++ b/.github/workflows/linux-check.yaml @@ -156,6 +156,19 @@ jobs: working-directory: build run: ninja + - name: Generate locales + shell: bash + run: | + sudo locale-gen en_US + sudo locale-gen en_US.UTF-8 + sudo locale-gen es_ES + sudo locale-gen es_ES.UTF-8 + sudo locale-gen fr_FR + sudo locale-gen fr_FR.UTF-8 + sudo locale-gen ru_RU + sudo locale-gen ru_RU.UTF-8 + sudo update-locale + - name: Tests shell: bash working-directory: build @@ -169,14 +182,5 @@ jobs: # world_feed_integration_tests - https://github.com/organicmaps/organicmaps/issues/215 CTEST_EXCLUDE_REGEX: "drape_tests|generator_integration_tests|opening_hours_integration_tests|opening_hours_supported_features_tests|routing_benchmarks|routing_integration_tests|routing_quality_tests|search_quality_tests|storage_integration_tests|shaders_tests|world_feed_integration_tests" run: | - sudo locale-gen en_US - sudo locale-gen en_US.UTF-8 - sudo locale-gen es_ES - sudo locale-gen es_ES.UTF-8 - sudo locale-gen fr_FR - sudo locale-gen fr_FR.UTF-8 - sudo locale-gen ru_RU - sudo locale-gen ru_RU.UTF-8 - sudo update-locale ln -s ../data data - ctest -LE "fixture" -E "$CTEST_EXCLUDE_REGEX" --output-on-failure + ctest -L "omim-test" -E "$CTEST_EXCLUDE_REGEX" --output-on-failure diff --git a/.github/workflows/macos-check.yaml b/.github/workflows/macos-check.yaml index 63e4c55a56e7d..4d146a39a0a20 100644 --- a/.github/workflows/macos-check.yaml +++ b/.github/workflows/macos-check.yaml @@ -91,4 +91,4 @@ jobs: CTEST_EXCLUDE_REGEX: "drape_tests|generator_integration_tests|opening_hours_integration_tests|opening_hours_supported_features_tests|routing_benchmarks|routing_integration_tests|routing_quality_tests|search_quality_tests|storage_integration_tests|shaders_tests|world_feed_integration_tests" run: | ln -s ../data data - ctest -LE "fixture" -E "$CTEST_EXCLUDE_REGEX" --output-on-failure + ctest -L "omim-test" -E "$CTEST_EXCLUDE_REGEX" --output-on-failure diff --git a/CMakeLists.txt b/CMakeLists.txt index a506404a0a118..153fd42aee9d2 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -22,7 +22,9 @@ endif() message(STATUS "Using compiler ${CMAKE_CXX_COMPILER_ID} ${CMAKE_CXX_COMPILER_VERSION}") -option(UNITY_DISABLE "Disable unity build" OFF) +option(COVERAGE_REPORT "Configure for coverage report" OFF) + +option(UNITY_DISABLE "Disable unity build" ON) if (NOT UNITY_DISABLE AND NOT DEFINED ENV{UNITY_DISABLE}) set(CMAKE_UNITY_BUILD ON) if (DEFINED ENV{UNITY_BUILD_BATCH_SIZE}) @@ -212,6 +214,9 @@ if (NOT SKIP_TESTS) enable_testing() # Enables ctest -T memcheck with valgrind include(CTest) + if (COVERAGE_REPORT) + include(OmimCoverage) + endif () endif() if (NOT PYTHON_VERSION) diff --git a/cmake/OmimCoverage.cmake b/cmake/OmimCoverage.cmake new file mode 100644 index 0000000000000..a82b923bfad11 --- /dev/null +++ b/cmake/OmimCoverage.cmake @@ -0,0 +1,41 @@ +add_compile_options(-O0 --coverage) +add_link_options(--coverage) + +set(COVERAGE_REPORT_DIR ${CMAKE_BINARY_DIR}/coverage_report) + +find_program(GCOVR_EXECUTABLE_PATH gcovr) +if (NOT GCOVR_EXECUTABLE_PATH) + message(FATAL_ERROR "'gcovr' is required to generate test coverage report. Details: gcovr.com.") +endif () + +if (${CMAKE_CXX_COMPILER_ID} MATCHES "GNU|AppleClang") + set(GCOV_EXECUTABLE "gcov") +elseif (${CMAKE_CXX_COMPILER_ID} MATCHES "Clang") + set(GCOV_EXECUTABLE "llvm-cov") +endif () + +find_program(GCOV_EXECUTABLE_PATH ${GCOV_EXECUTABLE}) +if (NOT GCOV_EXECUTABLE_PATH) + message(FATAL_ERROR "'${GCOV_EXECUTABLE}' is required to generate test coverage report.") +endif () + +if (${GCOV_EXECUTABLE_PATH} MATCHES "llvm-cov") + set(GCOV_EXECUTABLE_PATH "${GCOV_EXECUTABLE_PATH} gcov") +endif () + +add_custom_target(omim_coverage + # Remove harfbuzz.cc.* files because they reference .rl files that do not exist and cannot be excluded by gcovr. + COMMAND rm -f ${CMAKE_BINARY_DIR}/3party/harfbuzz/CMakeFiles/harfbuzz.dir/harfbuzz/src/harfbuzz.cc.* + # Recreate coverage_report folder + COMMAND rm -rf ${COVERAGE_REPORT_DIR} && mkdir ${COVERAGE_REPORT_DIR} + # Run gcovr + COMMAND ${GCOVR_EXECUTABLE_PATH} + --config=${OMIM_ROOT}/gcovr.cfg + --root=${OMIM_ROOT} + --object-directory=${CMAKE_BINARY_DIR} + --exclude=${CMAKE_BINARY_DIR} # Exclude autogenerated files from Qt and some 3party libraries. + --gcov-executable=${GCOV_EXECUTABLE_PATH} + --html-nested=${COVERAGE_REPORT_DIR}/html/ >> ${COVERAGE_REPORT_DIR}/summary.txt + WORKING_DIRECTORY ${CMAKE_BINARY_DIR} + COMMENT "Generating coverage report..." +) diff --git a/cmake/OmimTesting.cmake b/cmake/OmimTesting.cmake index 91adbebcb8189..5b673ac764f87 100644 --- a/cmake/OmimTesting.cmake +++ b/cmake/OmimTesting.cmake @@ -13,7 +13,7 @@ add_test( ) set_tests_properties(OmimStartTestServer PROPERTIES FIXTURES_SETUP TestServer) set_tests_properties(OmimStopTestServer PROPERTIES FIXTURES_CLEANUP TestServer) -set_tests_properties(OmimStartTestServer OmimStopTestServer PROPERTIES LABELS "fixture") +set_tests_properties(OmimStartTestServer OmimStopTestServer PROPERTIES LABELS "omim-fixture") # Options: # * REQUIRE_QT - requires QT event loop @@ -80,6 +80,7 @@ function(omim_add_ctest name require_server boost_test) WORKING_DIRECTORY ${CMAKE_BINARY_DIR} ) if (require_server) - set_tests_properties(${TEST_NAME} PROPERTIES FIXTURES_REQUIRED TestServer) + set_tests_properties(${name} PROPERTIES FIXTURES_REQUIRED TestServer) endif() + set_tests_properties(${name} PROPERTIES LABELS "omim-test") endfunction() diff --git a/gcovr.cfg b/gcovr.cfg new file mode 100644 index 0000000000000..c2c79ee15740a --- /dev/null +++ b/gcovr.cfg @@ -0,0 +1,22 @@ +# gcovr configuration file +# Details: https://gcovr.com/en/master/manpage.html + +# Exclude all subfolders of "3party/" except for "opening_hours" +exclude = ^3party\/(?!opening_hours($|\/)).* +# Exclude testing folders +exclude = testing/ +exclude = qt_tstfrm/ +# Exclude all "*tests*" subfolders +exclude = .*tests.* + +exclude-unreachable-branches = yes +exclude-throw-branches = yes + +print-summary = yes + +gcov-parallel = 20 +gcov-ignore-errors = all +gcov-ignore-parse-errors = all + +html-title = Organic Maps Code Coverage Report +html-self-contained = yes