[ci] Add test coverage check

Signed-off-by: Andrew Shkrob <[email protected]>
AndrewShkrob · Oct 29, 2023 · 165fd91 · 165fd91
1 parent 10f0eaf
commit 165fd91
Show file tree

Hide file tree

Showing 7 changed files with 272 additions and 14 deletions.
diff --git a/.github/workflows/coverage-check.yaml b/.github/workflows/coverage-check.yaml
@@ -0,0 +1,185 @@
+name: Coverage Report
+on:
+  workflow_dispatch: # Manual trigger
+  pull_request:
+    types:
+      - opened
+      - synchronize
+      - labeled
+      - unlabeled
+
+# Cancels previous jobs if the same branch or PR was updated again.
+concurrency:
+  group: ${{ github.workflow }}-coverage-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  should-run-check:
+    name: Should run coverage
+    runs-on: ubuntu-22.04
+    outputs:
+      run-from-pr: ${{ steps.run-from-pr.outputs.run-from-pr }}
+      manually-triggered: ${{ steps.manually-triggered.outputs.manually-triggered }}
+    steps:
+      - name: Check if PR has 'Coverage' label
+        id: run-from-pr
+        if: github.event_name == 'pull_request'
+        env:
+          PR_NUMBER: ${{ github.event.pull_request.number }}
+          GH_TOKEN: ${{ github.token }}
+        run: |
+          LABEL_NAME="Coverage"
+          LABELS=$(gh pr view https://github.com/$GITHUB_REPOSITORY/pull/$PR_NUMBER --json labels)
+          if echo "$LABELS" | jq -e '.labels[].name' | grep -q "$LABEL_NAME"; then
+            echo "run-from-pr=true" >> $GITHUB_OUTPUT
+            echo "'Coverage' label found in PR."
+          fi
+      - name: Check if manually triggered
+        id: manually-triggered
+        if: github.event_name == 'workflow_dispatch'
+        run: echo "manually-triggered=true" >> $GITHUB_OUTPUT
+
+  coverage:
+    needs: should-run-check
+    name: Run coverage
+    runs-on: ubuntu-22.04
+    if: ${{ needs.should-run-check.outputs.run-from-pr == 'true' || needs.should-run-check.outputs.manually-triggered == 'true'}}
+    outputs:
+      coverage-summary: ${{ steps.export.outputs.coverage-summary }}
+    steps:
+      - name: Free disk space by removing .NET, Android and Haskell
+        shell: bash
+        run: |
+          sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc
+
+      - name: Checkout sources
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 100 # enough to get all commits for the current day
+
+      - name: Parallel submodules checkout
+        shell: bash
+        run: git submodule update --depth 1 --init --recursive --jobs=$(($(nproc) * 20))
+
+      - name: Install build tools and dependencies
+        shell: bash
+        run: |
+          sudo apt update -y
+          sudo apt install -y \
+              ninja-build \
+              libgl1-mesa-dev \
+              libglvnd-dev \
+              qt6-base-dev \
+              libqt6svg6-dev \
+              qt6-positioning-dev \
+              libqt6positioning6-plugins \
+              libqt6positioning6 \
+              llvm
+          pip install gcovr
+
+      - name: Configure
+        shell: bash
+        run: ./configure.sh
+
+      - name: Configure ccache
+        uses: hendrikmuhs/[email protected]
+        with:
+          key: ${{ github.workflow }}-coverage
+
+      - name: CMake
+        shell: bash
+        env:
+          CC: clang-14
+          CXX: clang++-14
+          CMAKE_C_COMPILER_LAUNCHER: ccache
+          CMAKE_CXX_COMPILER_LAUNCHER: ccache
+        # -g1 should slightly reduce build time.
+        run: |
+          cmake . -B build -G Ninja -DCMAKE_BUILD_TYPE=Debug \
+            -DCMAKE_CXX_FLAGS=-g1 -DCOVERAGE_REPORT=ON
+
+      - name: Compile
+        shell: bash
+        working-directory: build
+        run: ninja
+
+      - name: Generate locales
+        shell: bash
+        run: |
+          sudo locale-gen en_US
+          sudo locale-gen en_US.UTF-8
+          sudo locale-gen es_ES
+          sudo locale-gen es_ES.UTF-8
+          sudo locale-gen fr_FR
+          sudo locale-gen fr_FR.UTF-8
+          sudo locale-gen ru_RU
+          sudo locale-gen ru_RU.UTF-8
+          sudo update-locale
+
+      - name: Tests
+        shell: bash
+        working-directory: build
+        env:
+          # drape_tests - requires X Window
+          # generator_integration_tests - https://github.com/organicmaps/organicmaps/issues/225
+          # opening_hours_integration_tests - https://github.com/organicmaps/organicmaps/issues/219
+          # opening_hours_supported_features_tests - https://github.com/organicmaps/organicmaps/issues/219
+          # routing_integration_tests - https://github.com/organicmaps/organicmaps/issues/221
+          # shaders_tests - https://github.com/organicmaps/organicmaps/issues/223
+          # world_feed_integration_tests - https://github.com/organicmaps/organicmaps/issues/215
+          CTEST_EXCLUDE_REGEX: "drape_tests|generator_integration_tests|opening_hours_integration_tests|opening_hours_supported_features_tests|routing_benchmarks|routing_integration_tests|routing_quality_tests|search_quality_tests|storage_integration_tests|shaders_tests|world_feed_integration_tests"
+        run: |
+          ln -s ../data data
+          ctest -L "omim-test" -E "$CTEST_EXCLUDE_REGEX" --output-on-failure
+
+      - name: Run coverage report generation
+        shell: bash
+        working-directory: build
+        run: |
+          cmake --build . --target omim_coverage
+          cat coverage_report/summary.txt
+
+      - name: Archive the coverage report
+        working-directory: build/coverage_report
+        run: zip -r coverage_report.zip html/
+
+      - name: Upload artifact
+        uses: actions/upload-artifact@v2
+        with:
+          name: coverage-report
+          path: build/coverage_report/coverage_report.zip
+
+      - name: Export coverage summary
+        id: export
+        working-directory: build/coverage_report
+        run: |
+          echo coverage-summary=$(echo 'Hello, World!' | base64) >> $GITHUB_OUTPUT
+
+  post-coverage-comment:
+    needs: [should-run-check]
+    name: Post comment with coverage info
+    if: ${{ needs.should-run-check.outputs.run-from-pr == 'true' }}
+    runs-on: ubuntu-22.04
+    steps:
+      - name: Post comment with coverage info
+        shell: bash
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+# SUMMARY=$(echo "${{ needs.coverage.outputs.coverage-summary }}" | base64 -d)
+          SUMMARY='Hello, World!'
+          ACTION_RUN_URL="https://github.com/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID"
+          PR_NUMBER="${{ github.event.pull_request.number }}"
+          COMMENT_ID=$(gh api --method GET repos/:owner/:repo/issues/${PR_NUMBER}/comments | jq -r '.[] | select(.user.login == "github-actions" and .body | contains("Coverage Summary:")) | .id')
+
+          COMMENT_BODY="Coverage Summary:\n \\
+          \`\`\`\n \\
+          ${SUMMARY}\n \\
+          \`\`\`\n \\
+          Coverage report can be found [here](${ACTION_RUN_URL})."
+
+          if [ -n "$COMMENT_ID" ]; then
+            gh api --method PATCH repos/:owner/:repo/issues/comments/${COMMENT_ID} --field body="$COMMENT_BODY"
+          else
+            gh api --method POST repos/:owner/:repo/issues/${PR_NUMBER}/comments --field body="$COMMENT_BODY"
+          fi
diff --git a/.github/workflows/linux-check.yaml b/.github/workflows/linux-check.yaml
@@ -156,6 +156,19 @@ jobs:
         working-directory: build
         run: ninja
 
+      - name: Generate locales
+        shell: bash
+        run: |
+          sudo locale-gen en_US
+          sudo locale-gen en_US.UTF-8
+          sudo locale-gen es_ES
+          sudo locale-gen es_ES.UTF-8
+          sudo locale-gen fr_FR
+          sudo locale-gen fr_FR.UTF-8
+          sudo locale-gen ru_RU
+          sudo locale-gen ru_RU.UTF-8
+          sudo update-locale
+
       - name: Tests
         shell: bash
         working-directory: build
@@ -169,14 +182,5 @@ jobs:
           # world_feed_integration_tests - https://github.com/organicmaps/organicmaps/issues/215
           CTEST_EXCLUDE_REGEX: "drape_tests|generator_integration_tests|opening_hours_integration_tests|opening_hours_supported_features_tests|routing_benchmarks|routing_integration_tests|routing_quality_tests|search_quality_tests|storage_integration_tests|shaders_tests|world_feed_integration_tests"
         run: |
-          sudo locale-gen en_US
-          sudo locale-gen en_US.UTF-8
-          sudo locale-gen es_ES
-          sudo locale-gen es_ES.UTF-8
-          sudo locale-gen fr_FR
-          sudo locale-gen fr_FR.UTF-8
-          sudo locale-gen ru_RU
-          sudo locale-gen ru_RU.UTF-8
-          sudo update-locale
           ln -s ../data data
-          ctest -LE "fixture" -E "$CTEST_EXCLUDE_REGEX" --output-on-failure
+          ctest -L "omim-test" -E "$CTEST_EXCLUDE_REGEX" --output-on-failure
diff --git a/.github/workflows/macos-check.yaml b/.github/workflows/macos-check.yaml
@@ -91,4 +91,4 @@ jobs:
           CTEST_EXCLUDE_REGEX: "drape_tests|generator_integration_tests|opening_hours_integration_tests|opening_hours_supported_features_tests|routing_benchmarks|routing_integration_tests|routing_quality_tests|search_quality_tests|storage_integration_tests|shaders_tests|world_feed_integration_tests"
         run: |
           ln -s ../data data
-          ctest -LE "fixture" -E "$CTEST_EXCLUDE_REGEX" --output-on-failure
+          ctest -L "omim-test" -E "$CTEST_EXCLUDE_REGEX" --output-on-failure
diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -22,7 +22,9 @@ endif()
 
 message(STATUS "Using compiler ${CMAKE_CXX_COMPILER_ID} ${CMAKE_CXX_COMPILER_VERSION}")
 
-option(UNITY_DISABLE "Disable unity build" OFF)
+option(COVERAGE_REPORT "Configure for coverage report" OFF)
+
+option(UNITY_DISABLE "Disable unity build" ON)
 if (NOT UNITY_DISABLE AND NOT DEFINED ENV{UNITY_DISABLE})
   set(CMAKE_UNITY_BUILD ON)
   if (DEFINED ENV{UNITY_BUILD_BATCH_SIZE})
@@ -212,6 +214,9 @@ if (NOT SKIP_TESTS)
   enable_testing()
   # Enables ctest -T memcheck with valgrind
   include(CTest)
+  if (COVERAGE_REPORT)
+    include(OmimCoverage)
+  endif ()
 endif()
 
 if (NOT PYTHON_VERSION)

diff --git a/cmake/OmimCoverage.cmake b/cmake/OmimCoverage.cmake
@@ -0,0 +1,41 @@
+add_compile_options(-O0 --coverage)
+add_link_options(--coverage)
+
+set(COVERAGE_REPORT_DIR ${CMAKE_BINARY_DIR}/coverage_report)
+
+find_program(GCOVR_EXECUTABLE_PATH gcovr)
+if (NOT GCOVR_EXECUTABLE_PATH)
+  message(FATAL_ERROR "'gcovr' is required to generate test coverage report. Details: gcovr.com.")
+endif ()
+
+if (${CMAKE_CXX_COMPILER_ID} MATCHES "GNU|AppleClang")
+  set(GCOV_EXECUTABLE "gcov")
+elseif (${CMAKE_CXX_COMPILER_ID} MATCHES "Clang")
+  set(GCOV_EXECUTABLE "llvm-cov")
+endif ()
+
+find_program(GCOV_EXECUTABLE_PATH ${GCOV_EXECUTABLE})
+if (NOT GCOV_EXECUTABLE_PATH)
+  message(FATAL_ERROR "'${GCOV_EXECUTABLE}' is required to generate test coverage report.")
+endif ()
+
+if (${GCOV_EXECUTABLE_PATH} MATCHES "llvm-cov")
+  set(GCOV_EXECUTABLE_PATH "${GCOV_EXECUTABLE_PATH} gcov")
+endif ()
+
+add_custom_target(omim_coverage
+  # Remove harfbuzz.cc.* files because they reference .rl files that do not exist and cannot be excluded by gcovr.
+  COMMAND rm -f ${CMAKE_BINARY_DIR}/3party/harfbuzz/CMakeFiles/harfbuzz.dir/harfbuzz/src/harfbuzz.cc.*
+  # Recreate coverage_report folder
+  COMMAND rm -rf ${COVERAGE_REPORT_DIR} && mkdir ${COVERAGE_REPORT_DIR}
+  # Run gcovr
+  COMMAND ${GCOVR_EXECUTABLE_PATH}
+  --config=${OMIM_ROOT}/gcovr.cfg
+  --root=${OMIM_ROOT}
+  --object-directory=${CMAKE_BINARY_DIR}
+  --exclude=${CMAKE_BINARY_DIR} # Exclude autogenerated files from Qt and some 3party libraries.
+  --gcov-executable=${GCOV_EXECUTABLE_PATH}
+  --html-nested=${COVERAGE_REPORT_DIR}/html/ >> ${COVERAGE_REPORT_DIR}/summary.txt
+  WORKING_DIRECTORY ${CMAKE_BINARY_DIR}
+  COMMENT "Generating coverage report..."
+)
diff --git a/cmake/OmimTesting.cmake b/cmake/OmimTesting.cmake
@@ -13,7 +13,7 @@ add_test(
 )
 set_tests_properties(OmimStartTestServer PROPERTIES FIXTURES_SETUP TestServer)
 set_tests_properties(OmimStopTestServer PROPERTIES FIXTURES_CLEANUP TestServer)
-set_tests_properties(OmimStartTestServer OmimStopTestServer PROPERTIES LABELS "fixture")
+set_tests_properties(OmimStartTestServer OmimStopTestServer PROPERTIES LABELS "omim-fixture")
 
 # Options:
 # * REQUIRE_QT - requires QT event loop
@@ -80,6 +80,7 @@ function(omim_add_ctest name require_server boost_test)
     WORKING_DIRECTORY ${CMAKE_BINARY_DIR}
   )
   if (require_server)
-    set_tests_properties(${TEST_NAME} PROPERTIES FIXTURES_REQUIRED TestServer)
+    set_tests_properties(${name} PROPERTIES FIXTURES_REQUIRED TestServer)
   endif()
+  set_tests_properties(${name} PROPERTIES LABELS "omim-test")
 endfunction()
diff --git a/gcovr.cfg b/gcovr.cfg
@@ -0,0 +1,22 @@
+# gcovr configuration file
+# Details: https://gcovr.com/en/master/manpage.html
+
+# Exclude all subfolders of "3party/" except for "opening_hours"
+exclude = ^3party\/(?!opening_hours($|\/)).*
+# Exclude testing folders
+exclude = testing/
+exclude = qt_tstfrm/
+# Exclude all "*tests*" subfolders
+exclude = .*tests.*
+
+exclude-unreachable-branches = yes
+exclude-throw-branches = yes
+
+print-summary = yes
+
+gcov-parallel = 20
+gcov-ignore-errors = all
+gcov-ignore-parse-errors = all
+
+html-title = Organic Maps Code Coverage Report
+html-self-contained = yes