From 93b230bb820c2b8ccc5ede79d7cd358ca165b6aa Mon Sep 17 00:00:00 2001 From: UebelAndre Date: Tue, 13 Jun 2023 10:34:14 -0700 Subject: [PATCH] Fix code coverage collection. (#2001) * Minor CI and test cleanup * Bump min tested Bazel version to 5.2.0 * Fix code coverage collection. --- .bazelci/presubmit.yml | 33 ++++++++++++++++++++++++++------ .bazelrc | 11 +++++++---- docs/index.md | 2 +- rust/private/rust.bzl | 41 ++++++++++++++++++++++++++++++++-------- util/collect_coverage.sh | 15 ++++++++++++--- 5 files changed, 80 insertions(+), 22 deletions(-) diff --git a/.bazelci/presubmit.yml b/.bazelci/presubmit.yml index 7b3afaa950..5a17ccc9a2 100644 --- a/.bazelci/presubmit.yml +++ b/.bazelci/presubmit.yml @@ -35,11 +35,19 @@ crate_universe_vendor_example_targets: &crate_universe_vendor_example_targets crate_universe_unnamed_vendor_example_targets: &crate_universe_unnamed_vendor_example_targets - "//vendor_remote_manifests:crates_vendor" - "//vendor_remote_pkgs:crates_vendor" +coverage_validation_post_shell_commands: &coverage_validation_post_shell_commands + - | + grep -q '^SF:.*\.rs$' bazel-out/_coverage/_coverage_report.dat \ + || { 1>&2 echo "Failed to find any Rust coverage" \ + ; 1>&2 cat bazel-out/_coverage/_coverage_report.dat \ + ; exit 1 \ + ; } tasks: ubuntu2004: build_targets: *default_linux_targets test_targets: *default_linux_targets coverage_targets: *default_linux_targets + post_shell_commands: *coverage_validation_post_shell_commands rbe_ubuntu1604: shell_commands: - sed -i 's/^# load("@bazelci_rules/load("@bazelci_rules/' WORKSPACE.bazel @@ -57,6 +65,7 @@ tasks: build_targets: *default_macos_targets test_targets: *default_macos_targets coverage_targets: *default_macos_targets + post_shell_commands: *coverage_validation_post_shell_commands windows: build_targets: *default_windows_targets test_targets: *default_windows_targets @@ -90,9 +99,11 @@ tasks: ubuntu2004_with_aspects: name: With Aspects platform: ubuntu2004 + build_flags: *aspects_flags build_targets: *default_linux_targets test_targets: *default_linux_targets - build_flags: *aspects_flags + coverage_targets: *default_linux_targets + post_shell_commands: *coverage_validation_post_shell_commands rbe_ubuntu1604_with_aspects: name: With Aspects platform: rbe_ubuntu1604 @@ -126,15 +137,19 @@ tasks: macos_with_aspects: name: With Aspects platform: macos + build_flags: *aspects_flags build_targets: *default_macos_targets test_targets: *default_macos_targets - build_flags: *aspects_flags + coverage_targets: *default_macos_targets + post_shell_commands: *coverage_validation_post_shell_commands macos_rolling_with_aspects: name: "Macos Rolling Bazel Version With Aspects" platform: macos + build_flags: *aspects_flags build_targets: *default_macos_targets test_targets: *default_macos_targets - build_flags: *aspects_flags + coverage_targets: *default_macos_targets + post_shell_commands: *coverage_validation_post_shell_commands soft_fail: yes bazel: "rolling" windows_with_aspects: @@ -190,17 +205,21 @@ tasks: ubuntu1804: name: "Min Bazel Version" # If updating the minimum bazel version, please also update /docs/index.md - bazel: "5.0.0" + bazel: "5.2.0" platform: ubuntu1804 build_targets: *default_linux_targets test_targets: *default_linux_targets + coverage_targets: *default_linux_targets + post_shell_commands: *coverage_validation_post_shell_commands ubuntu1804_with_aspects: name: "Min Bazel Version With Aspects" - bazel: "5.0.0" + bazel: "5.2.0" platform: ubuntu1804 build_targets: *default_linux_targets test_targets: *default_linux_targets build_flags: *aspects_flags + coverage_targets: *default_linux_targets + post_shell_commands: *coverage_validation_post_shell_commands ubuntu2004_min_rust_version: name: "Min Rust Version" platform: ubuntu2004 @@ -257,10 +276,12 @@ tasks: name: "Rolling Bazel Version With Aspects" bazel: "rolling" platform: ubuntu2004 + build_flags: *aspects_flags soft_fail: yes build_targets: *default_linux_targets test_targets: *default_linux_targets - build_flags: *aspects_flags + coverage_targets: *default_linux_targets + post_shell_commands: *coverage_validation_post_shell_commands linux_docs: name: Docs platform: ubuntu2004 diff --git a/.bazelrc b/.bazelrc index 1a11f65a62..b4d00296fd 100644 --- a/.bazelrc +++ b/.bazelrc @@ -5,14 +5,17 @@ ## https://bazel.build/docs/best-practices#bazelrc-file ############################################################################### +# https://bazel.build/reference/command-line-reference#flag--enable_platform_specific_config +common --enable_platform_specific_config + # https://bazel.build/docs/windows#symlink startup --windows_enable_symlinks - -# Required on windows -common --enable_platform_specific_config -startup:windows --windows_enable_symlinks build:windows --enable_runfiles +# Enable the only currently supported report type +# https://bazel.build/reference/command-line-reference#flag--combined_report +coverage --combined_report=lcov + ############################################################################### ## Unique configuration groups ############################################################################### diff --git a/docs/index.md b/docs/index.md index 78fdd1d5ca..78ed9a77cb 100644 --- a/docs/index.md +++ b/docs/index.md @@ -93,7 +93,7 @@ Failure to do so will result in rules attempting to match a `stable` toolchain w ## Supported bazel versions -The oldest version of Bazel the `main` branch is tested against is `5.0.0`. Previous versions may still be functional in certain environments, but this is the minimum version we strive to fully support. +The oldest version of Bazel the `main` branch is tested against is `5.2.0`. Previous versions may still be functional in certain environments, but this is the minimum version we strive to fully support. We test these rules against the latest rolling releases of Bazel, and aim for compatibility with them, but prioritise stable releases over rolling releases where necessary. diff --git a/rust/private/rust.bzl b/rust/private/rust.bzl index 0cc569ecaf..2316981435 100644 --- a/rust/private/rust.bzl +++ b/rust/private/rust.bzl @@ -474,8 +474,16 @@ def _rust_test_impl(ctx): if not toolchain.llvm_profdata: fail("toolchain.llvm_profdata is required if toolchain.llvm_cov is set.") - env["RUST_LLVM_COV"] = toolchain.llvm_cov.path - env["RUST_LLVM_PROFDATA"] = toolchain.llvm_profdata.path + llvm_cov_path = toolchain.llvm_cov.short_path + if llvm_cov_path.startswith("../"): + llvm_cov_path = llvm_cov_path[len("../"):] + + llvm_profdata_path = toolchain.llvm_profdata.short_path + if llvm_profdata_path.startswith("../"): + llvm_profdata_path = llvm_profdata_path[len("../"):] + + env["RUST_LLVM_COV"] = llvm_cov_path + env["RUST_LLVM_PROFDATA"] = llvm_profdata_path components = "{}/{}".format(ctx.label.workspace_root, ctx.label.package).split("/") env["CARGO_MANIFEST_DIR"] = "/".join([c for c in components if c]) providers.append(testing.TestEnvironment(env)) @@ -652,11 +660,6 @@ _common_attrs = { ), default = Label("@bazel_tools//tools/cpp:current_cc_toolchain"), ), - "_collect_cc_coverage": attr.label( - default = Label("//util:collect_coverage"), - executable = True, - cfg = "exec", - ), "_error_format": attr.label( default = Label("//:error_format"), ), @@ -698,6 +701,28 @@ _common_attrs = { ), } +_coverage_attrs = { + "_collect_cc_coverage": attr.label( + default = Label("//util:collect_coverage"), + executable = True, + cfg = "exec", + ), + # Bazel’s coverage runner + # (https://github.com/bazelbuild/bazel/blob/6.0.0/tools/test/collect_coverage.sh) + # needs a binary called “lcov_merge.” Its location is passed in the + # LCOV_MERGER environmental variable. For builtin rules, this variable + # is set automatically based on a magic “$lcov_merger” or + # “:lcov_merger” attribute, but it’s not possible to create such + # attributes in Starlark. Therefore we specify the variable ourselves. + # Note that the coverage runner runs in the runfiles root instead of + # the execution root, therefore we use “path” instead of “short_path.” + "_lcov_merger": attr.label( + default = configuration_field(fragment = "coverage", name = "output_generator"), + executable = True, + cfg = "exec", + ), +} + _experimental_use_cc_common_link_attrs = { "experimental_use_cc_common_link": attr.int( doc = ( @@ -765,7 +790,7 @@ _rust_test_attrs = dict({ default = Label("@bazel_tools//tools/cpp:grep-includes"), executable = True, ), -}.items() + _experimental_use_cc_common_link_attrs.items()) +}.items() + _coverage_attrs.items() + _experimental_use_cc_common_link_attrs.items()) _common_providers = [ rust_common.crate_info, diff --git a/util/collect_coverage.sh b/util/collect_coverage.sh index 328a5c6f01..648de5eeef 100755 --- a/util/collect_coverage.sh +++ b/util/collect_coverage.sh @@ -6,14 +6,20 @@ if [[ -n "${VERBOSE_COVERAGE:-}" ]]; then set -x fi +if [[ "${RUNFILES_DIR:0:1}" != "/" ]]; then + if [[ -n "${ROOT}" ]]; then + RUNFILES_DIR="${ROOT}/${RUNFILES_DIR}" + fi +fi + readonly profdata_file=$COVERAGE_DIR/coverage.profdata -"$RUNFILES_DIR/$TEST_WORKSPACE/$RUST_LLVM_PROFDATA" \ +"$RUNFILES_DIR/$RUST_LLVM_PROFDATA" \ merge \ --sparse "$COVERAGE_DIR"/*.profraw \ -output "$profdata_file" -"$RUNFILES_DIR/$TEST_WORKSPACE/$RUST_LLVM_COV" \ +"$RUNFILES_DIR/$RUST_LLVM_COV" \ export \ -format=lcov \ -instr-profile "$profdata_file" \ @@ -22,4 +28,7 @@ readonly profdata_file=$COVERAGE_DIR/coverage.profdata -path-equivalence=.,"$ROOT" \ "$RUNFILES_DIR/$TEST_WORKSPACE/$TEST_BINARY" \ @"$COVERAGE_MANIFEST" \ - | sed 's#/proc/self/cwd/##' > "$COVERAGE_OUTPUT_FILE" + | sed 's#/proc/self/cwd/##' > "$COVERAGE_DIR/rust_coverage.dat" + +# Bazel doesn't support LLVM profdata coverage amongst other coverage formats. +rm "$profdata_file"