From 632db36240b23143d682630dee34160bbc2ccd39 Mon Sep 17 00:00:00 2001 From: Filip Niksic Date: Wed, 16 Oct 2024 13:24:22 -0700 Subject: [PATCH] Add an end-to-end test for updating corpus database. More tests will be added in upcoming CLs. PiperOrigin-RevId: 686619758 --- .github/workflows/bazel_test.yml | 12 ++- .github/workflows/bazel_test_centipede.yml | 4 +- .github/workflows/cmake_test.yml | 4 +- e2e_tests/BUILD | 32 +++++++ e2e_tests/corpus_database_test.cc | 94 +++++++++++++++++++ e2e_tests/testdata/BUILD | 10 ++ e2e_tests/testdata/CMakeLists.txt | 13 ++- .../fuzz_tests_for_corpus_database_testing.cc | 32 +++++++ 8 files changed, 194 insertions(+), 7 deletions(-) create mode 100644 e2e_tests/corpus_database_test.cc create mode 100644 e2e_tests/testdata/fuzz_tests_for_corpus_database_testing.cc diff --git a/.github/workflows/bazel_test.yml b/.github/workflows/bazel_test.yml index 93e32d9a..564f8b76 100644 --- a/.github/workflows/bazel_test.yml +++ b/.github/workflows/bazel_test.yml @@ -27,7 +27,7 @@ on: jobs: run_tests: name: Run tests - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 timeout-minutes: 60 strategy: matrix: @@ -44,7 +44,7 @@ jobs: - name: Install dependencies run: | sudo apt-get update && sudo apt-get install -yq \ - clang + clang-18 - name: Setup --config=fuzztest for Bazel run: | bazel run //bazel:setup_configs > fuzztest.bazelrc @@ -64,6 +64,14 @@ jobs: run: | bazel test --build_tests_only --test_output=errors \ -c ${{ matrix.compilation_mode }} --config=fuzztest //e2e_tests:all + - name: Run end-to-end tests with --config=fuzztest-experimental + if: matrix.config == 'fuzztest' + run: | + bazel test --build_tests_only --test_output=errors \ + -c ${{ matrix.compilation_mode }} \ + --config=fuzztest-experimental --config=asan \ + --platform_suffix=fuzztest-experimental-asan \ + //e2e_tests:corpus_database_test - name: Save new cache based on main if: github.ref == 'refs/heads/main' uses: actions/cache/save@v4 diff --git a/.github/workflows/bazel_test_centipede.yml b/.github/workflows/bazel_test_centipede.yml index 8c416bbb..09b2f00d 100644 --- a/.github/workflows/bazel_test_centipede.yml +++ b/.github/workflows/bazel_test_centipede.yml @@ -27,7 +27,7 @@ on: jobs: run_tests: name: Run Centipede tests - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 timeout-minutes: 60 strategy: matrix: @@ -43,7 +43,7 @@ jobs: - name: Install dependencies run: | sudo apt-get update && sudo apt-get install -yq \ - clang llvm libssl-dev + clang-18 llvm-18 libssl-dev - name: Restore latest cache uses: actions/cache/restore@v4 with: diff --git a/.github/workflows/cmake_test.yml b/.github/workflows/cmake_test.yml index d04aa0d7..d7bb8a32 100644 --- a/.github/workflows/cmake_test.yml +++ b/.github/workflows/cmake_test.yml @@ -27,7 +27,7 @@ on: jobs: run_tests: name: Run CMake tests - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 timeout-minutes: 30 env: CCACHE_BASEDIR: ${{ github.workspace }} @@ -46,7 +46,7 @@ jobs: - name: Install dependencies run: | sudo apt-get update && sudo apt-get install -yq \ - ccache clang cmake ninja-build libprotobuf-dev protobuf-compiler + ccache clang-18 cmake ninja-build libprotobuf-dev protobuf-compiler - name: Restore latest cache uses: actions/cache/restore@v4 with: diff --git a/e2e_tests/BUILD b/e2e_tests/BUILD index bb593463..5e6403ee 100644 --- a/e2e_tests/BUILD +++ b/e2e_tests/BUILD @@ -54,6 +54,10 @@ cc_test( "//conditions:default": [], }), shard_count = 50, + tags = [ + # Don't cache the results. + "external", + ], deps = [ ":test_binary_util", "@com_google_absl//absl/container:flat_hash_map", @@ -91,3 +95,31 @@ cc_binary( "@com_googlesource_code_re2//:re2", ], ) + +# Must be run with `--config=fuzztest-experimental --config=asan`. +cc_test( + name = "corpus_database_test", + srcs = ["corpus_database_test.cc"], + data = [ + "@com_google_fuzztest//centipede:centipede_uninstrumented", + "@com_google_fuzztest//e2e_tests/testdata:fuzz_tests_for_corpus_database_testing.stripped", + ], + local_defines = select({ + "@com_google_fuzztest//fuzztest:use_centipede": ["FUZZTEST_USE_CENTIPEDE"], + "//conditions:default": [], + }), + tags = [ + # Don't cache the results. + "external", + # Don't include in wildcard expansions (:..., :all). + "manual", + ], + deps = [ + ":test_binary_util", + "@com_google_absl//absl/base:no_destructor", + "@com_google_absl//absl/log:check", + "@com_google_absl//absl/strings", + "@com_google_fuzztest//centipede:weak_sancov_stubs", + "@com_google_googletest//:gtest_main", + ], +) diff --git a/e2e_tests/corpus_database_test.cc b/e2e_tests/corpus_database_test.cc new file mode 100644 index 00000000..a4563dc4 --- /dev/null +++ b/e2e_tests/corpus_database_test.cc @@ -0,0 +1,94 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include // NOLINT +#include +#include + +#include "gmock/gmock.h" +#include "gtest/gtest.h" +#include "absl/base/no_destructor.h" +#include "absl/log/check.h" +#include "absl/strings/str_cat.h" +#include "absl/strings/string_view.h" +#include "./e2e_tests/test_binary_util.h" + +namespace fuzztest::internal { +namespace { + +using ::testing::HasSubstr; + +class UpdateCorpusDatabaseTest : public testing::Test { + protected: + static void SetUpTestSuite() { +#if defined(__has_feature) +#if !__has_feature(address_sanitizer) + CHECK(false) << "The test binary is not built with ASAN. Please run with " + "--config=asan."; +#elif !__has_feature(coverage_sanitizer) || !defined(FUZZTEST_USE_CENTIPEDE) + CHECK(false) << "The test binary is not built with coverage " + "instrumentation for Centipede. " + "Please run with --config=fuzztest-experimental."; +#endif +#endif + + temp_dir_ = new TempDir(); + + auto [status, std_out, std_err] = RunBinary( + CentipedePath(), + {.flags = { + {"binary", + absl::StrCat(BinaryPath((std::filesystem::path("testdata") / + "fuzz_tests_for_corpus_database_testing") + .c_str()), + " ", + CreateFuzzTestFlag("corpus_database", + GetCorpusDatabasePath()), + " ", CreateFuzzTestFlag("fuzz_for", "30s"))}}}); + + *centipede_std_out_ = std::move(std_out); + *centipede_std_err_ = std::move(std_err); + } + + static void TearDownTestSuite() { + delete temp_dir_; + temp_dir_ = nullptr; + } + + static std::string GetCorpusDatabasePath() { + CHECK(temp_dir_ != nullptr); + return std::filesystem::path(temp_dir_->dirname()) / "corpus_database"; + } + + static absl::string_view GetCentipedeStdOut() { return *centipede_std_out_; } + + static absl::string_view GetCentipedeStdErr() { return *centipede_std_err_; } + + private: + static TempDir *temp_dir_; + static absl::NoDestructor centipede_std_out_; + static absl::NoDestructor centipede_std_err_; +}; + +TempDir *UpdateCorpusDatabaseTest::temp_dir_ = nullptr; +absl::NoDestructor UpdateCorpusDatabaseTest::centipede_std_out_{}; +absl::NoDestructor UpdateCorpusDatabaseTest::centipede_std_err_{}; + +TEST_F(UpdateCorpusDatabaseTest, RunsFuzzTests) { + EXPECT_THAT(GetCentipedeStdErr(), + HasSubstr("Fuzzing FuzzTest.FailsInTwoWays")); +} + +} // namespace +} // namespace fuzztest::internal diff --git a/e2e_tests/testdata/BUILD b/e2e_tests/testdata/BUILD index e747e61a..e9a6d770 100644 --- a/e2e_tests/testdata/BUILD +++ b/e2e_tests/testdata/BUILD @@ -142,3 +142,13 @@ cc_binary( "@com_google_fuzztest//fuzztest:fuzztest_gtest_main", ], ) + +cc_binary( + name = "fuzz_tests_for_corpus_database_testing", + testonly = 1, + srcs = ["fuzz_tests_for_corpus_database_testing.cc"], + deps = [ + "@com_google_fuzztest//fuzztest", + "@com_google_fuzztest//fuzztest:fuzztest_gtest_main", + ], +) diff --git a/e2e_tests/testdata/CMakeLists.txt b/e2e_tests/testdata/CMakeLists.txt index 2e29242a..03831059 100644 --- a/e2e_tests/testdata/CMakeLists.txt +++ b/e2e_tests/testdata/CMakeLists.txt @@ -95,4 +95,15 @@ set_target_properties( unit_test_and_fuzz_tests.stripped PROPERTIES RUNTIME_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/_main/e2e_tests/testdata" -) \ No newline at end of file +) + +add_executable( + fuzz_tests_for_corpus_database_testing.stripped + fuzz_tests_for_corpus_database_testing.cc +) +link_fuzztest(fuzz_tests_for_corpus_database_testing.stripped) +set_target_properties( + fuzz_tests_for_corpus_database_testing.stripped + PROPERTIES RUNTIME_OUTPUT_DIRECTORY + "${CMAKE_BINARY_DIR}/_main/e2e_tests/testdata" +) diff --git a/e2e_tests/testdata/fuzz_tests_for_corpus_database_testing.cc b/e2e_tests/testdata/fuzz_tests_for_corpus_database_testing.cc new file mode 100644 index 00000000..a2e1359e --- /dev/null +++ b/e2e_tests/testdata/fuzz_tests_for_corpus_database_testing.cc @@ -0,0 +1,32 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "gtest/gtest.h" +#include "./fuzztest/fuzztest.h" + +namespace { + +volatile int force_write = 0; + +// This test fails in two ways: +// 1. It fails with an assertion failure, e.g., when `v == {2025}`. +// 2. It fails with a heap buffer overflow, e.g., when `v == {4050}`. +void FailsInTwoWays(const std::vector& v) { + if (v.size() % 7 != 1) return; + ASSERT_NE(v[0], 2025); + if (v[0] == 2 * 2025) force_write = v.data()[v.size()]; +} +FUZZ_TEST(FuzzTest, FailsInTwoWays); + +} // namespace