Skip to content

Commit

Permalink
Add an end-to-end test for updating corpus database.
Browse files Browse the repository at this point in the history
More tests will be added in upcoming CLs.

PiperOrigin-RevId: 686619758
  • Loading branch information
fniksic authored and copybara-github committed Oct 22, 2024
1 parent c942c29 commit 78ddddb
Show file tree
Hide file tree
Showing 7 changed files with 191 additions and 2 deletions.
8 changes: 8 additions & 0 deletions .github/workflows/bazel_test.yml
Original file line number Diff line number Diff line change
Expand Up @@ -64,6 +64,14 @@ jobs:
run: |
bazel test --build_tests_only --test_output=errors \
-c ${{ matrix.compilation_mode }} --config=fuzztest //e2e_tests:all
- name: Run end-to-end tests with --config=fuzztest-experimental
if: matrix.config == 'fuzztest'
run: |
bazel test --build_tests_only --test_output=errors \
-c ${{ matrix.compilation_mode }} \
--config=fuzztest-experimental --config=asan \
--platform_suffix=fuzztest-experimental-asan \
//e2e_tests:corpus_database_test
- name: Save new cache based on main
if: github.ref == 'refs/heads/main'
uses: actions/cache/save@v4
Expand Down
4 changes: 3 additions & 1 deletion bazel/setup_configs.sh
Original file line number Diff line number Diff line change
Expand Up @@ -116,7 +116,9 @@ build:fuzztest-experimental --dynamic_mode=off
# We apply coverage tracking instrumentation to everything but Centipede and the
# FuzzTest framework itself (including GoogleTest and GoogleMock).
build:fuzztest-experimental --copt=-fsanitize-coverage=trace-pc-guard,pc-table,trace-loads,trace-cmp,control-flow
# TODO(b/374840534): Add -fsanitize-coverage=control-flow once we start building
# with clang 16+.
build:fuzztest-experimental --copt=-fsanitize-coverage=trace-pc-guard,pc-table,trace-loads,trace-cmp
build:fuzztest-experimental --per_file_copt=${COMMON_FILTER},${FUZZTEST_FILTER},${CENTIPEDE_FILTER},googletest/.*,googlemock/.*@-fsanitize-coverage=0
EOF

Expand Down
32 changes: 32 additions & 0 deletions e2e_tests/BUILD
Original file line number Diff line number Diff line change
Expand Up @@ -54,6 +54,10 @@ cc_test(
"//conditions:default": [],
}),
shard_count = 50,
tags = [
# Don't cache the results.
"external",
],
deps = [
":test_binary_util",
"@com_google_absl//absl/container:flat_hash_map",
Expand Down Expand Up @@ -91,3 +95,31 @@ cc_binary(
"@com_googlesource_code_re2//:re2",
],
)

# Must be run with `--config=fuzztest-experimental --config=asan`.
cc_test(
name = "corpus_database_test",
srcs = ["corpus_database_test.cc"],
data = [
"@com_google_fuzztest//centipede:centipede_uninstrumented",
"@com_google_fuzztest//e2e_tests/testdata:fuzz_tests_for_corpus_database_testing.stripped",
],
local_defines = select({
"@com_google_fuzztest//fuzztest:use_centipede": ["FUZZTEST_USE_CENTIPEDE"],
"//conditions:default": [],
}),
tags = [
# Don't cache the results.
"external",
# Don't include in wildcard expansions (:..., :all).
"manual",
],
deps = [
":test_binary_util",
"@com_google_absl//absl/base:no_destructor",
"@com_google_absl//absl/log:check",
"@com_google_absl//absl/strings",
"@com_google_fuzztest//centipede:weak_sancov_stubs",
"@com_google_googletest//:gtest_main",
],
)
94 changes: 94 additions & 0 deletions e2e_tests/corpus_database_test.cc
Original file line number Diff line number Diff line change
@@ -0,0 +1,94 @@
// Copyright 2024 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

#include <filesystem> // NOLINT
#include <string>
#include <utility>

#include "gmock/gmock.h"
#include "gtest/gtest.h"
#include "absl/base/no_destructor.h"
#include "absl/log/check.h"
#include "absl/strings/str_cat.h"
#include "absl/strings/string_view.h"
#include "./e2e_tests/test_binary_util.h"

namespace fuzztest::internal {
namespace {

using ::testing::HasSubstr;

class UpdateCorpusDatabaseTest : public testing::Test {
protected:
static void SetUpTestSuite() {
#if defined(__has_feature)
#if !__has_feature(address_sanitizer)
CHECK(false) << "The test binary is not built with ASAN. Please run with "
"--config=asan.";
#elif !__has_feature(coverage_sanitizer) || !defined(FUZZTEST_USE_CENTIPEDE)
CHECK(false) << "The test binary is not built with coverage "
"instrumentation for Centipede. "
"Please run with --config=fuzztest-experimental.";
#endif
#endif

temp_dir_ = new TempDir();

auto [status, std_out, std_err] = RunBinary(
CentipedePath(),
{.flags = {
{"binary",
absl::StrCat(BinaryPath((std::filesystem::path("testdata") /
"fuzz_tests_for_corpus_database_testing")
.c_str()),
" ",
CreateFuzzTestFlag("corpus_database",
GetCorpusDatabasePath()),
" ", CreateFuzzTestFlag("fuzz_for", "30s"))}}});

*centipede_std_out_ = std::move(std_out);
*centipede_std_err_ = std::move(std_err);
}

static void TearDownTestSuite() {
delete temp_dir_;
temp_dir_ = nullptr;
}

static std::string GetCorpusDatabasePath() {
CHECK(temp_dir_ != nullptr);
return std::filesystem::path(temp_dir_->dirname()) / "corpus_database";
}

static absl::string_view GetCentipedeStdOut() { return *centipede_std_out_; }

static absl::string_view GetCentipedeStdErr() { return *centipede_std_err_; }

private:
static TempDir *temp_dir_;
static absl::NoDestructor<std::string> centipede_std_out_;
static absl::NoDestructor<std::string> centipede_std_err_;
};

TempDir *UpdateCorpusDatabaseTest::temp_dir_ = nullptr;
absl::NoDestructor<std::string> UpdateCorpusDatabaseTest::centipede_std_out_{};
absl::NoDestructor<std::string> UpdateCorpusDatabaseTest::centipede_std_err_{};

TEST_F(UpdateCorpusDatabaseTest, RunsFuzzTests) {
EXPECT_THAT(GetCentipedeStdErr(),
HasSubstr("Fuzzing FuzzTest.FailsInTwoWays"));
}

} // namespace
} // namespace fuzztest::internal
10 changes: 10 additions & 0 deletions e2e_tests/testdata/BUILD
Original file line number Diff line number Diff line change
Expand Up @@ -142,3 +142,13 @@ cc_binary(
"@com_google_fuzztest//fuzztest:fuzztest_gtest_main",
],
)

cc_binary(
name = "fuzz_tests_for_corpus_database_testing",
testonly = 1,
srcs = ["fuzz_tests_for_corpus_database_testing.cc"],
deps = [
"@com_google_fuzztest//fuzztest",
"@com_google_fuzztest//fuzztest:fuzztest_gtest_main",
],
)
13 changes: 12 additions & 1 deletion e2e_tests/testdata/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -95,4 +95,15 @@ set_target_properties(
unit_test_and_fuzz_tests.stripped
PROPERTIES RUNTIME_OUTPUT_DIRECTORY
"${CMAKE_BINARY_DIR}/_main/e2e_tests/testdata"
)
)

add_executable(
fuzz_tests_for_corpus_database_testing.stripped
fuzz_tests_for_corpus_database_testing.cc
)
link_fuzztest(fuzz_tests_for_corpus_database_testing.stripped)
set_target_properties(
fuzz_tests_for_corpus_database_testing.stripped
PROPERTIES RUNTIME_OUTPUT_DIRECTORY
"${CMAKE_BINARY_DIR}/_main/e2e_tests/testdata"
)
32 changes: 32 additions & 0 deletions e2e_tests/testdata/fuzz_tests_for_corpus_database_testing.cc
Original file line number Diff line number Diff line change
@@ -0,0 +1,32 @@
// Copyright 2024 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

#include "gtest/gtest.h"
#include "./fuzztest/fuzztest.h"

namespace {

volatile int force_write = 0;

// This test fails in two ways:
// 1. It fails with an assertion failure, e.g., when `v == {2025}`.
// 2. It fails with a heap buffer overflow, e.g., when `v == {4050}`.
void FailsInTwoWays(const std::vector<int>& v) {
if (v.size() % 7 != 1) return;
ASSERT_NE(v[0], 2025);
if (v[0] == 2 * 2025) force_write = v.data()[v.size()];
}
FUZZ_TEST(FuzzTest, FailsInTwoWays);

} // namespace

0 comments on commit 78ddddb

Please sign in to comment.