Skip to content

Commit

Permalink
Add an end-to-end test for updating corpus database.
Browse files Browse the repository at this point in the history
More tests will be added in upcoming CLs.

PiperOrigin-RevId: 686619758
  • Loading branch information
fniksic authored and copybara-github committed Oct 21, 2024
1 parent c942c29 commit 632db36
Show file tree
Hide file tree
Showing 8 changed files with 194 additions and 7 deletions.
12 changes: 10 additions & 2 deletions .github/workflows/bazel_test.yml
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@ on:
jobs:
run_tests:
name: Run tests
runs-on: ubuntu-latest
runs-on: ubuntu-24.04
timeout-minutes: 60
strategy:
matrix:
Expand All @@ -44,7 +44,7 @@ jobs:
- name: Install dependencies
run: |
sudo apt-get update && sudo apt-get install -yq \
clang
clang-18
- name: Setup --config=fuzztest for Bazel
run: |
bazel run //bazel:setup_configs > fuzztest.bazelrc
Expand All @@ -64,6 +64,14 @@ jobs:
run: |
bazel test --build_tests_only --test_output=errors \
-c ${{ matrix.compilation_mode }} --config=fuzztest //e2e_tests:all
- name: Run end-to-end tests with --config=fuzztest-experimental
if: matrix.config == 'fuzztest'
run: |
bazel test --build_tests_only --test_output=errors \
-c ${{ matrix.compilation_mode }} \
--config=fuzztest-experimental --config=asan \
--platform_suffix=fuzztest-experimental-asan \
//e2e_tests:corpus_database_test
- name: Save new cache based on main
if: github.ref == 'refs/heads/main'
uses: actions/cache/save@v4
Expand Down
4 changes: 2 additions & 2 deletions .github/workflows/bazel_test_centipede.yml
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@ on:
jobs:
run_tests:
name: Run Centipede tests
runs-on: ubuntu-latest
runs-on: ubuntu-24.04
timeout-minutes: 60
strategy:
matrix:
Expand All @@ -43,7 +43,7 @@ jobs:
- name: Install dependencies
run: |
sudo apt-get update && sudo apt-get install -yq \
clang llvm libssl-dev
clang-18 llvm-18 libssl-dev
- name: Restore latest cache
uses: actions/cache/restore@v4
with:
Expand Down
4 changes: 2 additions & 2 deletions .github/workflows/cmake_test.yml
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@ on:
jobs:
run_tests:
name: Run CMake tests
runs-on: ubuntu-latest
runs-on: ubuntu-24.04
timeout-minutes: 30
env:
CCACHE_BASEDIR: ${{ github.workspace }}
Expand All @@ -46,7 +46,7 @@ jobs:
- name: Install dependencies
run: |
sudo apt-get update && sudo apt-get install -yq \
ccache clang cmake ninja-build libprotobuf-dev protobuf-compiler
ccache clang-18 cmake ninja-build libprotobuf-dev protobuf-compiler
- name: Restore latest cache
uses: actions/cache/restore@v4
with:
Expand Down
32 changes: 32 additions & 0 deletions e2e_tests/BUILD
Original file line number Diff line number Diff line change
Expand Up @@ -54,6 +54,10 @@ cc_test(
"//conditions:default": [],
}),
shard_count = 50,
tags = [
# Don't cache the results.
"external",
],
deps = [
":test_binary_util",
"@com_google_absl//absl/container:flat_hash_map",
Expand Down Expand Up @@ -91,3 +95,31 @@ cc_binary(
"@com_googlesource_code_re2//:re2",
],
)

# Must be run with `--config=fuzztest-experimental --config=asan`.
cc_test(
name = "corpus_database_test",
srcs = ["corpus_database_test.cc"],
data = [
"@com_google_fuzztest//centipede:centipede_uninstrumented",
"@com_google_fuzztest//e2e_tests/testdata:fuzz_tests_for_corpus_database_testing.stripped",
],
local_defines = select({
"@com_google_fuzztest//fuzztest:use_centipede": ["FUZZTEST_USE_CENTIPEDE"],
"//conditions:default": [],
}),
tags = [
# Don't cache the results.
"external",
# Don't include in wildcard expansions (:..., :all).
"manual",
],
deps = [
":test_binary_util",
"@com_google_absl//absl/base:no_destructor",
"@com_google_absl//absl/log:check",
"@com_google_absl//absl/strings",
"@com_google_fuzztest//centipede:weak_sancov_stubs",
"@com_google_googletest//:gtest_main",
],
)
94 changes: 94 additions & 0 deletions e2e_tests/corpus_database_test.cc
Original file line number Diff line number Diff line change
@@ -0,0 +1,94 @@
// Copyright 2024 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

#include <filesystem> // NOLINT
#include <string>
#include <utility>

#include "gmock/gmock.h"
#include "gtest/gtest.h"
#include "absl/base/no_destructor.h"
#include "absl/log/check.h"
#include "absl/strings/str_cat.h"
#include "absl/strings/string_view.h"
#include "./e2e_tests/test_binary_util.h"

namespace fuzztest::internal {
namespace {

using ::testing::HasSubstr;

class UpdateCorpusDatabaseTest : public testing::Test {
protected:
static void SetUpTestSuite() {
#if defined(__has_feature)
#if !__has_feature(address_sanitizer)
CHECK(false) << "The test binary is not built with ASAN. Please run with "
"--config=asan.";
#elif !__has_feature(coverage_sanitizer) || !defined(FUZZTEST_USE_CENTIPEDE)
CHECK(false) << "The test binary is not built with coverage "
"instrumentation for Centipede. "
"Please run with --config=fuzztest-experimental.";
#endif
#endif

temp_dir_ = new TempDir();

auto [status, std_out, std_err] = RunBinary(
CentipedePath(),
{.flags = {
{"binary",
absl::StrCat(BinaryPath((std::filesystem::path("testdata") /
"fuzz_tests_for_corpus_database_testing")
.c_str()),
" ",
CreateFuzzTestFlag("corpus_database",
GetCorpusDatabasePath()),
" ", CreateFuzzTestFlag("fuzz_for", "30s"))}}});

*centipede_std_out_ = std::move(std_out);
*centipede_std_err_ = std::move(std_err);
}

static void TearDownTestSuite() {
delete temp_dir_;
temp_dir_ = nullptr;
}

static std::string GetCorpusDatabasePath() {
CHECK(temp_dir_ != nullptr);
return std::filesystem::path(temp_dir_->dirname()) / "corpus_database";
}

static absl::string_view GetCentipedeStdOut() { return *centipede_std_out_; }

static absl::string_view GetCentipedeStdErr() { return *centipede_std_err_; }

private:
static TempDir *temp_dir_;
static absl::NoDestructor<std::string> centipede_std_out_;
static absl::NoDestructor<std::string> centipede_std_err_;
};

TempDir *UpdateCorpusDatabaseTest::temp_dir_ = nullptr;
absl::NoDestructor<std::string> UpdateCorpusDatabaseTest::centipede_std_out_{};
absl::NoDestructor<std::string> UpdateCorpusDatabaseTest::centipede_std_err_{};

TEST_F(UpdateCorpusDatabaseTest, RunsFuzzTests) {
EXPECT_THAT(GetCentipedeStdErr(),
HasSubstr("Fuzzing FuzzTest.FailsInTwoWays"));
}

} // namespace
} // namespace fuzztest::internal
10 changes: 10 additions & 0 deletions e2e_tests/testdata/BUILD
Original file line number Diff line number Diff line change
Expand Up @@ -142,3 +142,13 @@ cc_binary(
"@com_google_fuzztest//fuzztest:fuzztest_gtest_main",
],
)

cc_binary(
name = "fuzz_tests_for_corpus_database_testing",
testonly = 1,
srcs = ["fuzz_tests_for_corpus_database_testing.cc"],
deps = [
"@com_google_fuzztest//fuzztest",
"@com_google_fuzztest//fuzztest:fuzztest_gtest_main",
],
)
13 changes: 12 additions & 1 deletion e2e_tests/testdata/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -95,4 +95,15 @@ set_target_properties(
unit_test_and_fuzz_tests.stripped
PROPERTIES RUNTIME_OUTPUT_DIRECTORY
"${CMAKE_BINARY_DIR}/_main/e2e_tests/testdata"
)
)

add_executable(
fuzz_tests_for_corpus_database_testing.stripped
fuzz_tests_for_corpus_database_testing.cc
)
link_fuzztest(fuzz_tests_for_corpus_database_testing.stripped)
set_target_properties(
fuzz_tests_for_corpus_database_testing.stripped
PROPERTIES RUNTIME_OUTPUT_DIRECTORY
"${CMAKE_BINARY_DIR}/_main/e2e_tests/testdata"
)
32 changes: 32 additions & 0 deletions e2e_tests/testdata/fuzz_tests_for_corpus_database_testing.cc
Original file line number Diff line number Diff line change
@@ -0,0 +1,32 @@
// Copyright 2024 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

#include "gtest/gtest.h"
#include "./fuzztest/fuzztest.h"

namespace {

volatile int force_write = 0;

// This test fails in two ways:
// 1. It fails with an assertion failure, e.g., when `v == {2025}`.
// 2. It fails with a heap buffer overflow, e.g., when `v == {4050}`.
void FailsInTwoWays(const std::vector<int>& v) {
if (v.size() % 7 != 1) return;
ASSERT_NE(v[0], 2025);
if (v[0] == 2 * 2025) force_write = v.data()[v.size()];
}
FUZZ_TEST(FuzzTest, FailsInTwoWays);

} // namespace

0 comments on commit 632db36

Please sign in to comment.