#Centipede Copy the crossover algorithm of the legacy mutator to the …

…new mutator.

Also share the crossover knobs across the two mutators.

PiperOrigin-RevId: 579319218

centipede/BUILD

@@ -837,8 +837,10 @@ cc_library(
     srcs = ["fuzztest_mutator.cc"],
     hdrs = ["fuzztest_mutator.h"],
     deps = [
+        ":byte_array_mutator",
         ":defs",
         ":execution_metadata",
+        ":knobs",
         ":mutation_input",
         "@com_google_absl//absl/random",
         "@com_google_fuzztest//fuzztest:domain_core",
@@ -1369,6 +1371,7 @@ cc_test(
     deps = [
         ":defs",
         ":fuzztest_mutator",
+        ":knobs",
         "@com_google_absl//absl/container:flat_hash_set",
         "@com_google_absl//absl/strings",
         "@com_google_googletest//:gtest_main",

centipede/byte_array_mutator.cc

@@ -298,7 +298,7 @@ void ByteArrayMutator::CrossOverOverwrite(ByteArray &data,
             data.begin() + pos);
 }
 
-static const KnobId knob_cross_over_insert_or_overwrite =
+const KnobId knob_cross_over_insert_or_overwrite =
     Knobs::NewId("cross_over_insert_or_overwrite");
 
 void ByteArrayMutator::CrossOver(ByteArray &data, const ByteArray &other) {
@@ -316,8 +316,7 @@ void ByteArrayMutator::CrossOver(ByteArray &data, const ByteArray &other) {
 // Controls how much crossover is used during mutations.
 // https://en.wikipedia.org/wiki/Crossover_(genetic_algorithm)
 // TODO(kcc): add tests with different values of knobs.
-static const KnobId knob_mutate_or_crossover =
-    Knobs::NewId("mutate_or_crossover");
+const KnobId knob_mutate_or_crossover = Knobs::NewId("mutate_or_crossover");
 
 void ByteArrayMutator::MutateMany(const std::vector<MutationInputRef> &inputs,
                                   size_t num_mutants,

centipede/byte_array_mutator.h

@@ -243,6 +243,14 @@ class ByteArrayMutator {
   CmpDictionary cmp_dictionary_;
 };
 
+// Controls how much crossover is used during mutations.
+// https://en.wikipedia.org/wiki/Crossover_(genetic_algorithm)
+// TODO(kcc): add tests with different values of knobs.
+extern const KnobId knob_mutate_or_crossover;
+// Controls how much crossver inserts data from the other input instead of
+// overwriting.
+extern const KnobId knob_cross_over_insert_or_overwrite;
+
 }  // namespace centipede
 
 #endif  // THIRD_PARTY_CENTIPEDE_BYTE_ARRAY_MUTATOR_H_

centipede/centipede_callbacks.h

@@ -51,7 +51,7 @@ class CentipedeCallbacks {
   CentipedeCallbacks(const Environment &env)
       : env_(env),
         byte_array_mutator_(env.knobs, GetRandomSeed(env.seed)),
-        fuzztest_mutator_(GetRandomSeed(env.seed)),
+        fuzztest_mutator_(env.knobs, GetRandomSeed(env.seed)),
         inputs_blobseq_(shmem_name1_.c_str(), env.shmem_size_mb << 20,
                         env.use_posix_shmem),
         outputs_blobseq_(shmem_name2_.c_str(), env.shmem_size_mb << 20,

centipede/fuzztest_mutator.cc

@@ -14,6 +14,15 @@
 
 #include "./centipede/fuzztest_mutator.h"
 
+#include <algorithm>
+#include <cstddef>
+#include <cstdint>
+#include <random>
+
+#include "absl/random/random.h"
+#include "./centipede/byte_array_mutator.h"
+#include "./centipede/defs.h"
+#include "./centipede/knobs.h"
 #include "./fuzztest/domain_core.h"
 
 namespace centipede {
@@ -31,8 +40,9 @@ class FuzzTestMutator::MutatorDomain : public MutatorDomainBase {
       : MutatorDomainBase(fuzztest::VectorOf(fuzztest::Arbitrary<uint8_t>())) {}
 };
 
-FuzzTestMutator::FuzzTestMutator(uint64_t seed)
-    : prng_(std::seed_seq({seed, seed >> 32})),
+FuzzTestMutator::FuzzTestMutator(const Knobs &knobs, uint64_t seed)
+    : knobs_(knobs),
+      prng_(std::seed_seq({seed, seed >> 32})),
       domain_(std::make_unique<MutatorDomain>()) {
   domain_->WithMinSize(1).WithMaxSize(max_len_);
   if (fuzztest::internal::GetExecutionCoverage() == nullptr) {
@@ -44,6 +54,41 @@ FuzzTestMutator::FuzzTestMutator(uint64_t seed)
 
 FuzzTestMutator::~FuzzTestMutator() = default;
 
+void FuzzTestMutator::CrossOverInsert(ByteArray &data, const ByteArray &other) {
+  // insert other[first:first+size] at data[pos]
+  const auto size = absl::Uniform<size_t>(
+      prng_, 1, std::min(max_len_ - data.size(), other.size()) + 1);
+  const auto first = absl::Uniform<size_t>(prng_, 0, other.size() - size + 1);
+  const auto pos = absl::Uniform<size_t>(prng_, 0, data.size() + 1);
+  data.insert(data.begin() + pos, other.begin() + first,
+              other.begin() + first + size);
+}
+
+void FuzzTestMutator::CrossOverOverwrite(ByteArray &data,
+                                         const ByteArray &other) {
+  // Overwrite data[pos:pos+size] with other[first:first+size].
+  // Overwrite no more than half of data.
+  size_t max_size = std::max(1UL, data.size() / 2);
+  const auto first = absl::Uniform<size_t>(prng_, 0, other.size());
+  max_size = std::min(max_size, other.size() - first);
+  const auto size = absl::Uniform<size_t>(prng_, 1, max_size + 1);
+  const auto pos = absl::Uniform<size_t>(prng_, 0, data.size() - size + 1);
+  std::copy(other.begin() + first, other.begin() + first + size,
+            data.begin() + pos);
+}
+
+void FuzzTestMutator::CrossOver(ByteArray &data, const ByteArray &other) {
+  if (data.size() >= max_len_) {
+    CrossOverOverwrite(data, other);
+  } else {
+    if (knobs_.GenerateBool(knob_cross_over_insert_or_overwrite, prng_())) {
+      CrossOverInsert(data, other);
+    } else {
+      CrossOverOverwrite(data, other);
+    }
+  }
+}
+
 void FuzzTestMutator::MutateMany(const std::vector<MutationInputRef>& inputs,
                                  size_t num_mutants,
                                  std::vector<ByteArray>& mutants) {
@@ -57,7 +102,14 @@ void FuzzTestMutator::MutateMany(const std::vector<MutationInputRef>& inputs,
   for (int i = 0; i < num_mutants; ++i) {
     auto mutant = inputs[absl::Uniform<size_t>(prng_, 0, inputs.size())].data;
     if (mutant.size() > max_len_) mutant.resize(max_len_);
-    domain_->Mutate(mutant, prng_, /*only_shrink=*/false);
+    if (knobs_.GenerateBool(knob_mutate_or_crossover, prng_())) {
+      // Perform crossover with some other input. It may be the same input.
+      const auto &other_input =
+          inputs[absl::Uniform<size_t>(prng_, 0, inputs.size())].data;
+      CrossOver(mutant, other_input);
+    } else {
+      domain_->Mutate(mutant, prng_, /*only_shrink=*/false);
+    }
     mutants.push_back(std::move(mutant));
   }
 }

centipede/fuzztest_mutator.h

@@ -20,6 +20,7 @@
 #include "absl/random/random.h"
 #include "./centipede/defs.h"
 #include "./centipede/execution_metadata.h"
+#include "./centipede/knobs.h"
 #include "./centipede/mutation_input.h"
 
 namespace centipede {
@@ -31,8 +32,8 @@ namespace centipede {
 // This class is thread-compatible.
 class FuzzTestMutator {
  public:
-  // Initialize the mutator with the given RNG `seed`.
-  explicit FuzzTestMutator(uint64_t seed);
+  // Initialize the mutator with the given `knobs` and RNG `seed`.
+  explicit FuzzTestMutator(const Knobs &knobs, uint64_t seed);
   ~FuzzTestMutator();
 
   // Takes non-empty `inputs`, produces `num_mutants` mutations in `mutants`.
@@ -56,10 +57,17 @@ class FuzzTestMutator {
   // Propagates the execution `metadata` to the internal mutation dictionary.
   void SetMetadata(const ExecutionMetadata& metadata);
 
+  // The crossover algorithm based on the legacy ByteArrayMutator.
+  // TODO: Implement and use the domain level crossover.
+  void CrossOverInsert(ByteArray &data, const ByteArray &other);
+  void CrossOverOverwrite(ByteArray &data, const ByteArray &other);
+  void CrossOver(ByteArray &data, const ByteArray &other);
+
   // Size limits on the cmp entries to be used in mutation.
   static constexpr uint8_t kMaxCmpEntrySize = 15;
   static constexpr uint8_t kMinCmpEntrySize = 2;
 
+  const Knobs &knobs_;
   absl::BitGen prng_;
   size_t max_len_ = 1000;
   std::unique_ptr<MutatorDomain> domain_;

centipede/fuzztest_mutator_test.cc

@@ -24,20 +24,23 @@
 #include "absl/container/flat_hash_set.h"
 #include "absl/strings/str_join.h"
 #include "./centipede/defs.h"
+#include "./centipede/knobs.h"
 
 namespace centipede {
 
 namespace {
 
 using ::testing::AllOf;
 using ::testing::Each;
+using ::testing::IsSupersetOf;
 using ::testing::Le;
 using ::testing::SizeIs;
 using ::testing::Values;
 
 TEST(FuzzTestMutator, DifferentRngSeedsLeadToDifferentMutantSequences) {
-  FuzzTestMutator mutator[2]{FuzzTestMutator(/*seed=*/1),
-                             FuzzTestMutator(/*seed=*/2)};
+  const Knobs knobs;
+  FuzzTestMutator mutator[2]{FuzzTestMutator(knobs, /*seed=*/1),
+                             FuzzTestMutator(knobs, /*seed=*/2)};
 
   std::vector<ByteArray> res[2];
   for (size_t i = 0; i < 2; i++) {
@@ -56,7 +59,8 @@ TEST(FuzzTestMutator, DifferentRngSeedsLeadToDifferentMutantSequences) {
 
 TEST(FuzzTestMutator, MutateManyWorksWithInputsLargerThanMaxLen) {
   constexpr size_t kMaxLen = 4;
-  FuzzTestMutator mutator(/*seed=*/1);
+  const Knobs knobs;
+  FuzzTestMutator mutator(knobs, /*seed=*/1);
   EXPECT_TRUE(mutator.set_max_len(kMaxLen));
   constexpr size_t kNumMutantsToGenerate = 10000;
   std::vector<ByteArray> mutants;
@@ -75,6 +79,72 @@ TEST(FuzzTestMutator, MutateManyWorksWithInputsLargerThanMaxLen) {
               AllOf(SizeIs(kNumMutantsToGenerate), Each(SizeIs(Le(kMaxLen)))));
 }
 
+TEST(FuzzTestMutator, CrossOverInsertsDataFromOtherInputs) {
+  const Knobs knobs;
+  FuzzTestMutator mutator(knobs, /*seed=*/1);
+  constexpr size_t kNumMutantsToGenerate = 100000;
+  std::vector<ByteArray> mutants;
+
+  mutator.MutateMany(
+      {
+          {.data = {0, 1, 2, 3}},
+          {.data = {4, 5, 6, 7}},
+      },
+      kNumMutantsToGenerate, mutants);
+
+  EXPECT_THAT(mutants, IsSupersetOf(std::vector<ByteArray>{
+                           // The entire other input
+                           {4, 5, 6, 7, 0, 1, 2, 3},
+                           {0, 1, 4, 5, 6, 7, 2, 3},
+                           {0, 1, 2, 3, 4, 5, 6, 7},
+                           // The prefix of other input
+                           {4, 5, 6, 0, 1, 2, 3},
+                           {0, 1, 4, 5, 6, 2, 3},
+                           {0, 1, 2, 3, 4, 5, 6},
+                           // The suffix of other input
+                           {5, 6, 7, 0, 1, 2, 3},
+                           {0, 1, 5, 6, 7, 2, 3},
+                           {0, 1, 2, 3, 5, 6, 7},
+                           // The middle of other input
+                           {5, 6, 0, 1, 2, 3},
+                           {0, 1, 5, 6, 2, 3},
+                           {0, 1, 2, 3, 5, 6},
+                       }));
+}
+
+TEST(FuzzTestMutator, CrossOverOverwritesDataFromOtherInputs) {
+  const Knobs knobs;
+  FuzzTestMutator mutator(knobs, /*seed=*/1);
+  constexpr size_t kNumMutantsToGenerate = 100000;
+  std::vector<ByteArray> mutants;
+
+  mutator.MutateMany(
+      {
+          {.data = {0, 1, 2, 3, 4, 5, 6, 7}},
+          {.data = {100, 101, 102, 103}},
+      },
+      kNumMutantsToGenerate, mutants);
+
+  EXPECT_THAT(mutants, IsSupersetOf(std::vector<ByteArray>{
+                           // The entire other input
+                           {100, 101, 102, 103, 4, 5, 6, 7},
+                           {0, 1, 100, 101, 102, 103, 6, 7},
+                           {0, 1, 2, 3, 100, 101, 102, 103},
+                           // The prefix of other input
+                           {100, 101, 102, 3, 4, 5, 6, 7},
+                           {0, 1, 2, 100, 101, 102, 6, 7},
+                           {0, 1, 2, 3, 4, 100, 101, 102},
+                           // The suffix of other input
+                           {101, 102, 103, 3, 4, 5, 6, 7},
+                           {0, 1, 2, 101, 102, 103, 6, 7},
+                           {0, 1, 2, 3, 4, 101, 102, 103},
+                           // The middle of other input
+                           {101, 102, 2, 3, 4, 5, 6, 7},
+                           {0, 1, 2, 101, 102, 5, 6, 7},
+                           {0, 1, 2, 3, 4, 5, 101, 102},
+                       }));
+}
+
 // Test parameter containing the mutation settings and the expectations of a
 // single mutation step.
 struct MutationStepTestParameter {
@@ -102,7 +172,8 @@ class MutationStepTest
     : public testing::TestWithParam<MutationStepTestParameter> {};
 
 TEST_P(MutationStepTest, GeneratesExpectedMutantsAndAvoidsUnexpectedMutants) {
-  FuzzTestMutator mutator(/*seed=*/1);
+  const Knobs knobs;
+  FuzzTestMutator mutator(knobs, /*seed=*/1);
   ASSERT_LE(GetParam().min_num_iterations, GetParam().max_num_iterations);
   if (GetParam().max_len.has_value())
     EXPECT_TRUE(mutator.set_max_len(*GetParam().max_len));