#Centipede Update ExportCorpusFromLocalDir and SaveCorpusToLocalDir t…

…o support remote files.

The names no longer make sense since the directories are no longer local. These
functions are also associated with similarly named flags. I've renamed
everything for consistency.

PiperOrigin-RevId: 577933102

centipede/BUILD

@@ -212,6 +212,7 @@ cc_library(
         ":defs",
         ":feature",
         ":logging",
+        ":remote_file",
         "@com_google_absl//absl/base:core_headers",
         "@com_google_absl//absl/log:check",
         "@com_google_absl//absl/strings",

centipede/centipede.cc

@@ -48,7 +48,7 @@
 #include <cstddef>
 #include <cstdint>
 #include <cstdlib>
-#include <filesystem>  // NOLINT
+#include <filesystem>
 #include <functional>
 #include <iostream>
 #include <memory>
@@ -129,9 +129,8 @@ Centipede::Centipede(const Environment &env, CentipedeCallbacks &user_callbacks,
     input_filter_cmd_.StartForkServer(TemporaryLocalDirPath(), "input_filter");
 }
 
-void Centipede::SaveCorpusToLocalDir(
-    const Environment &env, std::string_view save_corpus_to_local_dir) {
-  const WorkDir wd{env};
+void Centipede::CorpusToFiles(const Environment &env, std::string_view dir) {
+  WorkDir wd{env};
   for (size_t shard = 0; shard < env.total_shards; shard++) {
     auto reader = DefaultBlobFileReaderFactory();
     auto corpus_path = wd.CorpusPath(shard);
@@ -140,26 +139,23 @@ void Centipede::SaveCorpusToLocalDir(
     size_t num_read = 0;
     while (reader->Read(blob).ok()) {
       ++num_read;
-      WriteToLocalHashedFileInDir(save_corpus_to_local_dir, blob);
+      WriteToRemoteHashedFileInDir(dir, blob);
     }
     LOG(INFO) << "Read " << num_read << " from " << corpus_path;
   }
 }
 
-void Centipede::ExportCorpusFromLocalDir(const Environment &env,
-                                         std::string_view local_dir) {
-  const WorkDir wd{env};
-  // Shard the file paths in `local_dir` based on hashes of filenames.
+void Centipede::CorpusFromFiles(const Environment &env, std::string_view dir) {
+  WorkDir wd{env};
+  // Shard the file paths in `dir` based on hashes of filenames.
   // Such partition is stable: a given file always goes to a specific shard.
   std::vector<std::vector<std::string>> sharded_paths(env.total_shards);
+  std::vector<std::string> paths;
   size_t total_paths = 0;
-  for (const auto &entry :
-       std::filesystem::recursive_directory_iterator(local_dir)) {
-    if (entry.is_regular_file()) {
-      size_t filename_hash = std::hash<std::string>{}(entry.path().filename());
-      sharded_paths[filename_hash % env.total_shards].push_back(entry.path());
-      ++total_paths;
-    }
+  for (const std::string &path : RemoteListFilesRecursively(dir)) {
+    size_t filename_hash = std::hash<std::string>{}(path);
+    sharded_paths[filename_hash % env.total_shards].push_back(path);
+    ++total_paths;
   }
   // Iterate over all shards.
   size_t inputs_added = 0;
@@ -184,13 +180,13 @@ void Centipede::ExportCorpusFromLocalDir(const Environment &env,
         << "Failed to open corpus file: " << corpus_path;
     ByteArray shard_data;
     for (const auto &path : sharded_paths[shard]) {
-      ByteArray input;
-      ReadFromLocalFile(path, input);
+      std::string input;
+      RemoteFileGetContents(path, input);
       if (input.empty() || existing_hashes.contains(Hash(input))) {
         ++inputs_ignored;
         continue;
       }
-      CHECK_OK(appender->Write(input));
+      CHECK_OK(appender->Write(ByteArray{input.begin(), input.end()}));
       ++inputs_added;
     }
     LOG(INFO) << VV(shard) << VV(inputs_added) << VV(inputs_ignored)

centipede/centipede.h

@@ -59,14 +59,12 @@ class Centipede {
   void FuzzingLoop();
 
   // Saves the sharded corpus into `dir`, one file per input.
-  static void SaveCorpusToLocalDir(const Environment &env,
-                                   std::string_view dir);
+  static void CorpusToFiles(const Environment &env, std::string_view dir);
   // Exports the corpus from `dir` (one file per input) into the sharded corpus.
   // Reads `dir` recursively.
   // Ignores inputs that already exist in the shard they need to be added to.
   // Sharding is stable and depends only on env.total_shards and the file name.
-  static void ExportCorpusFromLocalDir(const Environment &env,
-                                       std::string_view dir);
+  static void CorpusFromFiles(const Environment &env, std::string_view dir);
 
  private:
   // Executes inputs from `input_vec`.

centipede/centipede_interface.cc

@@ -187,8 +187,8 @@ int CentipedeMain(const Environment &env,
                   CentipedeCallbacksFactory &callbacks_factory) {
   SetSignalHandlers(env.stop_at);
 
-  if (!env.save_corpus_to_local_dir.empty()) {
-    Centipede::SaveCorpusToLocalDir(env, env.save_corpus_to_local_dir);
+  if (!env.corpus_to_files.empty()) {
+    Centipede::CorpusToFiles(env, env.corpus_to_files);
     return EXIT_SUCCESS;
   }
 
@@ -201,15 +201,15 @@ int CentipedeMain(const Environment &env,
   }
 
   // Just export the corpus from a local dir and exit.
-  if (!env.export_corpus_from_local_dir.empty()) {
-    Centipede::ExportCorpusFromLocalDir(env, env.export_corpus_from_local_dir);
+  if (!env.corpus_from_files.empty()) {
+    Centipede::CorpusFromFiles(env, env.corpus_from_files);
     return EXIT_SUCCESS;
   }
 
   // Export the corpus from a local dir and then fuzz.
   if (!env.corpus_dir.empty()) {
     for (const auto &corpus_dir : env.corpus_dir) {
-      Centipede::ExportCorpusFromLocalDir(env, corpus_dir);
+      Centipede::CorpusFromFiles(env, corpus_dir);
     }
   }
 

centipede/environment.h

@@ -80,8 +80,8 @@ struct Environment {
   bool distill = false;
   size_t log_features_shards = 0;
   std::string knobs_file;
-  std::string save_corpus_to_local_dir;
-  std::string export_corpus_from_local_dir;
+  std::string corpus_to_files;
+  std::string corpus_from_files;
   std::vector<std::string> corpus_dir;
   std::string symbolizer_path = "llvm-symbolizer";
   std::string objdump_path = "objdump";

centipede/environment_flags.cc

@@ -263,12 +263,10 @@ ABSL_FLAG(bool, print_runner_log, default_env->print_runner_log,
 ABSL_FLAG(std::string, knobs_file, default_env->knobs_file,
           "If not empty, knobs will be read from this (possibly remote) file."
           " The feature is experimental, not yet fully functional.");
-ABSL_FLAG(std::string, save_corpus_to_local_dir,
-          default_env->save_corpus_to_local_dir,
+ABSL_FLAG(std::string, corpus_to_files, default_env->corpus_to_files,
           "Save the remote corpus from working to the given directory, one "
           "file per corpus.");
-ABSL_FLAG(std::string, export_corpus_from_local_dir,
-          default_env->export_corpus_from_local_dir,
+ABSL_FLAG(std::string, corpus_from_files, default_env->corpus_from_files,
           "Export a corpus from a local directory with one file per input into "
           "the sharded remote corpus in workdir. Not recursive.");
 ABSL_FLAG(std::vector<std::string>, corpus_dir, default_env->corpus_dir,
@@ -470,9 +468,8 @@ Environment CreateEnvironmentFromFlags(const std::vector<std::string> &argv) {
       .distill = absl::GetFlag(FLAGS_distill),
       .log_features_shards = absl::GetFlag(FLAGS_log_features_shards),
       .knobs_file = absl::GetFlag(FLAGS_knobs_file),
-      .save_corpus_to_local_dir = absl::GetFlag(FLAGS_save_corpus_to_local_dir),
-      .export_corpus_from_local_dir =
-          absl::GetFlag(FLAGS_export_corpus_from_local_dir),
+      .corpus_to_files = absl::GetFlag(FLAGS_corpus_to_files),
+      .corpus_from_files = absl::GetFlag(FLAGS_corpus_from_files),
       .corpus_dir = absl::GetFlag(FLAGS_corpus_dir),
       .symbolizer_path = absl::GetFlag(FLAGS_symbolizer_path),
       .objdump_path = absl::GetFlag(FLAGS_objdump_path),

centipede/test_fuzzing_util.sh

@@ -73,7 +73,7 @@ function centipede::test_crashing_target() {
   # Create a corpus with one crasher and one other input.
   echo -n "${crash_input}" >"${TMPCORPUS}/${crash_input}" # induces abort in the target.
   echo -n "${nice_input}" >"${TMPCORPUS}/${nice_input}"     # just some input.
-  ${target} --workdir="${WD}" --export_corpus_from_local_dir="${TMPCORPUS}"
+  ${target} --workdir="${WD}" --corpus_from_files="${TMPCORPUS}"
 
   # Run fuzzing with num_runs=0, i.e. only run the inputs from the corpus.
   # Expecting a crash to be observed and reported.

centipede/testing/centipede_main_test.sh

@@ -75,7 +75,7 @@ test_debug_symbols() {
   centipede::assert_regex_in_file "EDGE: LLVMFuzzerTestOneInput .*testing/test_fuzz_target.cc" "${LOG}"
 
   echo "============ ${FUNC}: add func1/func2-A inputs to the corpus."
-  test_fuzz --workdir="${WD}" --export_corpus_from_local_dir="${TMPCORPUS}"
+  test_fuzz --workdir="${WD}" --corpus_from_files="${TMPCORPUS}"
 
   echo "============ ${FUNC}: run again, append to the same LOG file."
   # TODO(b/282845630): Passing `--num_runs=1` only to trigger telemetry dumping.
@@ -129,7 +129,7 @@ test_dictionary() {
   echo "foo" >"${TMPCORPUS}"/foo
   echo "bat" >"${TMPCORPUS}"/binary
   centipede::ensure_empty_dir "${WD}"
-  test_fuzz --workdir="${WD}" --export_corpus_from_local_dir "${TMPCORPUS}"
+  test_fuzz --workdir="${WD}" --corpus_from_files "${TMPCORPUS}"
   cp "${WD}/corpus.000000" "${DICT}"
 
   echo "============ ${FUNC}: testing binary dictionary file"
@@ -150,7 +150,7 @@ test_for_each_blob() {
   echo "FoO" >"${TMPCORPUS}"/a
   echo "bAr" >"${TMPCORPUS}"/b
 
-  test_fuzz --workdir="${WD}" --export_corpus_from_local_dir "${TMPCORPUS}"
+  test_fuzz --workdir="${WD}" --corpus_from_files "${TMPCORPUS}"
   echo "============ ${FUNC}: test for_each_blob"
   test_fuzz --for_each_blob="cat %P" "${WD}"/corpus.000000 | tee "${LOG}"
   centipede::assert_regex_in_file "Running 'cat %P' on ${WD}/corpus.000000" "${LOG}"

centipede/testing/clusterfuzz_format_test.sh

@@ -66,7 +66,7 @@ test_crashing_target() {
   # Create a corpus with one crasher and one other input.
   cp "$1" "${TMPCORPUS}"  # Triggers an error.
   echo -n "foo" >"${TMPCORPUS}/foo"     # Just some input.
-  abort_test_fuzz --export_corpus_from_local_dir="${TMPCORPUS}"
+  abort_test_fuzz --corpus_from_files="${TMPCORPUS}"
 
   # Run fuzzing with num_runs=0, i.e. only run the inputs from the corpus.
   # Expecting a crash to be observed and reported.

centipede/util.cc

@@ -56,6 +56,7 @@
 #include "./centipede/defs.h"
 #include "./centipede/feature.h"
 #include "./centipede/logging.h"
+#include "./centipede/remote_file.h"
 
 namespace centipede {
 
@@ -142,6 +143,13 @@ void WriteToLocalHashedFileInDir(std::string_view dir_path,
   WriteToLocalFile(file_path, data);
 }
 
+void WriteToRemoteHashedFileInDir(std::string_view dir_path,
+                                  absl::Span<const uint8_t> data) {
+  if (dir_path.empty()) return;
+  std::string file_path = std::filesystem::path(dir_path).append(Hash(data));
+  RemoteFileSetContents(file_path, std::string(data.begin(), data.end()));
+}
+
 std::string HashOfFileContents(std::string_view file_path) {
   ByteArray ba;
   ReadFromLocalFile(file_path, ba);

centipede/util.h

@@ -60,6 +60,9 @@ void WriteToLocalFile(std::string_view file_path, const FeatureVec &data);
 // Writes `data` to `dir_path`/Hash(`data`). Does nothing if `dir_path.empty()`.
 void WriteToLocalHashedFileInDir(std::string_view dir_path,
                                  absl::Span<const uint8_t> data);
+// Same as `WriteToLocalHashedFileInDir` except supports remote files.
+void WriteToRemoteHashedFileInDir(std::string_view dir_path,
+                                  absl::Span<const uint8_t> data);
 // Returns a path string suitable to create a temporary local directory.
 // Will return the same value every time it is called within one thread,
 // but different values for different threads and difference processes.