perf: better mutualization of data structures across threads (#1)

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "grip-grab"
-version = "0.2.26"
+version = "0.2.27"
 edition = "2021"
 authors = ["Alexandre Pasmantier <[email protected]>"]
 license = "Apache-2.0"

src/cli.rs

@@ -32,10 +32,6 @@ pub struct Cli {
     #[clap(short = 'G', long, default_value_t = false)]
     pub disregard_gitignore: bool,
 
-    /// upper boundary for the number of results to expect (will panic if #results > max_results)
-    #[clap(short = 'M', long, default_value_t = 5000)]
-    pub max_results: usize,
-
     /// number of threads to use
     #[clap(short = 'T', long, default_value_t = 4)]
     pub n_threads: usize,
@@ -99,7 +95,6 @@ pub struct PostProcessedCli {
     pub patterns: Vec<String>,
     pub paths: Vec<PathBuf>,
     pub ignored_paths: Vec<PathBuf>,
-    pub max_results: usize,
     pub n_threads: usize,
     pub disregard_gitignore: bool,
     pub multiline: bool,
@@ -123,7 +118,6 @@ pub fn process_cli_args(mut cli: Cli) -> anyhow::Result<PostProcessedCli> {
         },
         paths: utils::resolve_paths(cli.paths),
         ignored_paths: utils::resolve_paths(cli.ignore_paths),
-        max_results: cli.max_results,
         n_threads: cli.n_threads,
         disregard_gitignore: cli.disregard_gitignore,
         multiline: cli.multiline,

src/main.rs

@@ -1,6 +1,9 @@
+use std::sync::Arc;
+
 use clap::Parser;
 
-use crossbeam::queue::ArrayQueue;
+use crossbeam::queue::SegQueue;
+use grep::regex::RegexMatcher;
 use ignore::DirEntry;
 use printer::PrinterConfig;
 use search::build_searcher;
@@ -18,27 +21,29 @@ mod utils;
 
 pub fn main() -> anyhow::Result<()> {
     let cli_args = process_cli_args(Cli::parse())?;
-    let queue: ArrayQueue<FileResults> = ArrayQueue::new(cli_args.max_results);
+    let queue: Arc<SegQueue<FileResults>> = Arc::new(SegQueue::new());
 
-    let matcher = build_matcher(&cli_args.patterns)?;
     let haystack_builder = walk_builder(
         cli_args.paths.iter().map(|p| p.as_path()).collect(),
         &cli_args.ignored_paths,
         cli_args.n_threads,
         !cli_args.disregard_gitignore,
         cli_args.filter_filetypes,
     );
+    let matcher: Arc<RegexMatcher> = Arc::new(build_matcher(&cli_args.patterns)?);
     haystack_builder.build_parallel().run(|| {
-        Box::new(|entry: Result<DirEntry, ignore::Error>| match entry {
+        let matcher = Arc::clone(&matcher);
+        let mut searcher = build_searcher(cli_args.multiline);
+        let queue = Arc::clone(&queue);
+        Box::new(move |entry: Result<DirEntry, ignore::Error>| match entry {
             Ok(entry) => {
                 let file_type = entry.file_type().unwrap();
                 if !file_type.is_dir() {
                     let path = entry.path().to_path_buf();
-                    let mut searcher = build_searcher(cli_args.multiline);
                     match search_file(path, &matcher, &mut searcher) {
                         Ok(file_results) => {
                             if !file_results.is_empty() {
-                                queue.push(file_results).unwrap();
+                                queue.push(file_results);
                             }
                         }
                         Err(_err) => (),
@@ -61,7 +66,8 @@ pub fn main() -> anyhow::Result<()> {
         ..Default::default()
     };
     let mut printer = Printer::new(printer_config);
-    queue
+    let printer_queue = Arc::into_inner(queue).unwrap();
+    printer_queue
         .into_iter()
         .for_each(|file_results| printer.write(file_results).unwrap());