experiment: try avoiding multi-thread with xapian

Try to avoid multi-threaded operation with Xapian. This remove the thread workers during indexing, and avoids the indexing background thread. So, mu4e has to wait once again during indexing. We can improve upon that, but first we need to know if it avoids the problem of issue #2756.
djcb · Sep 16, 2024 · c99d1c8 · c99d1c8
1 parent 61ba929
commit c99d1c8
Show file tree

Hide file tree

Showing 4 changed files with 106 additions and 42 deletions.
diff --git a/lib/mu-indexer.cc b/lib/mu-indexer.cc
@@ -105,6 +105,7 @@ struct Indexer::Private {
 	bool handler(const std::string& fullpath, struct stat* statbuf, Scanner::HandleType htype);
 
 	void maybe_start_worker();
+
 	void item_worker();
 	void scan_worker();
 
@@ -135,6 +136,8 @@ struct Indexer::Private {
 		Type type;
 	};
 
+	void handle_item(WorkItem&& item);
+
 	AsyncQueue<WorkItem> todos_;
 
 	Progress   progress_{};
@@ -193,7 +196,11 @@ Indexer::Private::handler(const std::string& fullpath, struct stat* statbuf,
 		return true;
 	}
 	case Scanner::HandleType::LeaveDir: {
+#ifdef XAPIAN_SINGLE_THREADED
+		handle_item({fullpath, WorkItem::Type::Dir});
+#else
 		todos_.push({fullpath, WorkItem::Type::Dir});
+#endif /*XAPIAN_SINGLE_THREADED*/
 		return true;
 	}
 
@@ -210,9 +217,13 @@ Indexer::Private::handler(const std::string& fullpath, struct stat* statbuf,
 		if (statbuf->st_ctime <= dirstamp_ && store_.contains_message(fullpath))
 			return false;
 
+#ifdef XAPIAN_SINGLE_THREADED
+		handle_item({fullpath, WorkItem::Type::File});
+#else
 		// push the remaining messages to our "todo" queue for
 		// (re)parsing and adding/updating to the database.
 		todos_.push({fullpath, WorkItem::Type::File});
+#endif
 		return true;
 	}
 	default:
@@ -260,6 +271,30 @@ Indexer::Private::add_message(const std::string& path)
 	return true;
 }
 
+
+void
+Indexer::Private::handle_item(WorkItem&& item)
+{
+	try {
+		switch (item.type) {
+		case WorkItem::Type::File: {
+			if (G_LIKELY(add_message(item.full_path)))
+				++progress_.updated;
+		} break;
+		case WorkItem::Type::Dir:
+			store_.set_dirstamp(item.full_path, ::time(NULL));
+			break;
+		default:
+			g_warn_if_reached();
+			break;
+		}
+	} catch (const Mu::Error& er) {
+		mu_warning("error adding message @ {}: {}", item.full_path, er.what());
+	}
+}
+
+
+
 void
 Indexer::Private::item_worker()
 {
@@ -270,22 +305,8 @@ Indexer::Private::item_worker()
 	while (state_ == IndexState::Scanning) {
 		if (!todos_.pop(item, 250ms))
 			continue;
-		try {
-			switch (item.type) {
-			case WorkItem::Type::File: {
-				if (G_LIKELY(add_message(item.full_path)))
-					++progress_.updated;
-			} break;
-			case WorkItem::Type::Dir:
-				store_.set_dirstamp(item.full_path, ::time(NULL));
-				break;
-			default:
-				g_warn_if_reached();
-				break;
-			}
-		} catch (const Mu::Error& er) {
-			mu_warning("error adding message @ {}: {}", item.full_path, er.what());
-		}
+
+		handle_item(std::move(item));
 
 		maybe_start_worker();
 		std::this_thread::yield();

diff --git a/lib/mu-server.cc b/lib/mu-server.cc
@@ -149,6 +149,7 @@ struct Server::Private {
 	Store&            store() { return store_; }
 	const Store&      store() const { return store_; }
 	Indexer&          indexer() { return store().indexer(); }
+	void              do_index(const Indexer::Config& conf);
 	//CommandMap&       command_map() const { return command_map_; }
 
 	//
@@ -761,6 +762,20 @@ get_stats(const Indexer::Progress& stats, const std::string& state)
 	return sexp;
 }
 
+void
+Server::Private::do_index(const Indexer::Config& conf)
+{
+	StopWatch sw{"indexing"};
+	indexer().start(conf);
+	while (indexer().is_running()) {
+		std::this_thread::sleep_for(std::chrono::milliseconds(2000));
+		output_sexp(get_stats(indexer().progress(), "running"),
+			    Server::OutputFlags::Flush);
+	}
+	output_sexp(get_stats(indexer().progress(), "complete"),
+		    Server::OutputFlags::Flush);
+}
+
 void
 Server::Private::index_handler(const Command& cmd)
 {
@@ -770,22 +785,22 @@ Server::Private::index_handler(const Command& cmd)
 	// ignore .noupdate with an empty store.
 	conf.ignore_noupdate = store().empty();
 
+#ifdef XAPIAN_SINGLE_THREADED
+	// nothing to do
+	if (indexer().is_running())
+		return;
+	do_index(conf);
+#else
 	indexer().stop();
 	if (index_thread_.joinable())
 		index_thread_.join();
 
 	// start a background track.
 	index_thread_ = std::thread([this, conf = std::move(conf)] {
-		StopWatch sw{"indexing"};
-		indexer().start(conf);
-		while (indexer().is_running()) {
-			std::this_thread::sleep_for(std::chrono::milliseconds(2000));
-			output_sexp(get_stats(indexer().progress(), "running"),
-				    Server::OutputFlags::Flush);
-		}
-		output_sexp(get_stats(indexer().progress(), "complete"),
-			    Server::OutputFlags::Flush);
+		do_index(conf);
 	});
+#endif /*XAPIAN_SINGLE_THREADED */
+
 }
 
 void

diff --git a/meson.build b/meson.build
@@ -133,7 +133,13 @@ add_project_arguments(['-DHAVE_CONFIG_H'], language: 'cpp')
 config_h_dep=declare_dependency(
   include_directories: include_directories(['.']))
 
-
+#
+# single-threaded Xapian access?
+#
+if get_option('xapian-single-threaded')
+  config_h_data.set('XAPIAN_SINGLE_THREADED', true)
+  message('use Xapian only in a single thread')
+endif
 #
 # d_type, d_ino are not available universally, so let's check
 # (we use them for optimizations in mu-scanner
@@ -322,6 +328,8 @@ if gmime_dep.version() == '3.2.13'
   warning('See: https://github.com/jstedfast/gmime/issues/133')
 endif
 
+
+
 # Local Variables:
 # indent-tabs-mode: nil
 # End:
diff --git a/meson_options.txt b/meson_options.txt
@@ -14,36 +14,56 @@
 ## along with this program; if not, write to the Free Software Foundation,
 ## Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
 
-option('tests',
+
+option('cld2',
        type : 'feature',
        value: 'auto',
-       description: 'build unit tests')
+       description: 'Add support for language-detection through cld2')
+
+#
+# emacs
+#
+
+option('emacs',
+       type: 'string',
+       value: 'emacs',
+       description: 'name/path of the emacs executable (for byte-compilation)')
+
+option('lispdir',
+       type: 'string',
+       description: 'path under which to install emacs-lisp files')
+
+
+#
+# guile
+#
 
 option('guile',
        type : 'feature',
        value: 'auto',
        description: 'build the guile scripting support (requires guile-3.x)')
 
-option('cld2',
-       type : 'feature',
-       value: 'auto',
-       description: 'Compact Language Detector2')
-
 # by default, this uses guile_dep.get_variable(pkgconfig: 'extensiondir')
 option('guile-extension-dir',
        type: 'string',
        description: 'custom install path for the guile extension module')
 
+
+#
+# misc
+#
+
+option('tests',
+       type : 'feature',
+       value: 'auto',
+       description: 'build unit tests')
+
+option('xapian-single-threaded',
+       type : 'boolean',
+       value: true,
+       description: 'only use Xapian from a single thread')
+
 option('readline',
        type: 'feature',
        value: 'auto',
        description: 'enable readline support for the mu4e repl')
-
-option('emacs',
-       type: 'string',
-       value: 'emacs',
-       description: 'name/path of the emacs executable')
-
-option('lispdir',
-       type: 'string',
-       description: 'path under which to install emacs-lisp files')