Add multilang in tokenizer manager.

quickwit-oss · Jul 17, 2023 · da127ae · da127ae
1 parent c7475e0
commit da127ae
Show file tree

Hide file tree

Showing 2 changed files with 10 additions and 8 deletions.
diff --git a/quickwit/quickwit-doc-mapper/src/default_doc_mapper/tokenizer_entry.rs b/quickwit/quickwit-doc-mapper/src/default_doc_mapper/tokenizer_entry.rs
@@ -51,14 +51,10 @@ impl TokenizerConfig {
     pub fn text_analyzer(&self) -> anyhow::Result<TextAnalyzer> {
         let mut text_analyzer_builder = match &self.tokenizer_type {
             TokenizerType::Simple => TextAnalyzer::builder(SimpleTokenizer::default()).dynamic(),
-            // Note(fmassot): `multilang` is currently an "all-in-one" tokenizer with default
-            // filter. Static filters allows better performance which is a requirement
-            // for the `happy-plazza` project. We may want to revisit that later.
             #[cfg(feature = "multilang")]
-            TokenizerType::Multilang => TextAnalyzer::builder(MultiLangTokenizer::default())
-                .filter(RemoveLongFilter::limit(DEFAULT_REMOVE_TOKEN_LENGTH))
-                .filter(LowerCaser)
-                .dynamic(),
+            TokenizerType::Multilang => {
+                TextAnalyzer::builder(MultiLangTokenizer::default()).dynamic()
+            }
             TokenizerType::Ngram(options) => {
                 let tokenizer =
                     NgramTokenizer::new(options.min_gram, options.max_gram, options.prefix_only)

diff --git a/quickwit/quickwit-query/src/tokenizers/mod.rs b/quickwit/quickwit-query/src/tokenizers/mod.rs
@@ -55,7 +55,13 @@ pub fn create_default_quickwit_tokenizer_manager() -> TokenizerManager {
             .build(),
     );
     #[cfg(feature = "multilang")]
-    tokenizer_manager.register("multilang", MultiLangTokenizer::default());
+    tokenizer_manager.register(
+        "multilang_default",
+        TextAnalyzer::builder(MultiLangTokenizer::default())
+            .filter(RemoveLongFilter::limit(DEFAULT_REMOVE_TOKEN_LENGTH))
+            .filter(LowerCaser)
+            .build(),
+    );
     tokenizer_manager
 }