[opt](invert index) Empty strings are not written to the index in the…

… case of TOKENIZED
apache · Dec 21, 2023 · f659e47 · f659e47
1 parent 4ddef31
commit f659e47
Showing 1 changed file with 6 additions and 0 deletions.
diff --git a/be/src/olap/rowset/segment_v2/inverted_index_writer.cpp b/be/src/olap/rowset/segment_v2/inverted_index_writer.cpp
@@ -304,6 +304,9 @@ class InvertedIndexColumnWriterImpl : public InvertedIndexColumnWriter {
                                << "value length:" << v->get_size() << ", ignore this value";
                     new_fulltext_field(empty_value.c_str(), 0);
                     RETURN_IF_ERROR(add_null_document());
+                } else if (_parser_type != InvertedIndexParserType::PARSER_NONE && v->empty()) {
+                    new_fulltext_field(empty_value.c_str(), 0);
+                    RETURN_IF_ERROR(add_null_document());
                 } else {
                     new_fulltext_field(v->get_data(), v->get_size());
                     RETURN_IF_ERROR(add_document());
@@ -354,6 +357,9 @@ class InvertedIndexColumnWriterImpl : public InvertedIndexColumnWriter {
                                << "value length:" << value.length() << ", ignore this value";
                     new_fulltext_field(empty_value.c_str(), 0);
                     RETURN_IF_ERROR(add_null_document());
+                } else if (_parser_type != InvertedIndexParserType::PARSER_NONE && v->empty()) {
+                    new_fulltext_field(empty_value.c_str(), 0);
+                    RETURN_IF_ERROR(add_null_document());
                 } else {
                     new_fulltext_field(value.c_str(), value.length());
                     RETURN_IF_ERROR(add_document());