diff --git a/be/src/olap/rowset/segment_v2/inverted_index_writer.cpp b/be/src/olap/rowset/segment_v2/inverted_index_writer.cpp index 851826027210fde..c34024edd6db18e 100644 --- a/be/src/olap/rowset/segment_v2/inverted_index_writer.cpp +++ b/be/src/olap/rowset/segment_v2/inverted_index_writer.cpp @@ -299,13 +299,10 @@ class InvertedIndexColumnWriterImpl : public InvertedIndexColumnWriter { get_parser_ignore_above_value_from_properties(_index_meta->properties()); auto ignore_above = std::stoi(ignore_above_value); for (int i = 0; i < count; ++i) { - // only ignore_above UNTOKENIZED strings - if (_parser_type == InvertedIndexParserType::PARSER_NONE && - v->get_size() > ignore_above) { - VLOG_DEBUG << "fulltext index value length can be at most " - << ignore_above_value << ", but got " - << "value length:" << v->get_size() << ", ignore this value"; - new_fulltext_field(empty_value.c_str(), 0); + // only ignore_above UNTOKENIZED strings and empty strings not tokenized + if ((_parser_type == InvertedIndexParserType::PARSER_NONE && + v->get_size() > ignore_above) || + (_parser_type != InvertedIndexParserType::PARSER_NONE && v->empty())) { RETURN_IF_ERROR(add_null_document()); } else { new_fulltext_field(v->get_data(), v->get_size()); @@ -352,13 +349,10 @@ class InvertedIndexColumnWriterImpl : public InvertedIndexColumnWriter { } auto value = join(strings, " "); - // only ignore_above UNTOKENIZED strings - if (_parser_type == InvertedIndexParserType::PARSER_NONE && - value.length() > ignore_above) { - VLOG_DEBUG << "fulltext index value length can be at most " - << ignore_above_value << ", but got " - << "value length:" << value.length() << ", ignore this value"; - new_fulltext_field(empty_value.c_str(), 0); + // only ignore_above UNTOKENIZED strings and empty strings not tokenized + if ((_parser_type == InvertedIndexParserType::PARSER_NONE && + value.length() > ignore_above) || + (_parser_type != InvertedIndexParserType::PARSER_NONE && value.empty())) { RETURN_IF_ERROR(add_null_document()); } else { new_fulltext_field(value.c_str(), value.length());