[fix](inverted index) cloud mode supports lowercase

apache · Mar 29, 2024 · a07bea2 · a07bea2
1 parent 3ff63ed
commit a07bea2
Show file tree

Hide file tree

Showing 5 changed files with 30 additions and 12 deletions.
diff --git a/be/src/olap/inverted_index_parser.h b/be/src/olap/inverted_index_parser.h
@@ -99,12 +99,11 @@ std::string get_parser_ignore_above_value_from_properties(
 template <bool ReturnTrue = false>
 std::string get_parser_lowercase_from_properties(
         const std::map<std::string, std::string>& properties) {
+    DBUG_EXECUTE_IF("inverted_index_parser.get_parser_lowercase_from_properties", { return ""; })
+
     if (properties.find(INVERTED_INDEX_PARSER_LOWERCASE_KEY) != properties.end()) {
         return properties.at(INVERTED_INDEX_PARSER_LOWERCASE_KEY);
     } else {
-        DBUG_EXECUTE_IF("inverted_index_parser.get_parser_lowercase_from_properties",
-                        { return ""; })
-
         if constexpr (ReturnTrue) {
             return INVERTED_INDEX_PARSER_TRUE;
         } else {

diff --git a/be/src/olap/tablet_schema.cpp b/be/src/olap/tablet_schema.cpp
@@ -796,16 +796,23 @@ void TabletIndex::to_schema_pb(TabletIndexPB* index) const {
     }
     index->set_index_type(_index_type);
     for (const auto& kv : _properties) {
+        DBUG_EXECUTE_IF("tablet_schema.to_schema_pb", {
+            if (kv.first == INVERTED_INDEX_PARSER_LOWERCASE_KEY) {
+                continue;
+            }
+        })
         (*index->mutable_properties())[kv.first] = kv.second;
     }
     index->set_index_suffix_name(_escaped_index_suffix_path);
 
     DBUG_EXECUTE_IF("tablet_schema.to_schema_pb", { return; })
 
     // lowercase by default
-    if (!_properties.contains(INVERTED_INDEX_PARSER_LOWERCASE_KEY)) {
-        (*index->mutable_properties())[INVERTED_INDEX_PARSER_LOWERCASE_KEY] =
-                INVERTED_INDEX_PARSER_TRUE;
+    if (!_properties.empty()) {
+        if (!_properties.contains(INVERTED_INDEX_PARSER_LOWERCASE_KEY)) {
+            (*index->mutable_properties())[INVERTED_INDEX_PARSER_LOWERCASE_KEY] =
+                    INVERTED_INDEX_PARSER_TRUE;
+        }
     }
 }
 

diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Index.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/Index.java
@@ -70,6 +70,14 @@ public Index(long indexId, String indexName, List<String> columns,
         this.indexType = indexType;
         this.properties = properties;
         this.comment = comment;
+        if (indexType == IndexDef.IndexType.INVERTED) {
+            if (!this.properties.isEmpty()) {
+                String key = InvertedIndexUtil.INVERTED_INDEX_PARSER_LOWERCASE_KEY;
+                if (!properties.containsKey(key)) {
+                    this.properties.put(key, "true");
+                }
+            }
+        }
     }
 
     public Index() {

diff --git a/regression-test/pipeline/cloud_p0/conf/regression-conf-custom.groovy b/regression-test/pipeline/cloud_p0/conf/regression-conf-custom.groovy
@@ -44,8 +44,8 @@ excludeSuites = "000_the_start_sentinel_do_not_touch," + // keep this line as th
     "test_spark_load," +
     "test_stream_load_move_memtable," +
     "test_stream_load_new_move_memtable," +
-    "zzz_the_end_sentinel_do_not_touch" // keep this line as the last line
-
+    "zzz_the_end_sentinel_do_not_touch" + // keep this line as the last line
+    "test_index_lowercase_fault_injection"
 
 excludeDirectories = "000_the_start_sentinel_do_not_touch," + // keep this line as the first line
     "cloud/multi_cluster," + // run in specific regression pipeline

diff --git a/regression-test/suites/fault_injection_p0/test_index_lowercase_fault_injection.groovy b/regression-test/suites/fault_injection_p0/test_index_lowercase_fault_injection.groovy
@@ -35,7 +35,11 @@ suite("test_index_lowercase_fault_injection") {
           COMMENT "OLAP"
           DISTRIBUTED BY HASH(`@timestamp`) BUCKETS 1
           PROPERTIES (
-          "replication_allocation" = "tag.location.default: 1"
+          "replication_allocation" = "tag.location.default: 1",
+          "compaction_policy" = "time_series",
+          "time_series_compaction_goal_size_mbytes" = "1024",
+          "time_series_compaction_file_count_threshold" = "20",
+          "time_series_compaction_time_threshold_seconds" = "3600"
         );
       """
     }
@@ -55,14 +59,14 @@ suite("test_index_lowercase_fault_injection") {
         sql """ INSERT INTO ${testTable} VALUES (893964653, '232.0.0.0', 'GET /images/hm_bg.jpg HTTP/1.0', 200, 3781); """
 
         sql 'sync'
+
+        qt_sql """ select count() from ${testTable} where (request match 'HTTP');  """
+        qt_sql """ select count() from ${testTable} where (request match 'http');  """
       } finally {
         GetDebugPoint().disableDebugPointForAllBEs("inverted_index_parser.get_parser_lowercase_from_properties")
         GetDebugPoint().disableDebugPointForAllBEs("tablet_schema.to_schema_pb")
       }
 
-      qt_sql """ select count() from ${testTable} where (request match 'HTTP');  """
-      qt_sql """ select count() from ${testTable} where (request match 'http');  """
-
       sql """ INSERT INTO ${testTable} VALUES (893964672, '26.1.0.0', 'GET /images/hm_bg.jpg HTTP/1.0', 304, 0); """
       sql """ INSERT INTO ${testTable} VALUES (893964672, '26.1.0.0', 'GET /images/hm_bg.jpg HTTP/1.0', 304, 0); """
       sql """ INSERT INTO ${testTable} VALUES (893964653, '232.0.0.0', 'GET /images/hm_bg.jpg HTTP/1.0', 200, 3781); """