[feature](Nereids): add is null in predicate and put or expansion rul…

…e in rewriter (#28348) add is null in predicate put or expansion rule in rewrite job
apache · Dec 20, 2023 · a1fa91d · a1fa91d
1 parent 1142942
commit a1fa91d
Show file tree

Hide file tree

Showing 6 changed files with 39 additions and 14 deletions.
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Rewriter.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Rewriter.java
@@ -78,6 +78,7 @@
 import org.apache.doris.nereids.rules.rewrite.MergeProjects;
 import org.apache.doris.nereids.rules.rewrite.MergeSetOperations;
 import org.apache.doris.nereids.rules.rewrite.NormalizeSort;
+import org.apache.doris.nereids.rules.rewrite.OrExpansion;
 import org.apache.doris.nereids.rules.rewrite.PruneEmptyPartition;
 import org.apache.doris.nereids.rules.rewrite.PruneFileScanPartition;
 import org.apache.doris.nereids.rules.rewrite.PruneOlapScanPartition;
@@ -377,6 +378,8 @@ private static List<RewriteJob> getWholeTreeRewriteJobs(List<RewriteJob> jobs) {
                 topic("rewrite cte sub-tree",
                         custom(RuleType.REWRITE_CTE_CHILDREN, () -> new RewriteCteChildren(jobs))
                 ),
+                topic("or expansion",
+                        topDown(new OrExpansion())),
                 topic("whole plan check",
                         custom(RuleType.ADJUST_NULLABLE, AdjustNullable::new)
                 )

diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleSet.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleSet.java
@@ -18,7 +18,6 @@
 package org.apache.doris.nereids.rules;
 
 import org.apache.doris.nereids.rules.exploration.MergeProjectsCBO;
-import org.apache.doris.nereids.rules.exploration.OrExpansion;
 import org.apache.doris.nereids.rules.exploration.TransposeAggSemiJoin;
 import org.apache.doris.nereids.rules.exploration.TransposeAggSemiJoinProject;
 import org.apache.doris.nereids.rules.exploration.join.InnerJoinLAsscom;
@@ -119,7 +118,6 @@ public class RuleSet {
             .add(PushdownProjectThroughSemiJoin.INSTANCE)
             .add(TransposeAggSemiJoin.INSTANCE)
             .add(TransposeAggSemiJoinProject.INSTANCE)
-            .add(OrExpansion.INSTANCE)
             .build();
 
     public static final List<RuleFactory> PUSH_DOWN_FILTERS = ImmutableList.of(

diff --git a/...ereids/rules/exploration/OrExpansion.java → ...is/nereids/rules/rewrite/OrExpansion.java b/...ereids/rules/exploration/OrExpansion.java → ...is/nereids/rules/rewrite/OrExpansion.java
@@ -15,15 +15,16 @@
 // specific language governing permissions and limitations
 // under the License.
 
-package org.apache.doris.nereids.rules.exploration;
+package org.apache.doris.nereids.rules.rewrite;
 
 import org.apache.doris.common.Pair;
 import org.apache.doris.nereids.CascadesContext;
 import org.apache.doris.nereids.rules.Rule;
 import org.apache.doris.nereids.rules.RuleType;
-import org.apache.doris.nereids.rules.rewrite.PushdownExpressionsInHashCondition;
+import org.apache.doris.nereids.rules.exploration.OneExplorationRuleFactory;
 import org.apache.doris.nereids.trees.expressions.Alias;
 import org.apache.doris.nereids.trees.expressions.Expression;
+import org.apache.doris.nereids.trees.expressions.IsNull;
 import org.apache.doris.nereids.trees.expressions.NamedExpression;
 import org.apache.doris.nereids.trees.expressions.Not;
 import org.apache.doris.nereids.trees.expressions.Slot;
@@ -72,7 +73,7 @@ public class OrExpansion extends OneExplorationRuleFactory {
 
     @Override
     public Rule build() {
-        return logicalJoin().when(JoinUtils::shouldNestedLoopJoin)
+        return logicalJoin(any(), any()).when(JoinUtils::shouldNestedLoopJoin)
                 .when(join -> supportJoinType.contains(join.getJoinType())
                         && ConnectContext.get().getSessionVariable().getEnablePipelineEngine())
                 .thenApply(ctx -> {
@@ -82,7 +83,7 @@ public Rule build() {
 
                     //1. Try to split or conditions
                     Pair<List<Expression>, List<Expression>> hashOtherConditions = splitOrCondition(join);
-                    if (hashOtherConditions == null) {
+                    if (hashOtherConditions == null || hashOtherConditions.first.size() <= 1) {
                         return join;
                     }
 
@@ -221,7 +222,11 @@ private List<Plan> expandInnerJoin(CascadesContext ctx, Pair<List<Expression>,
             LogicalCTEProducer<? extends Plan> rightProducer) {
         List<Expression> disjunctions = hashOtherConditions.first;
         List<Expression> otherConditions = hashOtherConditions.second;
-        List<Expression> notExprs = disjunctions.stream().map(Not::new).collect(Collectors.toList());
+        // For null values, equalTo and not equalTo both return false
+        // To avoid it, we always return true when there is null
+        List<Expression> notExprs = disjunctions.stream()
+                .map(e -> ExpressionUtils.or(new Not(e), new IsNull(e)))
+                .collect(ImmutableList.toImmutableList());
         List<Plan> joins = Lists.newArrayList();
 
         for (int hashCondIdx = 0; hashCondIdx < disjunctions.size(); hashCondIdx++) {

diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/FilterEstimation.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/FilterEstimation.java
@@ -367,7 +367,7 @@ public Statistics visitNot(Not not, EstimationContext context) {
                 // 4. not A like XXX
                 colBuilder.setNumNulls(0);
                 Preconditions.checkArgument(
-                        child instanceof EqualTo
+                        child instanceof EqualPredicate
                                 || child instanceof InPredicate
                                 || child instanceof IsNull
                                 || child instanceof Like,

diff --git a/regression-test/data/nereids_p0/union/or_expansion.out b/regression-test/data/nereids_p0/union/or_expansion.out
@@ -1,9 +1,13 @@
 -- This file is automatically generated. You should know what you did if you want to edit this
 -- !order_ij --
+\N	\N
+1	\N
+1	1
+1	1
 
 -- !order_laj --
+\N
 0
-1
 2
 3
 4
@@ -14,6 +18,8 @@
 9
 
 -- !order_loj --
+\N	\N
+\N	\N
 0	\N
 1	\N
 2	\N
@@ -24,8 +30,13 @@
 7	\N
 8	\N
 9	\N
+1	1
+1	1
 
 -- !order_foj --
+\N	\N
+\N	\N
+\N	\N
 0	\N
 1	\N
 2	\N
@@ -36,6 +47,8 @@
 7	\N
 8	\N
 9	\N
+1	1
+1	1
 \N	20
 \N	21
 \N	22

diff --git a/regression-test/suites/nereids_p0/union/or_expansion.groovy b/regression-test/suites/nereids_p0/union/or_expansion.groovy
@@ -45,11 +45,6 @@ suite("or_expansion") {
         )
     """
 
-    for (int i = 0; i < 10; i++) {
-        sql "insert into oe1 values(${i}, ${i})"
-        sql "insert into oe2 values(${i+20}, ${i+20})"
-    }
-
     sql """
     alter table oe1 modify column k0 set stats ('row_count'='1000', 'ndv'='1000', 'min_value'='1', 'max_value'='1000', 'avg_size'='1000', 'max_size'='1000' )
     """
@@ -99,6 +94,17 @@ suite("or_expansion") {
         contains "VHASH JOIN"
     }
 
+    for (int i = 0; i < 10; i++) {
+        sql "insert into oe1 values(${i}, ${i})"
+        sql "insert into oe2 values(${i+20}, ${i+20})"
+    }
+    sql "insert into oe1 values(null, 1)"
+    sql "insert into oe1 values(1, null)"
+    sql "insert into oe1 values(null, null)"
+    sql "insert into oe2 values(null, 1)"
+    sql "insert into oe2 values(1, null)"
+    sql "insert into oe2 values(null, null)"
+
     qt_order_ij """
         select oe1.k0, oe2.k0
         from oe1 inner join oe2