diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Rewriter.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Rewriter.java index fcbbdee6ee67b6..24495ccc620d13 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Rewriter.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Rewriter.java @@ -78,6 +78,7 @@ import org.apache.doris.nereids.rules.rewrite.MergeProjects; import org.apache.doris.nereids.rules.rewrite.MergeSetOperations; import org.apache.doris.nereids.rules.rewrite.NormalizeSort; +import org.apache.doris.nereids.rules.rewrite.OrExpansion; import org.apache.doris.nereids.rules.rewrite.PruneEmptyPartition; import org.apache.doris.nereids.rules.rewrite.PruneFileScanPartition; import org.apache.doris.nereids.rules.rewrite.PruneOlapScanPartition; @@ -377,6 +378,8 @@ private static List getWholeTreeRewriteJobs(List jobs) { topic("rewrite cte sub-tree", custom(RuleType.REWRITE_CTE_CHILDREN, () -> new RewriteCteChildren(jobs)) ), + topic("or expansion", + topDown(new OrExpansion())), topic("whole plan check", custom(RuleType.ADJUST_NULLABLE, AdjustNullable::new) ) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleSet.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleSet.java index dfb00b08b84a70..b7a43d66984c8a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleSet.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleSet.java @@ -18,7 +18,6 @@ package org.apache.doris.nereids.rules; import org.apache.doris.nereids.rules.exploration.MergeProjectsCBO; -import org.apache.doris.nereids.rules.exploration.OrExpansion; import org.apache.doris.nereids.rules.exploration.TransposeAggSemiJoin; import org.apache.doris.nereids.rules.exploration.TransposeAggSemiJoinProject; import org.apache.doris.nereids.rules.exploration.join.InnerJoinLAsscom; @@ -119,7 +118,6 @@ public class RuleSet { .add(PushdownProjectThroughSemiJoin.INSTANCE) .add(TransposeAggSemiJoin.INSTANCE) .add(TransposeAggSemiJoinProject.INSTANCE) - .add(OrExpansion.INSTANCE) .build(); public static final List PUSH_DOWN_FILTERS = ImmutableList.of( diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/OrExpansion.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/OrExpansion.java similarity index 95% rename from fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/OrExpansion.java rename to fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/OrExpansion.java index 9019565142bfa3..fd7e7e844ad862 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/OrExpansion.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/OrExpansion.java @@ -15,15 +15,16 @@ // specific language governing permissions and limitations // under the License. -package org.apache.doris.nereids.rules.exploration; +package org.apache.doris.nereids.rules.rewrite; import org.apache.doris.common.Pair; import org.apache.doris.nereids.CascadesContext; import org.apache.doris.nereids.rules.Rule; import org.apache.doris.nereids.rules.RuleType; -import org.apache.doris.nereids.rules.rewrite.PushdownExpressionsInHashCondition; +import org.apache.doris.nereids.rules.exploration.OneExplorationRuleFactory; import org.apache.doris.nereids.trees.expressions.Alias; import org.apache.doris.nereids.trees.expressions.Expression; +import org.apache.doris.nereids.trees.expressions.IsNull; import org.apache.doris.nereids.trees.expressions.NamedExpression; import org.apache.doris.nereids.trees.expressions.Not; import org.apache.doris.nereids.trees.expressions.Slot; @@ -72,7 +73,7 @@ public class OrExpansion extends OneExplorationRuleFactory { @Override public Rule build() { - return logicalJoin().when(JoinUtils::shouldNestedLoopJoin) + return logicalJoin(any(), any()).when(JoinUtils::shouldNestedLoopJoin) .when(join -> supportJoinType.contains(join.getJoinType()) && ConnectContext.get().getSessionVariable().getEnablePipelineEngine()) .thenApply(ctx -> { @@ -82,7 +83,7 @@ public Rule build() { //1. Try to split or conditions Pair, List> hashOtherConditions = splitOrCondition(join); - if (hashOtherConditions == null) { + if (hashOtherConditions == null || hashOtherConditions.first.size() <= 1) { return join; } @@ -221,7 +222,11 @@ private List expandInnerJoin(CascadesContext ctx, Pair, LogicalCTEProducer rightProducer) { List disjunctions = hashOtherConditions.first; List otherConditions = hashOtherConditions.second; - List notExprs = disjunctions.stream().map(Not::new).collect(Collectors.toList()); + // For null values, equalTo and not equalTo both return false + // To avoid it, we always return true when there is null + List notExprs = disjunctions.stream() + .map(e -> ExpressionUtils.or(new Not(e), new IsNull(e))) + .collect(ImmutableList.toImmutableList()); List joins = Lists.newArrayList(); for (int hashCondIdx = 0; hashCondIdx < disjunctions.size(); hashCondIdx++) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/FilterEstimation.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/FilterEstimation.java index 0a7d6b70c39e3d..9c04b59ce6df6b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/FilterEstimation.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/FilterEstimation.java @@ -367,7 +367,7 @@ public Statistics visitNot(Not not, EstimationContext context) { // 4. not A like XXX colBuilder.setNumNulls(0); Preconditions.checkArgument( - child instanceof EqualTo + child instanceof EqualPredicate || child instanceof InPredicate || child instanceof IsNull || child instanceof Like, diff --git a/regression-test/data/nereids_p0/union/or_expansion.out b/regression-test/data/nereids_p0/union/or_expansion.out index dd82dbbcab71d4..4779bf2ace1b45 100644 --- a/regression-test/data/nereids_p0/union/or_expansion.out +++ b/regression-test/data/nereids_p0/union/or_expansion.out @@ -1,9 +1,13 @@ -- This file is automatically generated. You should know what you did if you want to edit this -- !order_ij -- +\N \N +1 \N +1 1 +1 1 -- !order_laj -- +\N 0 -1 2 3 4 @@ -14,6 +18,8 @@ 9 -- !order_loj -- +\N \N +\N \N 0 \N 1 \N 2 \N @@ -24,8 +30,13 @@ 7 \N 8 \N 9 \N +1 1 +1 1 -- !order_foj -- +\N \N +\N \N +\N \N 0 \N 1 \N 2 \N @@ -36,6 +47,8 @@ 7 \N 8 \N 9 \N +1 1 +1 1 \N 20 \N 21 \N 22 diff --git a/regression-test/suites/nereids_p0/union/or_expansion.groovy b/regression-test/suites/nereids_p0/union/or_expansion.groovy index 9139ea65c26ddf..2d68098cd2d03f 100644 --- a/regression-test/suites/nereids_p0/union/or_expansion.groovy +++ b/regression-test/suites/nereids_p0/union/or_expansion.groovy @@ -45,11 +45,6 @@ suite("or_expansion") { ) """ - for (int i = 0; i < 10; i++) { - sql "insert into oe1 values(${i}, ${i})" - sql "insert into oe2 values(${i+20}, ${i+20})" - } - sql """ alter table oe1 modify column k0 set stats ('row_count'='1000', 'ndv'='1000', 'min_value'='1', 'max_value'='1000', 'avg_size'='1000', 'max_size'='1000' ) """ @@ -99,6 +94,17 @@ suite("or_expansion") { contains "VHASH JOIN" } + for (int i = 0; i < 10; i++) { + sql "insert into oe1 values(${i}, ${i})" + sql "insert into oe2 values(${i+20}, ${i+20})" + } + sql "insert into oe1 values(null, 1)" + sql "insert into oe1 values(1, null)" + sql "insert into oe1 values(null, null)" + sql "insert into oe2 values(null, 1)" + sql "insert into oe2 values(1, null)" + sql "insert into oe2 values(null, null)" + qt_order_ij """ select oe1.k0, oe2.k0 from oe1 inner join oe2