From 343c0c1a1ae6b12ae03d0eea898ee937646aac11 Mon Sep 17 00:00:00 2001 From: minghong Date: Wed, 3 Apr 2024 15:38:47 +0800 Subject: [PATCH] [fix](nereids) runtime filter with probe expr should be pushed though set operator #33010 --- .../post/RuntimeFilterGenerator.java | 18 +++++++++++++ .../runtime_filter/test_pushdown_setop.out | 25 +++++++++++++++++++ .../runtime_filter/test_pushdown_setop.groovy | 7 ++++-- 3 files changed, 48 insertions(+), 2 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/RuntimeFilterGenerator.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/RuntimeFilterGenerator.java index 421a90fb24966a..d7e3f95be6a4d8 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/RuntimeFilterGenerator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/RuntimeFilterGenerator.java @@ -51,6 +51,7 @@ import org.apache.doris.nereids.trees.plans.physical.PhysicalPlan; import org.apache.doris.nereids.trees.plans.physical.PhysicalProject; import org.apache.doris.nereids.trees.plans.physical.PhysicalRelation; +import org.apache.doris.nereids.trees.plans.physical.PhysicalSetOperation; import org.apache.doris.nereids.trees.plans.physical.PhysicalTopN; import org.apache.doris.nereids.trees.plans.physical.PhysicalWindow; import org.apache.doris.nereids.trees.plans.physical.RuntimeFilter; @@ -497,6 +498,23 @@ public PhysicalRelation visitPhysicalRelation(PhysicalRelation relation, Cascade return relation; } + @Override + public PhysicalSetOperation visitPhysicalSetOperation(PhysicalSetOperation setOperation, CascadesContext context) { + setOperation.children().forEach(child -> child.accept(this, context)); + RuntimeFilterContext ctx = context.getRuntimeFilterContext(); + if (!setOperation.getRegularChildrenOutputs().isEmpty()) { + // example: RegularChildrenOutputs is empty + // "select 1 a, 2 b union all select 3, 4 union all select 10 e, 20 f;" + for (int i = 0; i < setOperation.getOutput().size(); i++) { + Pair childSlotPair = ctx.getAliasTransferPair(setOperation.getRegularChildOutput(0).get(0)); + if (childSlotPair != null) { + ctx.aliasTransferMapPut(setOperation.getOutput().get(i), childSlotPair); + } + } + } + return setOperation; + } + // runtime filter build side ndv private long getBuildSideNdv(AbstractPhysicalJoin join, ComparisonPredicate compare) { diff --git a/regression-test/data/nereids_tpch_shape_sf1000_p0/runtime_filter/test_pushdown_setop.out b/regression-test/data/nereids_tpch_shape_sf1000_p0/runtime_filter/test_pushdown_setop.out index d794a8635f2bf1..5d0e6c3143d8d1 100644 --- a/regression-test/data/nereids_tpch_shape_sf1000_p0/runtime_filter/test_pushdown_setop.out +++ b/regression-test/data/nereids_tpch_shape_sf1000_p0/runtime_filter/test_pushdown_setop.out @@ -22,3 +22,28 @@ PhysicalResultSink --------------PhysicalProject ----------------PhysicalOlapScan[region] +-- !rf_setop_expr -- +PhysicalResultSink +--hashAgg[GLOBAL] +----PhysicalDistribute[DistributionSpecGather] +------hashAgg[LOCAL] +--------PhysicalProject +----------hashJoin[INNER_JOIN] hashCondition=((expr_abs(l_linenumber) = expr_cast(r_regionkey as LARGEINT))) otherCondition=() build RFs:RF0 expr_cast(r_regionkey as LARGEINT)->[abs(l_linenumber),abs(o_orderkey)] +------------PhysicalDistribute[DistributionSpecHash] +--------------PhysicalProject +----------------PhysicalExcept +------------------PhysicalProject +--------------------hashAgg[GLOBAL] +----------------------PhysicalDistribute[DistributionSpecHash] +------------------------hashAgg[LOCAL] +--------------------------PhysicalProject +----------------------------PhysicalOlapScan[lineitem] apply RFs: RF0 +------------------PhysicalDistribute[DistributionSpecHash] +--------------------PhysicalProject +----------------------hashAgg[LOCAL] +------------------------PhysicalProject +--------------------------PhysicalOlapScan[orders] apply RFs: RF0 +------------PhysicalDistribute[DistributionSpecHash] +--------------PhysicalProject +----------------PhysicalOlapScan[region] + diff --git a/regression-test/suites/nereids_tpch_shape_sf1000_p0/runtime_filter/test_pushdown_setop.groovy b/regression-test/suites/nereids_tpch_shape_sf1000_p0/runtime_filter/test_pushdown_setop.groovy index e4dc3395af8999..04ad8e20bddb24 100644 --- a/regression-test/suites/nereids_tpch_shape_sf1000_p0/runtime_filter/test_pushdown_setop.groovy +++ b/regression-test/suites/nereids_tpch_shape_sf1000_p0/runtime_filter/test_pushdown_setop.groovy @@ -30,10 +30,13 @@ suite("test_pushdown_setop") { sql 'set enable_nereids_timeout = false' sql 'set enable_runtime_filter_prune=false' sql 'set runtime_filter_type=8' - def query = """ select count() from ((select l_linenumber from lineitem) except (select o_orderkey from orders)) T join region on T.l_linenumber = r_regionkey;""" qt_rf_setop """ explain shape plan - ${query} + select count() from ((select l_linenumber from lineitem) except (select o_orderkey from orders)) T join region on T.l_linenumber = r_regionkey; + """ + + qt_rf_setop_expr """ + explain shape plan select count() from ((select l_linenumber from lineitem) except (select o_orderkey from orders)) T join region on abs(T.l_linenumber) = r_regionkey; """ }