Skip to content

Commit

Permalink
[Bug](materialized-view) make mv matched when preagg have value colum…
Browse files Browse the repository at this point in the history
…n predicate contained in mv'where clause (#22779)

1. make mv matched when preagg have value column predicate contained in mv
'where clause
2. fix `org.apache.doris.common.AnalysisException: errCode = 2, detailMessage = BITMAP_UNION need input a bitmap column, but input INVALID_TYPE`
3. make the error message more detailed when create mv stmt parse failed
  • Loading branch information
BiteTheDDDDt authored Aug 9, 2023
1 parent 5147c09 commit 89dc1f7
Show file tree
Hide file tree
Showing 5 changed files with 94 additions and 9 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -452,14 +452,14 @@ private MVColumnItem buildMVColumnItem(Analyzer analyzer, FunctionCallExpr funct
break;
case FunctionSet.BITMAP_UNION:
type = Type.BITMAP;
if (analyzer != null && !baseType.isBitmapType()) {
if (!isReplay && analyzer != null && !baseType.isBitmapType()) {
throw new AnalysisException(
"BITMAP_UNION need input a bitmap column, but input " + baseType.toString());
}
break;
case FunctionSet.HLL_UNION:
type = Type.HLL;
if (analyzer != null && !baseType.isHllType()) {
if (!isReplay && analyzer != null && !baseType.isHllType()) {
throw new AnalysisException("HLL_UNION need input a hll column, but input " + baseType.toString());
}
break;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -316,7 +316,8 @@ public void parseStmt(Analyzer analyzer) throws IOException {
try {
stmt.analyze(analyzer);
} catch (Exception e) {
LOG.warn("CreateMaterializedViewStmt analyze failed, reason=" + e.getMessage());
LOG.warn("CreateMaterializedViewStmt analyze failed, reason=", e);
return;
}
}

Expand All @@ -327,7 +328,7 @@ public void parseStmt(Analyzer analyzer) throws IOException {
Map<String, Expr> columnNameToDefineExpr = stmt.parseDefineExpr(analyzer);
setColumnsDefineExpr(columnNameToDefineExpr);
} catch (Exception e) {
LOG.warn("CreateMaterializedViewStmt parseDefineExpr failed, reason=" + e.getMessage());
LOG.warn("CreateMaterializedViewStmt parseDefineExpr failed, reason=", e);
}

} catch (Exception e) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -21,10 +21,12 @@
import org.apache.doris.catalog.AggregateType;
import org.apache.doris.catalog.Column;
import org.apache.doris.catalog.MaterializedIndex;
import org.apache.doris.catalog.MaterializedIndexMeta;
import org.apache.doris.catalog.OlapTable;
import org.apache.doris.common.Pair;
import org.apache.doris.nereids.annotation.Developing;
import org.apache.doris.nereids.exceptions.AnalysisException;
import org.apache.doris.nereids.parser.NereidsParser;
import org.apache.doris.nereids.rules.Rule;
import org.apache.doris.nereids.rules.RuleType;
import org.apache.doris.nereids.rules.rewrite.RewriteRuleFactory;
Expand Down Expand Up @@ -63,6 +65,7 @@
import org.apache.doris.nereids.trees.plans.logical.LogicalProject;
import org.apache.doris.nereids.trees.plans.logical.LogicalRepeat;
import org.apache.doris.nereids.util.ExpressionUtils;
import org.apache.doris.planner.PlanNode;

import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
Expand Down Expand Up @@ -1014,6 +1017,10 @@ public CheckContext(LogicalOlapScan scan, long indexId) {
public boolean isBaseIndex() {
return index == scan.getTable().getBaseIndexId();
}

public MaterializedIndexMeta getMeta() {
return scan.getTable().getIndexMetaByIndexId(index);
}
}

/**
Expand All @@ -1029,11 +1036,13 @@ private PreAggStatus checkGroupingExprs(
/**
* Predicates should not have value type columns.
*/
private PreAggStatus checkPredicates(
List<Expression> predicates,
CheckContext checkContext) {
return disablePreAggIfContainsAnyValueColumn(predicates, checkContext,
"Predicate %s contains value column %s");
private PreAggStatus checkPredicates(List<Expression> predicates, CheckContext checkContext) {
Set<String> indexConjuncts = PlanNode
.splitAndCompoundPredicateToConjuncts(checkContext.getMeta().getWhereClause()).stream()
.map(e -> new NereidsParser().parseExpression(e.toSql()).toSql()).collect(Collectors.toSet());
return disablePreAggIfContainsAnyValueColumn(
predicates.stream().filter(e -> !indexConjuncts.contains(e.toSql())).collect(Collectors.toList()),
checkContext, "Predicate %s contains value column %s");
}

/**
Expand Down
19 changes: 19 additions & 0 deletions regression-test/data/mv_p0/where/mvljc/mvljc.out
Original file line number Diff line number Diff line change
@@ -0,0 +1,19 @@
-- This file is automatically generated. You should know what you did if you want to edit this
-- !select_mv --
1 2
3 -6

-- !select_mv --
2
2
2
2
2
2
4
4
4
4
4
4

56 changes: 56 additions & 0 deletions regression-test/suites/mv_p0/where/mvljc/mvljc.groovy
Original file line number Diff line number Diff line change
@@ -0,0 +1,56 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.

import org.codehaus.groovy.runtime.IOGroovyMethods

suite ("mvljc") {
sql """ DROP TABLE IF EXISTS d_table; """
sql """set enable_nereids_planner=true"""
sql """
create table d_table(
k1 int null,
k2 int not null,
k3 bigint null,
k4 varchar(100) null
)
duplicate key (k1,k2,k3)
distributed BY hash(k1) buckets 3
properties("replication_num" = "1");
"""

sql "insert into d_table select 1,1,1,'a';"
sql "insert into d_table select 2,2,2,'b';"
sql "insert into d_table select 3,-3,null,'c';"

createMV ("""CREATE MATERIALIZED VIEW mvljc AS SELECT k1, SUM(k2) FROM d_table WHERE k3 < 2 or k2 < 0 GROUP BY k1;""")

sql "insert into d_table select 1,1,1,'a';"
sql "insert into d_table select 2,2,2,'b';"
sql "insert into d_table select 3,-3,null,'c';"

explain {
sql("SELECT k1, SUM(k2) FROM d_table WHERE k3 < 2 or k2 < 0 GROUP BY k1 order by 1,2;")
contains "(mvljc)"
}
qt_select_mv "SELECT k1, SUM(k2) FROM d_table WHERE k3 < 2 or k2 < 0 GROUP BY k1 order by 1,2;"

explain {
sql("SELECT a.k1 + 1 FROM ( SELECT k1, SUM(k2) FROM d_table WHERE k3 < 2 or k2 < 0 GROUP BY k1 ) a, d_table order by 1;")
contains "(mvljc)"
}
qt_select_mv "SELECT a.k1 + 1 FROM ( SELECT k1, SUM(k2) FROM d_table WHERE k3 < 2 or k2 < 0 GROUP BY k1 ) a, d_table order by 1;"
}

0 comments on commit 89dc1f7

Please sign in to comment.