Skip to content

Commit

Permalink
[Feat](Nereids) add use mv hint (#40167)
Browse files Browse the repository at this point in the history
support hint like:
/*+ no_use_mv(tablename mvname) / which forbid tablename(indexname) to
be choose
or /+ use_mv(tablename mvname) */ which choose specific materialize view
  • Loading branch information
LiBinfeng-01 authored Sep 3, 2024
1 parent 5e93a71 commit d0654cd
Show file tree
Hide file tree
Showing 10 changed files with 490 additions and 35 deletions.
95 changes: 95 additions & 0 deletions fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java
Original file line number Diff line number Diff line change
Expand Up @@ -54,6 +54,8 @@
import org.apache.doris.mtmv.MTMVRelatedTableIf;
import org.apache.doris.mtmv.MTMVSnapshotIf;
import org.apache.doris.mtmv.MTMVVersionSnapshot;
import org.apache.doris.nereids.hint.Hint;
import org.apache.doris.nereids.hint.UseMvHint;
import org.apache.doris.persist.gson.GsonPostProcessable;
import org.apache.doris.persist.gson.GsonUtils;
import org.apache.doris.qe.ConnectContext;
Expand Down Expand Up @@ -565,6 +567,99 @@ public Map<Long, MaterializedIndexMeta> getVisibleIndexIdToMeta() {
return visibleMVs;
}

public Long getBestMvIdWithHint(List<Long> orderedMvs) {
Optional<UseMvHint> useMvHint = getUseMvHint("USE_MV");
Optional<UseMvHint> noUseMvHint = getUseMvHint("NO_USE_MV");
if (useMvHint.isPresent() && noUseMvHint.isPresent()) {
if (noUseMvHint.get().getNoUseMVName(this.name).contains(useMvHint.get().getUseMvName(this.name))) {
String errorMsg = "conflict mv exist in use_mv and no_use_mv in the same time"
+ useMvHint.get().getUseMvName(this.name);
useMvHint.get().setStatus(Hint.HintStatus.SYNTAX_ERROR);
useMvHint.get().setErrorMessage(errorMsg);
noUseMvHint.get().setStatus(Hint.HintStatus.SYNTAX_ERROR);
noUseMvHint.get().setErrorMessage(errorMsg);
}
return getMvIdWithUseMvHint(useMvHint.get(), orderedMvs);
} else if (useMvHint.isPresent()) {
return getMvIdWithUseMvHint(useMvHint.get(), orderedMvs);
} else if (noUseMvHint.isPresent()) {
return getMvIdWithNoUseMvHint(noUseMvHint.get(), orderedMvs);
}
return orderedMvs.get(0);
}

private Long getMvIdWithUseMvHint(UseMvHint useMvHint, List<Long> orderedMvs) {
if (useMvHint.isAllMv()) {
useMvHint.setStatus(Hint.HintStatus.SYNTAX_ERROR);
useMvHint.setErrorMessage("use_mv hint should only have one mv in one table: "
+ this.name);
return orderedMvs.get(0);
} else {
String mvName = useMvHint.getUseMvName(this.name);
if (mvName != null) {
if (mvName.equals("`*`")) {
useMvHint.setStatus(Hint.HintStatus.SYNTAX_ERROR);
useMvHint.setErrorMessage("use_mv hint should only have one mv in one table: "
+ this.name);
return orderedMvs.get(0);
}
Long choosedIndexId = indexNameToId.get(mvName);
if (orderedMvs.contains(choosedIndexId)) {
useMvHint.setStatus(Hint.HintStatus.SUCCESS);
return choosedIndexId;
} else {
useMvHint.setStatus(Hint.HintStatus.SYNTAX_ERROR);
useMvHint.setErrorMessage("do not have mv: " + mvName + " in table: " + this.name);
}
}
}
return orderedMvs.get(0);
}

private Long getMvIdWithNoUseMvHint(UseMvHint noUseMvHint, List<Long> orderedMvs) {
if (noUseMvHint.isAllMv()) {
noUseMvHint.setStatus(Hint.HintStatus.SUCCESS);
return getBaseIndex().getId();
} else {
List<String> mvNames = noUseMvHint.getNoUseMVName(this.name);
Set<Long> forbiddenIndexIds = Sets.newHashSet();
for (int i = 0; i < mvNames.size(); i++) {
if (mvNames.get(i).equals("`*`")) {
noUseMvHint.setStatus(Hint.HintStatus.SUCCESS);
return getBaseIndex().getId();
}
if (hasMaterializedIndex(mvNames.get(i))) {
Long forbiddenIndexId = indexNameToId.get(mvNames.get(i));
forbiddenIndexIds.add(forbiddenIndexId);
} else {
noUseMvHint.setStatus(Hint.HintStatus.SYNTAX_ERROR);
noUseMvHint.setErrorMessage("do not have mv: " + mvNames.get(i) + " in table: " + this.name);
break;
}
}
for (int i = 0; i < orderedMvs.size(); i++) {
if (forbiddenIndexIds.contains(orderedMvs.get(i))) {
noUseMvHint.setStatus(Hint.HintStatus.SUCCESS);
} else {
return orderedMvs.get(i);
}
}
}
return orderedMvs.get(0);
}

private Optional<UseMvHint> getUseMvHint(String useMvName) {
for (Hint hint : ConnectContext.get().getStatementContext().getHints()) {
if (hint.isSyntaxError()) {
continue;
}
if (hint.getHintName().equalsIgnoreCase(useMvName)) {
return Optional.of((UseMvHint) hint);
}
}
return Optional.empty();
}

public List<MaterializedIndex> getVisibleIndex() {
Optional<Partition> partition = idToPartition.values().stream().findFirst();
if (!partition.isPresent()) {
Expand Down
144 changes: 144 additions & 0 deletions fe/fe-core/src/main/java/org/apache/doris/nereids/hint/UseMvHint.java
Original file line number Diff line number Diff line change
@@ -0,0 +1,144 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.

package org.apache.doris.nereids.hint;

import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;

/**
* rule hint.
*/
public class UseMvHint extends Hint {

private final boolean isUseMv;

private final boolean isAllMv;

private final List<String> parameters;

private final Map<String, String> useMvTableColumnMap;

private final Map<String, List<String>> noUseMvTableColumnMap;

/**
* constructor of use mv hint
* @param hintName use mv
* @param parameters original parameters
* @param isUseMv use_mv hint or no_use_mv hint
* @param isAllMv should all mv be controlled
*/
public UseMvHint(String hintName, List<String> parameters, boolean isUseMv, boolean isAllMv) {
super(hintName);
this.isUseMv = isUseMv;
this.isAllMv = isAllMv;
this.parameters = parameters;
this.useMvTableColumnMap = initUseMvTableColumnMap(parameters);
this.noUseMvTableColumnMap = initNoUseMvTableColumnMap(parameters);
}

private Map<String, String> initUseMvTableColumnMap(List<String> parameters) {
Map<String, String> tempUseMvTableColumnMap = new HashMap<>();
if (!isUseMv) {
return tempUseMvTableColumnMap;
}
if (parameters.size() % 2 == 1) {
this.setStatus(HintStatus.SYNTAX_ERROR);
this.setErrorMessage("parameter of use_mv hint must be in pairs");
return tempUseMvTableColumnMap;
}
for (int i = 0; i < parameters.size(); i += 2) {
String tableName = parameters.get(i);
String columnName = parameters.get(i + 1);
if (tempUseMvTableColumnMap.containsKey(tableName)) {
this.setStatus(HintStatus.SYNTAX_ERROR);
this.setErrorMessage("use_mv hint should only have one mv in one table: "
+ tableName + "." + columnName);
break;
}
tempUseMvTableColumnMap.put(tableName, columnName);
}
return tempUseMvTableColumnMap;
}

private Map<String, List<String>> initNoUseMvTableColumnMap(List<String> parameters) {
Map<String, List<String>> tempNoUseMvTableColumnMap = new HashMap<>();
if (isUseMv) {
return tempNoUseMvTableColumnMap;
}
if (parameters.size() % 2 == 1) {
this.setStatus(HintStatus.SYNTAX_ERROR);
this.setErrorMessage("parameter of no_use_mv hint must be in pairs");
return tempNoUseMvTableColumnMap;
}
for (int i = 0; i < parameters.size(); i += 2) {
String tableName = parameters.get(i);
String columnName = parameters.get(i + 1);
if (tempNoUseMvTableColumnMap.containsKey(tableName)) {
tempNoUseMvTableColumnMap.get(tableName).add(columnName);
} else {
List<String> list = new ArrayList<>();
list.add(columnName);
tempNoUseMvTableColumnMap.put(tableName, list);
}
}
return tempNoUseMvTableColumnMap;
}

public boolean isUseMv() {
return isUseMv;
}

public boolean isAllMv() {
return isAllMv;
}

public String getUseMvName(String tableName) {
return useMvTableColumnMap.get(tableName);
}

public List<String> getNoUseMVName(String tableName) {
return noUseMvTableColumnMap.get(tableName);
}

@Override
public String getExplainString() {
StringBuilder out = new StringBuilder();
if (isUseMv) {
out.append("use_mv");
} else {
out.append("no_use_mv");
}
if (!parameters.isEmpty()) {
out.append("(");
for (int i = 0; i < parameters.size(); i++) {
if (i % 2 == 0) {
out.append(parameters.get(i));
} else {
out.append(".");
out.append(parameters.get(i));
out.append(" ");
}
}
out.append(")");
}

return out.toString();
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -232,6 +232,7 @@
import org.apache.doris.nereids.properties.SelectHintOrdered;
import org.apache.doris.nereids.properties.SelectHintSetVar;
import org.apache.doris.nereids.properties.SelectHintUseCboRule;
import org.apache.doris.nereids.properties.SelectHintUseMv;
import org.apache.doris.nereids.trees.TableSample;
import org.apache.doris.nereids.trees.expressions.Add;
import org.apache.doris.nereids.trees.expressions.And;
Expand Down Expand Up @@ -3165,7 +3166,7 @@ private LogicalPlan withSelectHint(LogicalPlan logicalPlan, List<ParserRuleConte
if (hintContexts.isEmpty()) {
return logicalPlan;
}
Map<String, SelectHint> hints = Maps.newLinkedHashMap();
ImmutableList.Builder<SelectHint> hints = ImmutableList.builder();
for (ParserRuleContext hintContext : hintContexts) {
SelectHintContext selectHintContext = (SelectHintContext) hintContext;
for (HintStatementContext hintStatement : selectHintContext.hintStatements) {
Expand All @@ -3187,7 +3188,7 @@ private LogicalPlan withSelectHint(LogicalPlan logicalPlan, List<ParserRuleConte
parameters.put(parameterName, value);
}
}
hints.put(hintName, new SelectHintSetVar(hintName, parameters));
hints.add(new SelectHintSetVar(hintName, parameters));
break;
case "leading":
List<String> leadingParameters = new ArrayList<>();
Expand All @@ -3197,10 +3198,10 @@ private LogicalPlan withSelectHint(LogicalPlan logicalPlan, List<ParserRuleConte
leadingParameters.add(parameterName);
}
}
hints.put(hintName, new SelectHintLeading(hintName, leadingParameters));
hints.add(new SelectHintLeading(hintName, leadingParameters));
break;
case "ordered":
hints.put(hintName, new SelectHintOrdered(hintName));
hints.add(new SelectHintOrdered(hintName));
break;
case "use_cbo_rule":
List<String> useRuleParameters = new ArrayList<>();
Expand All @@ -3210,7 +3211,7 @@ private LogicalPlan withSelectHint(LogicalPlan logicalPlan, List<ParserRuleConte
useRuleParameters.add(parameterName);
}
}
hints.put(hintName, new SelectHintUseCboRule(hintName, useRuleParameters, false));
hints.add(new SelectHintUseCboRule(hintName, useRuleParameters, false));
break;
case "no_use_cbo_rule":
List<String> noUseRuleParameters = new ArrayList<>();
Expand All @@ -3220,14 +3221,34 @@ private LogicalPlan withSelectHint(LogicalPlan logicalPlan, List<ParserRuleConte
noUseRuleParameters.add(parameterName);
}
}
hints.put(hintName, new SelectHintUseCboRule(hintName, noUseRuleParameters, true));
hints.add(new SelectHintUseCboRule(hintName, noUseRuleParameters, true));
break;
case "use_mv":
List<String> useIndexParameters = new ArrayList<String>();
for (HintAssignmentContext kv : hintStatement.parameters) {
String parameterName = visitIdentifierOrText(kv.key);
if (kv.key != null) {
useIndexParameters.add(parameterName);
}
}
hints.add(new SelectHintUseMv(hintName, useIndexParameters, true));
break;
case "no_use_mv":
List<String> noUseIndexParameters = new ArrayList<String>();
for (HintAssignmentContext kv : hintStatement.parameters) {
String parameterName = visitIdentifierOrText(kv.key);
if (kv.key != null) {
noUseIndexParameters.add(parameterName);
}
}
hints.add(new SelectHintUseMv(hintName, noUseIndexParameters, false));
break;
default:
break;
}
}
}
return new LogicalSelectHint<>(hints, logicalPlan);
return new LogicalSelectHint<>(hints.build(), logicalPlan);
}

@Override
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -59,7 +59,7 @@ public Plan visitUnboundResultSink(UnboundResultSink<? extends Plan> unboundResu
public Plan visitLogicalSubQueryAlias(LogicalSubQueryAlias<? extends Plan> alias,
StatementContext context) {
if (alias.child() instanceof LogicalSelectHint
&& ((LogicalSelectHint) alias.child()).isIncludeLeading()) {
&& ((LogicalSelectHint) alias.child()).isIncludeHint("Leading")) {
aliasQueries.add((LogicalSubQueryAlias<Plan>) alias);
List<String> tableName = new ArrayList<>();
tableName.add(alias.getAlias());
Expand Down
Loading

0 comments on commit d0654cd

Please sign in to comment.