Skip to content

Commit

Permalink
[feat](Nereids) support nereids hint position detaction (apache#39113)
Browse files Browse the repository at this point in the history 
When use hint in wrong position or use unsupport hint,
use channel(2) to filter it out
  • Loading branch information
@LiBinfeng-01
LiBinfeng-01 authored Aug 15, 2024
1 parent b2b15d3 commit 9f2756f
Show file tree
Hide file tree
Showing 11 changed files with 280 additions and 68 deletions.
18 changes: 18 additions & 0 deletions fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisLexer.g4
View file
Original file line number Diff line number Diff line change
Expand Up @@ -72,6 +72,19 @@ lexer grammar DorisLexer;
public void markUnclosedComment() {
has_unclosed_bracketed_comment = true;
}

// This variable will hold the external state
private boolean channel2;

// Method to set the external state
public void setChannel2(boolean value) {
this.channel2 = value;
}

// Method to decide the channel based on external state
private boolean isChannel2() {
return this.channel2;
}
}

SEMICOLON: ';';
Expand Down Expand Up @@ -686,6 +699,11 @@ BRACKETED_COMMENT
: '/*' {!isHint()}? ( BRACKETED_COMMENT | . )*? ('*/' | {markUnclosedComment();} EOF) -> channel(HIDDEN)
;

HINT_WITH_CHANNEL
: {isChannel2()}? HINT_START .*? HINT_END -> channel(2)
;


FROM_DUAL
: 'FROM' WS+ 'DUAL' -> channel(HIDDEN);

Expand Down
126 changes: 72 additions & 54 deletions fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -514,6 +514,12 @@ public class LogicalPlanBuilder extends DorisParserBaseVisitor<Object> {
return pos1.getCharPositionInLine() - pos2.getCharPositionInLine();
});

private final Map<Integer, ParserRuleContext> selectHintMap;

public LogicalPlanBuilder(Map<Integer, ParserRuleContext> selectHintMap) {
this.selectHintMap = selectHintMap;
}

@SuppressWarnings("unchecked")
protected <T> T typedVisit(ParseTree ctx) {
return (T) ctx.accept(this);
Expand Down Expand Up @@ -1352,7 +1358,16 @@ public LogicalPlan visitRegularQuerySpecification(RegularQuerySpecificationConte
Optional.ofNullable(ctx.aggClause()),
Optional.ofNullable(ctx.havingClause()));
selectPlan = withQueryOrganization(selectPlan, ctx.queryOrganization());
return withSelectHint(selectPlan, selectCtx.selectHint());
if ((selectHintMap == null) || selectHintMap.isEmpty()) {
return selectPlan;
}
List<ParserRuleContext> selectHintContexts = Lists.newArrayList();
for (Integer key : selectHintMap.keySet()) {
if (key > selectCtx.getStart().getStopIndex() && key < selectCtx.getStop().getStartIndex()) {
selectHintContexts.add(selectHintMap.get(key));
}
}
return withSelectHint(selectPlan, selectHintContexts);
});
}

Expand Down Expand Up @@ -3110,67 +3125,70 @@ private LogicalPlan withJoinRelations(LogicalPlan input, RelationContext ctx) {
return last;
}

private LogicalPlan withSelectHint(LogicalPlan logicalPlan, SelectHintContext hintContext) {
if (hintContext == null) {
private LogicalPlan withSelectHint(LogicalPlan logicalPlan, List<ParserRuleContext> hintContexts) {
if (hintContexts.isEmpty()) {
return logicalPlan;
}
Map<String, SelectHint> hints = Maps.newLinkedHashMap();
for (HintStatementContext hintStatement : hintContext.hintStatements) {
String hintName = hintStatement.hintName.getText().toLowerCase(Locale.ROOT);
switch (hintName) {
case "set_var":
Map<String, Optional<String>> parameters = Maps.newLinkedHashMap();
for (HintAssignmentContext kv : hintStatement.parameters) {
if (kv.key != null) {
String parameterName = visitIdentifierOrText(kv.key);
Optional<String> value = Optional.empty();
if (kv.constantValue != null) {
Literal literal = (Literal) visit(kv.constantValue);
value = Optional.ofNullable(literal.toLegacyLiteral().getStringValue());
} else if (kv.identifierValue != null) {
// maybe we should throw exception when the identifierValue is quoted identifier
value = Optional.ofNullable(kv.identifierValue.getText());
for (ParserRuleContext hintContext : hintContexts) {
SelectHintContext selectHintContext = (SelectHintContext) hintContext;
for (HintStatementContext hintStatement : selectHintContext.hintStatements) {
String hintName = hintStatement.hintName.getText().toLowerCase(Locale.ROOT);
switch (hintName) {
case "set_var":
Map<String, Optional<String>> parameters = Maps.newLinkedHashMap();
for (HintAssignmentContext kv : hintStatement.parameters) {
if (kv.key != null) {
String parameterName = visitIdentifierOrText(kv.key);
Optional<String> value = Optional.empty();
if (kv.constantValue != null) {
Literal literal = (Literal) visit(kv.constantValue);
value = Optional.ofNullable(literal.toLegacyLiteral().getStringValue());
} else if (kv.identifierValue != null) {
// maybe we should throw exception when the identifierValue is quoted identifier
value = Optional.ofNullable(kv.identifierValue.getText());
}
parameters.put(parameterName, value);
}
parameters.put(parameterName, value);
}
}
hints.put(hintName, new SelectHintSetVar(hintName, parameters));
break;
case "leading":
List<String> leadingParameters = new ArrayList<String>();
for (HintAssignmentContext kv : hintStatement.parameters) {
if (kv.key != null) {
String parameterName = visitIdentifierOrText(kv.key);
leadingParameters.add(parameterName);
hints.put(hintName, new SelectHintSetVar(hintName, parameters));
break;
case "leading":
List<String> leadingParameters = new ArrayList<String>();
for (HintAssignmentContext kv : hintStatement.parameters) {
if (kv.key != null) {
String parameterName = visitIdentifierOrText(kv.key);
leadingParameters.add(parameterName);
}
}
}
hints.put(hintName, new SelectHintLeading(hintName, leadingParameters));
break;
case "ordered":
hints.put(hintName, new SelectHintOrdered(hintName));
break;
case "use_cbo_rule":
List<String> useRuleParameters = new ArrayList<String>();
for (HintAssignmentContext kv : hintStatement.parameters) {
if (kv.key != null) {
String parameterName = visitIdentifierOrText(kv.key);
useRuleParameters.add(parameterName);
hints.put(hintName, new SelectHintLeading(hintName, leadingParameters));
break;
case "ordered":
hints.put(hintName, new SelectHintOrdered(hintName));
break;
case "use_cbo_rule":
List<String> useRuleParameters = new ArrayList<String>();
for (HintAssignmentContext kv : hintStatement.parameters) {
if (kv.key != null) {
String parameterName = visitIdentifierOrText(kv.key);
useRuleParameters.add(parameterName);
}
}
}
hints.put(hintName, new SelectHintUseCboRule(hintName, useRuleParameters, false));
break;
case "no_use_cbo_rule":
List<String> noUseRuleParameters = new ArrayList<String>();
for (HintAssignmentContext kv : hintStatement.parameters) {
String parameterName = visitIdentifierOrText(kv.key);
if (kv.key != null) {
noUseRuleParameters.add(parameterName);
hints.put(hintName, new SelectHintUseCboRule(hintName, useRuleParameters, false));
break;
case "no_use_cbo_rule":
List<String> noUseRuleParameters = new ArrayList<String>();
for (HintAssignmentContext kv : hintStatement.parameters) {
String parameterName = visitIdentifierOrText(kv.key);
if (kv.key != null) {
noUseRuleParameters.add(parameterName);
}
}
}
hints.put(hintName, new SelectHintUseCboRule(hintName, noUseRuleParameters, true));
break;
default:
break;
hints.put(hintName, new SelectHintUseCboRule(hintName, noUseRuleParameters, true));
break;
default:
break;
}
}
}
return new LogicalSelectHint<>(hints, logicalPlan);
Expand Down
5 changes: 5 additions & 0 deletions ...e-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilderForCreateView.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -52,10 +52,15 @@
import com.google.common.collect.ImmutableList;
import org.antlr.v4.runtime.ParserRuleContext;

import java.util.Map;
import java.util.Optional;

/**LogicalPlanBuilderForCreateView*/
public class LogicalPlanBuilderForCreateView extends LogicalPlanBuilder {
public LogicalPlanBuilderForCreateView(Map<Integer, ParserRuleContext> selectHintMap) {
super(selectHintMap);
}

@Override
protected LogicalPlan withGenerate(LogicalPlan plan, LateralViewContext ctx) {
ConnectContext.get().getStatementContext().addIndexInSqlToString(
Expand Down
5 changes: 5 additions & 0 deletions fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilderForSyncMv.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,7 @@
import org.apache.doris.nereids.util.PlanUtils;

import com.google.common.collect.ImmutableList;
import org.antlr.v4.runtime.ParserRuleContext;

import java.util.List;
import java.util.Map;
Expand All @@ -49,6 +50,10 @@
public class LogicalPlanBuilderForSyncMv extends LogicalPlanBuilder {
private Optional<String> querySql;

public LogicalPlanBuilderForSyncMv(Map<Integer, ParserRuleContext> selectHintMap) {
super(selectHintMap);
}

@Override
public Expression visitFunctionCallExpression(DorisParser.FunctionCallExpressionContext ctx) {
Expression expression = super.visitFunctionCallExpression(ctx);
Expand Down
37 changes: 34 additions & 3 deletions fe/fe-core/src/main/java/org/apache/doris/nereids/parser/NereidsParser.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,7 @@
import org.apache.doris.qe.SessionVariable;

import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import org.antlr.v4.runtime.CharStreams;
import org.antlr.v4.runtime.CommonTokenStream;
import org.antlr.v4.runtime.ParserRuleContext;
Expand Down Expand Up @@ -273,26 +274,56 @@ private <T> T parse(String sql, @Nullable LogicalPlanBuilder logicalPlanBuilder,
Function<DorisParser, ParserRuleContext> parseFunction) {
ParserRuleContext tree = toAst(sql, parseFunction);
LogicalPlanBuilder realLogicalPlanBuilder = logicalPlanBuilder == null
? new LogicalPlanBuilder() : logicalPlanBuilder;
? new LogicalPlanBuilder(getHintMap(sql, DorisParser::selectHint)) : logicalPlanBuilder;
return (T) realLogicalPlanBuilder.visit(tree);
}

public LogicalPlan parseForCreateView(String sql) {
ParserRuleContext tree = toAst(sql, DorisParser::singleStatement);
LogicalPlanBuilder realLogicalPlanBuilder = new LogicalPlanBuilderForCreateView();
LogicalPlanBuilder realLogicalPlanBuilder = new LogicalPlanBuilderForCreateView(
getHintMap(sql, DorisParser::selectHint));
return (LogicalPlan) realLogicalPlanBuilder.visit(tree);
}

public Optional<String> parseForSyncMv(String sql) {
ParserRuleContext tree = toAst(sql, DorisParser::singleStatement);
LogicalPlanBuilderForSyncMv logicalPlanBuilderForSyncMv = new LogicalPlanBuilderForSyncMv();
LogicalPlanBuilderForSyncMv logicalPlanBuilderForSyncMv = new LogicalPlanBuilderForSyncMv(
getHintMap(sql, DorisParser::selectHint));
logicalPlanBuilderForSyncMv.visit(tree);
return logicalPlanBuilderForSyncMv.getQuerySql();
}

/** get hint map */
public static Map<Integer, ParserRuleContext> getHintMap(String sql,
Function<DorisParser, ParserRuleContext> parseFunction) {
// parse hint first round
DorisLexer hintLexer = new DorisLexer(new CaseInsensitiveStream(CharStreams.fromString(sql)));
hintLexer.setChannel2(true);
CommonTokenStream hintTokenStream = new CommonTokenStream(hintLexer);

Map<Integer, ParserRuleContext> selectHintMap = Maps.newHashMap();

Token hintToken = hintTokenStream.getTokenSource().nextToken();
while (hintToken != null && hintToken.getType() != DorisLexer.EOF) {
int tokenType = hintToken.getType();
if (tokenType == DorisLexer.HINT_WITH_CHANNEL) {
String hintSql = sql.substring(hintToken.getStartIndex(), hintToken.getStopIndex() + 1);
DorisLexer newHintLexer = new DorisLexer(new CaseInsensitiveStream(CharStreams.fromString(hintSql)));
newHintLexer.setChannel2(false);
CommonTokenStream newHintTokenStream = new CommonTokenStream(newHintLexer);
DorisParser hintParser = new DorisParser(newHintTokenStream);
ParserRuleContext hintContext = parseFunction.apply(hintParser);
selectHintMap.put(hintToken.getStartIndex(), hintContext);
}
hintToken = hintTokenStream.getTokenSource().nextToken();
}
return selectHintMap;
}

/** toAst */
public static ParserRuleContext toAst(String sql, Function<DorisParser, ParserRuleContext> parseFunction) {
DorisLexer lexer = new DorisLexer(new CaseInsensitiveStream(CharStreams.fromString(sql)));
lexer.setChannel2(true);
CommonTokenStream tokenStream = new CommonTokenStream(lexer);
DorisParser parser = new DorisParser(tokenStream);

Expand Down
4 changes: 4 additions & 0 deletions fe/fe-core/src/main/java/org/apache/doris/nereids/parser/plsql/PLSqlLogicalPlanBuilder.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,10 @@
*/
public class PLSqlLogicalPlanBuilder extends LogicalPlanBuilder {

public PLSqlLogicalPlanBuilder() {
super(null);
}

public List<String> visitMultipartIdentifier(MultipartIdentifierContext ctx) {
return ctx.parts.stream()
.map(RuleContext::getText)
Expand Down
8 changes: 0 additions & 8 deletions fe/fe-core/src/test/java/org/apache/doris/nereids/parser/NereidsParserTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -352,16 +352,10 @@ public void testJoinHint() {
parsePlan("select * from t1 join [broadcast] t2 on t1.keyy=t2.keyy")
.matches(logicalJoin().when(j -> j.getDistributeHint().distributeType == DistributeType.BROADCAST_RIGHT));

parsePlan("select * from t1 join /*+ broadcast */ t2 on t1.keyy=t2.keyy")
.matches(logicalJoin().when(j -> j.getDistributeHint().distributeType == DistributeType.BROADCAST_RIGHT));

// invalid hint position
parsePlan("select * from [shuffle] t1 join t2 on t1.keyy=t2.keyy")
.assertThrowsExactly(ParseException.class);

parsePlan("select * from /*+ shuffle */ t1 join t2 on t1.keyy=t2.keyy")
.assertThrowsExactly(ParseException.class);

// invalid hint content
parsePlan("select * from t1 join [bucket] t2 on t1.keyy=t2.keyy")
.assertThrowsExactly(ParseException.class)
Expand All @@ -372,8 +366,6 @@ public void testJoinHint() {
+ "----------------------^^^");

// invalid multiple hints
parsePlan("select * from t1 join /*+ shuffle , broadcast */ t2 on t1.keyy=t2.keyy")
.assertThrowsExactly(ParseException.class);

parsePlan("select * from t1 join [shuffle,broadcast] t2 on t1.keyy=t2.keyy")
.assertThrowsExactly(ParseException.class);
Expand Down
Loading

0 comments on commit 9f2756f

Please sign in to comment.