Skip to content

Commit

Permalink
[chore](Nereids) turn off fallback
Browse files Browse the repository at this point in the history
  • Loading branch information
@morrySnow
morrySnow committed Apr 17, 2024
1 parent dbf6295 commit 7737b83
Show file tree
Hide file tree
Showing 8 changed files with 86 additions and 61 deletions.
18 changes: 9 additions & 9 deletions ...core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateViewInfo.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -97,15 +97,6 @@ public CreateViewInfo(boolean ifNotExists, TableNameInfo viewName, String commen

/** init */
public void init(ConnectContext ctx) throws UserException {
analyzeAndFillRewriteSqlMap(querySql, ctx);
OutermostPlanFinderContext outermostPlanFinderContext = new OutermostPlanFinderContext();
analyzedPlan.accept(OutermostPlanFinder.INSTANCE, outermostPlanFinderContext);
List<Slot> outputs = outermostPlanFinderContext.outermostPlan.getOutput();
createFinalCols(outputs);
}

/**validate*/
public void validate(ConnectContext ctx) throws UserException {
viewName.analyze(ctx);
FeNameFormat.checkTableName(viewName.getTbl());
// disallow external catalog
Expand All @@ -116,6 +107,15 @@ public void validate(ConnectContext ctx) throws UserException {
ErrorReport.reportAnalysisException(ErrorCode.ERR_TABLE_ACCESS_DENIED_ERROR,
PrivPredicate.CREATE.getPrivs().toString(), viewName.getTbl());
}
analyzeAndFillRewriteSqlMap(querySql, ctx);
OutermostPlanFinderContext outermostPlanFinderContext = new OutermostPlanFinderContext();
analyzedPlan.accept(OutermostPlanFinder.INSTANCE, outermostPlanFinderContext);
List<Slot> outputs = outermostPlanFinderContext.outermostPlan.getOutput();
createFinalCols(outputs);
}

/**validate*/
public void validate(ConnectContext ctx) throws UserException {
NereidsPlanner planner = new NereidsPlanner(ctx.getStatementContext());
planner.plan(new UnboundResultSink<>(logicalQuery), PhysicalProperties.ANY, ExplainLevel.NONE);
Set<String> colSets = Sets.newTreeSet(String.CASE_INSENSITIVE_ORDER);
Expand Down
41 changes: 38 additions & 3 deletions fe/fe-core/src/main/java/org/apache/doris/qe/ConnectProcessor.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,9 @@

package org.apache.doris.qe;

import org.apache.doris.analysis.CreateTableAsSelectStmt;
import org.apache.doris.analysis.CreateTableStmt;
import org.apache.doris.analysis.DeleteStmt;
import org.apache.doris.analysis.ExplainOptions;
import org.apache.doris.analysis.InsertStmt;
import org.apache.doris.analysis.KillStmt;
Expand All @@ -25,6 +28,7 @@
import org.apache.doris.analysis.SqlParser;
import org.apache.doris.analysis.SqlScanner;
import org.apache.doris.analysis.StatementBase;
import org.apache.doris.analysis.UpdateStmt;
import org.apache.doris.analysis.UserIdentity;
import org.apache.doris.catalog.Column;
import org.apache.doris.catalog.DatabaseIf;
Expand Down Expand Up @@ -246,6 +250,7 @@ public void executeQuery(MysqlCommand mysqlCommand, String originStmt) throws Ex
&& CacheAnalyzer.canUseSqlCache(sessionVariable);
List<StatementBase> stmts = null;
Exception nereidsParseException = null;
Exception nereidsSyntaxException = null;
long parseSqlStartTime = System.currentTimeMillis();
List<StatementBase> cachedStmts = null;
// Nereids do not support prepare and execute now, so forbid prepare command, only process query command
Expand Down Expand Up @@ -273,13 +278,14 @@ public void executeQuery(MysqlCommand mysqlCommand, String originStmt) throws Ex
// Because ParseException means the sql is not supported by Nereids.
// It should be parsed by old parser, so not setting nereidsParseException to avoid
// suppressing the exception thrown by old parser.
nereidsParseException = e;
} catch (Exception e) {
// TODO: We should catch all exception here until we support all query syntax.
if (LOG.isDebugEnabled()) {
LOG.debug("Nereids parse sql failed with other exception. Reason: {}. Statement: \"{}\".",
e.getMessage(), convertedStmt);
}
nereidsParseException = e;
nereidsSyntaxException = e;
}
}
}
Expand All @@ -292,15 +298,44 @@ public void executeQuery(MysqlCommand mysqlCommand, String originStmt) throws Ex
// if NereidsParser and oldParser both failed,
// prove is a new feature implemented only on the nereids,
// so an error message for the new nereids is thrown
if (nereidsParseException != null) {
throwable = nereidsParseException;
if (nereidsSyntaxException != null) {
throwable = nereidsSyntaxException;
}
// Parse sql failed, audit it and return
handleQueryException(throwable, convertedStmt, null, null);
return;
}
}

if (mysqlCommand == MysqlCommand.COM_QUERY
&& ctx.getSessionVariable().isEnableNereidsPlanner()
&& !ctx.getSessionVariable().enableFallbackToOriginalPlanner
&& stmts.stream().allMatch(s -> s instanceof QueryStmt
|| s instanceof InsertStmt
|| s instanceof UpdateStmt
|| s instanceof DeleteStmt
|| s instanceof CreateTableAsSelectStmt
|| s instanceof CreateTableStmt)) {
String errMsg;
Throwable exception = null;
if (nereidsParseException != null) {
errMsg = nereidsParseException.getMessage();
exception = nereidsParseException;
} else if (nereidsSyntaxException != null) {
errMsg = nereidsSyntaxException.getMessage();
exception = nereidsSyntaxException;
} else {
errMsg = "Nereids parse DQL failed. " + originStmt;
}
if (exception == null) {
exception = new AnalysisException(errMsg);
} else {
exception = new AnalysisException(errMsg, exception);
}
handleQueryException(exception, originStmt, null, null);
return;
}

List<String> origSingleStmtList = null;
// if stmts.size() > 1, split originStmt to multi singleStmts
if (stmts.size() > 1) {
Expand Down
2 changes: 1 addition & 1 deletion fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1262,7 +1262,7 @@ public void setEnableLeftZigZag(boolean enableLeftZigZag) {
// for nereids, fallback will cause the Doris return the correct result although the syntax is unsupported
// in nereids for some mistaken modification. You should set it on the
@VariableMgr.VarAttr(name = ENABLE_FALLBACK_TO_ORIGINAL_PLANNER, needForward = true)
public boolean enableFallbackToOriginalPlanner = true;
public boolean enableFallbackToOriginalPlanner = false;

@VariableMgr.VarAttr(name = ENABLE_NEREIDS_TIMEOUT, needForward = true)
public boolean enableNereidsTimeout = true;
Expand Down
16 changes: 5 additions & 11 deletions fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -586,23 +586,17 @@ public void execute(TUniqueId queryId) throws Exception {
}
// FIXME: Force fallback for:
// 1. group commit because nereids does not support it (see the following `isGroupCommit` variable)
// 2. insert into command because some nereids cases fail (including case1)
// Skip force fallback for:
// 1. Transaction insert because nereids support `insert into select` while legacy does not
// 2. Nereids support insert into external table while legacy does not
boolean isInsertCommand = parsedStmt != null
&& parsedStmt instanceof LogicalPlanAdapter
&& ((LogicalPlanAdapter) parsedStmt).getLogicalPlan() instanceof InsertIntoTableCommand;
/*boolean isGroupCommit = (Config.wait_internal_group_commit_finish
|| context.sessionVariable.isEnableInsertGroupCommit()) && isInsertCommand;*/
boolean isExternalTableInsert = false;
if (isInsertCommand) {
isExternalTableInsert = ((InsertIntoTableCommand) ((LogicalPlanAdapter) parsedStmt)
.getLogicalPlan()).isExternalTableSink();
}
boolean forceFallback = isInsertCommand && !isExternalTableInsert && !context.isTxnModel();
if (e instanceof NereidsException && !context.getSessionVariable().enableFallbackToOriginalPlanner
&& !forceFallback) {
boolean isGroupCommit = (Config.wait_internal_group_commit_finish
|| context.sessionVariable.isEnableInsertGroupCommit()) && isInsertCommand;
if (e instanceof NereidsException
&& !context.getSessionVariable().enableFallbackToOriginalPlanner
&& !isGroupCommit) {
LOG.warn("Analyze failed. {}", context.getQueryIdentifier(), e);
context.getState().setError(e.getMessage());
return;
Expand Down
8 changes: 4 additions & 4 deletions fe/fe-core/src/test/java/org/apache/doris/catalog/CreateFunctionTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -82,7 +82,7 @@ public void test() throws Exception {
createTable("create table db1.tbl1(k1 int, k2 bigint, k3 varchar(10), k4 char(5)) duplicate key(k1) "
+ "distributed by hash(k2) buckets 1 properties('replication_num' = '1');");

dorisAssert = new DorisAssert();
dorisAssert = new DorisAssert(ctx);
dorisAssert.useDatabase("db1");

Database db = Env.getCurrentInternalCatalog().getDbNullable("db1");
Expand Down Expand Up @@ -115,7 +115,7 @@ public void test() throws Exception {

queryStr = "select db1.id_masking(k1) from db1.tbl1";
Assert.assertTrue(containsIgnoreCase(dorisAssert.query(queryStr).explainQuery(),
"concat(left(CAST(CAST(k1 AS BIGINT) AS VARCHAR(65533)), 3), '****', right(CAST(CAST(k1 AS BIGINT) AS VARCHAR(65533)), 4))"));
"concat(left(CAST(`k1` AS VARCHAR(65533)), 3), '****', right(CAST(`k1` AS VARCHAR(65533)), 4))"));

// create alias function with cast
// cast any type to decimal with specific precision and scale
Expand Down Expand Up @@ -219,7 +219,7 @@ public void testCreateGlobalFunction() throws Exception {
createTable("create table db2.tbl1(k1 int, k2 bigint, k3 varchar(10), k4 char(5)) duplicate key(k1) "
+ "distributed by hash(k2) buckets 1 properties('replication_num' = '1');");

dorisAssert = new DorisAssert();
dorisAssert = new DorisAssert(ctx);
dorisAssert.useDatabase("db2");

Database db = Env.getCurrentInternalCatalog().getDbNullable("db2");
Expand All @@ -241,7 +241,7 @@ public void testCreateGlobalFunction() throws Exception {

queryStr = "select id_masking(k1) from db2.tbl1";
Assert.assertTrue(containsIgnoreCase(dorisAssert.query(queryStr).explainQuery(),
"concat(left(CAST(CAST(k1 AS BIGINT) AS VARCHAR(65533)), 3), '****', right(CAST(CAST(k1 AS BIGINT) AS VARCHAR(65533)), 4))"));
"concat(left(CAST(`k1` AS VARCHAR(65533)), 3), '****', right(CAST(`k1` AS VARCHAR(65533)), 4))"));

// 4. create alias function with cast
// cast any type to decimal with specific precision and scale
Expand Down
30 changes: 14 additions & 16 deletions regression-test/suites/external_table_p0/jdbc/test_mysql_jdbc_catalog.groovy
View file
Original file line number Diff line number Diff line change
Expand Up @@ -415,93 +415,91 @@ suite("test_mysql_jdbc_catalog", "p0,external,mysql,external_docker,external_doc
}
sql """ set enable_ext_func_pred_pushdown = "true"; """
// test date_add
sql """ set disable_nereids_rules='NORMALIZE_REWRITE_RULES'; """
order_qt_date_add_year """ select * from test_zd where date_add(d_z,interval 1 year) = '2023-01-01' order by 1; """
explain {
sql("select * from test_zd where date_add(d_z,interval 1 year) = '2023-01-01' order by 1;")

contains " QUERY: SELECT `id`, `d_z` FROM `doris_test`.`test_zd` WHERE (date_add(`d_z`, INTERVAL 1 year) = '2023-01-01')"
contains " QUERY: SELECT `id`, `d_z` FROM `doris_test`.`test_zd` WHERE (`d_z` = '2022-01-01')"
}
order_qt_date_add_month """ select * from test_zd where date_add(d_z,interval 1 month) = '2022-02-01' order by 1; """
explain {
sql("select * from test_zd where date_add(d_z,interval 1 month) = '2022-02-01' order by 1;")

contains " QUERY: SELECT `id`, `d_z` FROM `doris_test`.`test_zd` WHERE (date_add(`d_z`, INTERVAL 1 month) = '2022-02-01')"
contains " QUERY: SELECT `id`, `d_z` FROM `doris_test`.`test_zd` WHERE (`d_z` = '2022-01-01')"
}
order_qt_date_add_week """ select * from test_zd where date_add(d_z,interval 1 week) = '2022-01-08' order by 1; """
explain {
sql("select * from test_zd where date_add(d_z,interval 1 week) = '2022-01-08' order by 1;")

contains " QUERY: SELECT `id`, `d_z` FROM `doris_test`.`test_zd` WHERE (date_add(`d_z`, INTERVAL 1 week) = '2022-01-08')"
contains " QUERY: SELECT `id`, `d_z` FROM `doris_test`.`test_zd` WHERE (`d_z` = '2022-01-01')"
}
order_qt_date_add_day """ select * from test_zd where date_add(d_z,interval 1 day) = '2022-01-02' order by 1; """
explain {
sql("select * from test_zd where date_add(d_z,interval 1 day) = '2022-01-02' order by 1;")

contains " QUERY: SELECT `id`, `d_z` FROM `doris_test`.`test_zd` WHERE (date_add(`d_z`, INTERVAL 1 day) = '2022-01-02')"
contains " QUERY: SELECT `id`, `d_z` FROM `doris_test`.`test_zd` WHERE (`d_z` = '2022-01-01')"
}
order_qt_date_add_hour """ select * from test_zd where date_add(d_z,interval 1 hour) = '2022-01-01 01:00:00' order by 1; """
explain {
sql("select * from test_zd where date_add(d_z,interval 1 hour) = '2022-01-01 01:00:00' order by 1;")

contains " QUERY: SELECT `id`, `d_z` FROM `doris_test`.`test_zd` WHERE (date_add(`d_z`, INTERVAL 1 hour) = '2022-01-01 01:00:00')"
contains " QUERY: SELECT `id`, `d_z` FROM `doris_test`.`test_zd` WHERE (`d_z` = '2022-01-01')"
}
order_qt_date_add_min """ select * from test_zd where date_add(d_z,interval 1 minute) = '2022-01-01 00:01:00' order by 1; """
explain {
sql("select * from test_zd where date_add(d_z,interval 1 minute) = '2022-01-01 00:01:00' order by 1;")

contains " QUERY: SELECT `id`, `d_z` FROM `doris_test`.`test_zd` WHERE (date_add(`d_z`, INTERVAL 1 minute) = '2022-01-01 00:01:00')"
contains " QUERY: SELECT `id`, `d_z` FROM `doris_test`.`test_zd` WHERE (`d_z` = '2022-01-01')"
}
order_qt_date_add_sec """ select * from test_zd where date_add(d_z,interval 1 second) = '2022-01-01 00:00:01' order by 1; """
explain {
sql("select * from test_zd where date_add(d_z,interval 1 second) = '2022-01-01 00:00:01' order by 1;")

contains " QUERY: SELECT `id`, `d_z` FROM `doris_test`.`test_zd` WHERE (date_add(`d_z`, INTERVAL 1 second) = '2022-01-01 00:00:01')"
contains " QUERY: SELECT `id`, `d_z` FROM `doris_test`.`test_zd` WHERE (`d_z` = '2022-01-01')"
}
// date_sub
order_qt_date_sub_year """ select * from test_zd where date_sub(d_z,interval 1 year) = '2021-01-01' order by 1; """
explain {
sql("select * from test_zd where date_sub(d_z,interval 1 year) = '2021-01-01' order by 1;")

contains " QUERY: SELECT `id`, `d_z` FROM `doris_test`.`test_zd` WHERE (date_sub(`d_z`, INTERVAL 1 year) = '2021-01-01')"
contains " QUERY: SELECT `id`, `d_z` FROM `doris_test`.`test_zd` WHERE (`d_z` = '2022-01-01')"
}
order_qt_date_sub_month """ select * from test_zd where date_sub(d_z,interval 1 month) = '2021-12-01' order by 1; """
explain {
sql("select * from test_zd where date_sub(d_z,interval 1 month) = '2021-12-01' order by 1;")

contains " QUERY: SELECT `id`, `d_z` FROM `doris_test`.`test_zd` WHERE (date_sub(`d_z`, INTERVAL 1 month) = '2021-12-01')"
contains " QUERY: SELECT `id`, `d_z` FROM `doris_test`.`test_zd` WHERE (`d_z` = '2022-01-01')"
}
order_qt_date_sub_week """ select * from test_zd where date_sub(d_z,interval 1 week) = '2021-12-25' order by 1; """
explain {
sql("select * from test_zd where date_sub(d_z,interval 1 week) = '2021-12-25' order by 1;")

contains " QUERY: SELECT `id`, `d_z` FROM `doris_test`.`test_zd` WHERE (date_sub(`d_z`, INTERVAL 1 week) = '2021-12-25')"
contains " QUERY: SELECT `id`, `d_z` FROM `doris_test`.`test_zd` WHERE (`d_z` = '2022-01-01')"
}
order_qt_date_sub_day """ select * from test_zd where date_sub(d_z,interval 1 day) = '2021-12-31' order by 1; """
explain {
sql("select * from test_zd where date_sub(d_z,interval 1 day) = '2021-12-31' order by 1;")

contains " QUERY: SELECT `id`, `d_z` FROM `doris_test`.`test_zd` WHERE (date_sub(`d_z`, INTERVAL 1 day) = '2021-12-31')"
contains " QUERY: SELECT `id`, `d_z` FROM `doris_test`.`test_zd` WHERE (`d_z` = '2022-01-01')"
}
order_qt_date_sub_hour """ select * from test_zd where date_sub(d_z,interval 1 hour) = '2021-12-31 23:00:00' order by 1; """
explain {
sql("select * from test_zd where date_sub(d_z,interval 1 hour) = '2021-12-31 23:00:00' order by 1;")

contains " QUERY: SELECT `id`, `d_z` FROM `doris_test`.`test_zd` WHERE (date_sub(`d_z`, INTERVAL 1 hour) = '2021-12-31 23:00:00')"
contains " QUERY: SELECT `id`, `d_z` FROM `doris_test`.`test_zd` WHERE (`d_z` = '2022-01-01')"
}
order_qt_date_sub_min """ select * from test_zd where date_sub(d_z,interval 1 minute) = '2021-12-31 23:59:00' order by 1; """
explain {
sql("select * from test_zd where date_sub(d_z,interval 1 minute) = '2021-12-31 23:59:00' order by 1;")

contains " QUERY: SELECT `id`, `d_z` FROM `doris_test`.`test_zd` WHERE (date_sub(`d_z`, INTERVAL 1 minute) = '2021-12-31 23:59:00')"
contains " QUERY: SELECT `id`, `d_z` FROM `doris_test`.`test_zd` WHERE (`d_z` = '2022-01-01')"
}
order_qt_date_sub_sec """ select * from test_zd where date_sub(d_z,interval 1 second) = '2021-12-31 23:59:59' order by 1; """
explain {
sql("select * from test_zd where date_sub(d_z,interval 1 second) = '2021-12-31 23:59:59' order by 1;")

contains " QUERY: SELECT `id`, `d_z` FROM `doris_test`.`test_zd` WHERE (date_sub(`d_z`, INTERVAL 1 second) = '2021-12-31 23:59:59')"
contains " QUERY: SELECT `id`, `d_z` FROM `doris_test`.`test_zd` WHERE (`d_z` = '2022-01-01')"
}
sql """ set disable_nereids_rules=''; """

sql """ drop catalog if exists mysql_fun_push_catalog; """

Expand Down
Loading

0 comments on commit 7737b83

Please sign in to comment.