Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

[chore](Nereids) disable fallback_to_original_planner by default #32882

Merged
merged 1 commit into from
Apr 17, 2024
Merged

[chore](Nereids) disable fallback_to_original_planner by default #32882

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
18 changes: 9 additions & 9 deletions ...core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateViewInfo.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -97,15 +97,6 @@ public CreateViewInfo(boolean ifNotExists, TableNameInfo viewName, String commen

/** init */
public void init(ConnectContext ctx) throws UserException {
analyzeAndFillRewriteSqlMap(querySql, ctx);
OutermostPlanFinderContext outermostPlanFinderContext = new OutermostPlanFinderContext();
analyzedPlan.accept(OutermostPlanFinder.INSTANCE, outermostPlanFinderContext);
List<Slot> outputs = outermostPlanFinderContext.outermostPlan.getOutput();
createFinalCols(outputs);
}

/**validate*/
public void validate(ConnectContext ctx) throws UserException {
viewName.analyze(ctx);
FeNameFormat.checkTableName(viewName.getTbl());
// disallow external catalog
Expand All @@ -116,6 +107,15 @@ public void validate(ConnectContext ctx) throws UserException {
ErrorReport.reportAnalysisException(ErrorCode.ERR_TABLE_ACCESS_DENIED_ERROR,
PrivPredicate.CREATE.getPrivs().toString(), viewName.getTbl());
}
analyzeAndFillRewriteSqlMap(querySql, ctx);
OutermostPlanFinderContext outermostPlanFinderContext = new OutermostPlanFinderContext();
analyzedPlan.accept(OutermostPlanFinder.INSTANCE, outermostPlanFinderContext);
List<Slot> outputs = outermostPlanFinderContext.outermostPlan.getOutput();
createFinalCols(outputs);
}

/**validate*/
public void validate(ConnectContext ctx) throws UserException {
NereidsPlanner planner = new NereidsPlanner(ctx.getStatementContext());
planner.plan(new UnboundResultSink<>(logicalQuery), PhysicalProperties.ANY, ExplainLevel.NONE);
Set<String> colSets = Sets.newTreeSet(String.CASE_INSENSITIVE_ORDER);
Expand Down
41 changes: 38 additions & 3 deletions fe/fe-core/src/main/java/org/apache/doris/qe/ConnectProcessor.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,9 @@

package org.apache.doris.qe;

import org.apache.doris.analysis.CreateTableAsSelectStmt;
import org.apache.doris.analysis.CreateTableStmt;
import org.apache.doris.analysis.DeleteStmt;
import org.apache.doris.analysis.ExplainOptions;
import org.apache.doris.analysis.InsertStmt;
import org.apache.doris.analysis.KillStmt;
Expand All @@ -25,6 +28,7 @@
import org.apache.doris.analysis.SqlParser;
import org.apache.doris.analysis.SqlScanner;
import org.apache.doris.analysis.StatementBase;
import org.apache.doris.analysis.UpdateStmt;
import org.apache.doris.analysis.UserIdentity;
import org.apache.doris.catalog.Column;
import org.apache.doris.catalog.DatabaseIf;
Expand Down Expand Up @@ -246,6 +250,7 @@ public void executeQuery(MysqlCommand mysqlCommand, String originStmt) throws Ex
&& CacheAnalyzer.canUseSqlCache(sessionVariable);
List<StatementBase> stmts = null;
Exception nereidsParseException = null;
Exception nereidsSyntaxException = null;
long parseSqlStartTime = System.currentTimeMillis();
List<StatementBase> cachedStmts = null;
// Nereids do not support prepare and execute now, so forbid prepare command, only process query command
Expand Down Expand Up @@ -273,13 +278,14 @@ public void executeQuery(MysqlCommand mysqlCommand, String originStmt) throws Ex
// Because ParseException means the sql is not supported by Nereids.
// It should be parsed by old parser, so not setting nereidsParseException to avoid
// suppressing the exception thrown by old parser.
nereidsParseException = e;
} catch (Exception e) {
// TODO: We should catch all exception here until we support all query syntax.
if (LOG.isDebugEnabled()) {
LOG.debug("Nereids parse sql failed with other exception. Reason: {}. Statement: \"{}\".",
e.getMessage(), convertedStmt);
}
nereidsParseException = e;
nereidsSyntaxException = e;
}
}
}
Expand All @@ -292,15 +298,44 @@ public void executeQuery(MysqlCommand mysqlCommand, String originStmt) throws Ex
// if NereidsParser and oldParser both failed,
// prove is a new feature implemented only on the nereids,
// so an error message for the new nereids is thrown
if (nereidsParseException != null) {
throwable = nereidsParseException;
if (nereidsSyntaxException != null) {
throwable = nereidsSyntaxException;
}
// Parse sql failed, audit it and return
handleQueryException(throwable, convertedStmt, null, null);
return;
}
}

if (mysqlCommand == MysqlCommand.COM_QUERY
&& ctx.getSessionVariable().isEnableNereidsPlanner()
&& !ctx.getSessionVariable().enableFallbackToOriginalPlanner
&& stmts.stream().allMatch(s -> s instanceof QueryStmt
|| s instanceof InsertStmt
|| s instanceof UpdateStmt
|| s instanceof DeleteStmt
|| s instanceof CreateTableAsSelectStmt
|| s instanceof CreateTableStmt)) {
String errMsg;
Throwable exception = null;
if (nereidsParseException != null) {
errMsg = nereidsParseException.getMessage();
exception = nereidsParseException;
} else if (nereidsSyntaxException != null) {
errMsg = nereidsSyntaxException.getMessage();
exception = nereidsSyntaxException;
} else {
errMsg = "Nereids parse DQL failed. " + originStmt;
}
if (exception == null) {
exception = new AnalysisException(errMsg);
} else {
exception = new AnalysisException(errMsg, exception);
}
handleQueryException(exception, originStmt, null, null);
return;
}

List<String> origSingleStmtList = null;
// if stmts.size() > 1, split originStmt to multi singleStmts
if (stmts.size() > 1) {
Expand Down
2 changes: 1 addition & 1 deletion fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1262,7 +1262,7 @@ public void setEnableLeftZigZag(boolean enableLeftZigZag) {
// for nereids, fallback will cause the Doris return the correct result although the syntax is unsupported
// in nereids for some mistaken modification. You should set it on the
@VariableMgr.VarAttr(name = ENABLE_FALLBACK_TO_ORIGINAL_PLANNER, needForward = true)
public boolean enableFallbackToOriginalPlanner = true;
public boolean enableFallbackToOriginalPlanner = false;

@VariableMgr.VarAttr(name = ENABLE_NEREIDS_TIMEOUT, needForward = true)
public boolean enableNereidsTimeout = true;
Expand Down
16 changes: 5 additions & 11 deletions fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -586,23 +586,17 @@ public void execute(TUniqueId queryId) throws Exception {
}
// FIXME: Force fallback for:
// 1. group commit because nereids does not support it (see the following `isGroupCommit` variable)
// 2. insert into command because some nereids cases fail (including case1)
// Skip force fallback for:
// 1. Transaction insert because nereids support `insert into select` while legacy does not
// 2. Nereids support insert into external table while legacy does not
boolean isInsertCommand = parsedStmt != null
&& parsedStmt instanceof LogicalPlanAdapter
&& ((LogicalPlanAdapter) parsedStmt).getLogicalPlan() instanceof InsertIntoTableCommand;
/*boolean isGroupCommit = (Config.wait_internal_group_commit_finish
|| context.sessionVariable.isEnableInsertGroupCommit()) && isInsertCommand;*/
boolean isExternalTableInsert = false;
if (isInsertCommand) {
isExternalTableInsert = ((InsertIntoTableCommand) ((LogicalPlanAdapter) parsedStmt)
.getLogicalPlan()).isExternalTableSink();
}
boolean forceFallback = isInsertCommand && !isExternalTableInsert && !context.isTxnModel();
if (e instanceof NereidsException && !context.getSessionVariable().enableFallbackToOriginalPlanner
&& !forceFallback) {
boolean isGroupCommit = (Config.wait_internal_group_commit_finish
|| context.sessionVariable.isEnableInsertGroupCommit()) && isInsertCommand;
if (e instanceof NereidsException
&& !context.getSessionVariable().enableFallbackToOriginalPlanner
&& !isGroupCommit) {
LOG.warn("Analyze failed. {}", context.getQueryIdentifier(), e);
context.getState().setError(e.getMessage());
return;
Expand Down
8 changes: 4 additions & 4 deletions fe/fe-core/src/test/java/org/apache/doris/catalog/CreateFunctionTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -82,7 +82,7 @@ public void test() throws Exception {
createTable("create table db1.tbl1(k1 int, k2 bigint, k3 varchar(10), k4 char(5)) duplicate key(k1) "
+ "distributed by hash(k2) buckets 1 properties('replication_num' = '1');");

dorisAssert = new DorisAssert();
dorisAssert = new DorisAssert(ctx);
dorisAssert.useDatabase("db1");

Database db = Env.getCurrentInternalCatalog().getDbNullable("db1");
Expand Down Expand Up @@ -115,7 +115,7 @@ public void test() throws Exception {

queryStr = "select db1.id_masking(k1) from db1.tbl1";
Assert.assertTrue(containsIgnoreCase(dorisAssert.query(queryStr).explainQuery(),
"concat(left(CAST(CAST(k1 AS BIGINT) AS VARCHAR(65533)), 3), '****', right(CAST(CAST(k1 AS BIGINT) AS VARCHAR(65533)), 4))"));
"concat(left(CAST(`k1` AS VARCHAR(65533)), 3), '****', right(CAST(`k1` AS VARCHAR(65533)), 4))"));

// create alias function with cast
// cast any type to decimal with specific precision and scale
Expand Down Expand Up @@ -219,7 +219,7 @@ public void testCreateGlobalFunction() throws Exception {
createTable("create table db2.tbl1(k1 int, k2 bigint, k3 varchar(10), k4 char(5)) duplicate key(k1) "
+ "distributed by hash(k2) buckets 1 properties('replication_num' = '1');");

dorisAssert = new DorisAssert();
dorisAssert = new DorisAssert(ctx);
dorisAssert.useDatabase("db2");

Database db = Env.getCurrentInternalCatalog().getDbNullable("db2");
Expand All @@ -241,7 +241,7 @@ public void testCreateGlobalFunction() throws Exception {

queryStr = "select id_masking(k1) from db2.tbl1";
Assert.assertTrue(containsIgnoreCase(dorisAssert.query(queryStr).explainQuery(),
"concat(left(CAST(CAST(k1 AS BIGINT) AS VARCHAR(65533)), 3), '****', right(CAST(CAST(k1 AS BIGINT) AS VARCHAR(65533)), 4))"));
"concat(left(CAST(`k1` AS VARCHAR(65533)), 3), '****', right(CAST(`k1` AS VARCHAR(65533)), 4))"));

// 4. create alias function with cast
// cast any type to decimal with specific precision and scale
Expand Down
30 changes: 14 additions & 16 deletions regression-test/suites/external_table_p0/jdbc/test_mysql_jdbc_catalog.groovy
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -415,93 +415,91 @@ suite("test_mysql_jdbc_catalog", "p0,external,mysql,external_docker,external_doc
}
sql """ set enable_ext_func_pred_pushdown = "true"; """
// test date_add
sql """ set disable_nereids_rules='NORMALIZE_REWRITE_RULES'; """
order_qt_date_add_year """ select * from test_zd where date_add(d_z,interval 1 year) = '2023-01-01' order by 1; """
explain {
sql("select * from test_zd where date_add(d_z,interval 1 year) = '2023-01-01' order by 1;")

contains " QUERY: SELECT `id`, `d_z` FROM `doris_test`.`test_zd` WHERE (date_add(`d_z`, INTERVAL 1 year) = '2023-01-01')"
contains " QUERY: SELECT `id`, `d_z` FROM `doris_test`.`test_zd` WHERE (`d_z` = '2022-01-01')"
}
order_qt_date_add_month """ select * from test_zd where date_add(d_z,interval 1 month) = '2022-02-01' order by 1; """
explain {
sql("select * from test_zd where date_add(d_z,interval 1 month) = '2022-02-01' order by 1;")

contains " QUERY: SELECT `id`, `d_z` FROM `doris_test`.`test_zd` WHERE (date_add(`d_z`, INTERVAL 1 month) = '2022-02-01')"
contains " QUERY: SELECT `id`, `d_z` FROM `doris_test`.`test_zd` WHERE (`d_z` = '2022-01-01')"
}
order_qt_date_add_week """ select * from test_zd where date_add(d_z,interval 1 week) = '2022-01-08' order by 1; """
explain {
sql("select * from test_zd where date_add(d_z,interval 1 week) = '2022-01-08' order by 1;")

contains " QUERY: SELECT `id`, `d_z` FROM `doris_test`.`test_zd` WHERE (date_add(`d_z`, INTERVAL 1 week) = '2022-01-08')"
contains " QUERY: SELECT `id`, `d_z` FROM `doris_test`.`test_zd` WHERE (`d_z` = '2022-01-01')"
}
order_qt_date_add_day """ select * from test_zd where date_add(d_z,interval 1 day) = '2022-01-02' order by 1; """
explain {
sql("select * from test_zd where date_add(d_z,interval 1 day) = '2022-01-02' order by 1;")

contains " QUERY: SELECT `id`, `d_z` FROM `doris_test`.`test_zd` WHERE (date_add(`d_z`, INTERVAL 1 day) = '2022-01-02')"
contains " QUERY: SELECT `id`, `d_z` FROM `doris_test`.`test_zd` WHERE (`d_z` = '2022-01-01')"
}
order_qt_date_add_hour """ select * from test_zd where date_add(d_z,interval 1 hour) = '2022-01-01 01:00:00' order by 1; """
explain {
sql("select * from test_zd where date_add(d_z,interval 1 hour) = '2022-01-01 01:00:00' order by 1;")

contains " QUERY: SELECT `id`, `d_z` FROM `doris_test`.`test_zd` WHERE (date_add(`d_z`, INTERVAL 1 hour) = '2022-01-01 01:00:00')"
contains " QUERY: SELECT `id`, `d_z` FROM `doris_test`.`test_zd` WHERE (`d_z` = '2022-01-01')"
}
order_qt_date_add_min """ select * from test_zd where date_add(d_z,interval 1 minute) = '2022-01-01 00:01:00' order by 1; """
explain {
sql("select * from test_zd where date_add(d_z,interval 1 minute) = '2022-01-01 00:01:00' order by 1;")

contains " QUERY: SELECT `id`, `d_z` FROM `doris_test`.`test_zd` WHERE (date_add(`d_z`, INTERVAL 1 minute) = '2022-01-01 00:01:00')"
contains " QUERY: SELECT `id`, `d_z` FROM `doris_test`.`test_zd` WHERE (`d_z` = '2022-01-01')"
}
order_qt_date_add_sec """ select * from test_zd where date_add(d_z,interval 1 second) = '2022-01-01 00:00:01' order by 1; """
explain {
sql("select * from test_zd where date_add(d_z,interval 1 second) = '2022-01-01 00:00:01' order by 1;")

contains " QUERY: SELECT `id`, `d_z` FROM `doris_test`.`test_zd` WHERE (date_add(`d_z`, INTERVAL 1 second) = '2022-01-01 00:00:01')"
contains " QUERY: SELECT `id`, `d_z` FROM `doris_test`.`test_zd` WHERE (`d_z` = '2022-01-01')"
}
// date_sub
order_qt_date_sub_year """ select * from test_zd where date_sub(d_z,interval 1 year) = '2021-01-01' order by 1; """
explain {
sql("select * from test_zd where date_sub(d_z,interval 1 year) = '2021-01-01' order by 1;")

contains " QUERY: SELECT `id`, `d_z` FROM `doris_test`.`test_zd` WHERE (date_sub(`d_z`, INTERVAL 1 year) = '2021-01-01')"
contains " QUERY: SELECT `id`, `d_z` FROM `doris_test`.`test_zd` WHERE (`d_z` = '2022-01-01')"
}
order_qt_date_sub_month """ select * from test_zd where date_sub(d_z,interval 1 month) = '2021-12-01' order by 1; """
explain {
sql("select * from test_zd where date_sub(d_z,interval 1 month) = '2021-12-01' order by 1;")

contains " QUERY: SELECT `id`, `d_z` FROM `doris_test`.`test_zd` WHERE (date_sub(`d_z`, INTERVAL 1 month) = '2021-12-01')"
contains " QUERY: SELECT `id`, `d_z` FROM `doris_test`.`test_zd` WHERE (`d_z` = '2022-01-01')"
}
order_qt_date_sub_week """ select * from test_zd where date_sub(d_z,interval 1 week) = '2021-12-25' order by 1; """
explain {
sql("select * from test_zd where date_sub(d_z,interval 1 week) = '2021-12-25' order by 1;")

contains " QUERY: SELECT `id`, `d_z` FROM `doris_test`.`test_zd` WHERE (date_sub(`d_z`, INTERVAL 1 week) = '2021-12-25')"
contains " QUERY: SELECT `id`, `d_z` FROM `doris_test`.`test_zd` WHERE (`d_z` = '2022-01-01')"
}
order_qt_date_sub_day """ select * from test_zd where date_sub(d_z,interval 1 day) = '2021-12-31' order by 1; """
explain {
sql("select * from test_zd where date_sub(d_z,interval 1 day) = '2021-12-31' order by 1;")

contains " QUERY: SELECT `id`, `d_z` FROM `doris_test`.`test_zd` WHERE (date_sub(`d_z`, INTERVAL 1 day) = '2021-12-31')"
contains " QUERY: SELECT `id`, `d_z` FROM `doris_test`.`test_zd` WHERE (`d_z` = '2022-01-01')"
}
order_qt_date_sub_hour """ select * from test_zd where date_sub(d_z,interval 1 hour) = '2021-12-31 23:00:00' order by 1; """
explain {
sql("select * from test_zd where date_sub(d_z,interval 1 hour) = '2021-12-31 23:00:00' order by 1;")

contains " QUERY: SELECT `id`, `d_z` FROM `doris_test`.`test_zd` WHERE (date_sub(`d_z`, INTERVAL 1 hour) = '2021-12-31 23:00:00')"
contains " QUERY: SELECT `id`, `d_z` FROM `doris_test`.`test_zd` WHERE (`d_z` = '2022-01-01')"
}
order_qt_date_sub_min """ select * from test_zd where date_sub(d_z,interval 1 minute) = '2021-12-31 23:59:00' order by 1; """
explain {
sql("select * from test_zd where date_sub(d_z,interval 1 minute) = '2021-12-31 23:59:00' order by 1;")

contains " QUERY: SELECT `id`, `d_z` FROM `doris_test`.`test_zd` WHERE (date_sub(`d_z`, INTERVAL 1 minute) = '2021-12-31 23:59:00')"
contains " QUERY: SELECT `id`, `d_z` FROM `doris_test`.`test_zd` WHERE (`d_z` = '2022-01-01')"
}
order_qt_date_sub_sec """ select * from test_zd where date_sub(d_z,interval 1 second) = '2021-12-31 23:59:59' order by 1; """
explain {
sql("select * from test_zd where date_sub(d_z,interval 1 second) = '2021-12-31 23:59:59' order by 1;")

contains " QUERY: SELECT `id`, `d_z` FROM `doris_test`.`test_zd` WHERE (date_sub(`d_z`, INTERVAL 1 second) = '2021-12-31 23:59:59')"
contains " QUERY: SELECT `id`, `d_z` FROM `doris_test`.`test_zd` WHERE (`d_z` = '2022-01-01')"
}
sql """ set disable_nereids_rules=''; """

sql """ drop catalog if exists mysql_fun_push_catalog; """

Expand Down
Loading
Loading