Skip to content

Commit

Permalink
[Improve)(Variant) do not allow fall back to legacy planner (#30430)
Browse files Browse the repository at this point in the history
  • Loading branch information
eldenmoon authored Jan 29, 2024
1 parent 26c6ac6 commit 732194d
Show file tree
Hide file tree
Showing 270 changed files with 601 additions and 624 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -2499,6 +2499,9 @@ public static boolean isNotCloudMode() {

@ConfField
public static String cloud_sql_server_cluster_id = "RESERVED_CLUSTER_ID_FOR_SQL_SERVER";

@ConfField(mutable = true)
public static boolean enable_variant_access_in_original_planner = false;
//==========================================================================
// end of cloud config
//==========================================================================
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -1029,6 +1029,11 @@ public SlotDescriptor registerColumnRef(TableName tblName, String colName, List<

LOG.debug("register column ref table {}, colName {}, col {}", tblName, colName, col.toSql());
if (col.getType().isVariantType() || (subColNames != null && !subColNames.isEmpty())) {
if (!Config.enable_variant_access_in_original_planner
&& (subColNames != null && !subColNames.isEmpty())) {
ErrorReport.reportAnalysisException("Variant sub-column access is disabled in original planner,"
+ "set enable_variant_access_in_original_planner = true in session variable");
}
if (!col.getType().isVariantType()) {
ErrorReport.reportAnalysisException(ErrorCode.ERR_ILLEGAL_COLUMN_REFERENCE_ERROR,
Joiner.on(".").join(tblName.getTbl(), colName));
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -186,7 +186,7 @@ public Slot getRewrittenSlotRefByOriginalExpr(Expression originalExpr) {
* Add a slot ref attached with paths in context to avoid duplicated slot
*/
public void addPathSlotRef(Slot root, List<String> paths, SlotReference slotRef, Expression originalExpr) {
Comparator<List<String>> pathsComparator = new Comparator<List<String>>() {
subColumnSlotRefMap.computeIfAbsent(root, k -> Maps.newTreeMap(new Comparator<List<String>>() {
@Override
public int compare(List<String> lst1, List<String> lst2) {
Iterator<String> it1 = lst1.iterator();
Expand All @@ -199,8 +199,7 @@ public int compare(List<String> lst1, List<String> lst2) {
}
return Integer.compare(lst1.size(), lst2.size());
}
};
subColumnSlotRefMap.computeIfAbsent(root, k -> Maps.newTreeMap(pathsComparator));
}));
subColumnSlotRefMap.get(root).put(paths, slotRef);
subColumnOriginalExprMap.put(slotRef, originalExpr);
originalExprToRewrittenSubColumn.put(originalExpr, slotRef);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -219,6 +219,7 @@ public Expr visitMatch(Match match, PlanTranslatorContext context) {
String invertedIndexParser = InvertedIndexUtil.INVERTED_INDEX_PARSER_UNKNOWN;
String invertedIndexParserMode = InvertedIndexUtil.INVERTED_INDEX_PARSER_COARSE_GRANULARITY;
Map<String, String> invertedIndexCharFilter = new HashMap<>();
// Get the first slot from match's left expr
SlotRef left = (SlotRef) match.left().getInputSlots().stream().findFirst().get().accept(this, context);
OlapTable olapTbl = Optional.ofNullable(getOlapTableFromSlotDesc(left.getDesc()))
.orElse(getOlapTableDirectly(left));
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,7 @@ public Plan visitPhysicalFilter(PhysicalFilter<? extends Plan> filter, CascadesC
}

PhysicalProject<? extends Plan> project = (PhysicalProject<? extends Plan>) child;
if (project.isPulledUpProjectFromScan()) {
if (project.hasPushedDownToProjectionFunctions()) {
// ignore project which is pulled up from LogicalOlapScan
return filter;
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -59,7 +59,7 @@ public Plan visitPhysicalFilter(PhysicalFilter<? extends Plan> filter, CascadesC

Plan child = filter.child();
// Forbidden filter-project, we must make filter-project -> project-filter.
if (child instanceof PhysicalProject && !((PhysicalProject<?>) child).isPulledUpProjectFromScan()) {
if (child instanceof PhysicalProject && !((PhysicalProject<?>) child).hasPushedDownToProjectionFunctions()) {
throw new AnalysisException(
"Nereids generate a filter-project plan, but backend not support:\n" + filter.treeString());
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -49,10 +49,9 @@ public enum RuleType {
BINDING_UNBOUND_TVF_RELATION_FUNCTION(RuleTypeClass.REWRITE),
BINDING_SET_OPERATION_SLOT(RuleTypeClass.REWRITE),
BINDING_INLINE_TABLE_SLOT(RuleTypeClass.REWRITE),
COUNT_LITERAL_REWRITE(RuleTypeClass.REWRITE),
BINDING_SLOT_WITH_PATHS_PROJECT(RuleTypeClass.REWRITE),

BINDING_SLOT_WITH_PATHS_SCAN(RuleTypeClass.REWRITE),
COUNT_LITERAL_REWRITE(RuleTypeClass.REWRITE),

REPLACE_SORT_EXPRESSION_BY_CHILD_OUTPUT(RuleTypeClass.REWRITE),

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -184,7 +184,7 @@ private void checkMatchIsUsedCorrectly(Plan plan) {
if (plan instanceof LogicalFilter && (plan.child(0) instanceof LogicalOlapScan
|| plan.child(0) instanceof LogicalDeferMaterializeOlapScan
|| plan.child(0) instanceof LogicalProject
&& ((LogicalProject<?>) plan.child(0)).isPulledUpProjectFromScan())) {
&& ((LogicalProject<?>) plan.child(0)).hasPushedDownToProjectionFunctions())) {
return;
} else {
throw new AnalysisException(String.format(
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,7 @@ public class PullUpProjectUnderTopN extends OneRewriteRuleFactory {
@Override
public Rule build() {
return logicalTopN(logicalProject().whenNot(p -> p.isAllSlots()))
.whenNot(topN -> topN.child().hasPushedDownToProjectionFunctions())
.then(topN -> {
LogicalProject<Plan> project = topN.child();
Set<Slot> outputSet = project.child().getOutputSet();
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -44,15 +44,15 @@ public List<Rule> buildRules() {
logicalFilter(logicalProject())
.whenNot(filter -> filter.child().getProjects().stream().anyMatch(
expr -> expr.anyMatch(WindowExpression.class::isInstance)))
.whenNot(filter -> filter.child().isPulledUpProjectFromScan())
.whenNot(filter -> filter.child().hasPushedDownToProjectionFunctions())
.then(PushDownFilterThroughProject::pushdownFilterThroughProject)
.toRule(RuleType.PUSH_DOWN_FILTER_THROUGH_PROJECT),
// filter(project(limit)) will change to filter(limit(project)) by PushdownProjectThroughLimit,
// then we should change filter(limit(project)) to project(filter(limit))
logicalFilter(logicalLimit(logicalProject()))
.whenNot(filter -> filter.child().child().getProjects().stream()
.anyMatch(expr -> expr.anyMatch(WindowExpression.class::isInstance)))
.whenNot(filter -> filter.child().child().isPulledUpProjectFromScan())
.whenNot(filter -> filter.child().child().hasPushedDownToProjectionFunctions())
.then(filter -> {
LogicalLimit<LogicalProject<Plan>> limit = filter.child();
LogicalProject<Plan> project = limit.child();
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -19,13 +19,22 @@

import org.apache.doris.nereids.trees.expressions.Expression;

import java.util.stream.Collectors;

/**
* Function that could be rewritten and pushed down to projection
*/
public interface PushDownToProjectionFunction {
// check if specified function could be pushed down to project
/**
* check if specified function could be pushed down to project
* @param pushDownExpr expr to check
* @return if it is valid to push down input expr
*/
static boolean validToPushDown(Expression pushDownExpr) {
// Currently only Variant type could be pushed down
return pushDownExpr instanceof PushDownToProjectionFunction && pushDownExpr.getDataType().isVariantType();
// Currently only element at for variant type could be pushed down
return !pushDownExpr.collectToList(
PushDownToProjectionFunction.class::isInstance).stream().filter(
x -> ((Expression) x).getDataType().isVariantType()).collect(
Collectors.toList()).isEmpty();
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -71,8 +71,9 @@ default List<NamedExpression> mergeProjections(Project childProject) {
/**
* Check if it is a project that is pull up from scan in analyze rule
* e.g. BindSlotWithPaths
* And check if contains PushDownToProjectionFunction that can pushed down to project
*/
default boolean isPulledUpProjectFromScan() {
default boolean hasPushedDownToProjectionFunctions() {
return ConnectContext.get() != null
&& ConnectContext.get().getSessionVariable() != null
&& ConnectContext.get().getSessionVariable().isEnableRewriteElementAtToSlot()
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -47,7 +47,7 @@ public boolean acceptsType(DataType other) {

@Override
public String simpleString() {
return "map";
return "variant";
}

@Override
Expand Down
2 changes: 1 addition & 1 deletion regression-test/data/variant_p0/column_name.out
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
-- This file is automatically generated. You should know what you did if you want to edit this
-- !sql --
中文 \N
中文 unicode

-- !sql --
""
Expand Down
Original file line number Diff line number Diff line change
@@ -1,14 +1,14 @@
SELECT
cast(v:repo.name as string),
cast(v["repo"]["name"] as string),
count() AS prs,
count(distinct cast(v:actor.login as string)) AS authors
count(distinct cast(v["actor"]["login"] as string)) AS authors
FROM github_events
WHERE (cast(v:type as string) = 'PullRequestEvent') AND (cast(v:payload.action as string) = 'opened') AND (cast(v:actor.login as string) IN
WHERE (cast(v["type"] as string) = 'PullRequestEvent') AND (cast(v["payload"]["action"] as string) = 'opened') AND (cast(v["actor"]["login"] as string) IN
(
SELECT cast(v:actor.login as string)
SELECT cast(v["actor"]["login"] as string)
FROM github_events
WHERE (cast(v:type as string) = 'PullRequestEvent') AND (cast(v:payload.action as string)= 'opened') AND (cast(v:repo.name as string) IN ('rspec/rspec-core', 'golden-warning/giraffedraft-server', 'apache/spark'))
)) AND (lower(cast(v:repo.name as string)) NOT LIKE '%clickhouse%')
GROUP BY cast(v:repo.name as string)
ORDER BY authors DESC, prs DESC, cast(v:repo.name as string) DESC
WHERE (cast(v["type"] as string) = 'PullRequestEvent') AND (cast(v["payload"]["action"] as string)= 'opened') AND (cast(v["repo"]["name"] as string) IN ('rspec/rspec-core', 'golden-warning/giraffedraft-server', 'apache/spark'))
)) AND (lower(cast(v["repo"]["name"] as string)) NOT LIKE '%clickhouse%')
GROUP BY cast(v["repo"]["name"] as string)
ORDER BY authors DESC, prs DESC, cast(v["repo"]["name"] as string) DESC
LIMIT 50
Original file line number Diff line number Diff line change
@@ -1,14 +1,14 @@
SELECT
cast(v:repo.name as string),
cast(v["repo"]["name"] as string),
count() AS prs,
count(distinct cast(v:actor.login as string)) AS authors
count(distinct cast(v["actor"]["login"] as string)) AS authors
FROM github_events
WHERE (cast(v:type as string) = 'IssuesEvent') AND (cast(v:payload.action as string) = 'opened') AND (cast(v:actor.login as string) IN
WHERE (cast(v["type"] as string) = 'IssuesEvent') AND (cast(v["payload"]["action"] as string) = 'opened') AND (cast(v["actor"]["login"] as string) IN
(
SELECT cast(v:actor.login as string)
SELECT cast(v["actor"]["login"] as string)
FROM github_events
WHERE (cast(v:type as string) = 'IssuesEvent') AND (cast(v:payload.action as string) = 'opened') AND (cast(v:repo.name as string) IN ('No-CQRT/GooGuns', 'ivolunteerph/ivolunteerph', 'Tribler/tribler'))
)) AND (lower(cast(v:repo.name as string)) NOT LIKE '%clickhouse%')
GROUP BY cast(v:repo.name as string)
ORDER BY authors DESC, prs DESC, cast(v:repo.name as string) ASC
WHERE (cast(v["type"] as string) = 'IssuesEvent') AND (cast(v["payload"]["action"] as string) = 'opened') AND (cast(v["repo"]["name"] as string) IN ('No-CQRT/GooGuns', 'ivolunteerph/ivolunteerph', 'Tribler/tribler'))
)) AND (lower(cast(v["repo"]["name"] as string)) NOT LIKE '%clickhouse%')
GROUP BY cast(v["repo"]["name"] as string)
ORDER BY authors DESC, prs DESC, cast(v["repo"]["name"] as string) ASC
LIMIT 50
Original file line number Diff line number Diff line change
@@ -1,9 +1,9 @@
SELECT
cast(v:actor.login as string),
cast(v["actor"]["login"] as string),
count() AS c,
count(distinct cast(v:repo.name as string)) AS repos
count(distinct cast(v["repo"]["name"] as string)) AS repos
FROM github_events
WHERE cast(v:type as string) = 'PushEvent'
GROUP BY cast(v:actor.login as string)
WHERE cast(v["type"] as string) = 'PushEvent'
GROUP BY cast(v["actor"]["login"] as string)
ORDER BY c DESC, 1, 3
LIMIT 50
Original file line number Diff line number Diff line change
@@ -1 +1 @@
SELECT count() FROM github_events WHERE cast(v:type as string) = 'WatchEvent'
SELECT count() FROM github_events WHERE cast(v["type"] as string) = 'WatchEvent'
Original file line number Diff line number Diff line change
@@ -1 +1 @@
SELECT cast(v:payload.action as string), count() FROM github_events WHERE cast(v:type as string) = 'WatchEvent' GROUP BY cast(v:payload.action as string)
SELECT cast(v["payload"]["action"] as string), count() FROM github_events WHERE cast(v["type"] as string) = 'WatchEvent' GROUP BY cast(v["payload"]["action"] as string)
Original file line number Diff line number Diff line change
@@ -1 +1 @@
SELECT count() FROM github_events WHERE cast(v:type as string) = 'WatchEvent' AND cast(v:repo.name as string) IN ('apache/spark', 'GunZi200/Memory-Colour', 'isohuntto/openbay', 'wasabeef/awesome-android-ui') GROUP BY cast(v:payload.action as string)
SELECT count() FROM github_events WHERE cast(v["type"] as string) = 'WatchEvent' AND cast(v["repo"]["name"] as string) IN ('apache/spark', 'GunZi200/Memory-Colour', 'isohuntto/openbay', 'wasabeef/awesome-android-ui') GROUP BY cast(v["payload"]["action"] as string)
Original file line number Diff line number Diff line change
Expand Up @@ -4,11 +4,11 @@ SELECT
FROM
(
SELECT
cast(v:repo.name as string) as k,
cast(v["repo"]["name"] as string) as k,
count() AS c
FROM github_events
WHERE cast(v:type as string) = 'WatchEvent'
GROUP BY cast(v:repo.name as string)
WHERE cast(v["type"] as string) = 'WatchEvent'
GROUP BY cast(v["repo"]["name"] as string)
) t
GROUP BY stars
ORDER BY stars ASC
Original file line number Diff line number Diff line change
@@ -1 +1 @@
SELECT cast(v:repo.name as string) FROM github_events WHERE cast(v:type as string) = 'WatchEvent' ORDER BY cast(v:created_at as datetime), cast(v:repo.name as string) LIMIT 50
SELECT cast(v["repo"]["name"] as string) FROM github_events WHERE cast(v["type"] as string) = 'WatchEvent' ORDER BY cast(v["created_at"] as datetime), cast(v["repo"]["name"] as string) LIMIT 50
Original file line number Diff line number Diff line change
@@ -1 +1 @@
SELECT cast(v:repo.name as string), count() AS stars FROM github_events WHERE cast(v:type as string) = 'WatchEvent' AND year(cast(v:created_at as datetime)) = '2015' GROUP BY cast(v:repo.name as string) ORDER BY stars DESC, 1 LIMIT 50
SELECT cast(v["repo"]["name"] as string), count() AS stars FROM github_events WHERE cast(v["type"] as string) = 'WatchEvent' AND year(cast(v["created_at"] as datetime)) = '2015' GROUP BY cast(v["repo"]["name"] as string) ORDER BY stars DESC, 1 LIMIT 50
Original file line number Diff line number Diff line change
@@ -1 +1 @@
SELECT cast(v:repo.name as string), count() AS stars FROM github_events WHERE cast(v:type as string) = 'WatchEvent' AND year(cast(v:created_at as datetime)) = '2015' GROUP BY cast(v:repo.name as string) ORDER BY stars DESC, 1 LIMIT 50
SELECT cast(v["repo"]["name"] as string), count() AS stars FROM github_events WHERE cast(v["type"] as string) = 'WatchEvent' AND year(cast(v["created_at"] as datetime)) = '2015' GROUP BY cast(v["repo"]["name"] as string) ORDER BY stars DESC, 1 LIMIT 50
Original file line number Diff line number Diff line change
@@ -1 +1 @@
SELECT cast(v:repo.name as string), count() AS stars FROM github_events WHERE cast(v:type as string) = 'WatchEvent' AND year(cast(v:created_at as datetime)) = '2015' GROUP BY cast(v:repo.name as string) ORDER BY stars, cast(v:repo.name as string) DESC LIMIT 50
SELECT cast(v["repo"]["name"] as string), count() AS stars FROM github_events WHERE cast(v["type"] as string) = 'WatchEvent' AND year(cast(v["created_at"] as datetime)) = '2015' GROUP BY cast(v["repo"]["name"] as string) ORDER BY stars, cast(v["repo"]["name"] as string) DESC LIMIT 50
Original file line number Diff line number Diff line change
@@ -1 +1 @@
SELECT cast(v:repo.name as string), count() AS stars FROM github_events WHERE cast(v:type as string) = 'WatchEvent' AND year(cast(v:created_at as datetime)) = '2015' GROUP BY cast(v:repo.name as string) ORDER BY stars DESC, 1 LIMIT 50
SELECT cast(v["repo"]["name"] as string), count() AS stars FROM github_events WHERE cast(v["type"] as string) = 'WatchEvent' AND year(cast(v["created_at"] as datetime)) = '2015' GROUP BY cast(v["repo"]["name"] as string) ORDER BY stars DESC, 1 LIMIT 50
Original file line number Diff line number Diff line change
@@ -1 +1 @@
SELECT cast(v:repo.name as string), count() AS stars FROM github_events WHERE cast(v:type as string) = 'WatchEvent' AND year(cast(v:created_at as datetime)) = '2015' GROUP BY cast(v:repo.name as string) ORDER BY stars DESC, 1 LIMIT 50
SELECT cast(v["repo"]["name"] as string), count() AS stars FROM github_events WHERE cast(v["type"] as string) = 'WatchEvent' AND year(cast(v["created_at"] as datetime)) = '2015' GROUP BY cast(v["repo"]["name"] as string) ORDER BY stars DESC, 1 LIMIT 50
Original file line number Diff line number Diff line change
@@ -1 +1 @@
SELECT cast(v:repo.name as string), count() AS stars FROM github_events WHERE cast(v:type as string) = 'WatchEvent' AND year(cast(v:created_at as datetime)) = '2015' GROUP BY cast(v:repo.name as string) ORDER BY stars DESC, 1 LIMIT 50
SELECT cast(v["repo"]["name"] as string), count() AS stars FROM github_events WHERE cast(v["type"] as string) = 'WatchEvent' AND year(cast(v["created_at"] as datetime)) = '2015' GROUP BY cast(v["repo"]["name"] as string) ORDER BY stars DESC, 1 LIMIT 50
Original file line number Diff line number Diff line change
Expand Up @@ -13,11 +13,11 @@
-- FROM
-- (
-- SELECT
-- lower(cast(v:repo.name as string)) AS repo,
-- year(cast(v:created_at as datetime)) AS year,
-- lower(cast(v["repo"]["name"] as string)) AS repo,
-- year(cast(v["created_at"] as datetime)) AS year,
-- count() AS cnt
-- FROM github_events
-- WHERE (cast(v:type as string) = 'WatchEvent') AND (year(cast(v:created_at as datetime)) >= 2015)
-- WHERE (cast(v["type"] as string) = 'WatchEvent') AND (year(cast(v["created_at"] as datetime)) >= 2015)
-- GROUP BY
-- repo,
-- year
Expand Down
Original file line number Diff line number Diff line change
@@ -1,2 +1,2 @@
SELECT year(cast(v:created_at as datetime)) AS year, count() AS stars FROM github_events WHERE cast(v:type as string) = 'WatchEvent' GROUP BY year ORDER BY year
SELECT year(cast(v["created_at"] as datetime)) AS year, count() AS stars FROM github_events WHERE cast(v["type"] as string) = 'WatchEvent' GROUP BY year ORDER BY year

Original file line number Diff line number Diff line change
@@ -1 +1 @@
SELECT count() FROM github_events WHERE cast(v:type as string) = 'IssueCommentEvent'
SELECT count() FROM github_events WHERE cast(v["type"] as string) = 'IssueCommentEvent'
Original file line number Diff line number Diff line change
@@ -1 +1 @@
SELECT cast(v:repo.name as string), count() FROM github_events WHERE cast(v:type as string) = 'IssueCommentEvent' GROUP BY cast(v:repo.name as string) ORDER BY count() DESC, 1 LIMIT 50
SELECT cast(v["repo"]["name"] as string), count() FROM github_events WHERE cast(v["type"] as string) = 'IssueCommentEvent' GROUP BY cast(v["repo"]["name"] as string) ORDER BY count() DESC, 1 LIMIT 50
Original file line number Diff line number Diff line change
Expand Up @@ -6,12 +6,12 @@ SELECT
FROM
(
SELECT
cast(v:repo.name as string) as repo_name,
cast(v["repo"]["name"] as string) as repo_name,
count() AS comments,
count(distinct cast(v:payload.issue.`number` as int)) AS issues
count(distinct cast(v["payload"]["issue"]["number"] as int)) AS issues
FROM github_events
WHERE cast(v:type as string) = 'IssueCommentEvent'
GROUP BY cast(v:repo.name as string)
WHERE cast(v["type"] as string) = 'IssueCommentEvent'
GROUP BY cast(v["repo"]["name"] as string)
) t
ORDER BY comments DESC, 1, 3, 4
LIMIT 50
Loading

0 comments on commit 732194d

Please sign in to comment.