Skip to content

Commit

Permalink
[fix](Nereids) syntax of auto partition with date_trunc is wrong
Browse files Browse the repository at this point in the history
  • Loading branch information
morrySnow committed Mar 29, 2024
1 parent cc50d8c commit f0610c5
Show file tree
Hide file tree
Showing 44 changed files with 142 additions and 116 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -130,8 +130,9 @@ partitionSpec
;

partitionTable
: ((autoPartition=AUTO)? PARTITION BY (RANGE | LIST)? partitionList=identityOrFunctionList
(LEFT_PAREN (partitions=partitionsDef)? RIGHT_PAREN))
: ((autoPartition=AUTO)? PARTITION BY (RANGE | LIST)?
partitionList=identityOrFunctionList
(LEFT_PAREN (partitions=partitionsDef)? RIGHT_PAREN))
;

identityOrFunctionList
Expand Down
12 changes: 2 additions & 10 deletions fe/fe-core/src/main/cup/sql_parser.cup
Original file line number Diff line number Diff line change
Expand Up @@ -3191,29 +3191,21 @@ opt_partition ::=
RESULT = new ListPartitionDesc(columns, list);
:}
/* expr range partition */
| KW_AUTO KW_PARTITION KW_BY KW_RANGE function_call_expr:fnExpr
| KW_AUTO KW_PARTITION KW_BY KW_RANGE LPAREN function_call_expr:fnExpr RPAREN
LPAREN opt_all_partition_desc_list:list RPAREN
{:
ArrayList<Expr> exprs = new ArrayList<Expr>();
exprs.add(fnExpr);
RESULT = RangePartitionDesc.createRangePartitionDesc(exprs, list);
:}
| KW_AUTO KW_PARTITION KW_BY KW_RANGE function_name:functionName LPAREN expr_list:l COMMA
KW_INTERVAL expr:v ident:u RPAREN LPAREN opt_all_partition_desc_list:list RPAREN
{:
Expr fnExpr = FunctionCallExpr.functionWithIntervalConvert(functionName.getFunction().toLowerCase(), l.get(0), v, u);
ArrayList<Expr> exprs = new ArrayList<Expr>();
exprs.add(fnExpr);
RESULT = RangePartitionDesc.createRangePartitionDesc(exprs, list);
:}

/* expr list partition */
| KW_AUTO KW_PARTITION KW_BY KW_LIST LPAREN expr_list:exprs RPAREN
LPAREN opt_all_partition_desc_list:list RPAREN
{:
RESULT = ListPartitionDesc.createListPartitionDesc(exprs, list);
:}
| KW_AUTO KW_PARTITION KW_BY KW_LIST function_call_expr:fnExpr
| KW_AUTO KW_PARTITION KW_BY KW_LIST LPAREN function_call_expr:fnExpr RPAREN
LPAREN opt_all_partition_desc_list:list RPAREN
{:
ArrayList<Expr> exprs = new ArrayList<Expr>();
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -45,11 +45,8 @@ public class PartitionDesc {
protected ArrayList<Expr> partitionExprs; //eg: auto partition by range date_trunc(column, 'day')
protected boolean isAutoCreatePartitions;
protected PartitionType type;
public static final ImmutableSet<String> RANGE_PARTITION_FUNCTIONS = new ImmutableSortedSet.Builder<String>(
String.CASE_INSENSITIVE_ORDER).add("date_trunc").add("date_ceil").add("date_floor").add("second_floor")
.add("minute_floor").add("hour_floor").add("day_floor").add("month_floor").add("year_floor")
.add("second_ceil").add("minute_ceil").add("hour_ceil").add("day_ceil").add("month_ceil").add("year_ceil")
.build();
public static final ImmutableSet<String> RANGE_PARTITION_FUNCTIONS
= new ImmutableSortedSet.Builder<>(String.CASE_INSENSITIVE_ORDER).add("date_trunc").build();

public PartitionDesc() {}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2125,10 +2125,10 @@ private void createOlapTable(Database db, CreateTableStmt stmt) throws UserExcep
fn = func.getBuiltinFunction(func.getFnName().getFunction(), childTypes,
Function.CompareMode.IS_INDISTINGUISHABLE); // only for test
} catch (Exception e) {
throw new AnalysisException("partition expr " + func.getExprName() + " is illegal!");
throw new AnalysisException("partition expr " + func.getExprName() + " is illegal");
}
if (fn == null) {
throw new AnalysisException("partition expr " + func.getExprName() + " is illegal!");
throw new AnalysisException("partition expr " + func.getExprName() + " is illegal");
}
}
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,13 @@ public List<String> getNameParts() {
return nameParts;
}

public String getLastName() {
if (nameParts.isEmpty()) {
return "";
}
return nameParts.get(nameParts.size() - 1);
}

@Override
public String getName() {
return nameParts.stream().map(n -> {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2519,7 +2519,7 @@ public LogicalPlan visitCreateTable(CreateTableContext ctx) {
@Override
public PartitionTableInfo visitPartitionTable(DorisParser.PartitionTableContext ctx) {
boolean isAutoPartition = ctx.autoPartition != null;
ImmutableList<Expression> partitionList = ctx.partitionList.identityOrFunction().stream()
ImmutableList<Expression> partitionFields = ctx.partitionList.identityOrFunction().stream()
.map(partition -> {
IdentifierContext identifier = partition.identifier();
if (identifier != null) {
Expand All @@ -2530,10 +2530,10 @@ public PartitionTableInfo visitPartitionTable(DorisParser.PartitionTableContext
})
.collect(ImmutableList.toImmutableList());
return new PartitionTableInfo(
isAutoPartition,
ctx.RANGE() != null ? "RANGE" : "LIST",
ctx.partitions != null ? visitPartitionsDef(ctx.partitions) : null,
partitionList);
isAutoPartition,
ctx.RANGE() != null ? "RANGE" : "LIST",
ctx.partitions != null ? visitPartitionsDef(ctx.partitions) : null,
partitionFields);
}

@Override
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,7 @@
import org.apache.doris.nereids.trees.plans.commands.info.StepPartition;
import org.apache.doris.qe.ConnectContext;

import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import com.google.common.collect.Sets;

Expand All @@ -62,10 +63,10 @@ public class PartitionTableInfo {
null);

private boolean isAutoPartition;
private String partitionType;
private final String partitionType;
private List<String> partitionColumns;
private List<PartitionDefinition> partitionDefs;
private List<Expression> partitionList;
private final List<PartitionDefinition> partitionDefs;
private final List<Expression> partitionFields;

/**
* struct for partition definition
Expand All @@ -82,12 +83,40 @@ public PartitionTableInfo(
this.isAutoPartition = isAutoPartition;
this.partitionType = partitionType;
this.partitionDefs = partitionDefs;
this.partitionList = partitionFields;
if (this.partitionList != null) {
this.partitionColumns = this.partitionList.stream()
.filter(UnboundSlot.class::isInstance)
.map(partition -> ((UnboundSlot) partition).getName())
.collect(Collectors.toList());
this.partitionFields = partitionFields;
if (this.partitionFields != null) {
this.partitionColumns = Lists.newArrayListWithExpectedSize(partitionFields.size());
if (isAutoPartition && this.partitionFields.size() != 1) {
throw new AnalysisException("auto partition only support one function expression");
}
for (Expression partitionField : this.partitionFields) {
if (partitionField instanceof UnboundSlot) {
if (((UnboundSlot) partitionField).getNameParts().size() != 1) {
throw new AnalysisException("partition column " + partitionField.toSql()
+ " should not be qualified");
}
partitionColumns.add(((UnboundSlot) partitionField).getLastName());
} else if (partitionField instanceof UnboundFunction) {
UnboundFunction function = (UnboundFunction) partitionField;
if (isAutoPartition && !RangePartitionDesc.RANGE_PARTITION_FUNCTIONS.contains(function.getName())) {
throw new AnalysisException("auto create partition only support"
+ " function call expr is date_trunc. " + function.toSql());
}
function.children().stream()
.filter(UnboundSlot.class::isInstance)
.map(UnboundSlot.class::cast)
.peek(s -> {
if (s.getNameParts().size() != 1) {
throw new AnalysisException("partition column " + s.toSql()
+ " should not be qualified");
}
})
.map(UnboundSlot::getLastName)
.forEach(partitionColumns::add);
} else {
throw new AnalysisException("partition field not support expression " + partitionField.toSql());
}
}
}
}

Expand Down Expand Up @@ -158,7 +187,7 @@ public void validatePartitionInfo(

if (partitionColumns != null) {

if (partitionColumns.size() != partitionList.size()) {
if (this.partitionFields.stream().anyMatch(UnboundFunction.class::isInstance)) {
if (!isExternal && partitionType.equalsIgnoreCase(PartitionType.LIST.name())) {
throw new AnalysisException("internal catalog does not support functions in 'LIST' partition");
}
Expand Down Expand Up @@ -237,15 +266,15 @@ public PartitionDesc convertToPartitionDesc(boolean isExternal) {
if (partitionType.equals(PartitionType.RANGE.name())) {
if (isAutoPartition) {
partitionDesc = new RangePartitionDesc(
convertToLegacyAutoPartitionExprs(partitionList),
convertToLegacyAutoPartitionExprs(partitionFields),
partitionColumns, partitionDescs);
} else {
partitionDesc = new RangePartitionDesc(partitionColumns, partitionDescs);
}
} else {
if (isAutoPartition) {
partitionDesc = new ListPartitionDesc(
convertToLegacyAutoPartitionExprs(partitionList),
convertToLegacyAutoPartitionExprs(partitionFields),
partitionColumns, partitionDescs);
} else {
partitionDesc = new ListPartitionDesc(partitionColumns, partitionDescs);
Expand All @@ -259,7 +288,7 @@ public PartitionDesc convertToPartitionDesc(boolean isExternal) {
}

private static ArrayList<Expr> convertToLegacyAutoPartitionExprs(List<Expression> expressions) {
return new ArrayList<>(expressions.stream().map(expression -> {
return expressions.stream().map(expression -> {
if (expression instanceof UnboundSlot) {
return new SlotRef(null, ((UnboundSlot) expression).getName());
} else if (expression instanceof UnboundFunction) {
Expand All @@ -269,9 +298,9 @@ private static ArrayList<Expr> convertToLegacyAutoPartitionExprs(List<Expression
new FunctionParams(convertToLegacyArguments(function.children())));
} else {
throw new AnalysisException(
"unsupported auto partition expr " + expression.toString());
"unsupported auto partition expr " + expression.toString());
}
}).collect(Collectors.toList()));
}).collect(Collectors.toCollection(ArrayList::new));
}

private static List<Expr> convertToLegacyArguments(List<Expression> children) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -108,7 +108,7 @@ protected void runBeforeAll() throws Exception {
+ " event_day DATETIME NOT NULL\n"
+ ")\n"
+ "DUPLICATE KEY(event_day)\n"
+ "AUTO PARTITION BY range date_trunc(event_day, \"day\") (\n"
+ "AUTO PARTITION BY range (date_trunc(event_day, \"day\")) (\n"
+ "\tPARTITION `p20230807` values [(20230807 ), (20230808 )),\n"
+ "\tPARTITION `p20020106` values [(20020106 ), (20020107 ))\n"
+ ")\n"
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -194,7 +194,7 @@ protected void runBeforeAll() throws Exception {
+ " ) ENGINE=OLAP\n"
+ " DUPLICATE KEY(l_orderkey, l_linenumber, l_partkey, l_suppkey )\n"
+ " COMMENT 'OLAP'\n"
+ " AUTO PARTITION BY range date_trunc(`l_shipdate`, 'day') ()\n"
+ " AUTO PARTITION BY range (date_trunc(`l_shipdate`, 'day')) ()\n"
+ " DISTRIBUTED BY HASH(`l_orderkey`) BUCKETS 3\n"
+ " PROPERTIES (\n"
+ " \"replication_num\" = \"1\"\n"
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -59,7 +59,7 @@ protected void runBeforeAll() throws Exception {
@Override
public void createTable(String sql) throws Exception {
LogicalPlan plan = new NereidsParser().parseSingle(sql);
Assertions.assertTrue(plan instanceof CreateTableCommand);
Assertions.assertInstanceOf(CreateTableCommand.class, plan);
((CreateTableCommand) plan).run(connectContext, null);
}

Expand Down Expand Up @@ -827,7 +827,7 @@ public void testCreateTablePartitionForExternalCatalog() {

private PartitionDesc getCreateTableStmt(String sql) {
LogicalPlan plan = new NereidsParser().parseSingle(sql);
Assertions.assertTrue(plan instanceof CreateTableCommand);
Assertions.assertInstanceOf(CreateTableCommand.class, plan);
CreateTableInfo createTableInfo = ((CreateTableCommand) plan).getCreateTableInfo();
createTableInfo.validate(connectContext);
return createTableInfo.translateToLegacyStmt().getPartitionDesc();
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -51,7 +51,6 @@
import org.junit.rules.ExpectedException;

import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
import java.util.UUID;
Expand Down Expand Up @@ -99,7 +98,7 @@ public void testCreatePartitionRange() throws Exception {
+ " city_code VARCHAR(100)\n"
+ ")\n"
+ "DUPLICATE KEY(event_day, site_id, city_code)\n"
+ "AUTO PARTITION BY range date_trunc( event_day,'day') (\n"
+ "AUTO PARTITION BY range (date_trunc( event_day,'day')) (\n"
+ "\n"
+ ")\n"
+ "DISTRIBUTED BY HASH(event_day, site_id) BUCKETS 2\n"
Expand Down Expand Up @@ -183,7 +182,8 @@ public void testGetDBNames() throws Exception {
TGetDbsResult dbNames = impl.getDbNames(params);

Assert.assertEquals(dbNames.getDbs().size(), 2);
Assert.assertEquals(dbNames.getDbs(), Arrays.asList("test", "test_"));
Assert.assertTrue(dbNames.getDbs().contains("test"));
Assert.assertTrue(dbNames.getDbs().contains("test_"));
}

@Test
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -43,7 +43,7 @@ suite("partition_mv_rewrite_dimension_1") {
) ENGINE=OLAP
DUPLICATE KEY(`o_orderkey`, `o_custkey`)
COMMENT 'OLAP'
AUTO PARTITION BY range date_trunc(`o_orderdate`, 'day') ()
AUTO PARTITION BY range (date_trunc(`o_orderdate`, 'day')) ()
DISTRIBUTED BY HASH(`o_orderkey`) BUCKETS 96
PROPERTIES (
"replication_allocation" = "tag.location.default: 1"
Expand Down Expand Up @@ -73,7 +73,7 @@ suite("partition_mv_rewrite_dimension_1") {
) ENGINE=OLAP
DUPLICATE KEY(l_orderkey, l_linenumber, l_partkey, l_suppkey )
COMMENT 'OLAP'
AUTO PARTITION BY range date_trunc(`l_shipdate`, 'day') ()
AUTO PARTITION BY range (date_trunc(`l_shipdate`, 'day')) ()
DISTRIBUTED BY HASH(`l_orderkey`) BUCKETS 96
PROPERTIES (
"replication_allocation" = "tag.location.default: 1"
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,7 @@ suite("partition_mv_rewrite_dimension_2_3") {
) ENGINE=OLAP
DUPLICATE KEY(`o_orderkey`, `o_custkey`)
COMMENT 'OLAP'
AUTO PARTITION BY range date_trunc(`o_orderdate`, 'day') ()
AUTO PARTITION BY range (date_trunc(`o_orderdate`, 'day')) ()
DISTRIBUTED BY HASH(`o_orderkey`) BUCKETS 96
PROPERTIES (
"replication_allocation" = "tag.location.default: 1"
Expand Down Expand Up @@ -74,7 +74,7 @@ suite("partition_mv_rewrite_dimension_2_3") {
) ENGINE=OLAP
DUPLICATE KEY(l_orderkey, l_linenumber, l_partkey, l_suppkey )
COMMENT 'OLAP'
AUTO PARTITION BY range date_trunc(`l_shipdate`, 'day') ()
AUTO PARTITION BY range (date_trunc(`l_shipdate`, 'day')) ()
DISTRIBUTED BY HASH(`l_orderkey`) BUCKETS 96
PROPERTIES (
"replication_allocation" = "tag.location.default: 1"
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,7 @@ suite("partition_mv_rewrite_dimension_2_4") {
) ENGINE=OLAP
DUPLICATE KEY(`o_orderkey`, `o_custkey`)
COMMENT 'OLAP'
AUTO PARTITION BY range date_trunc(`o_orderdate`, 'day') ()
AUTO PARTITION BY range (date_trunc(`o_orderdate`, 'day')) ()
DISTRIBUTED BY HASH(`o_orderkey`) BUCKETS 96
PROPERTIES (
"replication_allocation" = "tag.location.default: 1"
Expand Down Expand Up @@ -74,7 +74,7 @@ suite("partition_mv_rewrite_dimension_2_4") {
) ENGINE=OLAP
DUPLICATE KEY(l_orderkey, l_linenumber, l_partkey, l_suppkey )
COMMENT 'OLAP'
AUTO PARTITION BY range date_trunc(`l_shipdate`, 'day') ()
AUTO PARTITION BY range (date_trunc(`l_shipdate`, 'day')) ()
DISTRIBUTED BY HASH(`l_orderkey`) BUCKETS 96
PROPERTIES (
"replication_allocation" = "tag.location.default: 1"
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,7 @@ suite("partition_mv_rewrite_dimension_2_5") {
) ENGINE=OLAP
DUPLICATE KEY(`o_orderkey`, `o_custkey`)
COMMENT 'OLAP'
AUTO PARTITION BY range date_trunc(`o_orderdate`, 'day') ()
AUTO PARTITION BY range (date_trunc(`o_orderdate`, 'day')) ()
DISTRIBUTED BY HASH(`o_orderkey`) BUCKETS 96
PROPERTIES (
"replication_allocation" = "tag.location.default: 1"
Expand Down Expand Up @@ -74,7 +74,7 @@ suite("partition_mv_rewrite_dimension_2_5") {
) ENGINE=OLAP
DUPLICATE KEY(l_orderkey, l_linenumber, l_partkey, l_suppkey )
COMMENT 'OLAP'
AUTO PARTITION BY range date_trunc(`l_shipdate`, 'day') ()
AUTO PARTITION BY range (date_trunc(`l_shipdate`, 'day')) ()
DISTRIBUTED BY HASH(`l_orderkey`) BUCKETS 96
PROPERTIES (
"replication_allocation" = "tag.location.default: 1"
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,7 @@ suite("partition_mv_rewrite_dimension_2_6") {
) ENGINE=OLAP
DUPLICATE KEY(`o_orderkey`, `o_custkey`)
COMMENT 'OLAP'
AUTO PARTITION BY range date_trunc(`o_orderdate`, 'day') ()
AUTO PARTITION BY range (date_trunc(`o_orderdate`, 'day')) ()
DISTRIBUTED BY HASH(`o_orderkey`) BUCKETS 96
PROPERTIES (
"replication_allocation" = "tag.location.default: 1"
Expand Down Expand Up @@ -74,7 +74,7 @@ suite("partition_mv_rewrite_dimension_2_6") {
) ENGINE=OLAP
DUPLICATE KEY(l_orderkey, l_linenumber, l_partkey, l_suppkey )
COMMENT 'OLAP'
AUTO PARTITION BY range date_trunc(`l_shipdate`, 'day') ()
AUTO PARTITION BY range (date_trunc(`l_shipdate`, 'day')) ()
DISTRIBUTED BY HASH(`l_orderkey`) BUCKETS 96
PROPERTIES (
"replication_allocation" = "tag.location.default: 1"
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,7 @@ suite("partition_mv_rewrite_dimension_2_full_join") {
) ENGINE=OLAP
DUPLICATE KEY(`o_orderkey`, `o_custkey`)
COMMENT 'OLAP'
AUTO PARTITION BY range date_trunc(`o_orderdate`, 'day') ()
AUTO PARTITION BY range (date_trunc(`o_orderdate`, 'day')) ()
DISTRIBUTED BY HASH(`o_orderkey`) BUCKETS 96
PROPERTIES (
"replication_allocation" = "tag.location.default: 1"
Expand Down Expand Up @@ -74,7 +74,7 @@ suite("partition_mv_rewrite_dimension_2_full_join") {
) ENGINE=OLAP
DUPLICATE KEY(l_orderkey, l_linenumber, l_partkey, l_suppkey )
COMMENT 'OLAP'
AUTO PARTITION BY range date_trunc(`l_shipdate`, 'day') ()
AUTO PARTITION BY range (date_trunc(`l_shipdate`, 'day')) ()
DISTRIBUTED BY HASH(`l_orderkey`) BUCKETS 96
PROPERTIES (
"replication_allocation" = "tag.location.default: 1"
Expand Down
Loading

0 comments on commit f0610c5

Please sign in to comment.