diff --git a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 index 58e755f7f453e3..2848736f172340 100644 --- a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 +++ b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 @@ -529,7 +529,7 @@ optScanParams relationPrimary : multipartIdentifier optScanParams? materializedViewName? specifiedPartition? - tabletList? tableAlias sample? relationHint? lateralView* #tableName + tabletList? tableAlias sample? tableSnapshot? relationHint? lateralView* #tableName | LEFT_PAREN query RIGHT_PAREN tableAlias lateralView* #aliasedQuery | tvfName=identifier LEFT_PAREN (properties=propertyItemList)? @@ -971,6 +971,11 @@ sampleMethod | INTEGER_VALUE ROWS #sampleByRows ; +tableSnapshot + : FOR VERSION AS OF version=INTEGER_VALUE + | FOR TIME AS OF time=STRING_LITERAL + ; + // this rule is used for explicitly capturing wrong identifiers such as test-table, which should actually be `test-table` // replace identifier with errorCapturingIdentifier where the immediate follow symbol is not an expression, otherwise // valid expressions such as "a-b" can be recognized as an identifier diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/FileQueryScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/FileQueryScanNode.java index 90c24989275787..174e93b21d5e36 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/FileQueryScanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/FileQueryScanNode.java @@ -21,6 +21,7 @@ import org.apache.doris.analysis.SlotDescriptor; import org.apache.doris.analysis.SlotId; import org.apache.doris.analysis.TableSample; +import org.apache.doris.analysis.TableSnapshot; import org.apache.doris.analysis.TupleDescriptor; import org.apache.doris.catalog.Column; import org.apache.doris.catalog.Env; @@ -96,6 +97,9 @@ public abstract class FileQueryScanNode extends FileScanNode { protected String brokerName; + @Getter + protected TableSnapshot tableSnapshot; + /** * External file scan node for Query hms table * needCheckColumnPriv: Some of ExternalFileScanNode do not need to check column priv diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergScanNode.java index aee9f0e0531f02..45d8d506f5daa1 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergScanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergScanNode.java @@ -283,6 +283,9 @@ private List doGetSplits() throws UserException { public Long getSpecifiedSnapshot() throws UserException { TableSnapshot tableSnapshot = source.getDesc().getRef().getTableSnapshot(); + if (tableSnapshot == null) { + tableSnapshot = this.tableSnapshot; + } if (tableSnapshot != null) { TableSnapshot.VersionType type = tableSnapshot.getType(); try { @@ -461,4 +464,8 @@ public String getNodeExplainString(String prefix, TExplainLevel detailLevel) { return super.getNodeExplainString(prefix, detailLevel) + String.format("%sicebergPredicatePushdown=\n%s\n", prefix, sb); } + + public void setTableSnapshot(TableSnapshot tableSnapshot) { + this.tableSnapshot = tableSnapshot; + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundRelation.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundRelation.java index 2dc81e01440731..8180fd8518ee01 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundRelation.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundRelation.java @@ -18,6 +18,7 @@ package org.apache.doris.nereids.analyzer; import org.apache.doris.analysis.TableScanParams; +import org.apache.doris.analysis.TableSnapshot; import org.apache.doris.common.Pair; import org.apache.doris.nereids.exceptions.UnboundException; import org.apache.doris.nereids.memo.GroupExpression; @@ -59,41 +60,47 @@ public class UnboundRelation extends LogicalRelation implements Unbound, BlockFu // the start and end position of the sql substring(e.g. "t1", "db1.t1", "ctl1.db1.t1") private final Optional> indexInSqlString; + private final Optional tableSnapshot; + public UnboundRelation(RelationId id, List nameParts) { this(id, nameParts, Optional.empty(), Optional.empty(), ImmutableList.of(), false, ImmutableList.of(), - ImmutableList.of(), Optional.empty(), Optional.empty(), null, Optional.empty()); + ImmutableList.of(), Optional.empty(), Optional.empty(), null, Optional.empty(), Optional.empty()); } public UnboundRelation(RelationId id, List nameParts, List partNames, boolean isTempPart) { this(id, nameParts, Optional.empty(), Optional.empty(), partNames, isTempPart, ImmutableList.of(), - ImmutableList.of(), Optional.empty(), Optional.empty(), null, Optional.empty()); + ImmutableList.of(), Optional.empty(), Optional.empty(), null, Optional.empty(), Optional.empty()); } public UnboundRelation(RelationId id, List nameParts, List partNames, boolean isTempPart, List tabletIds, List hints, Optional tableSample, Optional indexName) { this(id, nameParts, Optional.empty(), Optional.empty(), - partNames, isTempPart, tabletIds, hints, tableSample, indexName, null, Optional.empty()); + partNames, isTempPart, tabletIds, hints, tableSample, indexName, null, Optional.empty(), + Optional.empty()); } public UnboundRelation(RelationId id, List nameParts, List partNames, boolean isTempPart, List tabletIds, List hints, Optional tableSample, Optional indexName, - TableScanParams scanParams) { + TableScanParams scanParams, Optional tableSnapshot) { this(id, nameParts, Optional.empty(), Optional.empty(), - partNames, isTempPart, tabletIds, hints, tableSample, indexName, scanParams, Optional.empty()); + partNames, isTempPart, tabletIds, hints, tableSample, indexName, scanParams, Optional.empty(), + tableSnapshot); } public UnboundRelation(RelationId id, List nameParts, Optional groupExpression, Optional logicalProperties, List partNames, boolean isTempPart, List tabletIds, List hints, Optional tableSample, Optional indexName) { this(id, nameParts, groupExpression, logicalProperties, partNames, - isTempPart, tabletIds, hints, tableSample, indexName, null, Optional.empty()); + isTempPart, tabletIds, hints, tableSample, indexName, null, Optional.empty(), Optional.empty()); } public UnboundRelation(RelationId id, List nameParts, List partNames, boolean isTempPart, List tabletIds, List hints, Optional tableSample, Optional indexName, - TableScanParams scanParams, Optional> indexInSqlString) { + TableScanParams scanParams, Optional> indexInSqlString, + Optional tableSnapshot) { this(id, nameParts, Optional.empty(), Optional.empty(), - partNames, isTempPart, tabletIds, hints, tableSample, indexName, scanParams, indexInSqlString); + partNames, isTempPart, tabletIds, hints, tableSample, indexName, scanParams, indexInSqlString, + tableSnapshot); } /** @@ -102,7 +109,8 @@ public UnboundRelation(RelationId id, List nameParts, List partN public UnboundRelation(RelationId id, List nameParts, Optional groupExpression, Optional logicalProperties, List partNames, boolean isTempPart, List tabletIds, List hints, Optional tableSample, Optional indexName, - TableScanParams scanParams, Optional> indexInSqlString) { + TableScanParams scanParams, Optional> indexInSqlString, + Optional tableSnapshot) { super(id, PlanType.LOGICAL_UNBOUND_RELATION, groupExpression, logicalProperties); this.nameParts = ImmutableList.copyOf(Objects.requireNonNull(nameParts, "nameParts should not null")); this.partNames = ImmutableList.copyOf(Objects.requireNonNull(partNames, "partNames should not null")); @@ -113,6 +121,7 @@ public UnboundRelation(RelationId id, List nameParts, Optional getNameParts() { @@ -133,14 +142,14 @@ public LogicalProperties computeLogicalProperties() { public Plan withGroupExpression(Optional groupExpression) { return new UnboundRelation(relationId, nameParts, groupExpression, Optional.of(getLogicalProperties()), - partNames, isTempPart, tabletIds, hints, tableSample, indexName, null, indexInSqlString); + partNames, isTempPart, tabletIds, hints, tableSample, indexName, null, indexInSqlString, tableSnapshot); } @Override public Plan withGroupExprLogicalPropChildren(Optional groupExpression, Optional logicalProperties, List children) { return new UnboundRelation(relationId, nameParts, groupExpression, logicalProperties, partNames, - isTempPart, tabletIds, hints, tableSample, indexName, null, indexInSqlString); + isTempPart, tabletIds, hints, tableSample, indexName, null, indexInSqlString, tableSnapshot); } @Override @@ -207,4 +216,8 @@ public TableScanParams getScanParams() { public Optional> getIndexInSqlString() { return indexInSqlString; } + + public Optional getTableSnapshot() { + return tableSnapshot; + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java index 8a951f49a89289..e40eead5340878 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java @@ -573,6 +573,10 @@ public PlanFragment visitPhysicalFileScan(PhysicalFileScan fileScan, PlanTransla break; case ICEBERG: scanNode = new IcebergScanNode(context.nextPlanNodeId(), tupleDescriptor, false); + IcebergScanNode icebergScanNode = (IcebergScanNode) scanNode; + if (fileScan.getTableSnapshot().isPresent()) { + icebergScanNode.setTableSnapshot(fileScan.getTableSnapshot().get()); + } break; case HIVE: scanNode = new HiveScanNode(context.nextPlanNodeId(), tupleDescriptor, false); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java index 0089a25adbae35..0e244d794b0246 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java @@ -22,6 +22,7 @@ import org.apache.doris.analysis.StorageBackend; import org.apache.doris.analysis.TableName; import org.apache.doris.analysis.TableScanParams; +import org.apache.doris.analysis.TableSnapshot; import org.apache.doris.analysis.UserIdentity; import org.apache.doris.catalog.AggregateType; import org.apache.doris.catalog.BuiltinAggregateFunctions; @@ -1370,15 +1371,25 @@ public LogicalPlan visitTableName(TableNameContext ctx) { scanParams = new TableScanParams(ctx.optScanParams().funcName.getText(), map); } + TableSnapshot tableSnapshot = null; + if (ctx.tableSnapshot() != null) { + if (ctx.tableSnapshot().TIME() != null) { + tableSnapshot = new TableSnapshot(stripQuotes(ctx.tableSnapshot().time.getText())); + } else { + tableSnapshot = new TableSnapshot(Long.parseLong(ctx.tableSnapshot().version.getText())); + } + } + MultipartIdentifierContext identifier = ctx.multipartIdentifier(); TableSample tableSample = ctx.sample() == null ? null : (TableSample) visit(ctx.sample()); UnboundRelation relation = forCreateView ? new UnboundRelation(StatementScopeIdGenerator.newRelationId(), tableId, partitionNames, isTempPart, tabletIdLists, relationHints, Optional.ofNullable(tableSample), indexName, scanParams, - Optional.of(Pair.of(identifier.start.getStartIndex(), identifier.stop.getStopIndex()))) : + Optional.of(Pair.of(identifier.start.getStartIndex(), identifier.stop.getStopIndex())), + Optional.ofNullable(tableSnapshot)) : new UnboundRelation(StatementScopeIdGenerator.newRelationId(), tableId, partitionNames, isTempPart, tabletIdLists, relationHints, - Optional.ofNullable(tableSample), indexName, scanParams); + Optional.ofNullable(tableSample), indexName, scanParams, Optional.ofNullable(tableSnapshot)); LogicalPlan checkedRelation = LogicalPlanBuilderAssistant.withCheckPolicy(relation); LogicalPlan plan = withTableAlias(checkedRelation, ctx.tableAlias()); for (LateralViewContext lateralViewContext : ctx.lateralView()) { @@ -1387,6 +1398,14 @@ public LogicalPlan visitTableName(TableNameContext ctx) { return plan; } + public static String stripQuotes(String str) { + if ((str.charAt(0) == '\'' && str.charAt(str.length() - 1) == '\'') + || (str.charAt(0) == '\"' && str.charAt(str.length() - 1) == '\"')) { + str = str.substring(1, str.length() - 1); + } + return str; + } + @Override public LogicalPlan visitAliasedQuery(AliasedQueryContext ctx) { if (ctx.tableAlias().getText().equals("")) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindRelation.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindRelation.java index df3743928a9b96..cf12e80ff22c33 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindRelation.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindRelation.java @@ -274,13 +274,15 @@ private LogicalPlan getLogicalPlan(TableIf table, UnboundRelation unboundRelatio } hmsTable.setScanParams(unboundRelation.getScanParams()); return new LogicalFileScan(unboundRelation.getRelationId(), (HMSExternalTable) table, - qualifierWithoutTableName, unboundRelation.getTableSample()); + qualifierWithoutTableName, unboundRelation.getTableSample(), + unboundRelation.getTableSnapshot()); case ICEBERG_EXTERNAL_TABLE: case PAIMON_EXTERNAL_TABLE: case MAX_COMPUTE_EXTERNAL_TABLE: case TRINO_CONNECTOR_EXTERNAL_TABLE: return new LogicalFileScan(unboundRelation.getRelationId(), (ExternalTable) table, - qualifierWithoutTableName, unboundRelation.getTableSample()); + qualifierWithoutTableName, unboundRelation.getTableSample(), + unboundRelation.getTableSnapshot()); case SCHEMA: return new LogicalSchemaScan(unboundRelation.getRelationId(), table, qualifierWithoutTableName); case JDBC_EXTERNAL_TABLE: diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/LogicalFileScanToPhysicalFileScan.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/LogicalFileScanToPhysicalFileScan.java index d86e1d1667e18a..8edb683151e5c2 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/LogicalFileScanToPhysicalFileScan.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/LogicalFileScanToPhysicalFileScan.java @@ -40,7 +40,8 @@ public Rule build() { fileScan.getLogicalProperties(), fileScan.getConjuncts(), fileScan.getSelectedPartitions(), - fileScan.getTableSample()) + fileScan.getTableSample(), + fileScan.getTableSnapshot()) ).toRule(RuleType.LOGICAL_FILE_SCAN_TO_PHYSICAL_FILE_SCAN_RULE); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalFileScan.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalFileScan.java index f7bdae5c2f3bf3..06d349fe2a6314 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalFileScan.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalFileScan.java @@ -17,6 +17,7 @@ package org.apache.doris.nereids.trees.plans.logical; +import org.apache.doris.analysis.TableSnapshot; import org.apache.doris.catalog.PartitionItem; import org.apache.doris.datasource.ExternalTable; import org.apache.doris.nereids.memo.GroupExpression; @@ -46,22 +47,25 @@ public class LogicalFileScan extends LogicalExternalRelation { private final SelectedPartitions selectedPartitions; private final Optional tableSample; + private final Optional tableSnapshot; /** * Constructor for LogicalFileScan. */ public LogicalFileScan(RelationId id, ExternalTable table, List qualifier, Optional groupExpression, Optional logicalProperties, - Set conjuncts, SelectedPartitions selectedPartitions, Optional tableSample) { + Set conjuncts, SelectedPartitions selectedPartitions, Optional tableSample, + Optional tableSnapshot) { super(id, PlanType.LOGICAL_FILE_SCAN, table, qualifier, conjuncts, groupExpression, logicalProperties); this.selectedPartitions = selectedPartitions; this.tableSample = tableSample; + this.tableSnapshot = tableSnapshot; } public LogicalFileScan(RelationId id, ExternalTable table, List qualifier, - Optional tableSample) { + Optional tableSample, Optional tableSnapshot) { this(id, table, qualifier, Optional.empty(), Optional.empty(), - Sets.newHashSet(), SelectedPartitions.NOT_PRUNED, tableSample); + Sets.newHashSet(), SelectedPartitions.NOT_PRUNED, tableSample, tableSnapshot); } public SelectedPartitions getSelectedPartitions() { @@ -72,6 +76,10 @@ public Optional getTableSample() { return tableSample; } + public Optional getTableSnapshot() { + return tableSnapshot; + } + @Override public ExternalTable getTable() { Preconditions.checkArgument(table instanceof ExternalTable, @@ -90,31 +98,31 @@ public String toString() { @Override public LogicalFileScan withGroupExpression(Optional groupExpression) { return new LogicalFileScan(relationId, (ExternalTable) table, qualifier, groupExpression, - Optional.of(getLogicalProperties()), conjuncts, selectedPartitions, tableSample); + Optional.of(getLogicalProperties()), conjuncts, selectedPartitions, tableSample, tableSnapshot); } @Override public Plan withGroupExprLogicalPropChildren(Optional groupExpression, Optional logicalProperties, List children) { return new LogicalFileScan(relationId, (ExternalTable) table, qualifier, - groupExpression, logicalProperties, conjuncts, selectedPartitions, tableSample); + groupExpression, logicalProperties, conjuncts, selectedPartitions, tableSample, tableSnapshot); } @Override public LogicalFileScan withConjuncts(Set conjuncts) { return new LogicalFileScan(relationId, (ExternalTable) table, qualifier, Optional.empty(), - Optional.of(getLogicalProperties()), conjuncts, selectedPartitions, tableSample); + Optional.of(getLogicalProperties()), conjuncts, selectedPartitions, tableSample, tableSnapshot); } public LogicalFileScan withSelectedPartitions(SelectedPartitions selectedPartitions) { return new LogicalFileScan(relationId, (ExternalTable) table, qualifier, Optional.empty(), - Optional.of(getLogicalProperties()), conjuncts, selectedPartitions, tableSample); + Optional.of(getLogicalProperties()), conjuncts, selectedPartitions, tableSample, tableSnapshot); } @Override public LogicalFileScan withRelationId(RelationId relationId) { return new LogicalFileScan(relationId, (ExternalTable) table, qualifier, Optional.empty(), - Optional.empty(), conjuncts, selectedPartitions, tableSample); + Optional.empty(), conjuncts, selectedPartitions, tableSample, tableSnapshot); } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalFileScan.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalFileScan.java index 843a03f69f954a..8706db65f1e53e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalFileScan.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalFileScan.java @@ -17,6 +17,7 @@ package org.apache.doris.nereids.trees.plans.physical; +import org.apache.doris.analysis.TableSnapshot; import org.apache.doris.datasource.ExternalTable; import org.apache.doris.nereids.memo.GroupExpression; import org.apache.doris.nereids.properties.DistributionSpec; @@ -45,6 +46,7 @@ public class PhysicalFileScan extends PhysicalCatalogRelation { private final Set conjuncts; private final SelectedPartitions selectedPartitions; private final Optional tableSample; + private final Optional tableSnapshot; /** * Constructor for PhysicalFileScan. @@ -52,12 +54,14 @@ public class PhysicalFileScan extends PhysicalCatalogRelation { public PhysicalFileScan(RelationId id, ExternalTable table, List qualifier, DistributionSpec distributionSpec, Optional groupExpression, LogicalProperties logicalProperties, Set conjuncts, - SelectedPartitions selectedPartitions, Optional tableSample) { + SelectedPartitions selectedPartitions, Optional tableSample, + Optional tableSnapshot) { super(id, PlanType.PHYSICAL_FILE_SCAN, table, qualifier, groupExpression, logicalProperties); this.distributionSpec = distributionSpec; this.conjuncts = conjuncts; this.selectedPartitions = selectedPartitions; this.tableSample = tableSample; + this.tableSnapshot = tableSnapshot; } /** @@ -67,13 +71,14 @@ public PhysicalFileScan(RelationId id, ExternalTable table, List qualifi DistributionSpec distributionSpec, Optional groupExpression, LogicalProperties logicalProperties, PhysicalProperties physicalProperties, Statistics statistics, Set conjuncts, SelectedPartitions selectedPartitions, - Optional tableSample) { + Optional tableSample, Optional tableSnapshot) { super(id, PlanType.PHYSICAL_FILE_SCAN, table, qualifier, groupExpression, logicalProperties, physicalProperties, statistics); this.distributionSpec = distributionSpec; this.conjuncts = conjuncts; this.selectedPartitions = selectedPartitions; this.tableSample = tableSample; + this.tableSnapshot = tableSnapshot; } public DistributionSpec getDistributionSpec() { @@ -92,6 +97,10 @@ public Optional getTableSample() { return tableSample; } + public Optional getTableSnapshot() { + return tableSnapshot; + } + @Override public String toString() { return Utils.toSqlString("PhysicalFileScan", @@ -112,14 +121,14 @@ public R accept(PlanVisitor visitor, C context) { @Override public PhysicalFileScan withGroupExpression(Optional groupExpression) { return new PhysicalFileScan(relationId, getTable(), qualifier, distributionSpec, - groupExpression, getLogicalProperties(), conjuncts, selectedPartitions, tableSample); + groupExpression, getLogicalProperties(), conjuncts, selectedPartitions, tableSample, tableSnapshot); } @Override public Plan withGroupExprLogicalPropChildren(Optional groupExpression, Optional logicalProperties, List children) { return new PhysicalFileScan(relationId, getTable(), qualifier, distributionSpec, - groupExpression, logicalProperties.get(), conjuncts, selectedPartitions, tableSample); + groupExpression, logicalProperties.get(), conjuncts, selectedPartitions, tableSample, tableSnapshot); } @Override @@ -132,6 +141,6 @@ public PhysicalFileScan withPhysicalPropertiesAndStats(PhysicalProperties physic Statistics statistics) { return new PhysicalFileScan(relationId, getTable(), qualifier, distributionSpec, groupExpression, getLogicalProperties(), physicalProperties, statistics, conjuncts, - selectedPartitions, tableSample); + selectedPartitions, tableSample, tableSnapshot); } }