From 806e2418a5324b01e69422ada2d5fe95b758832e Mon Sep 17 00:00:00 2001 From: seawinde <149132972+seawinde@users.noreply.github.com> Date: Wed, 22 May 2024 19:18:35 +0800 Subject: [PATCH] [fix](mtmv) Fix table id overturn and optimize get table qualifier method (#34768) Table id may be the same but actually they are different tables. so we optimize the org.apache.doris.nereids.rules.exploration.mv.mapping.RelationMapping#getTableQualifier with following code: Objects.hash(table.getDatabase().getCatalog().getId(), table.getDatabase().getId(), table.getId()) table id is long, we identify the table used in mv rewrite is bitSet. the bitSet can only use int, so we mapping the long id to init id in every query when mv rewrite --- .../catalog/constraint/TableIdentifier.java | 10 ++-- .../apache/doris/nereids/NereidsPlanner.java | 2 +- .../doris/nereids/StatementContext.java | 24 ++++++++ .../doris/nereids/memo/StructInfoMap.java | 30 ++++++---- .../mv/MaterializationContext.java | 25 +++++--- .../exploration/mv/MaterializedViewUtils.java | 6 +- .../rules/exploration/mv/StructInfo.java | 29 +++++---- .../mv/mapping/RelationMapping.java | 21 +++---- .../StatementScopeIdGenerator.java | 8 +++ .../doris/nereids/trees/plans/TableId.java | 59 +++++++++++++++++++ .../hypergraph/CompareOuterJoinTest.java | 35 ++++------- .../joinorder/hypergraph/InferJoinTest.java | 24 ++------ .../hypergraph/InferPredicateTest.java | 19 +----- .../hypergraph/PullupExpressionTest.java | 25 ++------ .../doris/nereids/memo/StructInfoMapTest.java | 21 ++++--- .../exploration/mv/EliminateJoinTest.java | 29 +++------ .../mv/HyperGraphComparatorTest.java | 22 ++----- .../doris/nereids/sqltest/SqlTestBase.java | 19 ++++++ 18 files changed, 226 insertions(+), 182 deletions(-) create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/TableId.java diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/constraint/TableIdentifier.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/constraint/TableIdentifier.java index 2688fd5784db7a..8e510ec7a93ff5 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/constraint/TableIdentifier.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/constraint/TableIdentifier.java @@ -40,8 +40,9 @@ public TableIdentifier(TableIf tableIf) { Preconditions.checkArgument(tableIf != null, "Table can not be null in constraint"); tableId = tableIf.getId(); - databaseId = tableIf.getDatabase().getId(); - catalogId = tableIf.getDatabase().getCatalog().getId(); + databaseId = tableIf.getDatabase() == null ? 0L : tableIf.getDatabase().getId(); + catalogId = tableIf.getDatabase() == null || tableIf.getDatabase().getCatalog() == null + ? 0L : tableIf.getDatabase().getCatalog().getId(); } public TableIf toTableIf() { @@ -69,13 +70,14 @@ public boolean equals(Object o) { return false; } TableIdentifier that = (TableIdentifier) o; - return databaseId == that.databaseId + return catalogId == that.catalogId + && databaseId == that.databaseId && tableId == that.tableId; } @Override public int hashCode() { - return Objects.hash(databaseId, tableId); + return Objects.hash(catalogId, databaseId, tableId); } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/NereidsPlanner.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/NereidsPlanner.java index ac7a5ffbc586cd..ec54cc4cc4c953 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/NereidsPlanner.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/NereidsPlanner.java @@ -502,7 +502,7 @@ public String getExplainString(ExplainOptions explainOptions) { this.getPhysicalPlan()); if (statementContext != null) { if (statementContext.isHasUnknownColStats()) { - plan += "planed with unknown column statistics\n"; + plan += "\n\nStatistics\n planed with unknown column statistics\n"; } } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/StatementContext.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/StatementContext.java index 2ee253d9577c89..c2240e536d3f7d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/StatementContext.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/StatementContext.java @@ -20,6 +20,7 @@ import org.apache.doris.analysis.StatementBase; import org.apache.doris.catalog.Column; import org.apache.doris.catalog.TableIf; +import org.apache.doris.catalog.constraint.TableIdentifier; import org.apache.doris.common.IdGenerator; import org.apache.doris.common.Pair; import org.apache.doris.nereids.hint.Hint; @@ -30,8 +31,10 @@ import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.Slot; import org.apache.doris.nereids.trees.expressions.SlotReference; +import org.apache.doris.nereids.trees.expressions.StatementScopeIdGenerator; import org.apache.doris.nereids.trees.plans.ObjectId; import org.apache.doris.nereids.trees.plans.RelationId; +import org.apache.doris.nereids.trees.plans.TableId; import org.apache.doris.nereids.trees.plans.algebra.Relation; import org.apache.doris.nereids.trees.plans.logical.LogicalCTEConsumer; import org.apache.doris.nereids.trees.plans.logical.LogicalPlan; @@ -56,6 +59,7 @@ import java.util.Collection; import java.util.HashMap; import java.util.Iterator; +import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.Optional; @@ -100,6 +104,7 @@ public class StatementContext implements Closeable { private final IdGenerator objectIdGenerator = ObjectId.createGenerator(); private final IdGenerator relationIdGenerator = RelationId.createGenerator(); private final IdGenerator cteIdGenerator = CTEId.createGenerator(); + private final IdGenerator talbeIdGenerator = TableId.createGenerator(); private final Map> cteIdToConsumers = new HashMap<>(); private final Map> cteIdToOutputIds = new HashMap<>(); @@ -141,6 +146,9 @@ public class StatementContext implements Closeable { // and value is the new string used for replacement. private final TreeMap, String> indexInSqlToString = new TreeMap<>(new Pair.PairComparator<>()); + // Record table id mapping, the key is the hash code of union catalogId, databaseId, tableId + // the value is the auto-increment id in the cascades context + private final Map tableIdMapping = new LinkedHashMap<>(); public StatementContext() { this(ConnectContext.get(), null, 0); @@ -293,6 +301,10 @@ public RelationId getNextRelationId() { return relationIdGenerator.getNextId(); } + public TableId getNextTableId() { + return talbeIdGenerator.getNextId(); + } + public void setParsedStatement(StatementBase parsedStatement) { this.parsedStatement = parsedStatement; } @@ -496,4 +508,16 @@ public void addKeyColumn(Column column) { public boolean isKeyColumn(Column column) { return keyColumns.contains(column); } + + /** Get table id with lazy */ + public TableId getTableId(TableIf tableIf) { + TableIdentifier tableIdentifier = new TableIdentifier(tableIf); + TableId tableId = this.tableIdMapping.get(tableIdentifier); + if (tableId != null) { + return tableId; + } + tableId = StatementScopeIdGenerator.newTableId(); + this.tableIdMapping.put(tableIdentifier, tableId); + return tableId; + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/memo/StructInfoMap.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/memo/StructInfoMap.java index df41eb3eb48d07..0c11b5fbb224ba 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/memo/StructInfoMap.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/memo/StructInfoMap.java @@ -18,6 +18,7 @@ package org.apache.doris.nereids.memo; import org.apache.doris.common.Pair; +import org.apache.doris.nereids.CascadesContext; import org.apache.doris.nereids.rules.exploration.mv.StructInfo; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.logical.LogicalCatalogRelation; @@ -51,20 +52,21 @@ public class StructInfoMap { * @param group the group that the mv matched * @return struct info or null if not found */ - public @Nullable StructInfo getStructInfo(Memo memo, BitSet tableMap, Group group, Plan originPlan) { + public @Nullable StructInfo getStructInfo(CascadesContext cascadesContext, BitSet tableMap, Group group, + Plan originPlan) { StructInfo structInfo = infoMap.get(tableMap); if (structInfo != null) { return structInfo; } if (groupExpressionMap.isEmpty() || !groupExpressionMap.containsKey(tableMap)) { - refresh(group, memo.getRefreshVersion()); - group.getstructInfoMap().setRefreshVersion(memo.getRefreshVersion()); + refresh(group, cascadesContext); + group.getstructInfoMap().setRefreshVersion(cascadesContext.getMemo().getRefreshVersion()); } if (groupExpressionMap.containsKey(tableMap)) { Pair> groupExpressionBitSetPair = getGroupExpressionWithChildren( tableMap); - structInfo = constructStructInfo(groupExpressionBitSetPair.first, - groupExpressionBitSetPair.second, tableMap, originPlan); + structInfo = constructStructInfo(groupExpressionBitSetPair.first, groupExpressionBitSetPair.second, + tableMap, originPlan, cascadesContext); infoMap.put(tableMap, structInfo); } return structInfo; @@ -87,10 +89,11 @@ public void setRefreshVersion(long refreshVersion) { } private StructInfo constructStructInfo(GroupExpression groupExpression, List children, - BitSet tableMap, Plan originPlan) { + BitSet tableMap, Plan originPlan, CascadesContext cascadesContext) { // this plan is not origin plan, should record origin plan in struct info Plan plan = constructPlan(groupExpression, children, tableMap); - return originPlan == null ? StructInfo.of(plan) : StructInfo.of(plan, originPlan); + return originPlan == null ? StructInfo.of(plan, cascadesContext) + : StructInfo.of(plan, originPlan, cascadesContext); } private Plan constructPlan(GroupExpression groupExpression, List children, BitSet tableMap) { @@ -112,8 +115,9 @@ private Plan constructPlan(GroupExpression groupExpression, List childre * @param group the root group * */ - public void refresh(Group group, long memoVersion) { + public void refresh(Group group, CascadesContext cascadesContext) { StructInfoMap structInfoMap = group.getstructInfoMap(); + long memoVersion = cascadesContext.getMemo().getRefreshVersion(); if (!structInfoMap.getTableMaps().isEmpty() && memoVersion == structInfoMap.refreshVersion) { return; } @@ -121,14 +125,14 @@ public void refresh(Group group, long memoVersion) { for (GroupExpression groupExpression : group.getLogicalExpressions()) { List> childrenTableMap = new LinkedList<>(); if (groupExpression.children().isEmpty()) { - BitSet leaf = constructLeaf(groupExpression); + BitSet leaf = constructLeaf(groupExpression, cascadesContext); groupExpressionMap.put(leaf, Pair.of(groupExpression, new LinkedList<>())); continue; } for (Group child : groupExpression.children()) { StructInfoMap childStructInfoMap = child.getstructInfoMap(); if (!refreshedGroup.contains(child.getGroupId().asInt())) { - childStructInfoMap.refresh(child, memoVersion); + childStructInfoMap.refresh(child, cascadesContext); childStructInfoMap.setRefreshVersion(memoVersion); } refreshedGroup.add(child.getGroupId().asInt()); @@ -156,12 +160,12 @@ public void refresh(Group group, long memoVersion) { } } - private BitSet constructLeaf(GroupExpression groupExpression) { + private BitSet constructLeaf(GroupExpression groupExpression, CascadesContext cascadesContext) { Plan plan = groupExpression.getPlan(); BitSet tableMap = new BitSet(); if (plan instanceof LogicalCatalogRelation) { - // TODO: Bitset is not compatible with long, use tree map instead - tableMap.set((int) ((LogicalCatalogRelation) plan).getTable().getId()); + tableMap.set(cascadesContext.getStatementContext() + .getTableId(((LogicalCatalogRelation) plan).getTable()).asInt()); } // one row relation / CTE consumer return tableMap; diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializationContext.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializationContext.java index c2c2448b24c371..ebb231c3c680be 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializationContext.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializationContext.java @@ -107,13 +107,21 @@ public MaterializationContext(Plan mvPlan, Plan originalMvPlan, Plan mvScanPlan, // mv output expression shuttle, this will be used to expression rewrite this.mvExprToMvScanExprMapping = ExpressionMapping.generate(this.mvPlanOutputShuttledExpressions, this.mvScanPlan.getOutput()); - // copy the plan from cache, which the plan in cache may change - List viewStructInfos = MaterializedViewUtils.extractStructInfo( - mvPlan, cascadesContext, new BitSet()); - if (viewStructInfos.size() > 1) { - // view struct info should only have one, log error and use the first struct info - LOG.warn(String.format("view strut info is more than one, materialization name is %s, mv plan is %s", - getMaterializationQualifier(), getMvPlan().treeString())); + // Construct mv struct info, catch exception which may cause planner roll back + List viewStructInfos; + try { + viewStructInfos = MaterializedViewUtils.extractStructInfo(mvPlan, cascadesContext, new BitSet()); + if (viewStructInfos.size() > 1) { + // view struct info should only have one, log error and use the first struct info + LOG.warn(String.format("view strut info is more than one, materialization name is %s, mv plan is %s", + getMaterializationQualifier(), getMvPlan().treeString())); + } + } catch (Exception exception) { + LOG.warn(String.format("construct mv struct info fail, materialization name is %s, mv plan is %s", + getMaterializationQualifier(), getMvPlan().treeString()), exception); + this.available = false; + this.structInfo = null; + return; } this.structInfo = viewStructInfos.get(0); } @@ -276,9 +284,8 @@ public Void visitPhysicalRelation(PhysicalRelation physicalRelation, Void contex // rewrite success and chosen builder.append("\nMaterializedViewRewriteSuccessAndChose:\n"); if (!chosenMaterializationQualifiers.isEmpty()) { - builder.append(" Names: "); chosenMaterializationQualifiers.forEach(materializationQualifier -> - builder.append(generateQualifierName(materializationQualifier)).append(", ")); + builder.append(generateQualifierName(materializationQualifier)).append(", \n")); } // rewrite success but not chosen builder.append("\nMaterializedViewRewriteSuccessButNotChose:\n"); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewUtils.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewUtils.java index f1a90fb940c453..e4dc120c20e402 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewUtils.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewUtils.java @@ -149,7 +149,7 @@ public static List extractStructInfo(Plan plan, CascadesContext casc Group ownerGroup = plan.getGroupExpression().get().getOwnerGroup(); StructInfoMap structInfoMap = ownerGroup.getstructInfoMap(); // Refresh struct info in current level plan from top to bottom - structInfoMap.refresh(ownerGroup, cascadesContext.getMemo().getRefreshVersion()); + structInfoMap.refresh(ownerGroup, cascadesContext); structInfoMap.setRefreshVersion(cascadesContext.getMemo().getRefreshVersion()); Set queryTableSets = structInfoMap.getTableMaps(); @@ -161,7 +161,7 @@ public static List extractStructInfo(Plan plan, CascadesContext casc && !materializedViewTableSet.equals(queryTableSet)) { continue; } - StructInfo structInfo = structInfoMap.getStructInfo(cascadesContext.getMemo(), + StructInfo structInfo = structInfoMap.getStructInfo(cascadesContext, queryTableSet, ownerGroup, plan); if (structInfo != null) { structInfosBuilder.add(structInfo); @@ -171,7 +171,7 @@ public static List extractStructInfo(Plan plan, CascadesContext casc } } // if plan doesn't belong to any group, construct it directly - return ImmutableList.of(StructInfo.of(plan)); + return ImmutableList.of(StructInfo.of(plan, cascadesContext)); } /** diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/StructInfo.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/StructInfo.java index 55649c658b519d..fec66265880575 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/StructInfo.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/StructInfo.java @@ -96,7 +96,7 @@ public class StructInfo { // bottom plan which top plan only contain join or scan. this is needed by hyper graph private final Plan bottomPlan; private final List relations; - private final BitSet tableBitSet = new BitSet(); + private final BitSet tableBitSet; // this is for LogicalCompatibilityContext later private final Map relationIdStructInfoNodeMap; // this recorde the predicates which can pull up, not shuttled @@ -113,12 +113,13 @@ public class StructInfo { /** * The construct method for StructInfo */ - public StructInfo(Plan originalPlan, ObjectId originalPlanId, HyperGraph hyperGraph, boolean valid, Plan topPlan, + private StructInfo(Plan originalPlan, ObjectId originalPlanId, HyperGraph hyperGraph, boolean valid, Plan topPlan, Plan bottomPlan, List relations, Map relationIdStructInfoNodeMap, @Nullable Predicates predicates, Map> shuttledExpressionsToExpressionsMap, - Map namedExprIdAndExprMapping) { + Map namedExprIdAndExprMapping, + BitSet talbeIdSet) { this.originalPlan = originalPlan; this.originalPlanId = originalPlanId; this.hyperGraph = hyperGraph; @@ -127,7 +128,7 @@ public StructInfo(Plan originalPlan, ObjectId originalPlanId, HyperGraph hyperGr this.topPlan = topPlan; this.bottomPlan = bottomPlan; this.relations = relations; - relations.forEach(relation -> this.tableBitSet.set((int) (relation.getTable().getId()))); + this.tableBitSet = talbeIdSet; this.relationIdStructInfoNodeMap = relationIdStructInfoNodeMap; this.predicates = predicates; if (predicates == null) { @@ -150,7 +151,7 @@ public StructInfo(Plan originalPlan, ObjectId originalPlanId, HyperGraph hyperGr public StructInfo withPredicates(Predicates predicates) { return new StructInfo(this.originalPlan, this.originalPlanId, this.hyperGraph, this.valid, this.topPlan, this.bottomPlan, this.relations, this.relationIdStructInfoNodeMap, predicates, - this.shuttledExpressionsToExpressionsMap, this.namedExprIdAndExprMapping); + this.shuttledExpressionsToExpressionsMap, this.namedExprIdAndExprMapping, this.tableBitSet); } private static boolean collectStructInfoFromGraph(HyperGraph hyperGraph, @@ -265,15 +266,15 @@ private Pair predicatesDerive(Predicates predi * Build Struct info from plan. * Maybe return multi structInfo when original plan already be rewritten by mv */ - public static StructInfo of(Plan originalPlan) { - return of(originalPlan, originalPlan); + public static StructInfo of(Plan originalPlan, CascadesContext cascadesContext) { + return of(originalPlan, originalPlan, cascadesContext); } /** * Build Struct info from plan. * Maybe return multi structInfo when original plan already be rewritten by mv */ - public static StructInfo of(Plan derivedPlan, Plan originalPlan) { + public static StructInfo of(Plan derivedPlan, Plan originalPlan, CascadesContext cascadesContext) { // Split plan by the boundary which contains multi child LinkedHashSet> set = Sets.newLinkedHashSet(); set.add(LogicalJoin.class); @@ -281,14 +282,15 @@ public static StructInfo of(Plan derivedPlan, Plan originalPlan) { // if single table without join, the bottom is derivedPlan.accept(PLAN_SPLITTER, planSplitContext); return StructInfo.of(originalPlan, planSplitContext.getTopPlan(), planSplitContext.getBottomPlan(), - HyperGraph.builderForMv(planSplitContext.getBottomPlan()).build()); + HyperGraph.builderForMv(planSplitContext.getBottomPlan()).build(), cascadesContext); } /** * The construct method for init StructInfo */ public static StructInfo of(Plan originalPlan, @Nullable Plan topPlan, @Nullable Plan bottomPlan, - HyperGraph hyperGraph) { + HyperGraph hyperGraph, + CascadesContext cascadesContext) { ObjectId originalPlanId = originalPlan.getGroupExpression() .map(GroupExpression::getId).orElseGet(() -> new ObjectId(-1)); // if any of topPlan or bottomPlan is null, split the top plan to two parts by join node @@ -310,9 +312,14 @@ public static StructInfo of(Plan originalPlan, @Nullable Plan topPlan, @Nullable namedExprIdAndExprMapping, relationList, relationIdStructInfoNodeMap); + // Get mapped table id in relation and set + BitSet tableBitSet = new BitSet(); + for (CatalogRelation relation : relationList) { + tableBitSet.set(cascadesContext.getStatementContext().getTableId(relation.getTable()).asInt()); + } return new StructInfo(originalPlan, originalPlanId, hyperGraph, valid, topPlan, bottomPlan, relationList, relationIdStructInfoNodeMap, null, shuttledHashConjunctsToConjunctsMap, - namedExprIdAndExprMapping); + namedExprIdAndExprMapping, tableBitSet); } /** diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/mapping/RelationMapping.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/mapping/RelationMapping.java index eb53923da53cf8..a6f68d047bae35 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/mapping/RelationMapping.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/mapping/RelationMapping.java @@ -18,6 +18,7 @@ package org.apache.doris.nereids.rules.exploration.mv.mapping; import org.apache.doris.catalog.TableIf; +import org.apache.doris.catalog.constraint.TableIdentifier; import org.apache.doris.common.Pair; import org.apache.doris.nereids.trees.plans.algebra.CatalogRelation; @@ -62,22 +63,22 @@ public static RelationMapping of(ImmutableBiMap */ public static List generate(List sources, List targets) { // Construct tmp map, key is the table qualifier, value is the corresponding catalog relations - HashMultimap sourceTableRelationIdMap = HashMultimap.create(); + HashMultimap sourceTableRelationIdMap = HashMultimap.create(); for (CatalogRelation relation : sources) { - sourceTableRelationIdMap.put(getTableQualifier(relation.getTable()), + sourceTableRelationIdMap.put(getTableIdentifier(relation.getTable()), MappedRelation.of(relation.getRelationId(), relation)); } - HashMultimap targetTableRelationIdMap = HashMultimap.create(); + HashMultimap targetTableRelationIdMap = HashMultimap.create(); for (CatalogRelation relation : targets) { - targetTableRelationIdMap.put(getTableQualifier(relation.getTable()), + targetTableRelationIdMap.put(getTableIdentifier(relation.getTable()), MappedRelation.of(relation.getRelationId(), relation)); } - Set sourceTableKeySet = sourceTableRelationIdMap.keySet(); + Set sourceTableKeySet = sourceTableRelationIdMap.keySet(); List>> mappedRelations = new ArrayList<>(); - for (Long sourceTableId : sourceTableKeySet) { - Set sourceMappedRelations = sourceTableRelationIdMap.get(sourceTableId); - Set targetMappedRelations = targetTableRelationIdMap.get(sourceTableId); + for (TableIdentifier tableIdentifier : sourceTableKeySet) { + Set sourceMappedRelations = sourceTableRelationIdMap.get(tableIdentifier); + Set targetMappedRelations = targetTableRelationIdMap.get(tableIdentifier); if (targetMappedRelations.isEmpty()) { continue; } @@ -141,8 +142,8 @@ public static RelationMapping merge(List> return RelationMapping.of(mappingBuilder.build()); } - private static Long getTableQualifier(TableIf tableIf) { - return tableIf.getId(); + private static TableIdentifier getTableIdentifier(TableIf tableIf) { + return new TableIdentifier(tableIf); } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/StatementScopeIdGenerator.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/StatementScopeIdGenerator.java index 170c7cd5c70433..df7ef2ab69a100 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/StatementScopeIdGenerator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/StatementScopeIdGenerator.java @@ -20,6 +20,7 @@ import org.apache.doris.nereids.StatementContext; import org.apache.doris.nereids.trees.plans.ObjectId; import org.apache.doris.nereids.trees.plans.RelationId; +import org.apache.doris.nereids.trees.plans.TableId; import org.apache.doris.qe.ConnectContext; import com.google.common.annotations.VisibleForTesting; @@ -65,6 +66,13 @@ public static CTEId newCTEId() { return ConnectContext.get().getStatementContext().getNextCTEId(); } + public static TableId newTableId() { + if (ConnectContext.get() == null || ConnectContext.get().getStatementContext() == null) { + return statementContext.getNextTableId(); + } + return ConnectContext.get().getStatementContext().getNextTableId(); + } + /** * Reset Id Generator */ diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/TableId.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/TableId.java new file mode 100644 index 00000000000000..03e0ec450a3e73 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/TableId.java @@ -0,0 +1,59 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.trees.plans; + +import org.apache.doris.common.Id; +import org.apache.doris.common.IdGenerator; +import org.apache.doris.nereids.trees.expressions.StatementScopeIdGenerator; + +/** + * Table id + */ +public class TableId extends Id { + + public TableId(int id) { + super(id); + } + + /** + * Should be only called by {@link StatementScopeIdGenerator}. + */ + public static IdGenerator createGenerator() { + return new IdGenerator() { + @Override + public TableId getNextId() { + return new TableId(nextId++); + } + }; + } + + @Override + public String toString() { + return "TableId#" + id; + } + + @Override + public boolean equals(Object obj) { + return super.equals(obj); + } + + @Override + public int hashCode() { + return super.hashCode(); + } +} diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/CompareOuterJoinTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/CompareOuterJoinTest.java index ca543a0fd5249d..490090a43fbc60 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/CompareOuterJoinTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/CompareOuterJoinTest.java @@ -21,11 +21,6 @@ import org.apache.doris.nereids.rules.RuleSet; import org.apache.doris.nereids.rules.exploration.mv.ComparisonResult; import org.apache.doris.nereids.rules.exploration.mv.HyperGraphComparator; -import org.apache.doris.nereids.rules.exploration.mv.LogicalCompatibilityContext; -import org.apache.doris.nereids.rules.exploration.mv.MaterializedViewUtils; -import org.apache.doris.nereids.rules.exploration.mv.StructInfo; -import org.apache.doris.nereids.rules.exploration.mv.mapping.RelationMapping; -import org.apache.doris.nereids.rules.exploration.mv.mapping.SlotMapping; import org.apache.doris.nereids.sqltest.SqlTestBase; import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.plans.JoinType; @@ -37,7 +32,6 @@ import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; -import java.util.BitSet; import java.util.List; class CompareOuterJoinTest extends SqlTestBase { @@ -68,7 +62,7 @@ void testStarGraphWithInnerJoin() { HyperGraph h1 = HyperGraph.builderForMv(p1).build(); HyperGraph h2 = HyperGraph.builderForMv(p2).build(); Assertions.assertFalse( - HyperGraphComparator.isLogicCompatible(h1, h2, constructContext(p1, p2)).isInvalid()); + HyperGraphComparator.isLogicCompatible(h1, h2, constructContext(p1, p2, c1)).isInvalid()); } @Test @@ -76,9 +70,10 @@ void testRandomQuery() { connectContext.getSessionVariable().setDisableNereidsRules("PRUNE_EMPTY_PARTITION"); Plan p1 = new HyperGraphBuilder(Sets.newHashSet(JoinType.INNER_JOIN)) .randomBuildPlanWith(3, 3); - p1 = PlanChecker.from(connectContext, p1) + PlanChecker planChecker = PlanChecker.from(connectContext, p1) .analyze() - .rewrite() + .rewrite(); + p1 = planChecker .getPlan(); Plan p2 = PlanChecker.from(connectContext, p1) .analyze() @@ -88,7 +83,8 @@ void testRandomQuery() { HyperGraph h1 = HyperGraph.builderForMv(p1).build(); HyperGraph h2 = HyperGraph.builderForMv(p2).build(); Assertions.assertFalse( - HyperGraphComparator.isLogicCompatible(h1, h2, constructContext(p1, p2)).isInvalid()); + HyperGraphComparator.isLogicCompatible(h1, h2, + constructContext(p1, p2, planChecker.getCascadesContext())).isInvalid()); } @Test @@ -113,7 +109,7 @@ void testInnerJoinWithFilter() { .getAllPlan().get(0).child(0); HyperGraph h1 = HyperGraph.builderForMv(p1).build(); HyperGraph h2 = HyperGraph.builderForMv(p2).build(); - ComparisonResult res = HyperGraphComparator.isLogicCompatible(h1, h2, constructContext(p1, p2)); + ComparisonResult res = HyperGraphComparator.isLogicCompatible(h1, h2, constructContext(p1, p2, c1)); Assertions.assertEquals(1, res.getQueryExpressions().size()); Assertions.assertEquals("(id = 0)", res.getQueryExpressions().get(0).toSql()); } @@ -140,7 +136,8 @@ void testInnerJoinWithFilter2() { .getAllPlan().get(0).child(0); HyperGraph h1 = HyperGraph.builderForMv(p1).build(); HyperGraph h2 = HyperGraph.builderForMv(p2).build(); - List exprList = HyperGraphComparator.isLogicCompatible(h1, h2, constructContext(p1, p2)).getQueryExpressions(); + List exprList = HyperGraphComparator.isLogicCompatible(h1, h2, + constructContext(p1, p2, c1)).getQueryExpressions(); Assertions.assertEquals(0, exprList.size()); } @@ -166,7 +163,7 @@ void testLeftOuterJoinWithLeftFilter() { .getAllPlan().get(0).child(0); HyperGraph h1 = HyperGraph.builderForMv(p1).build(); HyperGraph h2 = HyperGraph.builderForMv(p2).build(); - ComparisonResult res = HyperGraphComparator.isLogicCompatible(h1, h2, constructContext(p1, p2)); + ComparisonResult res = HyperGraphComparator.isLogicCompatible(h1, h2, constructContext(p1, p2, c1)); Assertions.assertEquals(1, res.getQueryExpressions().size()); Assertions.assertEquals("(id = 0)", res.getQueryExpressions().get(0).toSql()); } @@ -193,17 +190,7 @@ void testLeftOuterJoinWithRightFilter() { .getAllPlan().get(0).child(0); HyperGraph h1 = HyperGraph.builderForMv(p1).build(); HyperGraph h2 = HyperGraph.builderForMv(p2).build(); - ComparisonResult res = HyperGraphComparator.isLogicCompatible(h1, h2, constructContext(p1, p2)); + ComparisonResult res = HyperGraphComparator.isLogicCompatible(h1, h2, constructContext(p1, p2, c1)); Assertions.assertTrue(res.isInvalid()); } - - LogicalCompatibilityContext constructContext(Plan p1, Plan p2) { - StructInfo st1 = MaterializedViewUtils.extractStructInfo(p1, - null, new BitSet()).get(0); - StructInfo st2 = MaterializedViewUtils.extractStructInfo(p2, - null, new BitSet()).get(0); - RelationMapping rm = RelationMapping.generate(st1.getRelations(), st2.getRelations()).get(0); - SlotMapping sm = SlotMapping.generate(rm); - return LogicalCompatibilityContext.from(rm, sm, st1, st2); - } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/InferJoinTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/InferJoinTest.java index c7d93a363e3926..026c354d787609 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/InferJoinTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/InferJoinTest.java @@ -21,11 +21,6 @@ import org.apache.doris.nereids.rules.RuleSet; import org.apache.doris.nereids.rules.exploration.mv.ComparisonResult; import org.apache.doris.nereids.rules.exploration.mv.HyperGraphComparator; -import org.apache.doris.nereids.rules.exploration.mv.LogicalCompatibilityContext; -import org.apache.doris.nereids.rules.exploration.mv.MaterializedViewUtils; -import org.apache.doris.nereids.rules.exploration.mv.StructInfo; -import org.apache.doris.nereids.rules.exploration.mv.mapping.RelationMapping; -import org.apache.doris.nereids.rules.exploration.mv.mapping.SlotMapping; import org.apache.doris.nereids.sqltest.SqlTestBase; import org.apache.doris.nereids.trees.expressions.functions.ExpressionTrait; import org.apache.doris.nereids.trees.plans.Plan; @@ -35,7 +30,6 @@ import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; -import java.util.BitSet; import java.util.stream.Collectors; class InferJoinTest extends SqlTestBase { @@ -61,7 +55,7 @@ void testInnerInferLeft() { .getAllPlan().get(0).child(0); HyperGraph h1 = HyperGraph.builderForMv(p1).build(); HyperGraph h2 = HyperGraph.builderForMv(p2).build(); - ComparisonResult res = HyperGraphComparator.isLogicCompatible(h1, h2, constructContext(p1, p2)); + ComparisonResult res = HyperGraphComparator.isLogicCompatible(h1, h2, constructContext(p1, p2, c1)); Assertions.assertFalse(res.isInvalid()); Assertions.assertEquals(1, res.getViewNoNullableSlot().size()); Assertions.assertEquals("[id, score]", @@ -90,7 +84,7 @@ void testInnerInferLeftWithFilter() { .getAllPlan().get(0).child(0); HyperGraph h1 = HyperGraph.builderForMv(p1).build(); HyperGraph h2 = HyperGraph.builderForMv(p2).build(); - ComparisonResult res = HyperGraphComparator.isLogicCompatible(h1, h2, constructContext(p1, p2)); + ComparisonResult res = HyperGraphComparator.isLogicCompatible(h1, h2, constructContext(p1, p2, c1)); Assertions.assertFalse(res.isInvalid()); Assertions.assertEquals(1, res.getViewNoNullableSlot().size()); Assertions.assertEquals("[id, score]", @@ -127,7 +121,7 @@ void testInnerInferLeftWithJoinCond() { .getAllPlan().get(0).child(0); HyperGraph h1 = HyperGraph.builderForMv(p1).build(); HyperGraph h2 = HyperGraph.builderForMv(p2).build(); - ComparisonResult res = HyperGraphComparator.isLogicCompatible(h1, h2, constructContext(p1, p2)); + ComparisonResult res = HyperGraphComparator.isLogicCompatible(h1, h2, constructContext(p1, p2, c1)); Assertions.assertFalse(res.isInvalid()); Assertions.assertEquals(1, res.getViewNoNullableSlot().size()); Assertions.assertEquals("[id, score]", @@ -157,17 +151,7 @@ void testLeftOuterJoinWithRightFilter() { .getAllPlan().get(0).child(0); HyperGraph h1 = HyperGraph.builderForMv(p1).build(); HyperGraph h2 = HyperGraph.builderForMv(p2).build(); - ComparisonResult res = HyperGraphComparator.isLogicCompatible(h1, h2, constructContext(p1, p2)); + ComparisonResult res = HyperGraphComparator.isLogicCompatible(h1, h2, constructContext(p1, p2, c1)); Assertions.assertTrue(res.isInvalid()); } - - LogicalCompatibilityContext constructContext(Plan p1, Plan p2) { - StructInfo st1 = MaterializedViewUtils.extractStructInfo(p1, - null, new BitSet()).get(0); - StructInfo st2 = MaterializedViewUtils.extractStructInfo(p2, - null, new BitSet()).get(0); - RelationMapping rm = RelationMapping.generate(st1.getRelations(), st2.getRelations()).get(0); - SlotMapping sm = SlotMapping.generate(rm); - return LogicalCompatibilityContext.from(rm, sm, st1, st2); - } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/InferPredicateTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/InferPredicateTest.java index b6f3fbb45b6186..acc4c90b67eaef 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/InferPredicateTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/InferPredicateTest.java @@ -21,11 +21,6 @@ import org.apache.doris.nereids.rules.RuleSet; import org.apache.doris.nereids.rules.exploration.mv.ComparisonResult; import org.apache.doris.nereids.rules.exploration.mv.HyperGraphComparator; -import org.apache.doris.nereids.rules.exploration.mv.LogicalCompatibilityContext; -import org.apache.doris.nereids.rules.exploration.mv.MaterializedViewUtils; -import org.apache.doris.nereids.rules.exploration.mv.StructInfo; -import org.apache.doris.nereids.rules.exploration.mv.mapping.RelationMapping; -import org.apache.doris.nereids.rules.exploration.mv.mapping.SlotMapping; import org.apache.doris.nereids.sqltest.SqlTestBase; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.util.PlanChecker; @@ -33,8 +28,6 @@ import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; -import java.util.BitSet; - class InferPredicateTest extends SqlTestBase { @Test void testPullUpQueryFilter() { @@ -58,18 +51,8 @@ void testPullUpQueryFilter() { .getAllPlan().get(0).child(0); HyperGraph h1 = HyperGraph.builderForMv(p1).build(); HyperGraph h2 = HyperGraph.builderForMv(p2).build(); - ComparisonResult res = HyperGraphComparator.isLogicCompatible(h1, h2, constructContext(p1, p2)); + ComparisonResult res = HyperGraphComparator.isLogicCompatible(h1, h2, constructContext(p1, p2, c1)); Assertions.assertFalse(res.isInvalid()); Assertions.assertEquals("(id = 1)", res.getQueryExpressions().get(0).toSql()); } - - LogicalCompatibilityContext constructContext(Plan p1, Plan p2) { - StructInfo st1 = MaterializedViewUtils.extractStructInfo(p1, - null, new BitSet()).get(0); - StructInfo st2 = MaterializedViewUtils.extractStructInfo(p2, - null, new BitSet()).get(0); - RelationMapping rm = RelationMapping.generate(st1.getRelations(), st2.getRelations()).get(0); - SlotMapping sm = SlotMapping.generate(rm); - return LogicalCompatibilityContext.from(rm, sm, st1, st2); - } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/PullupExpressionTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/PullupExpressionTest.java index def3281932a77d..26daa649ec029a 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/PullupExpressionTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/PullupExpressionTest.java @@ -21,11 +21,6 @@ import org.apache.doris.nereids.rules.RuleSet; import org.apache.doris.nereids.rules.exploration.mv.ComparisonResult; import org.apache.doris.nereids.rules.exploration.mv.HyperGraphComparator; -import org.apache.doris.nereids.rules.exploration.mv.LogicalCompatibilityContext; -import org.apache.doris.nereids.rules.exploration.mv.MaterializedViewUtils; -import org.apache.doris.nereids.rules.exploration.mv.StructInfo; -import org.apache.doris.nereids.rules.exploration.mv.mapping.RelationMapping; -import org.apache.doris.nereids.rules.exploration.mv.mapping.SlotMapping; import org.apache.doris.nereids.sqltest.SqlTestBase; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.util.PlanChecker; @@ -33,8 +28,6 @@ import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; -import java.util.BitSet; - class PullupExpressionTest extends SqlTestBase { @Test void testPullUpQueryFilter() { @@ -58,7 +51,7 @@ void testPullUpQueryFilter() { .getAllPlan().get(0).child(0); HyperGraph h1 = HyperGraph.builderForMv(p1).build(); HyperGraph h2 = HyperGraph.builderForMv(p2).build(); - ComparisonResult res = HyperGraphComparator.isLogicCompatible(h1, h2, constructContext(p1, p2)); + ComparisonResult res = HyperGraphComparator.isLogicCompatible(h1, h2, constructContext(p1, p2, c1)); Assertions.assertEquals(1, res.getQueryExpressions().size()); Assertions.assertEquals("(id = 1)", res.getQueryExpressions().get(0).toSql()); } @@ -85,7 +78,7 @@ void testPullUpQueryJoinCondition() { .getAllPlan().get(0).child(0); HyperGraph h1 = HyperGraph.builderForMv(p1).build(); HyperGraph h2 = HyperGraph.builderForMv(p2).build(); - ComparisonResult res = HyperGraphComparator.isLogicCompatible(h1, h2, constructContext(p1, p2)); + ComparisonResult res = HyperGraphComparator.isLogicCompatible(h1, h2, constructContext(p1, p2, c1)); Assertions.assertEquals(1, res.getQueryExpressions().size()); Assertions.assertEquals("(score = score)", res.getQueryExpressions().get(0).toSql()); } @@ -112,7 +105,7 @@ void testPullUpViewFilter() { .getAllPlan().get(0).child(0); HyperGraph h1 = HyperGraph.builderForMv(p1).build(); HyperGraph h2 = HyperGraph.builderForMv(p2).build(); - ComparisonResult res = HyperGraphComparator.isLogicCompatible(h1, h2, constructContext(p1, p2)); + ComparisonResult res = HyperGraphComparator.isLogicCompatible(h1, h2, constructContext(p1, p2, c1)); Assertions.assertEquals(2, res.getViewExpressions().size()); Assertions.assertEquals("(id = 1)", res.getViewExpressions().get(0).toSql()); Assertions.assertEquals("(id = 1)", res.getViewExpressions().get(1).toSql()); @@ -140,18 +133,8 @@ void testPullUpViewJoinCondition() { .getAllPlan().get(0).child(0); HyperGraph h1 = HyperGraph.builderForMv(p1).build(); HyperGraph h2 = HyperGraph.builderForMv(p2).build(); - ComparisonResult res = HyperGraphComparator.isLogicCompatible(h1, h2, constructContext(p1, p2)); + ComparisonResult res = HyperGraphComparator.isLogicCompatible(h1, h2, constructContext(p1, p2, c1)); Assertions.assertEquals(1, res.getViewExpressions().size()); Assertions.assertEquals("(score = score)", res.getViewExpressions().get(0).toSql()); } - - LogicalCompatibilityContext constructContext(Plan p1, Plan p2) { - StructInfo st1 = MaterializedViewUtils.extractStructInfo(p1, - null, new BitSet()).get(0); - StructInfo st2 = MaterializedViewUtils.extractStructInfo(p2, - null, new BitSet()).get(0); - RelationMapping rm = RelationMapping.generate(st1.getRelations(), st2.getRelations()).get(0); - SlotMapping sm = SlotMapping.generate(rm); - return LogicalCompatibilityContext.from(rm, sm, st1, st2); - } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/memo/StructInfoMapTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/memo/StructInfoMapTest.java index 80670dc366d7dc..0645cf8515b1f5 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/memo/StructInfoMapTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/memo/StructInfoMapTest.java @@ -22,6 +22,7 @@ import org.apache.doris.nereids.CascadesContext; import org.apache.doris.nereids.rules.exploration.mv.StructInfo; import org.apache.doris.nereids.sqltest.SqlTestBase; +import org.apache.doris.nereids.trees.plans.algebra.CatalogRelation; import org.apache.doris.nereids.util.PlanChecker; import org.apache.doris.qe.ConnectContext; import org.apache.doris.qe.SessionVariable; @@ -59,7 +60,7 @@ public BitSet getDisableNereidsRules() { Group root = c1.getMemo().getRoot(); Set tableMaps = root.getstructInfoMap().getTableMaps(); Assertions.assertTrue(tableMaps.isEmpty()); - root.getstructInfoMap().refresh(root, 1); + root.getstructInfoMap().refresh(root, c1); Assertions.assertEquals(1, tableMaps.size()); new MockUp() { @Mock @@ -87,7 +88,7 @@ public boolean isMVPartitionValid(MTMV mtmv, ConnectContext ctx) { .optimize() .printlnBestPlanTree(); root = c1.getMemo().getRoot(); - root.getstructInfoMap().refresh(root, 1); + root.getstructInfoMap().refresh(root, c1); tableMaps = root.getstructInfoMap().getTableMaps(); Assertions.assertEquals(2, tableMaps.size()); dropMvByNereids("drop materialized view mv1"); @@ -116,8 +117,8 @@ public BitSet getDisableNereidsRules() { Group root = c1.getMemo().getRoot(); Set tableMaps = root.getstructInfoMap().getTableMaps(); Assertions.assertTrue(tableMaps.isEmpty()); - root.getstructInfoMap().refresh(root, 1); - root.getstructInfoMap().refresh(root, 1); + root.getstructInfoMap().refresh(root, c1); + root.getstructInfoMap().refresh(root, c1); Assertions.assertEquals(1, tableMaps.size()); new MockUp() { @Mock @@ -144,8 +145,8 @@ public boolean isMVPartitionValid(MTMV mtmv, ConnectContext ctx) { .optimize() .printlnBestPlanTree(); root = c1.getMemo().getRoot(); - root.getstructInfoMap().refresh(root, 1); - root.getstructInfoMap().refresh(root, 1); + root.getstructInfoMap().refresh(root, c1); + root.getstructInfoMap().refresh(root, c1); tableMaps = root.getstructInfoMap().getTableMaps(); Assertions.assertEquals(2, tableMaps.size()); dropMvByNereids("drop materialized view mv1"); @@ -191,16 +192,18 @@ public boolean isMVPartitionValid(MTMV mtmv, ConnectContext ctx) { .rewrite() .optimize(); Group root = c1.getMemo().getRoot(); - root.getstructInfoMap().refresh(root, 1); + root.getstructInfoMap().refresh(root, c1); StructInfoMap structInfoMap = root.getstructInfoMap(); Assertions.assertEquals(2, structInfoMap.getTableMaps().size()); BitSet mvMap = structInfoMap.getTableMaps().stream() .filter(b -> b.cardinality() == 2) .collect(Collectors.toList()).get(0); - StructInfo structInfo = structInfoMap.getStructInfo(c1.getMemo(), mvMap, root, null); + StructInfo structInfo = structInfoMap.getStructInfo(c1, mvMap, root, null); System.out.println(structInfo.getOriginalPlan().treeString()); BitSet bitSet = new BitSet(); - structInfo.getRelations().forEach(r -> bitSet.set((int) r.getTable().getId())); + for (CatalogRelation relation : structInfo.getRelations()) { + bitSet.set(c1.getStatementContext().getTableId(relation.getTable()).asInt()); + } Assertions.assertEquals(bitSet, mvMap); dropMvByNereids("drop materialized view mv1"); } diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/mv/EliminateJoinTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/mv/EliminateJoinTest.java index 1756107b287942..2bec767d46f041 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/mv/EliminateJoinTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/mv/EliminateJoinTest.java @@ -20,8 +20,6 @@ import org.apache.doris.nereids.CascadesContext; import org.apache.doris.nereids.jobs.joinorder.hypergraph.HyperGraph; import org.apache.doris.nereids.rules.RuleSet; -import org.apache.doris.nereids.rules.exploration.mv.mapping.RelationMapping; -import org.apache.doris.nereids.rules.exploration.mv.mapping.SlotMapping; import org.apache.doris.nereids.sqltest.SqlTestBase; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.util.PlanChecker; @@ -30,8 +28,6 @@ import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; -import java.util.BitSet; - class EliminateJoinTest extends SqlTestBase { @Test void testLOJWithGroupBy() { @@ -67,9 +63,10 @@ void testLOJWithGroupBy() { HyperGraph h1 = HyperGraph.builderForMv(p1).build(); HyperGraph h2 = HyperGraph.builderForMv(p2).build(); HyperGraph h3 = HyperGraph.builderForMv(p3).build(); - ComparisonResult res = HyperGraphComparator.isLogicCompatible(h1, h2, constructContext(p1, p2)); + ComparisonResult res = HyperGraphComparator.isLogicCompatible(h1, h2, constructContext(p1, p2, c1)); Assertions.assertTrue(!res.isInvalid()); - Assertions.assertTrue(!HyperGraphComparator.isLogicCompatible(h1, h3, constructContext(p1, p2)).isInvalid()); + Assertions.assertTrue(!HyperGraphComparator.isLogicCompatible(h1, h3, + constructContext(p1, p2, c1)).isInvalid()); Assertions.assertTrue(res.getViewExpressions().isEmpty()); } @@ -97,7 +94,7 @@ void testLOJWithUK() throws Exception { .getAllPlan().get(0).child(0); HyperGraph h1 = HyperGraph.builderForMv(p1).build(); HyperGraph h2 = HyperGraph.builderForMv(p2).build(); - ComparisonResult res = HyperGraphComparator.isLogicCompatible(h1, h2, constructContext(p1, p2)); + ComparisonResult res = HyperGraphComparator.isLogicCompatible(h1, h2, constructContext(p1, p2, c1)); Assertions.assertTrue(!res.isInvalid()); Assertions.assertTrue(res.getViewExpressions().isEmpty()); dropConstraint("alter table T2 drop constraint uk"); @@ -139,10 +136,10 @@ void testLOJWithPKFK() throws Exception { HyperGraph h1 = HyperGraph.builderForMv(p1).build(); HyperGraph h2 = HyperGraph.builderForMv(p2).build(); HyperGraph h3 = HyperGraph.builderForMv(p3).build(); - ComparisonResult res = HyperGraphComparator.isLogicCompatible(h1, h2, constructContext(p1, p2)); + ComparisonResult res = HyperGraphComparator.isLogicCompatible(h1, h2, constructContext(p1, p2, c1)); Assertions.assertTrue(!res.isInvalid()); Assertions.assertTrue(res.getViewExpressions().isEmpty()); - Assertions.assertTrue(!HyperGraphComparator.isLogicCompatible(h1, h3, constructContext(p1, p2)).isInvalid()); + Assertions.assertTrue(!HyperGraphComparator.isLogicCompatible(h1, h3, constructContext(p1, p2, c1)).isInvalid()); dropConstraint("alter table T2 drop constraint pk"); } @@ -173,7 +170,7 @@ void testLOJWithPKFKAndUK1() throws Exception { .getAllPlan().get(0).child(0); HyperGraph h1 = HyperGraph.builderForMv(p1).build(); HyperGraph h2 = HyperGraph.builderForMv(p2).build(); - ComparisonResult res = HyperGraphComparator.isLogicCompatible(h1, h2, constructContext(p1, p2)); + ComparisonResult res = HyperGraphComparator.isLogicCompatible(h1, h2, constructContext(p1, p2, c1)); Assertions.assertTrue(!res.isInvalid()); Assertions.assertTrue(res.getViewExpressions().isEmpty()); dropConstraint("alter table T2 drop constraint pk"); @@ -206,20 +203,10 @@ void testLOJWithPKFKAndUK2() throws Exception { .getAllPlan().get(0).child(0); HyperGraph h1 = HyperGraph.builderForMv(p1).build(); HyperGraph h2 = HyperGraph.builderForMv(p2).build(); - ComparisonResult res = HyperGraphComparator.isLogicCompatible(h1, h2, constructContext(p1, p2)); + ComparisonResult res = HyperGraphComparator.isLogicCompatible(h1, h2, constructContext(p1, p2, c1)); Assertions.assertTrue(!res.isInvalid()); Assertions.assertTrue(res.getViewExpressions().isEmpty()); dropConstraint("alter table T2 drop constraint pk"); dropConstraint("alter table T3 drop constraint uk"); } - - LogicalCompatibilityContext constructContext(Plan p1, Plan p2) { - StructInfo st1 = MaterializedViewUtils.extractStructInfo(p1, - null, new BitSet()).get(0); - StructInfo st2 = MaterializedViewUtils.extractStructInfo(p2, - null, new BitSet()).get(0); - RelationMapping rm = RelationMapping.generate(st1.getRelations(), st2.getRelations()).get(0); - SlotMapping sm = SlotMapping.generate(rm); - return LogicalCompatibilityContext.from(rm, sm, st1, st2); - } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/mv/HyperGraphComparatorTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/mv/HyperGraphComparatorTest.java index 360f1310e5147f..6d752cd6cac401 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/mv/HyperGraphComparatorTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/mv/HyperGraphComparatorTest.java @@ -20,8 +20,6 @@ import org.apache.doris.nereids.CascadesContext; import org.apache.doris.nereids.jobs.joinorder.hypergraph.HyperGraph; import org.apache.doris.nereids.rules.RuleSet; -import org.apache.doris.nereids.rules.exploration.mv.mapping.RelationMapping; -import org.apache.doris.nereids.rules.exploration.mv.mapping.SlotMapping; import org.apache.doris.nereids.sqltest.SqlTestBase; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.util.PlanChecker; @@ -30,8 +28,6 @@ import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; -import java.util.BitSet; - class HyperGraphComparatorTest extends SqlTestBase { @Test void testInnerJoinAndLOJ() { @@ -59,7 +55,7 @@ void testInnerJoinAndLOJ() { .getAllPlan().get(0).child(0); HyperGraph h1 = HyperGraph.builderForMv(p1).build(); HyperGraph h2 = HyperGraph.builderForMv(p2).build(); - ComparisonResult res = HyperGraphComparator.isLogicCompatible(h1, h2, constructContext(p1, p2)); + ComparisonResult res = HyperGraphComparator.isLogicCompatible(h1, h2, constructContext(p1, p2, c1)); Assertions.assertTrue(!res.isInvalid()); Assertions.assertEquals(2, res.getViewNoNullableSlot().size()); } @@ -90,7 +86,7 @@ void testIJAndLojAssoc() { .getAllPlan().get(0).child(0); HyperGraph h1 = HyperGraph.builderForMv(p1).build(); HyperGraph h2 = HyperGraph.builderForMv(p2).build(); - ComparisonResult res = HyperGraphComparator.isLogicCompatible(h1, h2, constructContext(p1, p2)); + ComparisonResult res = HyperGraphComparator.isLogicCompatible(h1, h2, constructContext(p1, p2, c1)); Assertions.assertTrue(!res.isInvalid()); Assertions.assertEquals(2, res.getViewNoNullableSlot().size()); } @@ -122,7 +118,7 @@ void testIJAndLojAssocWithFilter() { .getAllPlan().get(0).child(0); HyperGraph h1 = HyperGraph.builderForMv(p1).build(); HyperGraph h2 = HyperGraph.builderForMv(p2).build(); - ComparisonResult res = HyperGraphComparator.isLogicCompatible(h1, h2, constructContext(p1, p2)); + ComparisonResult res = HyperGraphComparator.isLogicCompatible(h1, h2, constructContext(p1, p2, c1)); Assertions.assertTrue(!res.isInvalid()); Assertions.assertEquals(2, res.getViewNoNullableSlot().size()); } @@ -157,18 +153,8 @@ void testIJAndLojAssocWithJoinCond() { .getAllPlan().get(0).child(0); HyperGraph h1 = HyperGraph.builderForMv(p1).build(); HyperGraph h2 = HyperGraph.builderForMv(p2).build(); - ComparisonResult res = HyperGraphComparator.isLogicCompatible(h1, h2, constructContext(p1, p2)); + ComparisonResult res = HyperGraphComparator.isLogicCompatible(h1, h2, constructContext(p1, p2, c1)); Assertions.assertTrue(!res.isInvalid()); Assertions.assertEquals(2, res.getViewNoNullableSlot().size()); } - - LogicalCompatibilityContext constructContext(Plan p1, Plan p2) { - StructInfo st1 = MaterializedViewUtils.extractStructInfo(p1, - null, new BitSet()).get(0); - StructInfo st2 = MaterializedViewUtils.extractStructInfo(p2, - null, new BitSet()).get(0); - RelationMapping rm = RelationMapping.generate(st1.getRelations(), st2.getRelations()).get(0); - SlotMapping sm = SlotMapping.generate(rm); - return LogicalCompatibilityContext.from(rm, sm, st1, st2); - } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/sqltest/SqlTestBase.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/sqltest/SqlTestBase.java index 5c7aaed36e3fea..e879f461012063 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/sqltest/SqlTestBase.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/sqltest/SqlTestBase.java @@ -17,10 +17,19 @@ package org.apache.doris.nereids.sqltest; +import org.apache.doris.nereids.CascadesContext; +import org.apache.doris.nereids.rules.exploration.mv.LogicalCompatibilityContext; +import org.apache.doris.nereids.rules.exploration.mv.MaterializedViewUtils; +import org.apache.doris.nereids.rules.exploration.mv.StructInfo; +import org.apache.doris.nereids.rules.exploration.mv.mapping.RelationMapping; +import org.apache.doris.nereids.rules.exploration.mv.mapping.SlotMapping; import org.apache.doris.nereids.trees.expressions.StatementScopeIdGenerator; +import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.util.MemoPatternMatchSupported; import org.apache.doris.utframe.TestWithFeService; +import java.util.BitSet; + public abstract class SqlTestBase extends TestWithFeService implements MemoPatternMatchSupported { @Override protected void runBeforeAll() throws Exception { @@ -82,4 +91,14 @@ protected void runBeforeAll() throws Exception { protected void runBeforeEach() throws Exception { StatementScopeIdGenerator.clear(); } + + protected LogicalCompatibilityContext constructContext(Plan p1, Plan p2, CascadesContext context) { + StructInfo st1 = MaterializedViewUtils.extractStructInfo(p1, + context, new BitSet()).get(0); + StructInfo st2 = MaterializedViewUtils.extractStructInfo(p2, + context, new BitSet()).get(0); + RelationMapping rm = RelationMapping.generate(st1.getRelations(), st2.getRelations()).get(0); + SlotMapping sm = SlotMapping.generate(rm); + return LogicalCompatibilityContext.from(rm, sm, st1, st2); + } }