[fix](mtmv) Fix table id overturn and optimize get table qualifier me…

…thod (#34768)

Table id may be the same but actually they are different tables. so we optimize the
org.apache.doris.nereids.rules.exploration.mv.mapping.RelationMapping#getTableQualifier with following code:

Objects.hash(table.getDatabase().getCatalog().getId(), table.getDatabase().getId(), table.getId())

table id is long, we identify the table used in mv rewrite is bitSet. the bitSet can only use int, so we mapping the long id to init id in every query when mv rewrite

fe/fe-core/src/main/java/org/apache/doris/catalog/constraint/TableIdentifier.java

@@ -40,8 +40,9 @@ public TableIdentifier(TableIf tableIf) {
         Preconditions.checkArgument(tableIf != null,
                 "Table can not be null in constraint");
         tableId = tableIf.getId();
-        databaseId = tableIf.getDatabase().getId();
-        catalogId = tableIf.getDatabase().getCatalog().getId();
+        databaseId = tableIf.getDatabase() == null ? 0L : tableIf.getDatabase().getId();
+        catalogId = tableIf.getDatabase() == null || tableIf.getDatabase().getCatalog() == null
+                ? 0L : tableIf.getDatabase().getCatalog().getId();
     }
 
     public TableIf toTableIf() {
@@ -69,13 +70,14 @@ public boolean equals(Object o) {
             return false;
         }
         TableIdentifier that = (TableIdentifier) o;
-        return databaseId == that.databaseId
+        return catalogId == that.catalogId
+                && databaseId == that.databaseId
                 && tableId == that.tableId;
     }
 
     @Override
     public int hashCode() {
-        return Objects.hash(databaseId, tableId);
+        return Objects.hash(catalogId, databaseId, tableId);
     }
 
     @Override

fe/fe-core/src/main/java/org/apache/doris/nereids/NereidsPlanner.java

@@ -502,7 +502,7 @@ public String getExplainString(ExplainOptions explainOptions) {
                         this.getPhysicalPlan());
                 if (statementContext != null) {
                     if (statementContext.isHasUnknownColStats()) {
-                        plan += "planed with unknown column statistics\n";
+                        plan += "\n\nStatistics\n planed with unknown column statistics\n";
                     }
                 }
         }

fe/fe-core/src/main/java/org/apache/doris/nereids/StatementContext.java

@@ -20,6 +20,7 @@
 import org.apache.doris.analysis.StatementBase;
 import org.apache.doris.catalog.Column;
 import org.apache.doris.catalog.TableIf;
+import org.apache.doris.catalog.constraint.TableIdentifier;
 import org.apache.doris.common.IdGenerator;
 import org.apache.doris.common.Pair;
 import org.apache.doris.nereids.hint.Hint;
@@ -30,8 +31,10 @@
 import org.apache.doris.nereids.trees.expressions.Expression;
 import org.apache.doris.nereids.trees.expressions.Slot;
 import org.apache.doris.nereids.trees.expressions.SlotReference;
+import org.apache.doris.nereids.trees.expressions.StatementScopeIdGenerator;
 import org.apache.doris.nereids.trees.plans.ObjectId;
 import org.apache.doris.nereids.trees.plans.RelationId;
+import org.apache.doris.nereids.trees.plans.TableId;
 import org.apache.doris.nereids.trees.plans.algebra.Relation;
 import org.apache.doris.nereids.trees.plans.logical.LogicalCTEConsumer;
 import org.apache.doris.nereids.trees.plans.logical.LogicalPlan;
@@ -56,6 +59,7 @@
 import java.util.Collection;
 import java.util.HashMap;
 import java.util.Iterator;
+import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Optional;
@@ -100,6 +104,7 @@ public class StatementContext implements Closeable {
     private final IdGenerator<ObjectId> objectIdGenerator = ObjectId.createGenerator();
     private final IdGenerator<RelationId> relationIdGenerator = RelationId.createGenerator();
     private final IdGenerator<CTEId> cteIdGenerator = CTEId.createGenerator();
+    private final IdGenerator<TableId> talbeIdGenerator = TableId.createGenerator();
 
     private final Map<CTEId, Set<LogicalCTEConsumer>> cteIdToConsumers = new HashMap<>();
     private final Map<CTEId, Set<Slot>> cteIdToOutputIds = new HashMap<>();
@@ -141,6 +146,9 @@ public class StatementContext implements Closeable {
     // and value is the new string used for replacement.
     private final TreeMap<Pair<Integer, Integer>, String> indexInSqlToString
             = new TreeMap<>(new Pair.PairComparator<>());
+    // Record table id mapping, the key is the hash code of union catalogId, databaseId, tableId
+    // the value is the auto-increment id in the cascades context
+    private final Map<TableIdentifier, TableId> tableIdMapping = new LinkedHashMap<>();
 
     public StatementContext() {
         this(ConnectContext.get(), null, 0);
@@ -293,6 +301,10 @@ public RelationId getNextRelationId() {
         return relationIdGenerator.getNextId();
     }
 
+    public TableId getNextTableId() {
+        return talbeIdGenerator.getNextId();
+    }
+
     public void setParsedStatement(StatementBase parsedStatement) {
         this.parsedStatement = parsedStatement;
     }
@@ -496,4 +508,16 @@ public void addKeyColumn(Column column) {
     public boolean isKeyColumn(Column column) {
         return keyColumns.contains(column);
     }
+
+    /** Get table id with lazy */
+    public TableId getTableId(TableIf tableIf) {
+        TableIdentifier tableIdentifier = new TableIdentifier(tableIf);
+        TableId tableId = this.tableIdMapping.get(tableIdentifier);
+        if (tableId != null) {
+            return tableId;
+        }
+        tableId = StatementScopeIdGenerator.newTableId();
+        this.tableIdMapping.put(tableIdentifier, tableId);
+        return tableId;
+    }
 }

fe/fe-core/src/main/java/org/apache/doris/nereids/memo/StructInfoMap.java

@@ -18,6 +18,7 @@
 package org.apache.doris.nereids.memo;
 
 import org.apache.doris.common.Pair;
+import org.apache.doris.nereids.CascadesContext;
 import org.apache.doris.nereids.rules.exploration.mv.StructInfo;
 import org.apache.doris.nereids.trees.plans.Plan;
 import org.apache.doris.nereids.trees.plans.logical.LogicalCatalogRelation;
@@ -51,20 +52,21 @@ public class StructInfoMap {
      * @param group the group that the mv matched
      * @return struct info or null if not found
      */
-    public @Nullable StructInfo getStructInfo(Memo memo, BitSet tableMap, Group group, Plan originPlan) {
+    public @Nullable StructInfo getStructInfo(CascadesContext cascadesContext, BitSet tableMap, Group group,
+            Plan originPlan) {
         StructInfo structInfo = infoMap.get(tableMap);
         if (structInfo != null) {
             return structInfo;
         }
         if (groupExpressionMap.isEmpty() || !groupExpressionMap.containsKey(tableMap)) {
-            refresh(group, memo.getRefreshVersion());
-            group.getstructInfoMap().setRefreshVersion(memo.getRefreshVersion());
+            refresh(group, cascadesContext);
+            group.getstructInfoMap().setRefreshVersion(cascadesContext.getMemo().getRefreshVersion());
         }
         if (groupExpressionMap.containsKey(tableMap)) {
             Pair<GroupExpression, List<BitSet>> groupExpressionBitSetPair = getGroupExpressionWithChildren(
                     tableMap);
-            structInfo = constructStructInfo(groupExpressionBitSetPair.first,
-                    groupExpressionBitSetPair.second, tableMap, originPlan);
+            structInfo = constructStructInfo(groupExpressionBitSetPair.first, groupExpressionBitSetPair.second,
+                    tableMap, originPlan, cascadesContext);
             infoMap.put(tableMap, structInfo);
         }
         return structInfo;
@@ -87,10 +89,11 @@ public void setRefreshVersion(long refreshVersion) {
     }
 
     private StructInfo constructStructInfo(GroupExpression groupExpression, List<BitSet> children,
-            BitSet tableMap, Plan originPlan) {
+            BitSet tableMap, Plan originPlan, CascadesContext cascadesContext) {
         // this plan is not origin plan, should record origin plan in struct info
         Plan plan = constructPlan(groupExpression, children, tableMap);
-        return originPlan == null ? StructInfo.of(plan) : StructInfo.of(plan, originPlan);
+        return originPlan == null ? StructInfo.of(plan, cascadesContext)
+                : StructInfo.of(plan, originPlan, cascadesContext);
     }
 
     private Plan constructPlan(GroupExpression groupExpression, List<BitSet> children, BitSet tableMap) {
@@ -112,23 +115,24 @@ private Plan constructPlan(GroupExpression groupExpression, List<BitSet> childre
      * @param group the root group
      *
      */
-    public void refresh(Group group, long memoVersion) {
+    public void refresh(Group group, CascadesContext cascadesContext) {
         StructInfoMap structInfoMap = group.getstructInfoMap();
+        long memoVersion = cascadesContext.getMemo().getRefreshVersion();
         if (!structInfoMap.getTableMaps().isEmpty() && memoVersion == structInfoMap.refreshVersion) {
             return;
         }
         Set<Integer> refreshedGroup = new HashSet<>();
         for (GroupExpression groupExpression : group.getLogicalExpressions()) {
             List<Set<BitSet>> childrenTableMap = new LinkedList<>();
             if (groupExpression.children().isEmpty()) {
-                BitSet leaf = constructLeaf(groupExpression);
+                BitSet leaf = constructLeaf(groupExpression, cascadesContext);
                 groupExpressionMap.put(leaf, Pair.of(groupExpression, new LinkedList<>()));
                 continue;
             }
             for (Group child : groupExpression.children()) {
                 StructInfoMap childStructInfoMap = child.getstructInfoMap();
                 if (!refreshedGroup.contains(child.getGroupId().asInt())) {
-                    childStructInfoMap.refresh(child, memoVersion);
+                    childStructInfoMap.refresh(child, cascadesContext);
                     childStructInfoMap.setRefreshVersion(memoVersion);
                 }
                 refreshedGroup.add(child.getGroupId().asInt());
@@ -156,12 +160,12 @@ public void refresh(Group group, long memoVersion) {
         }
     }
 
-    private BitSet constructLeaf(GroupExpression groupExpression) {
+    private BitSet constructLeaf(GroupExpression groupExpression, CascadesContext cascadesContext) {
         Plan plan = groupExpression.getPlan();
         BitSet tableMap = new BitSet();
         if (plan instanceof LogicalCatalogRelation) {
-            // TODO: Bitset is not compatible with long, use tree map instead
-            tableMap.set((int) ((LogicalCatalogRelation) plan).getTable().getId());
+            tableMap.set(cascadesContext.getStatementContext()
+                    .getTableId(((LogicalCatalogRelation) plan).getTable()).asInt());
         }
         // one row relation / CTE consumer
         return tableMap;

fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializationContext.java

@@ -107,13 +107,21 @@ public MaterializationContext(Plan mvPlan, Plan originalMvPlan, Plan mvScanPlan,
         // mv output expression shuttle, this will be used to expression rewrite
         this.mvExprToMvScanExprMapping = ExpressionMapping.generate(this.mvPlanOutputShuttledExpressions,
                 this.mvScanPlan.getOutput());
-        // copy the plan from cache, which the plan in cache may change
-        List<StructInfo> viewStructInfos = MaterializedViewUtils.extractStructInfo(
-                mvPlan, cascadesContext, new BitSet());
-        if (viewStructInfos.size() > 1) {
-            // view struct info should only have one, log error and use the first struct info
-            LOG.warn(String.format("view strut info is more than one, materialization name is %s, mv plan is %s",
-                    getMaterializationQualifier(), getMvPlan().treeString()));
+        // Construct mv struct info, catch exception which may cause planner roll back
+        List<StructInfo> viewStructInfos;
+        try {
+            viewStructInfos = MaterializedViewUtils.extractStructInfo(mvPlan, cascadesContext, new BitSet());
+            if (viewStructInfos.size() > 1) {
+                // view struct info should only have one, log error and use the first struct info
+                LOG.warn(String.format("view strut info is more than one, materialization name is %s, mv plan is %s",
+                        getMaterializationQualifier(), getMvPlan().treeString()));
+            }
+        } catch (Exception exception) {
+            LOG.warn(String.format("construct mv struct info fail, materialization name is %s, mv plan is %s",
+                    getMaterializationQualifier(), getMvPlan().treeString()), exception);
+            this.available = false;
+            this.structInfo = null;
+            return;
         }
         this.structInfo = viewStructInfos.get(0);
     }
@@ -276,9 +284,8 @@ public Void visitPhysicalRelation(PhysicalRelation physicalRelation, Void contex
         // rewrite success and chosen
         builder.append("\nMaterializedViewRewriteSuccessAndChose:\n");
         if (!chosenMaterializationQualifiers.isEmpty()) {
-            builder.append("  Names: ");
             chosenMaterializationQualifiers.forEach(materializationQualifier ->
-                    builder.append(generateQualifierName(materializationQualifier)).append(", "));
+                    builder.append(generateQualifierName(materializationQualifier)).append(", \n"));
         }
         // rewrite success but not chosen
         builder.append("\nMaterializedViewRewriteSuccessButNotChose:\n");

fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewUtils.java

@@ -149,7 +149,7 @@ public static List<StructInfo> extractStructInfo(Plan plan, CascadesContext casc
             Group ownerGroup = plan.getGroupExpression().get().getOwnerGroup();
             StructInfoMap structInfoMap = ownerGroup.getstructInfoMap();
             // Refresh struct info in current level plan from top to bottom
-            structInfoMap.refresh(ownerGroup, cascadesContext.getMemo().getRefreshVersion());
+            structInfoMap.refresh(ownerGroup, cascadesContext);
             structInfoMap.setRefreshVersion(cascadesContext.getMemo().getRefreshVersion());
 
             Set<BitSet> queryTableSets = structInfoMap.getTableMaps();
@@ -161,7 +161,7 @@ public static List<StructInfo> extractStructInfo(Plan plan, CascadesContext casc
                             && !materializedViewTableSet.equals(queryTableSet)) {
                         continue;
                     }
-                    StructInfo structInfo = structInfoMap.getStructInfo(cascadesContext.getMemo(),
+                    StructInfo structInfo = structInfoMap.getStructInfo(cascadesContext,
                             queryTableSet, ownerGroup, plan);
                     if (structInfo != null) {
                         structInfosBuilder.add(structInfo);
@@ -171,7 +171,7 @@ public static List<StructInfo> extractStructInfo(Plan plan, CascadesContext casc
             }
         }
         // if plan doesn't belong to any group, construct it directly
-        return ImmutableList.of(StructInfo.of(plan));
+        return ImmutableList.of(StructInfo.of(plan, cascadesContext));
     }
 
     /**

fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/StructInfo.java

@@ -96,7 +96,7 @@ public class StructInfo {
     // bottom plan which top plan only contain join or scan. this is needed by hyper graph
     private final Plan bottomPlan;
     private final List<CatalogRelation> relations;
-    private final BitSet tableBitSet = new BitSet();
+    private final BitSet tableBitSet;
     // this is for LogicalCompatibilityContext later
     private final Map<RelationId, StructInfoNode> relationIdStructInfoNodeMap;
     // this recorde the predicates which can pull up, not shuttled
@@ -113,12 +113,13 @@ public class StructInfo {
     /**
      * The construct method for StructInfo
      */
-    public StructInfo(Plan originalPlan, ObjectId originalPlanId, HyperGraph hyperGraph, boolean valid, Plan topPlan,
+    private StructInfo(Plan originalPlan, ObjectId originalPlanId, HyperGraph hyperGraph, boolean valid, Plan topPlan,
             Plan bottomPlan, List<CatalogRelation> relations,
             Map<RelationId, StructInfoNode> relationIdStructInfoNodeMap,
             @Nullable Predicates predicates,
             Map<ExpressionPosition, Map<Expression, Expression>> shuttledExpressionsToExpressionsMap,
-            Map<ExprId, Expression> namedExprIdAndExprMapping) {
+            Map<ExprId, Expression> namedExprIdAndExprMapping,
+            BitSet talbeIdSet) {
         this.originalPlan = originalPlan;
         this.originalPlanId = originalPlanId;
         this.hyperGraph = hyperGraph;
@@ -127,7 +128,7 @@ public StructInfo(Plan originalPlan, ObjectId originalPlanId, HyperGraph hyperGr
         this.topPlan = topPlan;
         this.bottomPlan = bottomPlan;
         this.relations = relations;
-        relations.forEach(relation -> this.tableBitSet.set((int) (relation.getTable().getId())));
+        this.tableBitSet = talbeIdSet;
         this.relationIdStructInfoNodeMap = relationIdStructInfoNodeMap;
         this.predicates = predicates;
         if (predicates == null) {
@@ -150,7 +151,7 @@ public StructInfo(Plan originalPlan, ObjectId originalPlanId, HyperGraph hyperGr
     public StructInfo withPredicates(Predicates predicates) {
         return new StructInfo(this.originalPlan, this.originalPlanId, this.hyperGraph, this.valid, this.topPlan,
                 this.bottomPlan, this.relations, this.relationIdStructInfoNodeMap, predicates,
-                this.shuttledExpressionsToExpressionsMap, this.namedExprIdAndExprMapping);
+                this.shuttledExpressionsToExpressionsMap, this.namedExprIdAndExprMapping, this.tableBitSet);
     }
 
     private static boolean collectStructInfoFromGraph(HyperGraph hyperGraph,
@@ -265,30 +266,31 @@ private Pair<SplitPredicate, EquivalenceClass> predicatesDerive(Predicates predi
      * Build Struct info from plan.
      * Maybe return multi structInfo when original plan already be rewritten by mv
      */
-    public static StructInfo of(Plan originalPlan) {
-        return of(originalPlan, originalPlan);
+    public static StructInfo of(Plan originalPlan, CascadesContext cascadesContext) {
+        return of(originalPlan, originalPlan, cascadesContext);
     }
 
     /**
      * Build Struct info from plan.
      * Maybe return multi structInfo when original plan already be rewritten by mv
      */
-    public static StructInfo of(Plan derivedPlan, Plan originalPlan) {
+    public static StructInfo of(Plan derivedPlan, Plan originalPlan, CascadesContext cascadesContext) {
         // Split plan by the boundary which contains multi child
         LinkedHashSet<Class<? extends Plan>> set = Sets.newLinkedHashSet();
         set.add(LogicalJoin.class);
         PlanSplitContext planSplitContext = new PlanSplitContext(set);
         // if single table without join, the bottom is
         derivedPlan.accept(PLAN_SPLITTER, planSplitContext);
         return StructInfo.of(originalPlan, planSplitContext.getTopPlan(), planSplitContext.getBottomPlan(),
-                HyperGraph.builderForMv(planSplitContext.getBottomPlan()).build());
+                HyperGraph.builderForMv(planSplitContext.getBottomPlan()).build(), cascadesContext);
     }
 
     /**
      * The construct method for init StructInfo
      */
     public static StructInfo of(Plan originalPlan, @Nullable Plan topPlan, @Nullable Plan bottomPlan,
-            HyperGraph hyperGraph) {
+            HyperGraph hyperGraph,
+            CascadesContext cascadesContext) {
         ObjectId originalPlanId = originalPlan.getGroupExpression()
                 .map(GroupExpression::getId).orElseGet(() -> new ObjectId(-1));
         // if any of topPlan or bottomPlan is null, split the top plan to two parts by join node
@@ -310,9 +312,14 @@ public static StructInfo of(Plan originalPlan, @Nullable Plan topPlan, @Nullable
                 namedExprIdAndExprMapping,
                 relationList,
                 relationIdStructInfoNodeMap);
+        // Get mapped table id in relation and set
+        BitSet tableBitSet = new BitSet();
+        for (CatalogRelation relation : relationList) {
+            tableBitSet.set(cascadesContext.getStatementContext().getTableId(relation.getTable()).asInt());
+        }
         return new StructInfo(originalPlan, originalPlanId, hyperGraph, valid, topPlan, bottomPlan,
                 relationList, relationIdStructInfoNodeMap, null, shuttledHashConjunctsToConjunctsMap,
-                namedExprIdAndExprMapping);
+                namedExprIdAndExprMapping, tableBitSet);
     }
 
     /**