apache · englefly · Sep 7, 2024 · Sep 4, 2024
diff --git a/be/src/olap/tablet_manager.cpp b/be/src/olap/tablet_manager.cpp
@@ -1065,6 +1065,7 @@ void TabletManager::build_all_report_tablets_info(std::map<TTabletId, TTablet>*
         t_tablet_stat.__set_row_count(tablet_info.row_count);
         t_tablet_stat.__set_total_version_count(tablet_info.total_version_count);
         t_tablet_stat.__set_visible_version_count(tablet_info.visible_version_count);
+        t_tablet_stat.__set_visible_version(tablet_info.version);
     };
     for_each_tablet(handler, filter_all_tablets);
 

diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowTableStatsStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowTableStatsStmt.java
@@ -75,8 +75,9 @@ public class ShowTableStatsStmt extends ShowStmt implements NotFallbackInParser
             new ImmutableList.Builder<String>()
             .add("table_name")
             .add("index_name")
-            .add("row_count")
-            .add("update_time")
+            .add("analyze_row_count")
+            .add("report_row_count")
+            .add("report_row_count_for_nereids")
             .build();
 
     private static final ImmutableList<String> COLUMN_PARTITION_TITLE_NAMES =
@@ -200,47 +201,39 @@ public long getTableId() {
 
     public ShowResultSet constructResultSet(TableStatsMeta tableStatistic, TableIf table) {
         if (indexName != null) {
-            return constructIndexResultSet(tableStatistic);
+            return constructIndexResultSet(tableStatistic, table);
         }
         if (partitionNames == null) {
             return constructTableResultSet(tableStatistic, table);
         }
         if (columnNames == null) {
-            return constructPartitionResultSet(tableStatistic);
+            return constructPartitionResultSet(tableStatistic, table);
         } else {
-            return constructColumnPartitionResultSet(tableStatistic);
+            return constructColumnPartitionResultSet(tableStatistic, table);
         }
     }
 
     public ShowResultSet constructEmptyResultSet() {
         return new ShowResultSet(getMetaData(), new ArrayList<>());
     }
 
-    public ShowResultSet constructResultSet(TableIf table) {
-        List<List<String>> result = Lists.newArrayList();
-        if (partitionNames != null) {
-            // For partition, return empty result if table stats not exist.
-            return new ShowResultSet(getMetaData(), result);
-        }
-        List<String> row = Lists.newArrayList();
-        row.add("");
-        row.add("");
-        row.add(String.valueOf(table.getCachedRowCount()));
-        row.add("");
-        row.add("");
-        row.add("");
-        row.add("");
-        row.add("");
-        row.add(String.valueOf(table.autoAnalyzeEnabled()));
-        result.add(row);
-        return new ShowResultSet(getMetaData(), result);
-    }
-
     public ShowResultSet constructTableResultSet(TableStatsMeta tableStatistic, TableIf table) {
-        DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss");
         if (tableStatistic == null) {
-            return new ShowResultSet(getMetaData(), new ArrayList<>());
+            List<List<String>> result = Lists.newArrayList();
+            List<String> row = Lists.newArrayList();
+            row.add("");
+            row.add("");
+            row.add(String.valueOf(table.getCachedRowCount()));
+            row.add("");
+            row.add("");
+            row.add("");
+            row.add("");
+            row.add("");
+            row.add(String.valueOf(table.autoAnalyzeEnabled()));
+            result.add(row);
+            return new ShowResultSet(getMetaData(), result);
         }
+        DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss");
         List<List<String>> result = Lists.newArrayList();
         List<String> row = Lists.newArrayList();
         row.add(String.valueOf(tableStatistic.updatedRows));
@@ -260,9 +253,9 @@ public ShowResultSet constructTableResultSet(TableStatsMeta tableStatistic, Tabl
         return new ShowResultSet(getMetaData(), result);
     }
 
-    public ShowResultSet constructPartitionResultSet(TableStatsMeta tableStatistic) {
+    public ShowResultSet constructPartitionResultSet(TableStatsMeta tableStatistic, TableIf table) {
         List<List<String>> result = Lists.newArrayList();
-        if (!(table instanceof OlapTable)) {
+        if (!(table instanceof OlapTable) || tableStatistic == null) {
             return new ShowResultSet(getMetaData(), result);
         }
         Collection<String> partitions = partitionNames.isStar()
@@ -286,7 +279,7 @@ public ShowResultSet constructPartitionResultSet(TableStatsMeta tableStatistic)
         return new ShowResultSet(getMetaData(), result);
     }
 
-    public ShowResultSet constructIndexResultSet(TableStatsMeta tableStatistic) {
+    public ShowResultSet constructIndexResultSet(TableStatsMeta tableStatistic, TableIf table) {
         List<List<String>> result = Lists.newArrayList();
         if (!(table instanceof OlapTable)) {
             return new ShowResultSet(getMetaData(), result);
@@ -296,23 +289,20 @@ public ShowResultSet constructIndexResultSet(TableStatsMeta tableStatistic) {
         if (indexId == null) {
             throw new RuntimeException(String.format("Index %s not exist.", indexName));
         }
-        long rowCount = tableStatistic.getRowCount(olapTable.getIndexIdByName(indexName));
-        if (rowCount == -1) {
-            return new ShowResultSet(getMetaData(), result);
-        }
-        long updateTime = tableStatistic.getRowCountUpdateTime(olapTable.getIndexIdByName(indexName));
+        long rowCount = tableStatistic == null ? -1 : tableStatistic.getRowCount(olapTable.getIndexIdByName(indexName));
         List<String> row = Lists.newArrayList();
         row.add(table.getName());
         row.add(indexName);
         row.add(String.valueOf(rowCount));
-        row.add(String.valueOf(updateTime));
+        row.add(String.valueOf(olapTable.getRowCountForIndex(indexId, false)));
+        row.add(String.valueOf(olapTable.getRowCountForIndex(indexId, true)));
         result.add(row);
         return new ShowResultSet(getMetaData(), result);
     }
 
-    public ShowResultSet constructColumnPartitionResultSet(TableStatsMeta tableStatistic) {
+    public ShowResultSet constructColumnPartitionResultSet(TableStatsMeta tableStatistic, TableIf table) {
         List<List<String>> result = Lists.newArrayList();
-        if (!(table instanceof OlapTable)) {
+        if (!(table instanceof OlapTable) || tableStatistic == null) {
             return new ShowResultSet(getMetaData(), result);
         }
         OlapTable olapTable = (OlapTable) table;

diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/CloudTabletStatMgr.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/CloudTabletStatMgr.java
@@ -199,6 +199,7 @@ protected void runAfterCatalogReady() {
                                 tableRowsetCount += tabletRowsetCount;
                                 tableSegmentCount += tabletSegmentCount;
                             } // end for tablets
+                            index.setRowCountReported(true);
                             index.setRowCount(indexRowCount);
                         } // end for indices
                     } // end for partitions

diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/MaterializedIndex.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/MaterializedIndex.java
@@ -73,6 +73,8 @@ public enum IndexExtState {
     @SerializedName(value = "rollupFinishedVersion")
     private long rollupFinishedVersion;
 
+    private boolean rowCountReported = false;
+
     public MaterializedIndex() {
         this.state = IndexState.NORMAL;
         this.idToTablets = new HashMap<>();
@@ -206,6 +208,14 @@ public int getTabletOrderIdx(long tabletId) {
         return -1;
     }
 
+    public void setRowCountReported(boolean reported) {
+        this.rowCountReported = reported;
+    }
+
+    public boolean getRowCountReported() {
+        return this.rowCountReported;
+    }
+
     @Deprecated
     public void readFields(DataInput in) throws IOException {
         super.readFields(in);

diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java
@@ -1564,13 +1564,16 @@ public Set<Pair<String, String>> getColumnIndexPairs(Set<String> columns) {
 
     @Override
     public long fetchRowCount() {
-        return getRowCountForIndex(baseIndexId);
+        return getRowCountForIndex(baseIndexId, false);
     }
 
-    public long getRowCountForIndex(long indexId) {
+    public long getRowCountForIndex(long indexId, boolean strict) {
         long rowCount = 0;
         for (Map.Entry<Long, Partition> entry : idToPartition.entrySet()) {
             MaterializedIndex index = entry.getValue().getIndex(indexId);
+            if (strict && !index.getRowCountReported()) {
+                return -1;
+            }
             rowCount += (index == null || index.getRowCount() == -1) ? 0 : index.getRowCount();
         }
         return rowCount;

diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Replica.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/Replica.java
@@ -177,6 +177,8 @@ public static class ReplicaContext {
 
     private long userDropTime = -1;
 
+    private long lastReportVersion = 0;
+
     public Replica() {
     }
 
@@ -839,4 +841,12 @@ public boolean isScheduleAvailable() {
         return Env.getCurrentSystemInfo().checkBackendScheduleAvailable(backendId)
             && !isUserDrop();
     }
+
+    public void setLastReportVersion(long version) {
+        this.lastReportVersion = version;
+    }
+
+    public long getLastReportVersion() {
+        return lastReportVersion;
+    }
 }
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/TabletStatMgr.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/TabletStatMgr.java
@@ -128,16 +128,34 @@ protected void runAfterCatalogReady() {
                         long version = partition.getVisibleVersion();
                         for (MaterializedIndex index : partition.getMaterializedIndices(IndexExtState.VISIBLE)) {
                             long indexRowCount = 0L;
+                            boolean indexReported = true;
                             for (Tablet tablet : index.getTablets()) {
 
                                 Long tabletDataSize = 0L;
                                 Long tabletRemoteDataSize = 0L;
 
                                 Long tabletRowCount = 0L;
 
+                                boolean tabletReported = false;
                                 for (Replica replica : tablet.getReplicas()) {
+                                    LOG.debug("Table {} replica {} current version {}, report version {}",
+                                            olapTable.getName(), replica.getId(),
+                                            replica.getVersion(), replica.getLastReportVersion());
                                     if (replica.checkVersionCatchUp(version, false)
-                                            && replica.getRowCount() > tabletRowCount) {
+                                            && replica.getRowCount() >= tabletRowCount) {
+                                        // 1. If replica version and reported replica version are all equal to
+                                        // PARTITION_INIT_VERSION, set tabletReported to true, which indicates this
+                                        // tablet is empty for sure when previous report.
+                                        // 2. If last report version is larger than PARTITION_INIT_VERSION, set
+                                        // tabletReported to true as well. That is, we only guarantee all replicas of
+                                        // the tablet are reported for the init version.
+                                        // e.g. When replica version is 2, but last reported version is 1,
+                                        // tabletReported would be false.
+                                        if (replica.getVersion() == Partition.PARTITION_INIT_VERSION
+                                                && replica.getLastReportVersion() == Partition.PARTITION_INIT_VERSION
+                                                || replica.getLastReportVersion() > Partition.PARTITION_INIT_VERSION) {
+                                            tabletReported = true;
+                                        }
                                         tabletRowCount = replica.getRowCount();
                                     }
 
@@ -157,8 +175,14 @@ protected void runAfterCatalogReady() {
 
                                 tableRowCount += tabletRowCount;
                                 indexRowCount += tabletRowCount;
+                                // Only when all tablets of this index are reported, we set indexReported to true.
+                                indexReported = indexReported && tabletReported;
                             } // end for tablets
+                            index.setRowCountReported(indexReported);
                             index.setRowCount(indexRowCount);
+                            LOG.debug("Table {} index {} all tablets reported[{}], row count {}",
+                                    olapTable.getName(), olapTable.getIndexNameById(index.getId()),
+                                    indexReported, tableRowCount);
                         } // end for indices
                     } // end for partitions
 
@@ -193,6 +217,9 @@ private void updateTabletStat(Long beId, TTabletStatResult result) {
                         replica.setTotalVersionCount(stat.getTotalVersionCount());
                         replica.setVisibleVersionCount(stat.isSetVisibleVersionCount() ? stat.getVisibleVersionCount()
                                 : stat.getTotalVersionCount());
+                        // Older version BE doesn't set visible version. Set it to max for compatibility.
+                        replica.setLastReportVersion(stat.isSetVisibleVersion() ? stat.getVisibleVersion()
+                                : Long.MAX_VALUE);
                     }
                 }
             }

diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/StatsCalculator.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/StatsCalculator.java
@@ -366,7 +366,7 @@ private void checkIfUnknownStatsUsedAsKey(StatisticsBuilder builder) {
 
     private Statistics computeOlapScan(OlapScan olapScan) {
         OlapTable olapTable = olapScan.getTable();
-        double tableRowCount = olapTable.getRowCountForIndex(olapScan.getSelectedIndexId());
+        double tableRowCount = olapTable.getRowCountForIndex(olapScan.getSelectedIndexId(), true);
         if (tableRowCount <= 0) {
             AnalysisManager analysisManager = Env.getCurrentEnv().getAnalysisManager();
             TableStatsMeta tableMeta = analysisManager.findTableStatsStatus(olapScan.getTable().getId());

diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java b/fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java
@@ -2725,14 +2725,7 @@ private void handleShowTableStats() {
             return;
         }
         TableStatsMeta tableStats = Env.getCurrentEnv().getAnalysisManager().findTableStatsStatus(tableIf.getId());
-        /*
-           tableStats == null means it's not analyzed, in this case show the estimated row count.
-         */
-        if (tableStats == null) {
-            resultSet = showTableStatsStmt.constructResultSet(tableIf);
-        } else {
-            resultSet = showTableStatsStmt.constructResultSet(tableStats, tableIf);
-        }
+        resultSet = showTableStatsStmt.constructResultSet(tableStats, tableIf);
     }
 
     private void handleShowColumnStats() throws AnalysisException {

diff --git a/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java b/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java
@@ -3319,6 +3319,9 @@ public TStatus invalidateStatsCache(TInvalidateFollowerStatsCacheRequest request
             partitionNames = new PartitionNames(false, new ArrayList<>(target.partitions));
         }
         if (target.isTruncate) {
+            if (partitionNames == null || partitionNames.isStar() || partitionNames.getPartitionNames() == null) {
+                tableStats.clearIndexesRowCount();
+            }
             analysisManager.submitAsyncDropStatsTask(target.catalogId, target.dbId,
                     target.tableId, tableStats, partitionNames);
         } else {

diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisInfo.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisInfo.java
@@ -198,7 +198,6 @@ public enum ScheduleType {
     public final boolean enablePartition;
 
     public final ConcurrentMap<Long, Long> indexesRowCount = new ConcurrentHashMap<>();
-    public final ConcurrentMap<Long, Long> indexesRowCountUpdateTime = new ConcurrentHashMap<>();
 
     public AnalysisInfo(long jobId, long taskId, List<Long> taskIds, long catalogId, long dbId, long tblId,
             Set<Pair<String, String>> jobColumns, Set<String> partitionNames, String colName, Long indexId,
@@ -358,8 +357,4 @@ public TableIf getTable() {
     public void addIndexRowCount(long indexId, long rowCount) {
         indexesRowCount.put(indexId, rowCount);
     }
-
-    public void addIndexUpdateRowCountTime(long indexId, long time) {
-        indexesRowCountUpdateTime.put(indexId, time);
-    }
 }
diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisManager.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisManager.java
@@ -682,6 +682,9 @@ public void dropStats(TableIf table, PartitionNames partitionNames) throws DdlEx
         long catalogId = table.getDatabase().getCatalog().getId();
         long dbId = table.getDatabase().getId();
         long tableId = table.getId();
+        if (partitionNames == null || partitionNames.isStar() || partitionNames.getPartitionNames() == null) {
+            tableStats.clearIndexesRowCount();
+        }
         submitAsyncDropStatsTask(catalogId, dbId, tableId, tableStats, partitionNames);
         // Drop stats ddl is master only operation.
         Set<String> partitions = null;
@@ -810,6 +813,8 @@ public void invalidateLocalStats(long catalogId, long dbId, long tableId, Set<St
         }
         if (allColumn && allPartition) {
             tableStats.removeAllColumn();
+            tableStats.clearIndexesRowCount();
+            removeTableStats(tableId);
         }
         tableStats.updatedTime = 0;
         tableStats.userInjected = false;

diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/BaseAnalysisTask.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/BaseAnalysisTask.java
@@ -498,15 +498,13 @@ protected void runQuery(String sql) {
         try (AutoCloseConnectContext a  = StatisticsUtil.buildConnectContext()) {
             stmtExecutor = new StmtExecutor(a.connectContext, sql);
             ColStatsData colStatsData = new ColStatsData(stmtExecutor.executeInternalQuery().get(0));
-            long analyzeTimestamp = System.currentTimeMillis();
             // Update index row count after analyze.
             if (this instanceof OlapAnalysisTask) {
                 AnalysisInfo jobInfo = Env.getCurrentEnv().getAnalysisManager().findJobInfo(job.getJobInfo().jobId);
                 // For sync job, get jobInfo from job.jobInfo.
                 jobInfo = jobInfo == null ? job.jobInfo : jobInfo;
                 long indexId = info.indexId == -1 ? ((OlapTable) tbl).getBaseIndexId() : info.indexId;
                 jobInfo.addIndexRowCount(indexId, colStatsData.count);
-                jobInfo.addIndexUpdateRowCountTime(indexId, analyzeTimestamp);
             }
             Env.getCurrentEnv().getStatisticsCache().syncColStats(colStatsData);
             queryId = DebugUtil.printId(stmtExecutor.getContext().queryId());

diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/OlapAnalysisTask.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/OlapAnalysisTask.java
@@ -101,7 +101,7 @@ protected void doSample() {
         List<Long> tabletIds = pair.first;
         long totalRowCount = info.indexId == -1
                 ? tbl.getRowCount()
-                : ((OlapTable) tbl).getRowCountForIndex(info.indexId);
+                : ((OlapTable) tbl).getRowCountForIndex(info.indexId, false);
         double scaleFactor = (double) totalRowCount / (double) pair.second;
         // might happen if row count in fe metadata hasn't been updated yet
         if (Double.isInfinite(scaleFactor) || Double.isNaN(scaleFactor)) {

diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsRepository.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsRepository.java
@@ -392,7 +392,6 @@ public static void alterColumnStatistics(AlterColumnStatsStmt alterColumnStatsSt
             if (objects.table instanceof OlapTable) {
                 indexId = indexId == -1 ? ((OlapTable) objects.table).getBaseIndexId() : indexId;
                 mockedJobInfo.addIndexRowCount(indexId, (long) Double.parseDouble(rowCount));
-                mockedJobInfo.addIndexUpdateRowCountTime(indexId, timestamp);
             }
             Env.getCurrentEnv().getAnalysisManager().updateTableStatsForAlterStats(mockedJobInfo, objects.table);
         } else {