regression-test/framework/src/main/groovy/org/apache/doris/regression…

…/suite/Suite.groovy
apache · Aug 10, 2024 · 4f105fd · 4f105fd
1 parent bf11d48
commit 4f105fd
Show file tree

Hide file tree

Showing 4 changed files with 60 additions and 22 deletions.
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/FilterEstimation.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/FilterEstimation.java
@@ -86,12 +86,21 @@ public FilterEstimation(Set<Slot> aggSlots) {
     /**
      * This method will update the stats according to the selectivity.
      */
-    public Statistics estimate(Expression expression, Statistics statistics) {
+    public Statistics estimate(Expression expression, Statistics inputStats) {
         // For a comparison predicate, only when it's left side is a slot and right side is a literal, we would
         // consider is a valid predicate.
-        Statistics stats = expression.accept(this, new EstimationContext(statistics));
-        stats.enforceValid();
-        return stats;
+        Statistics outputStats = expression.accept(this, new EstimationContext(inputStats));
+        if (outputStats.getRowCount() == 0 && inputStats.getDeltaRowCount() > 0) {
+            StatisticsBuilder deltaStats = new StatisticsBuilder();
+            deltaStats.setDeltaRowCount(0);
+            deltaStats.setRowCount(inputStats.getDeltaRowCount());
+            for (Expression expr : inputStats.columnStatistics().keySet()) {
+                deltaStats.putColumnStatistics(expr, ColumnStatistic.UNKNOWN);
+            }
+            outputStats = expression.accept(this, new EstimationContext(deltaStats.build()));
+        }
+        outputStats.enforceValid();
+        return outputStats;
     }
 
     @Override

diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/StatsCalculator.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/StatsCalculator.java
@@ -646,10 +646,10 @@ private Statistics computeCatalogRelation(CatalogRelation catalogRelation) {
                 idxId = olapScan.getSelectedIndexId();
             }
         }
-        if (deltaRowCount > 0 && LOG.isDebugEnabled()) {
-            LOG.debug("{} is partially analyzed, clear min/max values in column stats",
-                    catalogRelation.getTable().getName());
-        }
+        // if (deltaRowCount > 0 && LOG.isDebugEnabled()) {
+        //     LOG.debug("{} is partially analyzed, clear min/max values in column stats",
+        //             catalogRelation.getTable().getName());
+        // }
         for (SlotReference slotReference : slotSet) {
             String colName = slotReference.getColumn().isPresent()
                     ? slotReference.getColumn().get().getName()
@@ -676,14 +676,14 @@ private Statistics computeCatalogRelation(CatalogRelation catalogRelation) {
                 hasUnknownCol = true;
             }
             if (ConnectContext.get() != null && ConnectContext.get().getSessionVariable().enableStats) {
-                if (deltaRowCount > 0) {
-                    // clear min-max to avoid error estimation
-                    // for example, after yesterday data loaded, user send query about yesterday immediately.
-                    // since yesterday data are not analyzed, the max date is before yesterday, and hence optimizer
-                    // estimates the filter result is zero
-                    colStatsBuilder.setMinExpr(null).setMinValue(Double.NEGATIVE_INFINITY)
-                            .setMaxExpr(null).setMaxValue(Double.POSITIVE_INFINITY);
-                }
+                // if (deltaRowCount > 0) {
+                //     // clear min-max to avoid error estimation
+                //     // for example, after yesterday data loaded, user send query about yesterday immediately.
+                //     // since yesterday data are not analyzed, the max date is before yesterday, and hence optimizer
+                //     // estimates the filter result is zero
+                //     colStatsBuilder.setMinExpr(null).setMinValue(Double.NEGATIVE_INFINITY)
+                //             .setMaxExpr(null).setMaxValue(Double.POSITIVE_INFINITY);
+                // }
                 columnStatisticBuilderMap.put(slotReference, colStatsBuilder);
             } else {
                 columnStatisticBuilderMap.put(slotReference, new ColumnStatisticBuilder(ColumnStatistic.UNKNOWN));
@@ -693,17 +693,18 @@ private Statistics computeCatalogRelation(CatalogRelation catalogRelation) {
         if (hasUnknownCol && ConnectContext.get() != null && ConnectContext.get().getStatementContext() != null) {
             ConnectContext.get().getStatementContext().setHasUnknownColStats(true);
         }
-        return normalizeCatalogRelationColumnStatsRowCount(rowCount, columnStatisticBuilderMap);
+        return normalizeCatalogRelationColumnStatsRowCount(rowCount, columnStatisticBuilderMap, deltaRowCount);
     }
 
     private Statistics normalizeCatalogRelationColumnStatsRowCount(double rowCount,
-            Map<Expression, ColumnStatisticBuilder> columnStatisticBuilderMap) {
+            Map<Expression, ColumnStatisticBuilder> columnStatisticBuilderMap,
+            double deltaRowCount) {
         Map<Expression, ColumnStatistic> columnStatisticMap = new HashMap<>();
         for (Expression slot : columnStatisticBuilderMap.keySet()) {
             columnStatisticMap.put(slot,
                     columnStatisticBuilderMap.get(slot).setCount(rowCount).build());
         }
-        return new Statistics(rowCount, columnStatisticMap);
+        return new Statistics(rowCount, columnStatisticMap, deltaRowCount);
     }
 
     private Statistics computeTopN(TopN topN) {

diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/Statistics.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/Statistics.java
@@ -37,15 +37,23 @@ public class Statistics {
     // the byte size of one tuple
     private double tupleSize;
 
+    private double deltaRowCount = 0.0;
+
     public Statistics(Statistics another) {
         this.rowCount = another.rowCount;
         this.expressionToColumnStats = new HashMap<>(another.expressionToColumnStats);
         this.tupleSize = another.tupleSize;
+        this.deltaRowCount = another.getDeltaRowCount();
     }
 
-    public Statistics(double rowCount, Map<Expression, ColumnStatistic> expressionToColumnStats) {
+    public Statistics(double rowCount, Map<Expression, ColumnStatistic> expressionToColumnStats, double deltaRowCount) {
         this.rowCount = rowCount;
         this.expressionToColumnStats = expressionToColumnStats;
+        this.deltaRowCount = deltaRowCount;
+    }
+
+    public Statistics(double rowCount, Map<Expression, ColumnStatistic> expressionToColumnStats) {
+        this(rowCount, expressionToColumnStats, 0);
     }
 
     public ColumnStatistic findColumnStatistics(Expression expression) {
@@ -150,7 +158,11 @@ public String toString() {
             return "-Infinite";
         }
         DecimalFormat format = new DecimalFormat("#,###.##");
-        return format.format(rowCount);
+        String rows = format.format(rowCount);
+        if (deltaRowCount > 0) {
+            rows = rows + "(" + format.format(deltaRowCount) + ")";
+        }
+        return rows;
     }
 
     public int getBENumber() {
@@ -209,4 +221,12 @@ public Statistics normalizeByRatio(double originRowCount) {
         }
         return builder.build();
     }
+
+    public double getDeltaRowCount() {
+        return deltaRowCount;
+    }
+
+    public void setDeltaRowCount(double deltaRowCount) {
+        this.deltaRowCount = deltaRowCount;
+    }
 }
diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsBuilder.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsBuilder.java
@@ -28,6 +28,8 @@ public class StatisticsBuilder {
 
     private final Map<Expression, ColumnStatistic> expressionToColumnStats;
 
+    private double deltaRowCount = 0.0;
+
     public StatisticsBuilder() {
         expressionToColumnStats = new HashMap<>();
     }
@@ -36,6 +38,7 @@ public StatisticsBuilder(Statistics statistics) {
         this.rowCount = statistics.getRowCount();
         expressionToColumnStats = new HashMap<>();
         expressionToColumnStats.putAll(statistics.columnStatistics());
+        deltaRowCount = statistics.getDeltaRowCount();
     }
 
     public StatisticsBuilder setRowCount(double rowCount) {
@@ -54,7 +57,12 @@ public StatisticsBuilder putColumnStatistics(Expression expression, ColumnStatis
         return this;
     }
 
+    public StatisticsBuilder setDeltaRowCount(double deltaRowCount) {
+        this.deltaRowCount = deltaRowCount;
+        return this;
+    }
+
     public Statistics build() {
-        return new Statistics(rowCount, expressionToColumnStats);
+        return new Statistics(rowCount, expressionToColumnStats, deltaRowCount);
     }
 }