[opt](nereids) refine expression estimation

apache · Sep 20, 2024 · 88e501e · 88e501e
1 parent 17da096
commit 88e501e
Show file tree

Hide file tree

Showing 2 changed files with 19 additions and 9 deletions.
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/ExpressionEstimation.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/ExpressionEstimation.java
@@ -128,49 +128,60 @@ public static ColumnStatistic estimate(Expression expression, Statistics stats)
 
     @Override
     public ColumnStatistic visit(Expression expr, Statistics context) {
+        ColumnStatistic stats = context.findColumnStatistics(expr);
+        if (stats != null) {
+            return stats;
+        }
         List<Expression> childrenExpr = expr.children();
         if (CollectionUtils.isEmpty(childrenExpr)) {
             return ColumnStatistic.UNKNOWN;
         }
         return expr.child(0).accept(this, context);
     }
 
-    //TODO: case-when need to re-implemented
     @Override
     public ColumnStatistic visitCaseWhen(CaseWhen caseWhen, Statistics context) {
         double ndv = caseWhen.getWhenClauses().size();
+        double width = 1;
         if (caseWhen.getDefaultValue().isPresent()) {
             ndv += 1;
         }
         for (WhenClause clause : caseWhen.getWhenClauses()) {
             ColumnStatistic colStats = ExpressionEstimation.estimate(clause.getResult(), context);
             ndv = Math.max(ndv, colStats.ndv);
+            width = Math.max(width, clause.getResult().getDataType().width());
         }
         if (caseWhen.getDefaultValue().isPresent()) {
             ColumnStatistic colStats = ExpressionEstimation.estimate(caseWhen.getDefaultValue().get(), context);
             ndv = Math.max(ndv, colStats.ndv);
+            width = Math.max(width, caseWhen.getDefaultValue().get().getDataType().width());
         }
         return new ColumnStatisticBuilder()
                 .setNdv(ndv)
                 .setMinValue(Double.NEGATIVE_INFINITY)
                 .setMaxValue(Double.POSITIVE_INFINITY)
-                .setAvgSizeByte(8)
+                .setAvgSizeByte(width)
                 .setNumNulls(0)
                 .build();
     }
 
     @Override
     public ColumnStatistic visitIf(If ifClause, Statistics context) {
         double ndv = 2;
+        double width = 1;
         ColumnStatistic colStatsThen = ExpressionEstimation.estimate(ifClause.child(1), context);
         ndv = Math.max(ndv, colStatsThen.ndv);
+        width = Math.max(width, ifClause.child(1).getDataType().width());
+
         ColumnStatistic colStatsElse = ExpressionEstimation.estimate(ifClause.child(2), context);
         ndv = Math.max(ndv, colStatsElse.ndv);
+        width = Math.max(width, ifClause.child(2).getDataType().width());
+
         return new ColumnStatisticBuilder()
                 .setNdv(ndv)
                 .setMinValue(Double.NEGATIVE_INFINITY)
                 .setMaxValue(Double.POSITIVE_INFINITY)
-                .setAvgSizeByte(8)
+                .setAvgSizeByte(width)
                 .setNumNulls(0)
                 .build();
     }
@@ -242,9 +253,9 @@ public ColumnStatistic visitLiteral(Literal literal, Statistics context) {
         return new ColumnStatisticBuilder()
                 .setMaxValue(literalVal)
                 .setMinValue(literalVal)
-                .setNdv(1)
+                .setNdv(literal.isNullLiteral() ? 0 : 1)
                 .setNumNulls(literal.isNullLiteral() ? 1 : 0)
-                .setAvgSizeByte(1)
+                .setAvgSizeByte(literal.getWidth())
                 .setMinExpr(literal.toLegacyLiteral())
                 .setMaxExpr(literal.toLegacyLiteral())
                 .build();
@@ -343,8 +354,7 @@ public ColumnStatistic visitMin(Min min, Statistics context) {
             return ColumnStatistic.UNKNOWN;
         }
         // if this is scalar agg, we will update count and ndv to 1 when visiting group clause
-        return new ColumnStatisticBuilder(columnStat)
-                .build();
+        return new ColumnStatisticBuilder(columnStat).build();
     }
 
     @Override
@@ -355,8 +365,7 @@ public ColumnStatistic visitMax(Max max, Statistics context) {
             return ColumnStatistic.UNKNOWN;
         }
         // if this is scalar agg, we will update count and ndv to 1 when visiting group clause
-        return new ColumnStatisticBuilder(columnStat)
-                .build();
+        return new ColumnStatisticBuilder(columnStat).build();
     }
 
     @Override

diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/FilterEstimation.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/FilterEstimation.java
@@ -334,6 +334,7 @@ private Statistics estimateEqualTo(ComparisonPredicate cp, ColumnStatistic stats
             } else {
                 double val = statsForRight.maxValue;
                 if (val > statsForLeft.maxValue || val < statsForLeft.minValue) {
+                    // TODO: will fix this in the next pr by adding RangeScalable protection
                     selectivity = 0.0;
                 } else if (ndv >= 1.0) {
                     selectivity = StatsMathUtil.minNonNaN(1.0, 1.0 / ndv);