[opt](nereids) refine expression estimation (#40698)

Stats deriving refinement step 2: refine expression estimation(part I) a. refine casewhen/if/literal's avgDataSize/numNull/ndv info. b. search column statistics cache at first during expression visiting. c. fix StringType's width() returning -1.
apache · Sep 24, 2024 · 7a7bc6a · 7a7bc6a
1 parent 839c446
commit 7a7bc6a
Show file tree

Hide file tree

Showing 7 changed files with 85 additions and 18 deletions.
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/ExpressionEstimation.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/ExpressionEstimation.java
@@ -128,49 +128,60 @@ public static ColumnStatistic estimate(Expression expression, Statistics stats)
 
     @Override
     public ColumnStatistic visit(Expression expr, Statistics context) {
+        ColumnStatistic stats = context.findColumnStatistics(expr);
+        if (stats != null) {
+            return stats;
+        }
         List<Expression> childrenExpr = expr.children();
         if (CollectionUtils.isEmpty(childrenExpr)) {
             return ColumnStatistic.UNKNOWN;
         }
         return expr.child(0).accept(this, context);
     }
 
-    //TODO: case-when need to re-implemented
     @Override
     public ColumnStatistic visitCaseWhen(CaseWhen caseWhen, Statistics context) {
         double ndv = caseWhen.getWhenClauses().size();
+        double width = 1;
         if (caseWhen.getDefaultValue().isPresent()) {
             ndv += 1;
         }
         for (WhenClause clause : caseWhen.getWhenClauses()) {
             ColumnStatistic colStats = ExpressionEstimation.estimate(clause.getResult(), context);
             ndv = Math.max(ndv, colStats.ndv);
+            width = Math.max(width, clause.getResult().getDataType().width());
         }
         if (caseWhen.getDefaultValue().isPresent()) {
             ColumnStatistic colStats = ExpressionEstimation.estimate(caseWhen.getDefaultValue().get(), context);
             ndv = Math.max(ndv, colStats.ndv);
+            width = Math.max(width, caseWhen.getDefaultValue().get().getDataType().width());
         }
         return new ColumnStatisticBuilder()
                 .setNdv(ndv)
                 .setMinValue(Double.NEGATIVE_INFINITY)
                 .setMaxValue(Double.POSITIVE_INFINITY)
-                .setAvgSizeByte(8)
+                .setAvgSizeByte(width)
                 .setNumNulls(0)
                 .build();
     }
 
     @Override
     public ColumnStatistic visitIf(If ifClause, Statistics context) {
         double ndv = 2;
+        double width = 1;
         ColumnStatistic colStatsThen = ExpressionEstimation.estimate(ifClause.child(1), context);
         ndv = Math.max(ndv, colStatsThen.ndv);
+        width = Math.max(width, ifClause.child(1).getDataType().width());
+
         ColumnStatistic colStatsElse = ExpressionEstimation.estimate(ifClause.child(2), context);
         ndv = Math.max(ndv, colStatsElse.ndv);
+        width = Math.max(width, ifClause.child(2).getDataType().width());
+
         return new ColumnStatisticBuilder()
                 .setNdv(ndv)
                 .setMinValue(Double.NEGATIVE_INFINITY)
                 .setMaxValue(Double.POSITIVE_INFINITY)
-                .setAvgSizeByte(8)
+                .setAvgSizeByte(width)
                 .setNumNulls(0)
                 .build();
     }
@@ -242,9 +253,9 @@ public ColumnStatistic visitLiteral(Literal literal, Statistics context) {
         return new ColumnStatisticBuilder()
                 .setMaxValue(literalVal)
                 .setMinValue(literalVal)
-                .setNdv(1)
+                .setNdv(literal.isNullLiteral() ? 0 : 1)
                 .setNumNulls(literal.isNullLiteral() ? 1 : 0)
-                .setAvgSizeByte(1)
+                .setAvgSizeByte(literal.getDataType().width())
                 .setMinExpr(literal.toLegacyLiteral())
                 .setMaxExpr(literal.toLegacyLiteral())
                 .build();
@@ -343,8 +354,7 @@ public ColumnStatistic visitMin(Min min, Statistics context) {
             return ColumnStatistic.UNKNOWN;
         }
         // if this is scalar agg, we will update count and ndv to 1 when visiting group clause
-        return new ColumnStatisticBuilder(columnStat)
-                .build();
+        return new ColumnStatisticBuilder(columnStat).build();
     }
 
     @Override
@@ -355,8 +365,7 @@ public ColumnStatistic visitMax(Max max, Statistics context) {
             return ColumnStatistic.UNKNOWN;
         }
         // if this is scalar agg, we will update count and ndv to 1 when visiting group clause
-        return new ColumnStatisticBuilder(columnStat)
-                .build();
+        return new ColumnStatisticBuilder(columnStat).build();
     }
 
     @Override

diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/FilterEstimation.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/FilterEstimation.java
@@ -334,6 +334,7 @@ private Statistics estimateEqualTo(ComparisonPredicate cp, ColumnStatistic stats
             } else {
                 double val = statsForRight.maxValue;
                 if (val > statsForLeft.maxValue || val < statsForLeft.minValue) {
+                    // TODO: will fix this in the next pr by adding RangeScalable protection
                     selectivity = 0.0;
                 } else if (ndv >= 1.0) {
                     selectivity = StatsMathUtil.minNonNaN(1.0, 1.0 / ndv);

diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/types/StringType.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/types/StringType.java
@@ -31,11 +31,6 @@ private StringType() {
         super(-1);
     }
 
-    @Override
-    public int width() {
-        return len;
-    }
-
     @Override
     public Type toCatalogDataType() {
         return Type.STRING;

diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/types/coercion/CharacterType.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/types/coercion/CharacterType.java
@@ -26,8 +26,8 @@
  */
 public abstract class CharacterType extends PrimitiveType {
 
-    public static final int DEFAULT_SLOT_SIZE = 20;
     private static final int WIDTH = 16;
+    public static final int DEFAULT_WIDTH = WIDTH;
 
     protected final int len;
 

diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/ColumnStatisticBuilder.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/ColumnStatisticBuilder.java
@@ -199,8 +199,7 @@ public void normalizeAvgSizeByte(SlotReference slot) {
         // When defining SQL schemas, users often tend to set the length of string \
         // fields much longer than actually needed for storage.
         if (slot.getDataType() instanceof CharacterType) {
-            avgSizeByte = Math.min(avgSizeByte,
-                    CharacterType.DEFAULT_SLOT_SIZE);
+            avgSizeByte = Math.min(avgSizeByte, CharacterType.DEFAULT_WIDTH);
         }
     }
 }
diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/Statistics.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/Statistics.java
@@ -151,7 +151,7 @@ public double computeTupleSize(List<Slot> slots) {
             for (Slot slot : slots) {
                 ColumnStatistic s = expressionToColumnStats.get(slot);
                 if (s != null) {
-                    tempSize += Math.max(1, Math.min(CharacterType.DEFAULT_SLOT_SIZE, s.avgSizeByte));
+                    tempSize += Math.max(1, Math.min(CharacterType.DEFAULT_WIDTH, s.avgSizeByte));
                 }
             }
             tupleSize = Math.max(1, tempSize);

diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/stats/ExpressionEstimationTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/stats/ExpressionEstimationTest.java
@@ -31,7 +31,14 @@
 import org.apache.doris.nereids.trees.expressions.functions.agg.Max;
 import org.apache.doris.nereids.trees.expressions.functions.agg.Min;
 import org.apache.doris.nereids.trees.expressions.functions.scalar.If;
+import org.apache.doris.nereids.trees.expressions.literal.BigIntLiteral;
 import org.apache.doris.nereids.trees.expressions.literal.BooleanLiteral;
+import org.apache.doris.nereids.trees.expressions.literal.DateTimeLiteral;
+import org.apache.doris.nereids.trees.expressions.literal.DateV2Literal;
+import org.apache.doris.nereids.trees.expressions.literal.DecimalLiteral;
+import org.apache.doris.nereids.trees.expressions.literal.DoubleLiteral;
+import org.apache.doris.nereids.trees.expressions.literal.NullLiteral;
+import org.apache.doris.nereids.trees.expressions.literal.VarcharLiteral;
 import org.apache.doris.nereids.types.DateType;
 import org.apache.doris.nereids.types.DoubleType;
 import org.apache.doris.nereids.types.IntegerType;
@@ -44,6 +51,7 @@
 import org.junit.jupiter.api.Assertions;
 import org.junit.jupiter.api.Test;
 
+import java.math.BigDecimal;
 import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.List;
@@ -357,6 +365,7 @@ public void testCaseWhen() {
         CaseWhen caseWhen = new CaseWhen(whens);
         ColumnStatistic est = ExpressionEstimation.estimate(caseWhen, stats);
         Assertions.assertEquals(est.ndv, 100);
+        Assertions.assertEquals(est.avgSizeByte, 16);
     }
 
     @Test
@@ -383,5 +392,59 @@ public void testIf() {
         If ifClause = new If(BooleanLiteral.TRUE, a, b);
         ColumnStatistic est = ExpressionEstimation.estimate(ifClause, stats);
         Assertions.assertEquals(est.ndv, 100);
+        Assertions.assertEquals(est.avgSizeByte, 16);
+    }
+
+    @Test
+    public void testLiteral() {
+        Statistics stats = new Statistics(1000, new HashMap<>());
+
+        BigIntLiteral l1 = new BigIntLiteral(1000000);
+        ColumnStatistic est = ExpressionEstimation.estimate(l1, stats);
+        Assertions.assertEquals(est.ndv, 1);
+        Assertions.assertEquals(est.avgSizeByte, 8);
+        Assertions.assertEquals(est.numNulls, 0);
+
+        VarcharLiteral l2 = new VarcharLiteral("abcdefghij");
+        est = ExpressionEstimation.estimate(l2, stats);
+        Assertions.assertEquals(est.ndv, 1);
+        Assertions.assertEquals(est.avgSizeByte, 10);
+        Assertions.assertEquals(est.numNulls, 0);
+
+        DoubleLiteral l3 = new DoubleLiteral(0.01);
+        est = ExpressionEstimation.estimate(l3, stats);
+        Assertions.assertEquals(est.ndv, 1);
+        Assertions.assertEquals(est.avgSizeByte, 8);
+        Assertions.assertEquals(est.numNulls, 0);
+
+        DateV2Literal l4 = new DateV2Literal("2024-09-10");
+        est = ExpressionEstimation.estimate(l4, stats);
+        Assertions.assertEquals(est.ndv, 1);
+        Assertions.assertEquals(est.avgSizeByte, 4);
+        Assertions.assertEquals(est.numNulls, 0);
+
+        DateTimeLiteral l5 = new DateTimeLiteral("2024-09-10 00:00:00");
+        est = ExpressionEstimation.estimate(l5, stats);
+        Assertions.assertEquals(est.ndv, 1);
+        Assertions.assertEquals(est.avgSizeByte, 16);
+        Assertions.assertEquals(est.numNulls, 0);
+
+        BooleanLiteral l6 = BooleanLiteral.TRUE;
+        est = ExpressionEstimation.estimate(l6, stats);
+        Assertions.assertEquals(est.ndv, 1);
+        Assertions.assertEquals(est.avgSizeByte, 1);
+        Assertions.assertEquals(est.numNulls, 0);
+
+        DecimalLiteral l7 = new DecimalLiteral(BigDecimal.valueOf(2024.0928));
+        est = ExpressionEstimation.estimate(l7, stats);
+        Assertions.assertEquals(est.ndv, 1);
+        Assertions.assertEquals(est.avgSizeByte, 16);
+        Assertions.assertEquals(est.numNulls, 0);
+
+        NullLiteral l8 = new NullLiteral();
+        est = ExpressionEstimation.estimate(l8, stats);
+        Assertions.assertEquals(est.ndv, 0);
+        Assertions.assertEquals(est.avgSizeByte, 1);
+        Assertions.assertEquals(est.numNulls, 1);
     }
 }