From 7a7bc6a7f73d81847f1416dca3485ed18589c274 Mon Sep 17 00:00:00 2001 From: xzj7019 <131111794+xzj7019@users.noreply.github.com> Date: Tue, 24 Sep 2024 14:15:22 +0800 Subject: [PATCH] [opt](nereids) refine expression estimation (#40698) Stats deriving refinement step 2: refine expression estimation(part I) a. refine casewhen/if/literal's avgDataSize/numNull/ndv info. b. search column statistics cache at first during expression visiting. c. fix StringType's width() returning -1. --- .../nereids/stats/ExpressionEstimation.java | 27 +++++--- .../doris/nereids/stats/FilterEstimation.java | 1 + .../doris/nereids/types/StringType.java | 5 -- .../nereids/types/coercion/CharacterType.java | 2 +- .../statistics/ColumnStatisticBuilder.java | 3 +- .../apache/doris/statistics/Statistics.java | 2 +- .../stats/ExpressionEstimationTest.java | 63 +++++++++++++++++++ 7 files changed, 85 insertions(+), 18 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/ExpressionEstimation.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/ExpressionEstimation.java index 126e9041721312..b26b0315047347 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/ExpressionEstimation.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/ExpressionEstimation.java @@ -128,6 +128,10 @@ public static ColumnStatistic estimate(Expression expression, Statistics stats) @Override public ColumnStatistic visit(Expression expr, Statistics context) { + ColumnStatistic stats = context.findColumnStatistics(expr); + if (stats != null) { + return stats; + } List childrenExpr = expr.children(); if (CollectionUtils.isEmpty(childrenExpr)) { return ColumnStatistic.UNKNOWN; @@ -135,26 +139,28 @@ public ColumnStatistic visit(Expression expr, Statistics context) { return expr.child(0).accept(this, context); } - //TODO: case-when need to re-implemented @Override public ColumnStatistic visitCaseWhen(CaseWhen caseWhen, Statistics context) { double ndv = caseWhen.getWhenClauses().size(); + double width = 1; if (caseWhen.getDefaultValue().isPresent()) { ndv += 1; } for (WhenClause clause : caseWhen.getWhenClauses()) { ColumnStatistic colStats = ExpressionEstimation.estimate(clause.getResult(), context); ndv = Math.max(ndv, colStats.ndv); + width = Math.max(width, clause.getResult().getDataType().width()); } if (caseWhen.getDefaultValue().isPresent()) { ColumnStatistic colStats = ExpressionEstimation.estimate(caseWhen.getDefaultValue().get(), context); ndv = Math.max(ndv, colStats.ndv); + width = Math.max(width, caseWhen.getDefaultValue().get().getDataType().width()); } return new ColumnStatisticBuilder() .setNdv(ndv) .setMinValue(Double.NEGATIVE_INFINITY) .setMaxValue(Double.POSITIVE_INFINITY) - .setAvgSizeByte(8) + .setAvgSizeByte(width) .setNumNulls(0) .build(); } @@ -162,15 +168,20 @@ public ColumnStatistic visitCaseWhen(CaseWhen caseWhen, Statistics context) { @Override public ColumnStatistic visitIf(If ifClause, Statistics context) { double ndv = 2; + double width = 1; ColumnStatistic colStatsThen = ExpressionEstimation.estimate(ifClause.child(1), context); ndv = Math.max(ndv, colStatsThen.ndv); + width = Math.max(width, ifClause.child(1).getDataType().width()); + ColumnStatistic colStatsElse = ExpressionEstimation.estimate(ifClause.child(2), context); ndv = Math.max(ndv, colStatsElse.ndv); + width = Math.max(width, ifClause.child(2).getDataType().width()); + return new ColumnStatisticBuilder() .setNdv(ndv) .setMinValue(Double.NEGATIVE_INFINITY) .setMaxValue(Double.POSITIVE_INFINITY) - .setAvgSizeByte(8) + .setAvgSizeByte(width) .setNumNulls(0) .build(); } @@ -242,9 +253,9 @@ public ColumnStatistic visitLiteral(Literal literal, Statistics context) { return new ColumnStatisticBuilder() .setMaxValue(literalVal) .setMinValue(literalVal) - .setNdv(1) + .setNdv(literal.isNullLiteral() ? 0 : 1) .setNumNulls(literal.isNullLiteral() ? 1 : 0) - .setAvgSizeByte(1) + .setAvgSizeByte(literal.getDataType().width()) .setMinExpr(literal.toLegacyLiteral()) .setMaxExpr(literal.toLegacyLiteral()) .build(); @@ -343,8 +354,7 @@ public ColumnStatistic visitMin(Min min, Statistics context) { return ColumnStatistic.UNKNOWN; } // if this is scalar agg, we will update count and ndv to 1 when visiting group clause - return new ColumnStatisticBuilder(columnStat) - .build(); + return new ColumnStatisticBuilder(columnStat).build(); } @Override @@ -355,8 +365,7 @@ public ColumnStatistic visitMax(Max max, Statistics context) { return ColumnStatistic.UNKNOWN; } // if this is scalar agg, we will update count and ndv to 1 when visiting group clause - return new ColumnStatisticBuilder(columnStat) - .build(); + return new ColumnStatisticBuilder(columnStat).build(); } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/FilterEstimation.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/FilterEstimation.java index b3576a0e58e61e..e7a62dcd484846 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/FilterEstimation.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/FilterEstimation.java @@ -334,6 +334,7 @@ private Statistics estimateEqualTo(ComparisonPredicate cp, ColumnStatistic stats } else { double val = statsForRight.maxValue; if (val > statsForLeft.maxValue || val < statsForLeft.minValue) { + // TODO: will fix this in the next pr by adding RangeScalable protection selectivity = 0.0; } else if (ndv >= 1.0) { selectivity = StatsMathUtil.minNonNaN(1.0, 1.0 / ndv); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/types/StringType.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/types/StringType.java index 935716e42bf05f..8e92f83274e2f5 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/types/StringType.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/types/StringType.java @@ -31,11 +31,6 @@ private StringType() { super(-1); } - @Override - public int width() { - return len; - } - @Override public Type toCatalogDataType() { return Type.STRING; diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/types/coercion/CharacterType.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/types/coercion/CharacterType.java index 446ccc7fd002c2..781b125702837c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/types/coercion/CharacterType.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/types/coercion/CharacterType.java @@ -26,8 +26,8 @@ */ public abstract class CharacterType extends PrimitiveType { - public static final int DEFAULT_SLOT_SIZE = 20; private static final int WIDTH = 16; + public static final int DEFAULT_WIDTH = WIDTH; protected final int len; diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/ColumnStatisticBuilder.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/ColumnStatisticBuilder.java index 47002355de907d..4e190ce388ee0a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/statistics/ColumnStatisticBuilder.java +++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/ColumnStatisticBuilder.java @@ -199,8 +199,7 @@ public void normalizeAvgSizeByte(SlotReference slot) { // When defining SQL schemas, users often tend to set the length of string \ // fields much longer than actually needed for storage. if (slot.getDataType() instanceof CharacterType) { - avgSizeByte = Math.min(avgSizeByte, - CharacterType.DEFAULT_SLOT_SIZE); + avgSizeByte = Math.min(avgSizeByte, CharacterType.DEFAULT_WIDTH); } } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/Statistics.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/Statistics.java index e18dc09792054e..7e539ef68f2755 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/statistics/Statistics.java +++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/Statistics.java @@ -151,7 +151,7 @@ public double computeTupleSize(List slots) { for (Slot slot : slots) { ColumnStatistic s = expressionToColumnStats.get(slot); if (s != null) { - tempSize += Math.max(1, Math.min(CharacterType.DEFAULT_SLOT_SIZE, s.avgSizeByte)); + tempSize += Math.max(1, Math.min(CharacterType.DEFAULT_WIDTH, s.avgSizeByte)); } } tupleSize = Math.max(1, tempSize); diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/stats/ExpressionEstimationTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/stats/ExpressionEstimationTest.java index 5f91e2b70d8ab0..91da5192b48c02 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/stats/ExpressionEstimationTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/stats/ExpressionEstimationTest.java @@ -31,7 +31,14 @@ import org.apache.doris.nereids.trees.expressions.functions.agg.Max; import org.apache.doris.nereids.trees.expressions.functions.agg.Min; import org.apache.doris.nereids.trees.expressions.functions.scalar.If; +import org.apache.doris.nereids.trees.expressions.literal.BigIntLiteral; import org.apache.doris.nereids.trees.expressions.literal.BooleanLiteral; +import org.apache.doris.nereids.trees.expressions.literal.DateTimeLiteral; +import org.apache.doris.nereids.trees.expressions.literal.DateV2Literal; +import org.apache.doris.nereids.trees.expressions.literal.DecimalLiteral; +import org.apache.doris.nereids.trees.expressions.literal.DoubleLiteral; +import org.apache.doris.nereids.trees.expressions.literal.NullLiteral; +import org.apache.doris.nereids.trees.expressions.literal.VarcharLiteral; import org.apache.doris.nereids.types.DateType; import org.apache.doris.nereids.types.DoubleType; import org.apache.doris.nereids.types.IntegerType; @@ -44,6 +51,7 @@ import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; +import java.math.BigDecimal; import java.util.ArrayList; import java.util.HashMap; import java.util.List; @@ -357,6 +365,7 @@ public void testCaseWhen() { CaseWhen caseWhen = new CaseWhen(whens); ColumnStatistic est = ExpressionEstimation.estimate(caseWhen, stats); Assertions.assertEquals(est.ndv, 100); + Assertions.assertEquals(est.avgSizeByte, 16); } @Test @@ -383,5 +392,59 @@ public void testIf() { If ifClause = new If(BooleanLiteral.TRUE, a, b); ColumnStatistic est = ExpressionEstimation.estimate(ifClause, stats); Assertions.assertEquals(est.ndv, 100); + Assertions.assertEquals(est.avgSizeByte, 16); + } + + @Test + public void testLiteral() { + Statistics stats = new Statistics(1000, new HashMap<>()); + + BigIntLiteral l1 = new BigIntLiteral(1000000); + ColumnStatistic est = ExpressionEstimation.estimate(l1, stats); + Assertions.assertEquals(est.ndv, 1); + Assertions.assertEquals(est.avgSizeByte, 8); + Assertions.assertEquals(est.numNulls, 0); + + VarcharLiteral l2 = new VarcharLiteral("abcdefghij"); + est = ExpressionEstimation.estimate(l2, stats); + Assertions.assertEquals(est.ndv, 1); + Assertions.assertEquals(est.avgSizeByte, 10); + Assertions.assertEquals(est.numNulls, 0); + + DoubleLiteral l3 = new DoubleLiteral(0.01); + est = ExpressionEstimation.estimate(l3, stats); + Assertions.assertEquals(est.ndv, 1); + Assertions.assertEquals(est.avgSizeByte, 8); + Assertions.assertEquals(est.numNulls, 0); + + DateV2Literal l4 = new DateV2Literal("2024-09-10"); + est = ExpressionEstimation.estimate(l4, stats); + Assertions.assertEquals(est.ndv, 1); + Assertions.assertEquals(est.avgSizeByte, 4); + Assertions.assertEquals(est.numNulls, 0); + + DateTimeLiteral l5 = new DateTimeLiteral("2024-09-10 00:00:00"); + est = ExpressionEstimation.estimate(l5, stats); + Assertions.assertEquals(est.ndv, 1); + Assertions.assertEquals(est.avgSizeByte, 16); + Assertions.assertEquals(est.numNulls, 0); + + BooleanLiteral l6 = BooleanLiteral.TRUE; + est = ExpressionEstimation.estimate(l6, stats); + Assertions.assertEquals(est.ndv, 1); + Assertions.assertEquals(est.avgSizeByte, 1); + Assertions.assertEquals(est.numNulls, 0); + + DecimalLiteral l7 = new DecimalLiteral(BigDecimal.valueOf(2024.0928)); + est = ExpressionEstimation.estimate(l7, stats); + Assertions.assertEquals(est.ndv, 1); + Assertions.assertEquals(est.avgSizeByte, 16); + Assertions.assertEquals(est.numNulls, 0); + + NullLiteral l8 = new NullLiteral(); + est = ExpressionEstimation.estimate(l8, stats); + Assertions.assertEquals(est.ndv, 0); + Assertions.assertEquals(est.avgSizeByte, 1); + Assertions.assertEquals(est.numNulls, 1); } }