From 4f105fd0d894f36c7268fc6e6e0e8f420f2726e5 Mon Sep 17 00:00:00 2001 From: englefly Date: Sun, 11 Aug 2024 00:32:45 +0800 Subject: [PATCH] regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/Suite.groovy --- .../doris/nereids/stats/FilterEstimation.java | 17 +++++++--- .../doris/nereids/stats/StatsCalculator.java | 31 ++++++++++--------- .../apache/doris/statistics/Statistics.java | 24 ++++++++++++-- .../doris/statistics/StatisticsBuilder.java | 10 +++++- 4 files changed, 60 insertions(+), 22 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/FilterEstimation.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/FilterEstimation.java index 0ce10ec0c3c61db..915a885a36ad870 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/FilterEstimation.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/FilterEstimation.java @@ -86,12 +86,21 @@ public FilterEstimation(Set aggSlots) { /** * This method will update the stats according to the selectivity. */ - public Statistics estimate(Expression expression, Statistics statistics) { + public Statistics estimate(Expression expression, Statistics inputStats) { // For a comparison predicate, only when it's left side is a slot and right side is a literal, we would // consider is a valid predicate. - Statistics stats = expression.accept(this, new EstimationContext(statistics)); - stats.enforceValid(); - return stats; + Statistics outputStats = expression.accept(this, new EstimationContext(inputStats)); + if (outputStats.getRowCount() == 0 && inputStats.getDeltaRowCount() > 0) { + StatisticsBuilder deltaStats = new StatisticsBuilder(); + deltaStats.setDeltaRowCount(0); + deltaStats.setRowCount(inputStats.getDeltaRowCount()); + for (Expression expr : inputStats.columnStatistics().keySet()) { + deltaStats.putColumnStatistics(expr, ColumnStatistic.UNKNOWN); + } + outputStats = expression.accept(this, new EstimationContext(deltaStats.build())); + } + outputStats.enforceValid(); + return outputStats; } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/StatsCalculator.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/StatsCalculator.java index d711b99655ba52c..2c954f9ee4382cd 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/StatsCalculator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/StatsCalculator.java @@ -646,10 +646,10 @@ private Statistics computeCatalogRelation(CatalogRelation catalogRelation) { idxId = olapScan.getSelectedIndexId(); } } - if (deltaRowCount > 0 && LOG.isDebugEnabled()) { - LOG.debug("{} is partially analyzed, clear min/max values in column stats", - catalogRelation.getTable().getName()); - } + // if (deltaRowCount > 0 && LOG.isDebugEnabled()) { + // LOG.debug("{} is partially analyzed, clear min/max values in column stats", + // catalogRelation.getTable().getName()); + // } for (SlotReference slotReference : slotSet) { String colName = slotReference.getColumn().isPresent() ? slotReference.getColumn().get().getName() @@ -676,14 +676,14 @@ private Statistics computeCatalogRelation(CatalogRelation catalogRelation) { hasUnknownCol = true; } if (ConnectContext.get() != null && ConnectContext.get().getSessionVariable().enableStats) { - if (deltaRowCount > 0) { - // clear min-max to avoid error estimation - // for example, after yesterday data loaded, user send query about yesterday immediately. - // since yesterday data are not analyzed, the max date is before yesterday, and hence optimizer - // estimates the filter result is zero - colStatsBuilder.setMinExpr(null).setMinValue(Double.NEGATIVE_INFINITY) - .setMaxExpr(null).setMaxValue(Double.POSITIVE_INFINITY); - } + // if (deltaRowCount > 0) { + // // clear min-max to avoid error estimation + // // for example, after yesterday data loaded, user send query about yesterday immediately. + // // since yesterday data are not analyzed, the max date is before yesterday, and hence optimizer + // // estimates the filter result is zero + // colStatsBuilder.setMinExpr(null).setMinValue(Double.NEGATIVE_INFINITY) + // .setMaxExpr(null).setMaxValue(Double.POSITIVE_INFINITY); + // } columnStatisticBuilderMap.put(slotReference, colStatsBuilder); } else { columnStatisticBuilderMap.put(slotReference, new ColumnStatisticBuilder(ColumnStatistic.UNKNOWN)); @@ -693,17 +693,18 @@ private Statistics computeCatalogRelation(CatalogRelation catalogRelation) { if (hasUnknownCol && ConnectContext.get() != null && ConnectContext.get().getStatementContext() != null) { ConnectContext.get().getStatementContext().setHasUnknownColStats(true); } - return normalizeCatalogRelationColumnStatsRowCount(rowCount, columnStatisticBuilderMap); + return normalizeCatalogRelationColumnStatsRowCount(rowCount, columnStatisticBuilderMap, deltaRowCount); } private Statistics normalizeCatalogRelationColumnStatsRowCount(double rowCount, - Map columnStatisticBuilderMap) { + Map columnStatisticBuilderMap, + double deltaRowCount) { Map columnStatisticMap = new HashMap<>(); for (Expression slot : columnStatisticBuilderMap.keySet()) { columnStatisticMap.put(slot, columnStatisticBuilderMap.get(slot).setCount(rowCount).build()); } - return new Statistics(rowCount, columnStatisticMap); + return new Statistics(rowCount, columnStatisticMap, deltaRowCount); } private Statistics computeTopN(TopN topN) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/Statistics.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/Statistics.java index da6bf93759308e4..9cec4cc18d4ba64 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/statistics/Statistics.java +++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/Statistics.java @@ -37,15 +37,23 @@ public class Statistics { // the byte size of one tuple private double tupleSize; + private double deltaRowCount = 0.0; + public Statistics(Statistics another) { this.rowCount = another.rowCount; this.expressionToColumnStats = new HashMap<>(another.expressionToColumnStats); this.tupleSize = another.tupleSize; + this.deltaRowCount = another.getDeltaRowCount(); } - public Statistics(double rowCount, Map expressionToColumnStats) { + public Statistics(double rowCount, Map expressionToColumnStats, double deltaRowCount) { this.rowCount = rowCount; this.expressionToColumnStats = expressionToColumnStats; + this.deltaRowCount = deltaRowCount; + } + + public Statistics(double rowCount, Map expressionToColumnStats) { + this(rowCount, expressionToColumnStats, 0); } public ColumnStatistic findColumnStatistics(Expression expression) { @@ -150,7 +158,11 @@ public String toString() { return "-Infinite"; } DecimalFormat format = new DecimalFormat("#,###.##"); - return format.format(rowCount); + String rows = format.format(rowCount); + if (deltaRowCount > 0) { + rows = rows + "(" + format.format(deltaRowCount) + ")"; + } + return rows; } public int getBENumber() { @@ -209,4 +221,12 @@ public Statistics normalizeByRatio(double originRowCount) { } return builder.build(); } + + public double getDeltaRowCount() { + return deltaRowCount; + } + + public void setDeltaRowCount(double deltaRowCount) { + this.deltaRowCount = deltaRowCount; + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsBuilder.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsBuilder.java index a0e75f7df380907..1e39957354607d1 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsBuilder.java +++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsBuilder.java @@ -28,6 +28,8 @@ public class StatisticsBuilder { private final Map expressionToColumnStats; + private double deltaRowCount = 0.0; + public StatisticsBuilder() { expressionToColumnStats = new HashMap<>(); } @@ -36,6 +38,7 @@ public StatisticsBuilder(Statistics statistics) { this.rowCount = statistics.getRowCount(); expressionToColumnStats = new HashMap<>(); expressionToColumnStats.putAll(statistics.columnStatistics()); + deltaRowCount = statistics.getDeltaRowCount(); } public StatisticsBuilder setRowCount(double rowCount) { @@ -54,7 +57,12 @@ public StatisticsBuilder putColumnStatistics(Expression expression, ColumnStatis return this; } + public StatisticsBuilder setDeltaRowCount(double deltaRowCount) { + this.deltaRowCount = deltaRowCount; + return this; + } + public Statistics build() { - return new Statistics(rowCount, expressionToColumnStats); + return new Statistics(rowCount, expressionToColumnStats, deltaRowCount); } }