From ab4f3482da4c95433a54ec6899edb77b459f5629 Mon Sep 17 00:00:00 2001 From: Jibing Li Date: Tue, 21 May 2024 17:27:39 +0800 Subject: [PATCH] Record partition update rows for each column. --- .../doris/analysis/ShowColumnStatsStmt.java | 48 +++++++++++++------ .../apache/doris/statistics/AnalysisInfo.java | 9 +++- .../doris/statistics/AnalysisInfoBuilder.java | 10 +++- .../doris/statistics/AnalysisManager.java | 4 ++ .../apache/doris/statistics/ColStatsMeta.java | 11 ++++- .../statistics/StatisticsAutoCollector.java | 1 + .../statistics/StatisticsRepository.java | 4 ++ .../doris/statistics/TableStatsMeta.java | 12 +++-- .../statistics/util/StatisticsUtilTest.java | 10 ++-- 9 files changed, 84 insertions(+), 25 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowColumnStatsStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowColumnStatsStmt.java index 01612af9f9f96b..6c54175784f951 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowColumnStatsStmt.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowColumnStatsStmt.java @@ -20,6 +20,8 @@ import org.apache.doris.catalog.Column; import org.apache.doris.catalog.DatabaseIf; import org.apache.doris.catalog.Env; +import org.apache.doris.catalog.OlapTable; +import org.apache.doris.catalog.Partition; import org.apache.doris.catalog.ScalarType; import org.apache.doris.catalog.TableIf; import org.apache.doris.common.ErrorCode; @@ -35,6 +37,7 @@ import org.apache.doris.statistics.ColStatsMeta; import org.apache.doris.statistics.ColumnStatistic; import org.apache.doris.statistics.ResultRow; +import org.apache.doris.statistics.TableStatsMeta; import com.google.common.collect.ImmutableList; import com.google.common.collect.Lists; @@ -186,22 +189,39 @@ public ShowResultSet constructResultSet(List, ColumnSt public ShowResultSet constructPartitionResultSet(List resultRows, TableIf tableIf) { List> result = Lists.newArrayList(); - resultRows.forEach(r -> { + for (ResultRow r : resultRows) { List row = Lists.newArrayList(); - row.add(r.get(0)); - row.add(r.get(1)); - row.add(r.get(2)); // TODO: Get index name. - row.add(r.get(3)); - row.add(r.get(4)); - row.add(r.get(5)); - row.add(r.get(6)); - row.add(r.get(7)); - row.add(r.get(8)); - row.add(r.get(9)); - row.add("N/A"); - row.add("Manual"); + row.add(r.get(0)); // column_name + row.add(r.get(1)); // partition_name + long indexId = Long.parseLong(r.get(2)); + String indexName = indexId == -1 ? tableIf.getName() : ((OlapTable) tableIf).getIndexNameById(indexId); + row.add(indexName); // index_name. + row.add(r.get(3)); // count + row.add(r.get(4)); // ndv + row.add(r.get(5)); // num_null + row.add(r.get(6)); // min + row.add(r.get(7)); // max + row.add(r.get(8)); // data_size + row.add(r.get(9)); // updated_time + String updateRows = "N/A"; + TableStatsMeta tableStats = Env.getCurrentEnv().getAnalysisManager().findTableStatsStatus(tableIf.getId()); + if (tableStats != null && tableIf instanceof OlapTable) { + OlapTable olapTable = (OlapTable) tableIf; + ColStatsMeta columnStatsMeta = tableStats.findColumnStatsMeta(indexName, r.get(0)); + if (columnStatsMeta != null && columnStatsMeta.partitionUpdateRows != null) { + Partition partition = olapTable.getPartition(r.get(1)); + if (partition != null) { + Long rows = columnStatsMeta.partitionUpdateRows.get(partition.getId()); + if (rows != null) { + updateRows = rows.toString(); + } + } + } + } + row.add(updateRows); // update_rows + row.add("Manual"); // trigger. Manual or System result.add(row); - }); + } return new ShowResultSet(getMetaData(), result); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisInfo.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisInfo.java index 4e4960f6ecf7cc..ef0d84e66c9b6c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisInfo.java +++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisInfo.java @@ -35,7 +35,9 @@ import java.io.DataOutput; import java.io.IOException; import java.text.ParseException; +import java.util.HashMap; import java.util.List; +import java.util.Map; import java.util.Set; import java.util.StringJoiner; @@ -188,6 +190,8 @@ public enum ScheduleType { @SerializedName("updateRows") public final long updateRows; + + public final Map partitionUpdateRows = new HashMap(); /** * * Used to store the newest partition version of tbl when creating this job. @@ -209,7 +213,7 @@ public AnalysisInfo(long jobId, long taskId, List taskIds, long catalogId, boolean partitionOnly, boolean samplingPartition, boolean isAllPartition, long partitionCount, CronExpression cronExpression, boolean forceFull, boolean usingSqlForExternalTable, long tblUpdateTime, long rowCount, boolean userInject, - long updateRows, JobPriority priority) { + long updateRows, JobPriority priority, Map partitionUpdateRows) { this.jobId = jobId; this.taskId = taskId; this.taskIds = taskIds; @@ -248,6 +252,9 @@ public AnalysisInfo(long jobId, long taskId, List taskIds, long catalogId, this.userInject = userInject; this.updateRows = updateRows; this.priority = priority; + if (partitionUpdateRows != null) { + this.partitionUpdateRows.putAll(partitionUpdateRows); + } } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisInfoBuilder.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisInfoBuilder.java index 97e96112d79798..e8cef0f64d6ff1 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisInfoBuilder.java +++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisInfoBuilder.java @@ -27,6 +27,7 @@ import org.apache.logging.log4j.core.util.CronExpression; import java.util.List; +import java.util.Map; import java.util.Set; public class AnalysisInfoBuilder { @@ -65,6 +66,7 @@ public class AnalysisInfoBuilder { private boolean userInject; private long updateRows; private JobPriority priority; + private Map partitionUpdateRows; public AnalysisInfoBuilder() { } @@ -105,6 +107,7 @@ public AnalysisInfoBuilder(AnalysisInfo info) { userInject = info.userInject; updateRows = info.updateRows; priority = info.priority; + partitionUpdateRows = info.partitionUpdateRows; } public AnalysisInfoBuilder setJobId(long jobId) { @@ -282,13 +285,18 @@ public AnalysisInfoBuilder setPriority(JobPriority priority) { return this; } + public AnalysisInfoBuilder setPartitionUpdateRows(Map partitionUpdateRows) { + this.partitionUpdateRows = partitionUpdateRows; + return this; + } + public AnalysisInfo build() { return new AnalysisInfo(jobId, taskId, taskIds, catalogId, dbId, tblId, jobColumns, partitionNames, colName, indexId, jobType, analysisMode, analysisMethod, analysisType, samplePercent, sampleRows, maxBucketNum, periodTimeInMs, message, lastExecTimeInMs, timeCostInMs, state, scheduleType, partitionOnly, samplingPartition, isAllPartition, partitionCount, cronExpression, forceFull, usingSqlForExternalTable, tblUpdateTime, rowCount, userInject, updateRows, - priority); + priority, partitionUpdateRows); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisManager.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisManager.java index 57109400ee7c5c..18f7f79b4c9b97 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisManager.java +++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisManager.java @@ -381,6 +381,7 @@ public AnalysisInfo buildAnalysisJobInfo(AnalyzeTblStmt stmt) { TableStatsMeta tableStatsStatus = findTableStatsStatus(table.getId()); infoBuilder.setUpdateRows(tableStatsStatus == null ? 0 : tableStatsStatus.updatedRows.get()); infoBuilder.setPriority(JobPriority.MANUAL); + infoBuilder.setPartitionUpdateRows(tableStatsStatus == null ? null : tableStatsStatus.partitionUpdateRows); return infoBuilder.build(); } @@ -514,6 +515,9 @@ public void updateTableStats(AnalysisInfo jobInfo) { if (jobInfo.partitionNames != null) { jobInfo.partitionNames.clear(); } + if (jobInfo.partitionUpdateRows != null) { + jobInfo.partitionUpdateRows.clear(); + } } @VisibleForTesting diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/ColStatsMeta.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/ColStatsMeta.java index 7e317d67bd740f..6cb2ced9286a65 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/statistics/ColStatsMeta.java +++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/ColStatsMeta.java @@ -23,6 +23,9 @@ import com.google.gson.annotations.SerializedName; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; import java.util.concurrent.atomic.AtomicLong; public class ColStatsMeta { @@ -49,8 +52,11 @@ public class ColStatsMeta { @SerializedName("rowCount") public long rowCount; + @SerializedName("pur") + public ConcurrentMap partitionUpdateRows = new ConcurrentHashMap<>(); + public ColStatsMeta(long updatedTime, AnalysisMethod analysisMethod, AnalysisType analysisType, JobType jobType, - long queriedTimes, long rowCount, long updatedRows) { + long queriedTimes, long rowCount, long updatedRows, Map partitionUpdateRows) { this.updatedTime = updatedTime; this.analysisMethod = analysisMethod; this.analysisType = analysisType; @@ -58,5 +64,8 @@ public ColStatsMeta(long updatedTime, AnalysisMethod analysisMethod, AnalysisTyp this.queriedTimes.addAndGet(queriedTimes); this.updatedRows = updatedRows; this.rowCount = rowCount; + if (partitionUpdateRows != null) { + this.partitionUpdateRows.putAll(partitionUpdateRows); + } } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsAutoCollector.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsAutoCollector.java index d64813afbc98a6..227184ba852e1a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsAutoCollector.java +++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsAutoCollector.java @@ -210,6 +210,7 @@ protected AnalysisInfo createAnalyzeJobForTbl( .setRowCount(rowCount) .setUpdateRows(tableStatsStatus == null ? 0 : tableStatsStatus.updatedRows.get()) .setPriority(priority) + .setPartitionUpdateRows(tableStatsStatus == null ? null : tableStatsStatus.partitionUpdateRows) .build(); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsRepository.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsRepository.java index a17b85002d2ce2..dee9cb042fee62 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsRepository.java +++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsRepository.java @@ -34,6 +34,7 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import java.util.ArrayList; import java.util.Collection; import java.util.Collections; import java.util.HashMap; @@ -122,6 +123,9 @@ public static ColumnStatistic queryColumnStatisticsByName( public static List queryColumnStatisticsByPartitions(TableIf table, Set columnNames, List partitionNames) { + if (!table.isPartitionedTable()) { + return new ArrayList<>(); + } long ctlId = table.getDatabase().getCatalog().getId(); long dbId = table.getDatabase().getId(); Map params = new HashMap<>(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/TableStatsMeta.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/TableStatsMeta.java index 3aa7decfed36f2..f9c633844cad01 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/statistics/TableStatsMeta.java +++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/TableStatsMeta.java @@ -134,16 +134,22 @@ public void update(AnalysisInfo analyzedJob, TableIf tableIf) { for (Pair colPair : analyzedJob.jobColumns) { ColStatsMeta colStatsMeta = colToColStatsMeta.get(colPair); if (colStatsMeta == null) { - colToColStatsMeta.put(colPair, new ColStatsMeta(updatedTime, analyzedJob.analysisMethod, + colToColStatsMeta.put(colPair, new ColStatsMeta(analyzedJob.createTime, analyzedJob.analysisMethod, analyzedJob.analysisType, analyzedJob.jobType, 0, analyzedJob.rowCount, - analyzedJob.updateRows)); + analyzedJob.updateRows, analyzedJob.partitionUpdateRows)); } else { - colStatsMeta.updatedTime = updatedTime; + colStatsMeta.updatedTime = analyzedJob.startTime; colStatsMeta.analysisType = analyzedJob.analysisType; colStatsMeta.analysisMethod = analyzedJob.analysisMethod; colStatsMeta.jobType = analyzedJob.jobType; colStatsMeta.updatedRows = analyzedJob.updateRows; colStatsMeta.rowCount = analyzedJob.rowCount; + if (colStatsMeta.partitionUpdateRows == null) { + colStatsMeta.partitionUpdateRows = new ConcurrentHashMap<>(); + } else { + colStatsMeta.partitionUpdateRows.clear(); + } + colStatsMeta.partitionUpdateRows.putAll(analyzedJob.partitionUpdateRows); } } jobType = analyzedJob.jobType; diff --git a/fe/fe-core/src/test/java/org/apache/doris/statistics/util/StatisticsUtilTest.java b/fe/fe-core/src/test/java/org/apache/doris/statistics/util/StatisticsUtilTest.java index 275471a66982b6..49e7751f26488c 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/statistics/util/StatisticsUtilTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/statistics/util/StatisticsUtilTest.java @@ -196,7 +196,7 @@ public TableStatsMeta findTableStatsStatus(long tblId) { new MockUp() { @Mock public ColStatsMeta findColumnStatsMeta(String indexName, String colName) { - return new ColStatsMeta(0, null, null, null, 0, 0, 0); + return new ColStatsMeta(0, null, null, null, 0, 0, 0, null); } }; @@ -244,7 +244,7 @@ public long getRowCount() { new MockUp() { @Mock public ColStatsMeta findColumnStatsMeta(String indexName, String colName) { - return new ColStatsMeta(0, null, null, null, 0, 100, 0); + return new ColStatsMeta(0, null, null, null, 0, 100, 0, null); } }; tableMeta.newPartitionLoaded.set(false); @@ -254,7 +254,7 @@ public ColStatsMeta findColumnStatsMeta(String indexName, String colName) { new MockUp() { @Mock public ColStatsMeta findColumnStatsMeta(String indexName, String colName) { - return new ColStatsMeta(0, null, null, null, 0, 0, 0); + return new ColStatsMeta(0, null, null, null, 0, 0, 0, null); } }; tableMeta.newPartitionLoaded.set(false); @@ -270,7 +270,7 @@ public long getRowCount() { new MockUp() { @Mock public ColStatsMeta findColumnStatsMeta(String indexName, String colName) { - return new ColStatsMeta(0, null, null, null, 0, 500, 0); + return new ColStatsMeta(0, null, null, null, 0, 500, 0, null); } }; tableMeta.newPartitionLoaded.set(false); @@ -286,7 +286,7 @@ public long getRowCount() { new MockUp() { @Mock public ColStatsMeta findColumnStatsMeta(String indexName, String colName) { - return new ColStatsMeta(0, null, null, null, 0, 100, 80); + return new ColStatsMeta(0, null, null, null, 0, 100, 80, null); } }; tableMeta.newPartitionLoaded.set(false);