Skip to content

Commit

Permalink
Support show partition stats. (#35114)
Browse files Browse the repository at this point in the history
  • Loading branch information
@Jibing-Li
Jibing-Li authored May 22, 2024
1 parent a33d2ee commit 8517b8f
Show file tree
Hide file tree
Showing 5 changed files with 246 additions and 91 deletions.
48 changes: 41 additions & 7 deletions fe/fe-core/src/main/java/org/apache/doris/analysis/ShowColumnStatsStmt.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,6 @@
import org.apache.doris.catalog.Env;
import org.apache.doris.catalog.ScalarType;
import org.apache.doris.catalog.TableIf;
import org.apache.doris.common.AnalysisException;
import org.apache.doris.common.ErrorCode;
import org.apache.doris.common.ErrorReport;
import org.apache.doris.common.Pair;
Expand All @@ -35,6 +34,7 @@
import org.apache.doris.statistics.AnalysisManager;
import org.apache.doris.statistics.ColStatsMeta;
import org.apache.doris.statistics.ColumnStatistic;
import org.apache.doris.statistics.ResultRow;

import com.google.common.collect.ImmutableList;
import com.google.common.collect.Lists;
Expand All @@ -46,7 +46,7 @@

public class ShowColumnStatsStmt extends ShowStmt {

private static final ImmutableList<String> TITLE_NAMES =
private static final ImmutableList<String> TABLE_COLUMN_TITLE_NAMES =
new ImmutableList.Builder<String>()
.add("column_name")
.add("index_name")
Expand All @@ -66,6 +66,22 @@ public class ShowColumnStatsStmt extends ShowStmt {
.add("last_analyze_row_count")
.build();

private static final ImmutableList<String> PARTITION_COLUMN_TITLE_NAMES =
new ImmutableList.Builder<String>()
.add("column_name")
.add("partition_name")
.add("index_name")
.add("count")
.add("ndv")
.add("num_null")
.add("min")
.add("max")
.add("data_size")
.add("updated_time")
.add("update_rows")
.add("trigger")
.build();

private final TableName tableName;

private final List<String> columnNames;
Expand All @@ -92,9 +108,6 @@ public void analyze(Analyzer analyzer) throws UserException {
tableName.analyze(analyzer);
if (partitionNames != null) {
partitionNames.analyze(analyzer);
if (partitionNames.getPartitionNames().size() > 1) {
throw new AnalysisException("Only one partition name could be specified");
}
}
CatalogIf<DatabaseIf> catalog = Env.getCurrentEnv().getCatalogMgr().getCatalog(tableName.getCtl());
if (catalog == null) {
Expand Down Expand Up @@ -129,8 +142,8 @@ public void analyze(Analyzer analyzer) throws UserException {
@Override
public ShowResultSetMetaData getMetaData() {
ShowResultSetMetaData.Builder builder = ShowResultSetMetaData.builder();

for (String title : TITLE_NAMES) {
ImmutableList<String> titles = partitionNames == null ? TABLE_COLUMN_TITLE_NAMES : PARTITION_COLUMN_TITLE_NAMES;
for (String title : titles) {
builder.addColumn(new Column(title, ScalarType.createVarchar(30)));
}
return builder.build();
Expand Down Expand Up @@ -171,6 +184,27 @@ public ShowResultSet constructResultSet(List<Pair<Pair<String, String>, ColumnSt
return new ShowResultSet(getMetaData(), result);
}

public ShowResultSet constructPartitionResultSet(List<ResultRow> resultRows, TableIf tableIf) {
List<List<String>> result = Lists.newArrayList();
resultRows.forEach(r -> {
List<String> row = Lists.newArrayList();
row.add(r.get(0));
row.add(r.get(1));
row.add(r.get(2)); // TODO: Get index name.
row.add(r.get(3));
row.add(r.get(4));
row.add(r.get(5));
row.add(r.get(6));
row.add(r.get(7));
row.add(r.get(8));
row.add(r.get(9));
row.add("N/A");
row.add("Manual");
result.add(row);
});
return new ShowResultSet(getMetaData(), result);
}

public PartitionNames getPartitionNames() {
return partitionNames;
}
Expand Down
94 changes: 65 additions & 29 deletions fe/fe-core/src/main/java/org/apache/doris/analysis/ShowTableStatsStmt.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,10 +20,10 @@
import org.apache.doris.catalog.Column;
import org.apache.doris.catalog.DatabaseIf;
import org.apache.doris.catalog.Env;
import org.apache.doris.catalog.OlapTable;
import org.apache.doris.catalog.Partition;
import org.apache.doris.catalog.ScalarType;
import org.apache.doris.catalog.TableIf;
import org.apache.doris.common.AnalysisException;
import org.apache.doris.common.ErrorCode;
import org.apache.doris.common.ErrorReport;
import org.apache.doris.common.UserException;
Expand All @@ -41,12 +41,12 @@
import java.time.LocalDateTime;
import java.time.format.DateTimeFormatter;
import java.util.ArrayList;
import java.util.Collection;
import java.util.List;

public class ShowTableStatsStmt extends ShowStmt {

// TODO add more columns
private static final ImmutableList<String> TITLE_NAMES =
private static final ImmutableList<String> TABLE_TITLE_NAMES =
new ImmutableList.Builder<String>()
.add("updated_rows")
.add("query_times")
Expand All @@ -58,6 +58,13 @@ public class ShowTableStatsStmt extends ShowStmt {
.add("user_inject")
.build();

private static final ImmutableList<String> PARTITION_TITLE_NAMES =
new ImmutableList.Builder<String>()
.add("partition_name")
.add("updated_rows")
.add("row_count")
.build();

private final TableName tableName;

private final PartitionNames partitionNames;
Expand All @@ -81,9 +88,6 @@ public void analyze(Analyzer analyzer) throws UserException {
tableName.analyze(analyzer);
if (partitionNames != null) {
partitionNames.analyze(analyzer);
if (partitionNames.getPartitionNames().size() > 1) {
throw new AnalysisException("Only one partition name could be specified");
}
}
CatalogIf<DatabaseIf> catalog = Env.getCurrentEnv().getCatalogMgr().getCatalog(tableName.getCtl());
if (catalog == null) {
Expand All @@ -97,11 +101,12 @@ public void analyze(Analyzer analyzer) throws UserException {
if (table == null) {
ErrorReport.reportAnalysisException(String.format("Table: %s not exists", tableName.getTbl()));
}
if (partitionNames != null) {
String partitionName = partitionNames.getPartitionNames().get(0);
Partition partition = table.getPartition(partitionName);
if (partition == null) {
ErrorReport.reportAnalysisException(String.format("Partition: %s not exists", partitionName));
if (partitionNames != null && !partitionNames.isStar()) {
for (String partitionName : partitionNames.getPartitionNames()) {
Partition partition = table.getPartition(partitionName);
if (partition == null) {
ErrorReport.reportAnalysisException(String.format("Partition: %s not exists", partitionName));
}
}
}
if (!Env.getCurrentEnv().getAccessManager()
Expand All @@ -117,7 +122,8 @@ public void analyze(Analyzer analyzer) throws UserException {
public ShowResultSetMetaData getMetaData() {
ShowResultSetMetaData.Builder builder = ShowResultSetMetaData.builder();

for (String title : TITLE_NAMES) {
ImmutableList<String> titles = partitionNames == null ? TABLE_TITLE_NAMES : PARTITION_TITLE_NAMES;
for (String title : titles) {
builder.addColumn(new Column(title, ScalarType.createVarchar(30)));
}
return builder.build();
Expand All @@ -127,15 +133,34 @@ public TableIf getTable() {
return table;
}

public long getPartitionId() {
public ShowResultSet constructResultSet(TableStatsMeta tableStatistic) {
if (partitionNames == null) {
return 0;
return constructTableResultSet(tableStatistic);
} else {
return constructPartitionResultSet(tableStatistic);
}
String partitionName = partitionNames.getPartitionNames().get(0);
return table.getPartition(partitionName).getId();
}

public ShowResultSet constructResultSet(TableStatsMeta tableStatistic) {
public ShowResultSet constructResultSet(long rowCount) {
List<List<String>> result = Lists.newArrayList();
if (partitionNames != null) {
// For partition, return empty result if table stats not exist.
return new ShowResultSet(getMetaData(), result);
}
List<String> row = Lists.newArrayList();
row.add("");
row.add("");
row.add(String.valueOf(rowCount));
row.add("");
row.add("");
row.add("");
row.add("");
row.add("");
result.add(row);
return new ShowResultSet(getMetaData(), result);
}

public ShowResultSet constructTableResultSet(TableStatsMeta tableStatistic) {
DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss");
if (tableStatistic == null) {
return new ShowResultSet(getMetaData(), new ArrayList<>());
Expand All @@ -147,7 +172,7 @@ public ShowResultSet constructResultSet(TableStatsMeta tableStatistic) {
row.add(String.valueOf(tableStatistic.rowCount));
LocalDateTime dateTime =
LocalDateTime.ofInstant(Instant.ofEpochMilli(tableStatistic.updatedTime),
java.time.ZoneId.systemDefault());
java.time.ZoneId.systemDefault());
String formattedDateTime = dateTime.format(formatter);
row.add(formattedDateTime);
row.add(tableStatistic.analyzeColumns().toString());
Expand All @@ -158,18 +183,29 @@ public ShowResultSet constructResultSet(TableStatsMeta tableStatistic) {
return new ShowResultSet(getMetaData(), result);
}

public ShowResultSet constructResultSet(long rowCount) {
public ShowResultSet constructPartitionResultSet(TableStatsMeta tableStatistic) {
List<List<String>> result = Lists.newArrayList();
List<String> row = Lists.newArrayList();
row.add("");
row.add("");
row.add(String.valueOf(rowCount));
row.add("");
row.add("");
row.add("");
row.add("");
row.add("");
result.add(row);
if (!(table instanceof OlapTable)) {
return new ShowResultSet(getMetaData(), result);
}
Collection<String> partitions = partitionNames.isStar()
? table.getPartitionNames()
: partitionNames.getPartitionNames();
for (String part : partitions) {
Partition partition = table.getPartition(part);
if (partition == null) {
continue;
}
Long updateRows = tableStatistic.partitionUpdateRows.get(partition.getId());
if (updateRows == null) {
continue;
}
List<String> row = Lists.newArrayList();
row.add(part);
row.add(String.valueOf(updateRows.longValue()));
row.add(String.valueOf(partition.getBaseIndex().getRowCount()));
result.add(row);
}
return new ShowResultSet(getMetaData(), result);
}

Expand Down
45 changes: 22 additions & 23 deletions fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -115,7 +115,6 @@
import org.apache.doris.analysis.ShowVariablesStmt;
import org.apache.doris.analysis.ShowViewStmt;
import org.apache.doris.analysis.ShowWorkloadGroupsStmt;
import org.apache.doris.analysis.TableName;
import org.apache.doris.analysis.UserIdentity;
import org.apache.doris.backup.AbstractJob;
import org.apache.doris.backup.BackupJob;
Expand Down Expand Up @@ -2627,19 +2626,27 @@ private void handleShowTableStats() {

private void handleShowColumnStats() throws AnalysisException {
ShowColumnStatsStmt showColumnStatsStmt = (ShowColumnStatsStmt) stmt;
TableName tableName = showColumnStatsStmt.getTableName();
TableIf tableIf = showColumnStatsStmt.getTable();
List<Pair<Pair<String, String>, ColumnStatistic>> columnStatistics = new ArrayList<>();
Set<String> columnNames = showColumnStatsStmt.getColumnNames();
PartitionNames partitionNames = showColumnStatsStmt.getPartitionNames();
boolean showCache = showColumnStatsStmt.isCached();
boolean isAllColumns = showColumnStatsStmt.isAllColumns();
if (isAllColumns && !showCache && partitionNames == null) {
getStatsForAllColumns(columnStatistics, tableIf);
if (partitionNames != null) {
List<String> partNames = partitionNames.getPartitionNames() == null
? new ArrayList<>(tableIf.getPartitionNames())
: partitionNames.getPartitionNames();
List<ResultRow> partitionColumnStats =
StatisticsRepository.queryColumnStatisticsByPartitions(tableIf, columnNames, partNames);
resultSet = showColumnStatsStmt.constructPartitionResultSet(partitionColumnStats, tableIf);
} else {
getStatsForSpecifiedColumns(columnStatistics, columnNames, tableIf, showCache, tableName, partitionNames);
if (isAllColumns && !showCache) {
getStatsForAllColumns(columnStatistics, tableIf);
} else {
getStatsForSpecifiedColumns(columnStatistics, columnNames, tableIf, showCache);
}
resultSet = showColumnStatsStmt.constructResultSet(columnStatistics);
}
resultSet = showColumnStatsStmt.constructResultSet(columnStatistics);
}

private void getStatsForAllColumns(List<Pair<Pair<String, String>, ColumnStatistic>> columnStatistics,
Expand All @@ -2662,8 +2669,7 @@ private void getStatsForAllColumns(List<Pair<Pair<String, String>, ColumnStatist
}

private void getStatsForSpecifiedColumns(List<Pair<Pair<String, String>, ColumnStatistic>> columnStatistics,
Set<String> columnNames, TableIf tableIf, boolean showCache,
TableName tableName, PartitionNames partitionNames)
Set<String> columnNames, TableIf tableIf, boolean showCache)
throws AnalysisException {
for (String colName : columnNames) {
// Olap base index use -1 as index id.
Expand All @@ -2683,24 +2689,17 @@ private void getStatsForSpecifiedColumns(List<Pair<Pair<String, String>, ColumnS
continue;
}
// Show column statistics in columnStatisticsCache.
ColumnStatistic columnStatistic;
if (showCache) {
ColumnStatistic columnStatistic = Env.getCurrentEnv().getStatisticsCache().getColumnStatistics(
tableIf.getDatabase().getCatalog().getId(),
tableIf.getDatabase().getId(), tableIf.getId(), indexId, colName);
columnStatistics.add(Pair.of(Pair.of(indexName, colName), columnStatistic));
} else if (partitionNames == null) {
ColumnStatistic columnStatistic =
StatisticsRepository.queryColumnStatisticsByName(
tableIf.getDatabase().getCatalog().getId(),
tableIf.getDatabase().getId(), tableIf.getId(), indexId, colName);
columnStatistics.add(Pair.of(Pair.of(indexName, colName), columnStatistic));
columnStatistic = Env.getCurrentEnv().getStatisticsCache().getColumnStatistics(
tableIf.getDatabase().getCatalog().getId(),
tableIf.getDatabase().getId(), tableIf.getId(), indexId, colName);
} else {
String finalIndexName = indexName;
columnStatistics.addAll(StatisticsRepository.queryColumnStatisticsByPartitions(tableName,
colName, partitionNames.getPartitionNames())
.stream().map(s -> Pair.of(Pair.of(finalIndexName, colName), s))
.collect(Collectors.toList()));
columnStatistic = StatisticsRepository.queryColumnStatisticsByName(
tableIf.getDatabase().getCatalog().getId(),
tableIf.getDatabase().getId(), tableIf.getId(), indexId, colName);
}
columnStatistics.add(Pair.of(Pair.of(indexName, colName), columnStatistic));
}
}
}
Expand Down
Loading

0 comments on commit 8517b8f

Please sign in to comment.