Skip to content

Commit

Permalink
1
Browse files Browse the repository at this point in the history
  • Loading branch information
@morningman
morningman committed Apr 21, 2024
1 parent cccffbc commit 7d24e36
Show file tree
Hide file tree
Showing 12 changed files with 61 additions and 44 deletions.
2 changes: 1 addition & 1 deletion fe/fe-core/src/main/java/org/apache/doris/common/ErrorCode.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -262,7 +262,7 @@ public enum ErrorCode {
ERR_VIEW_NO_EXPLAIN(1345, new byte[]{'H', 'Y', '0', '0', '0'}, "EXPLAIN/SHOW can not be issued; lacking "
+ "privileges for underlying table"),
ERR_FRM_UNKNOWN_TYPE(1346, new byte[]{'H', 'Y', '0', '0', '0'}, "File '%s' has unknown type '%s' in its header"),
ERR_WRONG_OBJECT(1347, new byte[]{'H', 'Y', '0', '0', '0'}, "'%s.%s' is not %s"),
ERR_WRONG_OBJECT(1347, new byte[]{'H', 'Y', '0', '0', '0'}, "'%s.%s' is not %s. %s."),
ERR_NONUPDATEABLE_COLUMN(1348, new byte[]{'H', 'Y', '0', '0', '0'}, "Column '%s' is not updatable"),
ERR_VIEW_SELECT_DERIVED(1349, new byte[]{'H', 'Y', '0', '0', '0'}, "View's SELECT contains a subquery in the FROM"
+ " clause"),
Expand Down
24 changes: 20 additions & 4 deletions fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -867,18 +867,22 @@ public void dropTable(DropTableStmt stmt) throws DdlException {
// Check if a view
if (stmt.isView()) {
if (!(table instanceof View)) {
ErrorReport.reportDdlException(ErrorCode.ERR_WRONG_OBJECT, dbName, tableName, "VIEW");
ErrorReport.reportDdlException(ErrorCode.ERR_WRONG_OBJECT, dbName, tableName, "VIEW",
genDropHint(table));
}
} else {
if (table instanceof View) {
ErrorReport.reportDdlException(ErrorCode.ERR_WRONG_OBJECT, dbName, tableName, "TABLE");
ErrorReport.reportDdlException(ErrorCode.ERR_WRONG_OBJECT, dbName, tableName, "TABLE",
genDropHint(table));
}
}

if (!stmt.isMaterializedView() && table instanceof MTMV) {
ErrorReport.reportDdlException(ErrorCode.ERR_WRONG_OBJECT, dbName, tableName, "TABLE");
ErrorReport.reportDdlException(ErrorCode.ERR_WRONG_OBJECT, dbName, tableName, "TABLE",
genDropHint(table));
} else if (stmt.isMaterializedView() && !(table instanceof MTMV)) {
ErrorReport.reportDdlException(ErrorCode.ERR_WRONG_OBJECT, dbName, tableName, "MTMV");
ErrorReport.reportDdlException(ErrorCode.ERR_WRONG_OBJECT, dbName, tableName, "MTMV",
genDropHint(table));
}

if (!stmt.isForceDrop()) {
Expand Down Expand Up @@ -941,6 +945,18 @@ public void dropTable(DropTableStmt stmt) throws DdlException {
tableName, dbName, stmt.isForceDrop(), costTimes);
}

private static String genDropHint(TableIf table) {
String type = "";
if (table instanceof View) {
type = "VIEW";
} else if (table instanceof OlapTable) {
type = "TABLE";
} else if (table instanceof MTMV) {
type = "MATERIALIZED VIEW";
}
return "Use 'DROP " + type + " " + table.getName();
}

public boolean unprotectDropTable(Database db, Table table, boolean isForceDrop, boolean isReplay,
long recycleTime) {
if (table.getType() == TableType.ELASTICSEARCH) {
Expand Down
3 changes: 2 additions & 1 deletion fe/fe-core/src/main/java/org/apache/doris/nereids/minidump/MinidumpUtils.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -247,7 +247,8 @@ private static ColumnStatistic getColumnStatistic(TableIf table, String colName)
}

private static Histogram getColumnHistogram(TableIf table, String colName) {
return Env.getCurrentEnv().getStatisticsCache().getHistogram(table.getId(), colName);
return Env.getCurrentEnv().getStatisticsCache().getHistogram(
table.getDatabase().getCatalog().getId(), table.getDatabase().getId(), table.getId(), colName);
}

/**
Expand Down
2 changes: 1 addition & 1 deletion fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1107,7 +1107,7 @@ private void handleShowCreateTable() throws AnalysisException {
} else {
if (showStmt.isView()) {
ErrorReport.reportAnalysisException(ErrorCode.ERR_WRONG_OBJECT, showStmt.getDb(),
showStmt.getTable(), "VIEW");
showStmt.getTable(), "VIEW", "Use 'SHOW CREATE TABLE '" + table.getName());
}
rows.add(Lists.newArrayList(table.getName(), createTableStmt.get(0)));
resultSet = table.getType() != TableType.MATERIALIZED_VIEW
Expand Down
5 changes: 3 additions & 2 deletions fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3143,9 +3143,10 @@ public TStatus updateStatsCache(TUpdateFollowerStatsCacheRequest request) throws
ColStatsData data = GsonUtils.GSON.fromJson(request.colStatsData, ColStatsData.class);
ColumnStatistic c = data.toColumnStatistic();
if (c == ColumnStatistic.UNKNOWN) {
Env.getCurrentEnv().getStatisticsCache().invalidate(k.tableId, k.idxId, k.colName);
Env.getCurrentEnv().getStatisticsCache().invalidate(k.catalogId, k.dbId, k.tableId, k.idxId, k.colName);
} else {
Env.getCurrentEnv().getStatisticsCache().updateColStatsCache(k.tableId, k.idxId, k.colName, c);
Env.getCurrentEnv().getStatisticsCache().updateColStatsCache(
k.catalogId, k.dbId, k.tableId, k.idxId, k.colName, c);
}
// Return Ok anyway
return new TStatus(TStatusCode.OK);
Expand Down
2 changes: 1 addition & 1 deletion fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisManager.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -669,7 +669,7 @@ public void invalidateLocalStats(long catalogId, long dbId, long tableId,
}
}
tableStats.removeColumn(indexName, column);
statisticsCache.invalidate(tableId, indexId, column);
statisticsCache.invalidate(catalogId, dbId, tableId, indexId, column);
}
}
tableStats.updatedTime = 0;
Expand Down
3 changes: 2 additions & 1 deletion fe/fe-core/src/main/java/org/apache/doris/statistics/HistogramTask.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -75,7 +75,8 @@ public void doExecute() throws Exception {

StringSubstitutor stringSubstitutor = new StringSubstitutor(params);
StatisticsUtil.execUpdate(stringSubstitutor.replace(ANALYZE_HISTOGRAM_SQL_TEMPLATE_TABLE));
Env.getCurrentEnv().getStatisticsCache().refreshHistogramSync(tbl.getId(), -1, col.getName());
Env.getCurrentEnv().getStatisticsCache().refreshHistogramSync(
tbl.getDatabase().getCatalog().getId(), tbl.getDatabase().getId(), tbl.getId(), -1, col.getName());
}

@Override
Expand Down
41 changes: 19 additions & 22 deletions fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsCache.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -91,16 +91,16 @@ public ColumnStatistic getColumnStatistics(long catalogId, long dbId, long tblId
return ColumnStatistic.UNKNOWN;
}

public Histogram getHistogram(long tblId, String colName) {
return getHistogram(tblId, -1, colName).orElse(null);
public Histogram getHistogram(long ctlId, long dbId, long tblId, String colName) {
return getHistogram(ctlId, dbId, tblId, -1, colName).orElse(null);
}

public Optional<Histogram> getHistogram(long tblId, long idxId, String colName) {
private Optional<Histogram> getHistogram(long ctlId, long dbId, long tblId, long idxId, String colName) {
ConnectContext ctx = ConnectContext.get();
if (ctx != null && ctx.getSessionVariable().internalSession) {
return Optional.empty();
}
StatisticsCacheKey k = new StatisticsCacheKey(tblId, idxId, colName);
StatisticsCacheKey k = new StatisticsCacheKey(ctlId, dbId, tblId, idxId, colName);
try {
CompletableFuture<Optional<Histogram>> f = histogramCache.get(k);
if (f.isDone()) {
Expand All @@ -112,24 +112,22 @@ public Optional<Histogram> getHistogram(long tblId, long idxId, String colName)
return Optional.empty();
}

public void invalidate(long tblId, long idxId, String colName) {
columnStatisticsCache.synchronous().invalidate(new StatisticsCacheKey(tblId, idxId, colName));
public void invalidate(long ctlId, long dbId, long tblId, long idxId, String colName) {
columnStatisticsCache.synchronous().invalidate(new StatisticsCacheKey(ctlId, dbId, tblId, idxId, colName));
}

public void updateColStatsCache(long tblId, long idxId, String colName, ColumnStatistic statistic) {
columnStatisticsCache.synchronous().put(new StatisticsCacheKey(tblId, idxId, colName), Optional.of(statistic));
public void updateColStatsCache(long ctlId, long dbId, long tblId, long idxId, String colName,
ColumnStatistic statistic) {
columnStatisticsCache.synchronous()
.put(new StatisticsCacheKey(ctlId, dbId, tblId, idxId, colName), Optional.of(statistic));
}

public void refreshColStatsSync(long tblId, long idxId, String colName) {
columnStatisticsCache.synchronous().refresh(new StatisticsCacheKey(-1, -1, tblId, idxId, colName));
public void refreshColStatsSync(long ctlId, long dbId, long tblId, long idxId, String colName) {
columnStatisticsCache.synchronous().refresh(new StatisticsCacheKey(ctlId, dbId, tblId, idxId, colName));
}

public void refreshColStatsSync(long catalogId, long dbId, long tblId, long idxId, String colName) {
columnStatisticsCache.synchronous().refresh(new StatisticsCacheKey(catalogId, dbId, tblId, idxId, colName));
}

public void refreshHistogramSync(long tblId, long idxId, String colName) {
histogramCache.synchronous().refresh(new StatisticsCacheKey(tblId, idxId, colName));
public void refreshHistogramSync(long ctlId, long dbId, long tblId, long idxId, String colName) {
histogramCache.synchronous().refresh(new StatisticsCacheKey(ctlId, dbId, tblId, idxId, colName));
}

public void preHeat() {
Expand Down Expand Up @@ -168,11 +166,9 @@ private void doPreHeat() {
for (ResultRow r : recentStatsUpdatedCols) {
try {
StatsId statsId = new StatsId(r);
long tblId = statsId.tblId;
long idxId = statsId.idxId;
String colId = statsId.colId;
final StatisticsCacheKey k =
new StatisticsCacheKey(tblId, idxId, colId);
new StatisticsCacheKey(statsId.catalogId, statsId.dbId, statsId.tblId, statsId.idxId,
statsId.colId);
ColumnStatistic c = ColumnStatistic.fromResultRow(r);
if (c.count > 0 && c.ndv == 0 && c.count != c.numNulls) {
c = ColumnStatistic.UNKNOWN;
Expand All @@ -189,10 +185,11 @@ private void doPreHeat() {
*/
public void syncColStats(ColStatsData data) {
StatsId statsId = data.statsId;
final StatisticsCacheKey k = new StatisticsCacheKey(statsId.tblId, statsId.idxId, statsId.colId);
final StatisticsCacheKey k = new StatisticsCacheKey(statsId.catalogId, statsId.dbId, statsId.tblId,
statsId.idxId, statsId.colId);
ColumnStatistic columnStatistic = data.toColumnStatistic();
if (columnStatistic == ColumnStatistic.UNKNOWN) {
invalidate(k.tableId, k.idxId, k.colName);
invalidate(k.catalogId, k.dbId, k.tableId, k.idxId, k.colName);
} else {
putCache(k, columnStatistic);
}
Expand Down
11 changes: 5 additions & 6 deletions fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsCacheKey.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -41,10 +41,6 @@ public class StatisticsCacheKey {

private static final String DELIMITER = "-";

public StatisticsCacheKey(long tableId, long idxId, String colName) {
this(-1, -1, tableId, idxId, colName);
}

public StatisticsCacheKey(long catalogId, long dbId, long tableId, long idxId, String colName) {
this.catalogId = catalogId;
this.dbId = dbId;
Expand All @@ -55,7 +51,7 @@ public StatisticsCacheKey(long catalogId, long dbId, long tableId, long idxId, S

@Override
public int hashCode() {
return Objects.hash(tableId, idxId, colName);
return Objects.hash(catalogId, dbId, tableId, idxId, colName);
}

@Override
Expand All @@ -67,13 +63,16 @@ public boolean equals(Object obj) {
return false;
}
StatisticsCacheKey k = (StatisticsCacheKey) obj;
return this.tableId == k.tableId && this.idxId == k.idxId && this.colName.equals(k.colName);
return this.catalogId == k.catalogId && this.dbId == k.dbId && this.tableId == k.tableId
&& this.idxId == k.idxId && this.colName.equals(k.colName);
}

@Override
public String toString() {
StringJoiner sj = new StringJoiner(DELIMITER);
sj.add("ColumnStats");
sj.add(String.valueOf(catalogId));
sj.add(String.valueOf(dbId));
sj.add(String.valueOf(tableId));
sj.add(String.valueOf(idxId));
sj.add(colName);
Expand Down
3 changes: 1 addition & 2 deletions fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsRepository.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -78,8 +78,7 @@ public class StatisticsRepository {
+ "'${colId}', ${partId}, ${count}, ${ndv}, ${nullCount}, ${min}, ${max}, ${dataSize}, NOW())";

private static final String DELETE_TABLE_STATISTICS_TEMPLATE = "DELETE FROM " + FeConstants.INTERNAL_DB_NAME
+ "." + "${tblName}" + " WHERE ${condition} AND `catalog_id` = '${catalogId}' AND `db_id` = '${dbId}'"
+ " AND `table_id` = '${tblId}'";
+ "." + "${tblName}" + " WHERE ${condition} AND `catalog_id` = '${catalogId}' AND `db_id` = '${dbId}'";

private static final String FETCH_RECENT_STATS_UPDATED_COL =
"SELECT * FROM "
Expand Down
5 changes: 4 additions & 1 deletion fe/fe-core/src/test/java/org/apache/doris/nereids/util/HyperGraphBuilder.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@
package org.apache.doris.nereids.util;

import org.apache.doris.catalog.Env;
import org.apache.doris.catalog.OlapTable;
import org.apache.doris.common.Pair;
import org.apache.doris.nereids.CascadesContext;
import org.apache.doris.nereids.hint.DistributeHint;
Expand Down Expand Up @@ -199,11 +200,13 @@ public void initStats(CascadesContext context) {
GroupExpression groupExpression = group.getLogicalExpression();
if (groupExpression.getPlan() instanceof LogicalOlapScan) {
LogicalOlapScan scan = (LogicalOlapScan) groupExpression.getPlan();
OlapTable table = scan.getTable();
Statistics stats = injectRowcount((LogicalOlapScan) groupExpression.getPlan());
for (Expression expr : stats.columnStatistics().keySet()) {
SlotReference slot = (SlotReference) expr;
Env.getCurrentEnv().getStatisticsCache().putCache(
new StatisticsCacheKey(scan.getTable().getId(), -1, slot.getName()),
new StatisticsCacheKey(table.getDatabase().getCatalog().getId(),
table.getDatabase().getId(), table.getId(), -1, slot.getName()),
stats.columnStatistics().get(expr));
}
}
Expand Down
4 changes: 2 additions & 2 deletions fe/fe-core/src/test/java/org/apache/doris/statistics/CacheTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -200,9 +200,9 @@ public List<ResultRow> execStatisticQuery(String sql) {
};

StatisticsCache statisticsCache = new StatisticsCache();
statisticsCache.refreshHistogramSync(0, -1, "col");
statisticsCache.refreshHistogramSync(0, 0, 0, -1, "col");
Thread.sleep(10000);
Histogram histogram = statisticsCache.getHistogram(0, "col");
Histogram histogram = statisticsCache.getHistogram(0, 0, 0, "col");
Assertions.assertNotNull(histogram);
}

Expand Down

0 comments on commit 7d24e36

Please sign in to comment.