From 7d24e360cf96e08233e8ea15aea0db1aa43acd1a Mon Sep 17 00:00:00 2001 From: morningman Date: Sun, 21 Apr 2024 00:14:33 +0800 Subject: [PATCH] 1 --- .../org/apache/doris/common/ErrorCode.java | 2 +- .../doris/datasource/InternalCatalog.java | 24 +++++++++-- .../doris/nereids/minidump/MinidumpUtils.java | 3 +- .../org/apache/doris/qe/ShowExecutor.java | 2 +- .../doris/service/FrontendServiceImpl.java | 5 ++- .../doris/statistics/AnalysisManager.java | 2 +- .../doris/statistics/HistogramTask.java | 3 +- .../doris/statistics/StatisticsCache.java | 41 +++++++++---------- .../doris/statistics/StatisticsCacheKey.java | 11 +++-- .../statistics/StatisticsRepository.java | 3 +- .../doris/nereids/util/HyperGraphBuilder.java | 5 ++- .../apache/doris/statistics/CacheTest.java | 4 +- 12 files changed, 61 insertions(+), 44 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/ErrorCode.java b/fe/fe-core/src/main/java/org/apache/doris/common/ErrorCode.java index 6474f208c6d958..efdf937701480b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/ErrorCode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/ErrorCode.java @@ -262,7 +262,7 @@ public enum ErrorCode { ERR_VIEW_NO_EXPLAIN(1345, new byte[]{'H', 'Y', '0', '0', '0'}, "EXPLAIN/SHOW can not be issued; lacking " + "privileges for underlying table"), ERR_FRM_UNKNOWN_TYPE(1346, new byte[]{'H', 'Y', '0', '0', '0'}, "File '%s' has unknown type '%s' in its header"), - ERR_WRONG_OBJECT(1347, new byte[]{'H', 'Y', '0', '0', '0'}, "'%s.%s' is not %s"), + ERR_WRONG_OBJECT(1347, new byte[]{'H', 'Y', '0', '0', '0'}, "'%s.%s' is not %s. %s."), ERR_NONUPDATEABLE_COLUMN(1348, new byte[]{'H', 'Y', '0', '0', '0'}, "Column '%s' is not updatable"), ERR_VIEW_SELECT_DERIVED(1349, new byte[]{'H', 'Y', '0', '0', '0'}, "View's SELECT contains a subquery in the FROM" + " clause"), diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java index 195fd1866f094c..f8da7d0715cc90 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java @@ -867,18 +867,22 @@ public void dropTable(DropTableStmt stmt) throws DdlException { // Check if a view if (stmt.isView()) { if (!(table instanceof View)) { - ErrorReport.reportDdlException(ErrorCode.ERR_WRONG_OBJECT, dbName, tableName, "VIEW"); + ErrorReport.reportDdlException(ErrorCode.ERR_WRONG_OBJECT, dbName, tableName, "VIEW", + genDropHint(table)); } } else { if (table instanceof View) { - ErrorReport.reportDdlException(ErrorCode.ERR_WRONG_OBJECT, dbName, tableName, "TABLE"); + ErrorReport.reportDdlException(ErrorCode.ERR_WRONG_OBJECT, dbName, tableName, "TABLE", + genDropHint(table)); } } if (!stmt.isMaterializedView() && table instanceof MTMV) { - ErrorReport.reportDdlException(ErrorCode.ERR_WRONG_OBJECT, dbName, tableName, "TABLE"); + ErrorReport.reportDdlException(ErrorCode.ERR_WRONG_OBJECT, dbName, tableName, "TABLE", + genDropHint(table)); } else if (stmt.isMaterializedView() && !(table instanceof MTMV)) { - ErrorReport.reportDdlException(ErrorCode.ERR_WRONG_OBJECT, dbName, tableName, "MTMV"); + ErrorReport.reportDdlException(ErrorCode.ERR_WRONG_OBJECT, dbName, tableName, "MTMV", + genDropHint(table)); } if (!stmt.isForceDrop()) { @@ -941,6 +945,18 @@ public void dropTable(DropTableStmt stmt) throws DdlException { tableName, dbName, stmt.isForceDrop(), costTimes); } + private static String genDropHint(TableIf table) { + String type = ""; + if (table instanceof View) { + type = "VIEW"; + } else if (table instanceof OlapTable) { + type = "TABLE"; + } else if (table instanceof MTMV) { + type = "MATERIALIZED VIEW"; + } + return "Use 'DROP " + type + " " + table.getName(); + } + public boolean unprotectDropTable(Database db, Table table, boolean isForceDrop, boolean isReplay, long recycleTime) { if (table.getType() == TableType.ELASTICSEARCH) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/minidump/MinidumpUtils.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/minidump/MinidumpUtils.java index 327b0aa6ffa53b..bd64da6d470b32 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/minidump/MinidumpUtils.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/minidump/MinidumpUtils.java @@ -247,7 +247,8 @@ private static ColumnStatistic getColumnStatistic(TableIf table, String colName) } private static Histogram getColumnHistogram(TableIf table, String colName) { - return Env.getCurrentEnv().getStatisticsCache().getHistogram(table.getId(), colName); + return Env.getCurrentEnv().getStatisticsCache().getHistogram( + table.getDatabase().getCatalog().getId(), table.getDatabase().getId(), table.getId(), colName); } /** diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java b/fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java index 11e62bdc5f0e27..b169c9d962cb6b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java @@ -1107,7 +1107,7 @@ private void handleShowCreateTable() throws AnalysisException { } else { if (showStmt.isView()) { ErrorReport.reportAnalysisException(ErrorCode.ERR_WRONG_OBJECT, showStmt.getDb(), - showStmt.getTable(), "VIEW"); + showStmt.getTable(), "VIEW", "Use 'SHOW CREATE TABLE '" + table.getName()); } rows.add(Lists.newArrayList(table.getName(), createTableStmt.get(0))); resultSet = table.getType() != TableType.MATERIALIZED_VIEW diff --git a/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java b/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java index 08d4c0ada6837c..18e6f416d55fd4 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java +++ b/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java @@ -3143,9 +3143,10 @@ public TStatus updateStatsCache(TUpdateFollowerStatsCacheRequest request) throws ColStatsData data = GsonUtils.GSON.fromJson(request.colStatsData, ColStatsData.class); ColumnStatistic c = data.toColumnStatistic(); if (c == ColumnStatistic.UNKNOWN) { - Env.getCurrentEnv().getStatisticsCache().invalidate(k.tableId, k.idxId, k.colName); + Env.getCurrentEnv().getStatisticsCache().invalidate(k.catalogId, k.dbId, k.tableId, k.idxId, k.colName); } else { - Env.getCurrentEnv().getStatisticsCache().updateColStatsCache(k.tableId, k.idxId, k.colName, c); + Env.getCurrentEnv().getStatisticsCache().updateColStatsCache( + k.catalogId, k.dbId, k.tableId, k.idxId, k.colName, c); } // Return Ok anyway return new TStatus(TStatusCode.OK); diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisManager.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisManager.java index 813dc4a9d51fc5..a2b30cce2ac5f4 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisManager.java +++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisManager.java @@ -669,7 +669,7 @@ public void invalidateLocalStats(long catalogId, long dbId, long tableId, } } tableStats.removeColumn(indexName, column); - statisticsCache.invalidate(tableId, indexId, column); + statisticsCache.invalidate(catalogId, dbId, tableId, indexId, column); } } tableStats.updatedTime = 0; diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/HistogramTask.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/HistogramTask.java index 3e02c47497b097..60da8f4d2a0803 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/statistics/HistogramTask.java +++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/HistogramTask.java @@ -75,7 +75,8 @@ public void doExecute() throws Exception { StringSubstitutor stringSubstitutor = new StringSubstitutor(params); StatisticsUtil.execUpdate(stringSubstitutor.replace(ANALYZE_HISTOGRAM_SQL_TEMPLATE_TABLE)); - Env.getCurrentEnv().getStatisticsCache().refreshHistogramSync(tbl.getId(), -1, col.getName()); + Env.getCurrentEnv().getStatisticsCache().refreshHistogramSync( + tbl.getDatabase().getCatalog().getId(), tbl.getDatabase().getId(), tbl.getId(), -1, col.getName()); } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsCache.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsCache.java index ab503cb4f7e9e9..d9539e41887822 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsCache.java +++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsCache.java @@ -91,16 +91,16 @@ public ColumnStatistic getColumnStatistics(long catalogId, long dbId, long tblId return ColumnStatistic.UNKNOWN; } - public Histogram getHistogram(long tblId, String colName) { - return getHistogram(tblId, -1, colName).orElse(null); + public Histogram getHistogram(long ctlId, long dbId, long tblId, String colName) { + return getHistogram(ctlId, dbId, tblId, -1, colName).orElse(null); } - public Optional getHistogram(long tblId, long idxId, String colName) { + private Optional getHistogram(long ctlId, long dbId, long tblId, long idxId, String colName) { ConnectContext ctx = ConnectContext.get(); if (ctx != null && ctx.getSessionVariable().internalSession) { return Optional.empty(); } - StatisticsCacheKey k = new StatisticsCacheKey(tblId, idxId, colName); + StatisticsCacheKey k = new StatisticsCacheKey(ctlId, dbId, tblId, idxId, colName); try { CompletableFuture> f = histogramCache.get(k); if (f.isDone()) { @@ -112,24 +112,22 @@ public Optional getHistogram(long tblId, long idxId, String colName) return Optional.empty(); } - public void invalidate(long tblId, long idxId, String colName) { - columnStatisticsCache.synchronous().invalidate(new StatisticsCacheKey(tblId, idxId, colName)); + public void invalidate(long ctlId, long dbId, long tblId, long idxId, String colName) { + columnStatisticsCache.synchronous().invalidate(new StatisticsCacheKey(ctlId, dbId, tblId, idxId, colName)); } - public void updateColStatsCache(long tblId, long idxId, String colName, ColumnStatistic statistic) { - columnStatisticsCache.synchronous().put(new StatisticsCacheKey(tblId, idxId, colName), Optional.of(statistic)); + public void updateColStatsCache(long ctlId, long dbId, long tblId, long idxId, String colName, + ColumnStatistic statistic) { + columnStatisticsCache.synchronous() + .put(new StatisticsCacheKey(ctlId, dbId, tblId, idxId, colName), Optional.of(statistic)); } - public void refreshColStatsSync(long tblId, long idxId, String colName) { - columnStatisticsCache.synchronous().refresh(new StatisticsCacheKey(-1, -1, tblId, idxId, colName)); + public void refreshColStatsSync(long ctlId, long dbId, long tblId, long idxId, String colName) { + columnStatisticsCache.synchronous().refresh(new StatisticsCacheKey(ctlId, dbId, tblId, idxId, colName)); } - public void refreshColStatsSync(long catalogId, long dbId, long tblId, long idxId, String colName) { - columnStatisticsCache.synchronous().refresh(new StatisticsCacheKey(catalogId, dbId, tblId, idxId, colName)); - } - - public void refreshHistogramSync(long tblId, long idxId, String colName) { - histogramCache.synchronous().refresh(new StatisticsCacheKey(tblId, idxId, colName)); + public void refreshHistogramSync(long ctlId, long dbId, long tblId, long idxId, String colName) { + histogramCache.synchronous().refresh(new StatisticsCacheKey(ctlId, dbId, tblId, idxId, colName)); } public void preHeat() { @@ -168,11 +166,9 @@ private void doPreHeat() { for (ResultRow r : recentStatsUpdatedCols) { try { StatsId statsId = new StatsId(r); - long tblId = statsId.tblId; - long idxId = statsId.idxId; - String colId = statsId.colId; final StatisticsCacheKey k = - new StatisticsCacheKey(tblId, idxId, colId); + new StatisticsCacheKey(statsId.catalogId, statsId.dbId, statsId.tblId, statsId.idxId, + statsId.colId); ColumnStatistic c = ColumnStatistic.fromResultRow(r); if (c.count > 0 && c.ndv == 0 && c.count != c.numNulls) { c = ColumnStatistic.UNKNOWN; @@ -189,10 +185,11 @@ private void doPreHeat() { */ public void syncColStats(ColStatsData data) { StatsId statsId = data.statsId; - final StatisticsCacheKey k = new StatisticsCacheKey(statsId.tblId, statsId.idxId, statsId.colId); + final StatisticsCacheKey k = new StatisticsCacheKey(statsId.catalogId, statsId.dbId, statsId.tblId, + statsId.idxId, statsId.colId); ColumnStatistic columnStatistic = data.toColumnStatistic(); if (columnStatistic == ColumnStatistic.UNKNOWN) { - invalidate(k.tableId, k.idxId, k.colName); + invalidate(k.catalogId, k.dbId, k.tableId, k.idxId, k.colName); } else { putCache(k, columnStatistic); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsCacheKey.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsCacheKey.java index fa924ab92842f4..7d5c7bccc52dab 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsCacheKey.java +++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsCacheKey.java @@ -41,10 +41,6 @@ public class StatisticsCacheKey { private static final String DELIMITER = "-"; - public StatisticsCacheKey(long tableId, long idxId, String colName) { - this(-1, -1, tableId, idxId, colName); - } - public StatisticsCacheKey(long catalogId, long dbId, long tableId, long idxId, String colName) { this.catalogId = catalogId; this.dbId = dbId; @@ -55,7 +51,7 @@ public StatisticsCacheKey(long catalogId, long dbId, long tableId, long idxId, S @Override public int hashCode() { - return Objects.hash(tableId, idxId, colName); + return Objects.hash(catalogId, dbId, tableId, idxId, colName); } @Override @@ -67,13 +63,16 @@ public boolean equals(Object obj) { return false; } StatisticsCacheKey k = (StatisticsCacheKey) obj; - return this.tableId == k.tableId && this.idxId == k.idxId && this.colName.equals(k.colName); + return this.catalogId == k.catalogId && this.dbId == k.dbId && this.tableId == k.tableId + && this.idxId == k.idxId && this.colName.equals(k.colName); } @Override public String toString() { StringJoiner sj = new StringJoiner(DELIMITER); sj.add("ColumnStats"); + sj.add(String.valueOf(catalogId)); + sj.add(String.valueOf(dbId)); sj.add(String.valueOf(tableId)); sj.add(String.valueOf(idxId)); sj.add(colName); diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsRepository.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsRepository.java index 789ae4e43501ab..87ce90c5300a2b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsRepository.java +++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsRepository.java @@ -78,8 +78,7 @@ public class StatisticsRepository { + "'${colId}', ${partId}, ${count}, ${ndv}, ${nullCount}, ${min}, ${max}, ${dataSize}, NOW())"; private static final String DELETE_TABLE_STATISTICS_TEMPLATE = "DELETE FROM " + FeConstants.INTERNAL_DB_NAME - + "." + "${tblName}" + " WHERE ${condition} AND `catalog_id` = '${catalogId}' AND `db_id` = '${dbId}'" - + " AND `table_id` = '${tblId}'"; + + "." + "${tblName}" + " WHERE ${condition} AND `catalog_id` = '${catalogId}' AND `db_id` = '${dbId}'"; private static final String FETCH_RECENT_STATS_UPDATED_COL = "SELECT * FROM " diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/util/HyperGraphBuilder.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/util/HyperGraphBuilder.java index 6a1da5a994ca4e..d45b4794f198be 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/util/HyperGraphBuilder.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/util/HyperGraphBuilder.java @@ -18,6 +18,7 @@ package org.apache.doris.nereids.util; import org.apache.doris.catalog.Env; +import org.apache.doris.catalog.OlapTable; import org.apache.doris.common.Pair; import org.apache.doris.nereids.CascadesContext; import org.apache.doris.nereids.hint.DistributeHint; @@ -199,11 +200,13 @@ public void initStats(CascadesContext context) { GroupExpression groupExpression = group.getLogicalExpression(); if (groupExpression.getPlan() instanceof LogicalOlapScan) { LogicalOlapScan scan = (LogicalOlapScan) groupExpression.getPlan(); + OlapTable table = scan.getTable(); Statistics stats = injectRowcount((LogicalOlapScan) groupExpression.getPlan()); for (Expression expr : stats.columnStatistics().keySet()) { SlotReference slot = (SlotReference) expr; Env.getCurrentEnv().getStatisticsCache().putCache( - new StatisticsCacheKey(scan.getTable().getId(), -1, slot.getName()), + new StatisticsCacheKey(table.getDatabase().getCatalog().getId(), + table.getDatabase().getId(), table.getId(), -1, slot.getName()), stats.columnStatistics().get(expr)); } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/statistics/CacheTest.java b/fe/fe-core/src/test/java/org/apache/doris/statistics/CacheTest.java index f23b93624b4dae..729291d5323ff0 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/statistics/CacheTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/statistics/CacheTest.java @@ -200,9 +200,9 @@ public List execStatisticQuery(String sql) { }; StatisticsCache statisticsCache = new StatisticsCache(); - statisticsCache.refreshHistogramSync(0, -1, "col"); + statisticsCache.refreshHistogramSync(0, 0, 0, -1, "col"); Thread.sleep(10000); - Histogram histogram = statisticsCache.getHistogram(0, "col"); + Histogram histogram = statisticsCache.getHistogram(0, 0, 0, "col"); Assertions.assertNotNull(histogram); }