diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Table.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/Table.java index 5d607cd226007c..cbac1b5d7650d0 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Table.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Table.java @@ -348,7 +348,7 @@ public void setName(String newName) { name = newName; } - void setQualifiedDbName(String qualifiedDbName) { + public void setQualifiedDbName(String qualifiedDbName) { this.qualifiedDbName = qualifiedDbName; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/ErrorCode.java b/fe/fe-core/src/main/java/org/apache/doris/common/ErrorCode.java index 6474f208c6d958..efdf937701480b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/ErrorCode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/ErrorCode.java @@ -262,7 +262,7 @@ public enum ErrorCode { ERR_VIEW_NO_EXPLAIN(1345, new byte[]{'H', 'Y', '0', '0', '0'}, "EXPLAIN/SHOW can not be issued; lacking " + "privileges for underlying table"), ERR_FRM_UNKNOWN_TYPE(1346, new byte[]{'H', 'Y', '0', '0', '0'}, "File '%s' has unknown type '%s' in its header"), - ERR_WRONG_OBJECT(1347, new byte[]{'H', 'Y', '0', '0', '0'}, "'%s.%s' is not %s"), + ERR_WRONG_OBJECT(1347, new byte[]{'H', 'Y', '0', '0', '0'}, "'%s.%s' is not %s. %s."), ERR_NONUPDATEABLE_COLUMN(1348, new byte[]{'H', 'Y', '0', '0', '0'}, "Column '%s' is not updatable"), ERR_VIEW_SELECT_DERIVED(1349, new byte[]{'H', 'Y', '0', '0', '0'}, "View's SELECT contains a subquery in the FROM" + " clause"), diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java index 195fd1866f094c..f8da7d0715cc90 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java @@ -867,18 +867,22 @@ public void dropTable(DropTableStmt stmt) throws DdlException { // Check if a view if (stmt.isView()) { if (!(table instanceof View)) { - ErrorReport.reportDdlException(ErrorCode.ERR_WRONG_OBJECT, dbName, tableName, "VIEW"); + ErrorReport.reportDdlException(ErrorCode.ERR_WRONG_OBJECT, dbName, tableName, "VIEW", + genDropHint(table)); } } else { if (table instanceof View) { - ErrorReport.reportDdlException(ErrorCode.ERR_WRONG_OBJECT, dbName, tableName, "TABLE"); + ErrorReport.reportDdlException(ErrorCode.ERR_WRONG_OBJECT, dbName, tableName, "TABLE", + genDropHint(table)); } } if (!stmt.isMaterializedView() && table instanceof MTMV) { - ErrorReport.reportDdlException(ErrorCode.ERR_WRONG_OBJECT, dbName, tableName, "TABLE"); + ErrorReport.reportDdlException(ErrorCode.ERR_WRONG_OBJECT, dbName, tableName, "TABLE", + genDropHint(table)); } else if (stmt.isMaterializedView() && !(table instanceof MTMV)) { - ErrorReport.reportDdlException(ErrorCode.ERR_WRONG_OBJECT, dbName, tableName, "MTMV"); + ErrorReport.reportDdlException(ErrorCode.ERR_WRONG_OBJECT, dbName, tableName, "MTMV", + genDropHint(table)); } if (!stmt.isForceDrop()) { @@ -941,6 +945,18 @@ public void dropTable(DropTableStmt stmt) throws DdlException { tableName, dbName, stmt.isForceDrop(), costTimes); } + private static String genDropHint(TableIf table) { + String type = ""; + if (table instanceof View) { + type = "VIEW"; + } else if (table instanceof OlapTable) { + type = "TABLE"; + } else if (table instanceof MTMV) { + type = "MATERIALIZED VIEW"; + } + return "Use 'DROP " + type + " " + table.getName(); + } + public boolean unprotectDropTable(Database db, Table table, boolean isForceDrop, boolean isReplay, long recycleTime) { if (table.getType() == TableType.ELASTICSEARCH) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/minidump/MinidumpUtils.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/minidump/MinidumpUtils.java index 327b0aa6ffa53b..bd64da6d470b32 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/minidump/MinidumpUtils.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/minidump/MinidumpUtils.java @@ -247,7 +247,8 @@ private static ColumnStatistic getColumnStatistic(TableIf table, String colName) } private static Histogram getColumnHistogram(TableIf table, String colName) { - return Env.getCurrentEnv().getStatisticsCache().getHistogram(table.getId(), colName); + return Env.getCurrentEnv().getStatisticsCache().getHistogram( + table.getDatabase().getCatalog().getId(), table.getDatabase().getId(), table.getId(), colName); } /** diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java b/fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java index 1f4fd368222610..d3f6bb0a605bd0 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java @@ -1107,7 +1107,7 @@ private void handleShowCreateTable() throws AnalysisException { } else { if (showStmt.isView()) { ErrorReport.reportAnalysisException(ErrorCode.ERR_WRONG_OBJECT, showStmt.getDb(), - showStmt.getTable(), "VIEW"); + showStmt.getTable(), "VIEW", "Use 'SHOW CREATE TABLE '" + table.getName()); } rows.add(Lists.newArrayList(table.getName(), createTableStmt.get(0))); resultSet = table.getType() != TableType.MATERIALIZED_VIEW @@ -2570,8 +2570,9 @@ private void handleShowColumnStats() throws AnalysisException { } private void getStatsForAllColumns(List, ColumnStatistic>> columnStatistics, - TableIf tableIf) throws AnalysisException { - List resultRows = StatisticsRepository.queryColumnStatisticsForTable(tableIf.getId()); + TableIf tableIf) { + List resultRows = StatisticsRepository.queryColumnStatisticsForTable( + tableIf.getDatabase().getCatalog().getId(), tableIf.getDatabase().getId(), tableIf.getId()); // row[4] is index id, row[5] is column name. for (ResultRow row : resultRows) { String indexName = tableIf.getName(); @@ -2616,7 +2617,9 @@ private void getStatsForSpecifiedColumns(List, ColumnS columnStatistics.add(Pair.of(Pair.of(indexName, colName), columnStatistic)); } else if (partitionNames == null) { ColumnStatistic columnStatistic = - StatisticsRepository.queryColumnStatisticsByName(tableIf.getId(), indexId, colName); + StatisticsRepository.queryColumnStatisticsByName( + tableIf.getDatabase().getCatalog().getId(), + tableIf.getDatabase().getId(), tableIf.getId(), indexId, colName); columnStatistics.add(Pair.of(Pair.of(indexName, colName), columnStatistic)); } else { String finalIndexName = indexName; diff --git a/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java b/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java index 08d4c0ada6837c..18e6f416d55fd4 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java +++ b/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java @@ -3143,9 +3143,10 @@ public TStatus updateStatsCache(TUpdateFollowerStatsCacheRequest request) throws ColStatsData data = GsonUtils.GSON.fromJson(request.colStatsData, ColStatsData.class); ColumnStatistic c = data.toColumnStatistic(); if (c == ColumnStatistic.UNKNOWN) { - Env.getCurrentEnv().getStatisticsCache().invalidate(k.tableId, k.idxId, k.colName); + Env.getCurrentEnv().getStatisticsCache().invalidate(k.catalogId, k.dbId, k.tableId, k.idxId, k.colName); } else { - Env.getCurrentEnv().getStatisticsCache().updateColStatsCache(k.tableId, k.idxId, k.colName, c); + Env.getCurrentEnv().getStatisticsCache().updateColStatsCache( + k.catalogId, k.dbId, k.tableId, k.idxId, k.colName, c); } // Return Ok anyway return new TStatus(TStatusCode.OK); diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisManager.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisManager.java index f13ba61f2a562f..a2b30cce2ac5f4 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisManager.java +++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisManager.java @@ -621,7 +621,7 @@ public void dropStats(DropStatsStmt dropStatsStmt) throws DdlException { invalidateLocalStats(catalogId, dbId, tblId, cols, tableStats); // Drop stats ddl is master only operation. invalidateRemoteStats(catalogId, dbId, tblId, cols, dropStatsStmt.isAllColumns()); - StatisticsRepository.dropStatistics(tblId, cols); + StatisticsRepository.dropStatisticsByColNames(catalogId, dbId, tblId, cols); } public void dropStats(TableIf table) throws DdlException { @@ -636,7 +636,7 @@ public void dropStats(TableIf table) throws DdlException { invalidateLocalStats(catalogId, dbId, tableId, cols, tableStats); // Drop stats ddl is master only operation. invalidateRemoteStats(catalogId, dbId, tableId, cols, true); - StatisticsRepository.dropStatistics(table.getId(), cols); + StatisticsRepository.dropStatisticsByColNames(catalogId, dbId, table.getId(), cols); } public void invalidateLocalStats(long catalogId, long dbId, long tableId, @@ -669,7 +669,7 @@ public void invalidateLocalStats(long catalogId, long dbId, long tableId, } } tableStats.removeColumn(indexName, column); - statisticsCache.invalidate(tableId, indexId, column); + statisticsCache.invalidate(catalogId, dbId, tableId, indexId, column); } } tableStats.updatedTime = 0; diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/ColumnStatisticsCacheLoader.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/ColumnStatisticsCacheLoader.java index 88ccd661b594c7..056ed7bcee5c7c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/statistics/ColumnStatisticsCacheLoader.java +++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/ColumnStatisticsCacheLoader.java @@ -71,7 +71,8 @@ protected Optional doLoad(StatisticsCacheKey key) { private Optional loadFromStatsTable(StatisticsCacheKey key) { List columnResults = null; try { - columnResults = StatisticsRepository.loadColStats(key.tableId, key.idxId, key.colName); + columnResults = StatisticsRepository.loadColStats( + key.catalogId, key.dbId, key.tableId, key.idxId, key.colName); } catch (InternalQueryExecutionException e) { LOG.info("Failed to load stats for table {} column {}. Reason:{}", key.tableId, key.colName, e.getMessage()); diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/HistogramTask.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/HistogramTask.java index 3e02c47497b097..60da8f4d2a0803 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/statistics/HistogramTask.java +++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/HistogramTask.java @@ -75,7 +75,8 @@ public void doExecute() throws Exception { StringSubstitutor stringSubstitutor = new StringSubstitutor(params); StatisticsUtil.execUpdate(stringSubstitutor.replace(ANALYZE_HISTOGRAM_SQL_TEMPLATE_TABLE)); - Env.getCurrentEnv().getStatisticsCache().refreshHistogramSync(tbl.getId(), -1, col.getName()); + Env.getCurrentEnv().getStatisticsCache().refreshHistogramSync( + tbl.getDatabase().getCatalog().getId(), tbl.getDatabase().getId(), tbl.getId(), -1, col.getName()); } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsCache.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsCache.java index ab503cb4f7e9e9..d9539e41887822 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsCache.java +++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsCache.java @@ -91,16 +91,16 @@ public ColumnStatistic getColumnStatistics(long catalogId, long dbId, long tblId return ColumnStatistic.UNKNOWN; } - public Histogram getHistogram(long tblId, String colName) { - return getHistogram(tblId, -1, colName).orElse(null); + public Histogram getHistogram(long ctlId, long dbId, long tblId, String colName) { + return getHistogram(ctlId, dbId, tblId, -1, colName).orElse(null); } - public Optional getHistogram(long tblId, long idxId, String colName) { + private Optional getHistogram(long ctlId, long dbId, long tblId, long idxId, String colName) { ConnectContext ctx = ConnectContext.get(); if (ctx != null && ctx.getSessionVariable().internalSession) { return Optional.empty(); } - StatisticsCacheKey k = new StatisticsCacheKey(tblId, idxId, colName); + StatisticsCacheKey k = new StatisticsCacheKey(ctlId, dbId, tblId, idxId, colName); try { CompletableFuture> f = histogramCache.get(k); if (f.isDone()) { @@ -112,24 +112,22 @@ public Optional getHistogram(long tblId, long idxId, String colName) return Optional.empty(); } - public void invalidate(long tblId, long idxId, String colName) { - columnStatisticsCache.synchronous().invalidate(new StatisticsCacheKey(tblId, idxId, colName)); + public void invalidate(long ctlId, long dbId, long tblId, long idxId, String colName) { + columnStatisticsCache.synchronous().invalidate(new StatisticsCacheKey(ctlId, dbId, tblId, idxId, colName)); } - public void updateColStatsCache(long tblId, long idxId, String colName, ColumnStatistic statistic) { - columnStatisticsCache.synchronous().put(new StatisticsCacheKey(tblId, idxId, colName), Optional.of(statistic)); + public void updateColStatsCache(long ctlId, long dbId, long tblId, long idxId, String colName, + ColumnStatistic statistic) { + columnStatisticsCache.synchronous() + .put(new StatisticsCacheKey(ctlId, dbId, tblId, idxId, colName), Optional.of(statistic)); } - public void refreshColStatsSync(long tblId, long idxId, String colName) { - columnStatisticsCache.synchronous().refresh(new StatisticsCacheKey(-1, -1, tblId, idxId, colName)); + public void refreshColStatsSync(long ctlId, long dbId, long tblId, long idxId, String colName) { + columnStatisticsCache.synchronous().refresh(new StatisticsCacheKey(ctlId, dbId, tblId, idxId, colName)); } - public void refreshColStatsSync(long catalogId, long dbId, long tblId, long idxId, String colName) { - columnStatisticsCache.synchronous().refresh(new StatisticsCacheKey(catalogId, dbId, tblId, idxId, colName)); - } - - public void refreshHistogramSync(long tblId, long idxId, String colName) { - histogramCache.synchronous().refresh(new StatisticsCacheKey(tblId, idxId, colName)); + public void refreshHistogramSync(long ctlId, long dbId, long tblId, long idxId, String colName) { + histogramCache.synchronous().refresh(new StatisticsCacheKey(ctlId, dbId, tblId, idxId, colName)); } public void preHeat() { @@ -168,11 +166,9 @@ private void doPreHeat() { for (ResultRow r : recentStatsUpdatedCols) { try { StatsId statsId = new StatsId(r); - long tblId = statsId.tblId; - long idxId = statsId.idxId; - String colId = statsId.colId; final StatisticsCacheKey k = - new StatisticsCacheKey(tblId, idxId, colId); + new StatisticsCacheKey(statsId.catalogId, statsId.dbId, statsId.tblId, statsId.idxId, + statsId.colId); ColumnStatistic c = ColumnStatistic.fromResultRow(r); if (c.count > 0 && c.ndv == 0 && c.count != c.numNulls) { c = ColumnStatistic.UNKNOWN; @@ -189,10 +185,11 @@ private void doPreHeat() { */ public void syncColStats(ColStatsData data) { StatsId statsId = data.statsId; - final StatisticsCacheKey k = new StatisticsCacheKey(statsId.tblId, statsId.idxId, statsId.colId); + final StatisticsCacheKey k = new StatisticsCacheKey(statsId.catalogId, statsId.dbId, statsId.tblId, + statsId.idxId, statsId.colId); ColumnStatistic columnStatistic = data.toColumnStatistic(); if (columnStatistic == ColumnStatistic.UNKNOWN) { - invalidate(k.tableId, k.idxId, k.colName); + invalidate(k.catalogId, k.dbId, k.tableId, k.idxId, k.colName); } else { putCache(k, columnStatistic); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsCacheKey.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsCacheKey.java index fa924ab92842f4..7d5c7bccc52dab 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsCacheKey.java +++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsCacheKey.java @@ -41,10 +41,6 @@ public class StatisticsCacheKey { private static final String DELIMITER = "-"; - public StatisticsCacheKey(long tableId, long idxId, String colName) { - this(-1, -1, tableId, idxId, colName); - } - public StatisticsCacheKey(long catalogId, long dbId, long tableId, long idxId, String colName) { this.catalogId = catalogId; this.dbId = dbId; @@ -55,7 +51,7 @@ public StatisticsCacheKey(long catalogId, long dbId, long tableId, long idxId, S @Override public int hashCode() { - return Objects.hash(tableId, idxId, colName); + return Objects.hash(catalogId, dbId, tableId, idxId, colName); } @Override @@ -67,13 +63,16 @@ public boolean equals(Object obj) { return false; } StatisticsCacheKey k = (StatisticsCacheKey) obj; - return this.tableId == k.tableId && this.idxId == k.idxId && this.colName.equals(k.colName); + return this.catalogId == k.catalogId && this.dbId == k.dbId && this.tableId == k.tableId + && this.idxId == k.idxId && this.colName.equals(k.colName); } @Override public String toString() { StringJoiner sj = new StringJoiner(DELIMITER); sj.add("ColumnStats"); + sj.add(String.valueOf(catalogId)); + sj.add(String.valueOf(dbId)); sj.add(String.valueOf(tableId)); sj.add(String.valueOf(idxId)); sj.add(colName); diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsRepository.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsRepository.java index c587f8fe87d8e8..87ce90c5300a2b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsRepository.java +++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsRepository.java @@ -63,22 +63,22 @@ public class StatisticsRepository { private static final String FETCH_COLUMN_STATISTIC_TEMPLATE = "SELECT * FROM " + FULL_QUALIFIED_COLUMN_STATISTICS_NAME - + " WHERE `id` = '${id}'"; + + " WHERE `id` = '${id}' AND `catalog_id` = '${catalogId}' AND `db_id` = '${dbId}'"; private static final String FETCH_PARTITIONS_STATISTIC_TEMPLATE = "SELECT * FROM " + FULL_QUALIFIED_COLUMN_STATISTICS_NAME - + " WHERE `id` IN (${idList})"; + + " WHERE `id` IN (${idList}) AND `catalog_id` = '${catalogId}' AND `db_id` = '${dbId}'"; private static final String FETCH_COLUMN_HISTOGRAM_TEMPLATE = "SELECT * FROM " + FULL_QUALIFIED_COLUMN_HISTOGRAM_NAME - + " WHERE `id` = '${id}'"; + + " WHERE `id` = '${id}' AND `catalog_id` = '${catalogId}' AND `db_id` = '${dbId}'"; - private static final String INSERT_INTO_COLUMN_STATISTICS = "INSERT INTO " + private static final String INSERT_INTO_COLUMN_STATISTICS_FOR_ALTER = "INSERT INTO " + FULL_QUALIFIED_COLUMN_STATISTICS_NAME + " VALUES('${id}', ${catalogId}, ${dbId}, ${tblId}, '${idxId}'," + "'${colId}', ${partId}, ${count}, ${ndv}, ${nullCount}, ${min}, ${max}, ${dataSize}, NOW())"; - private static final String DROP_TABLE_STATISTICS_TEMPLATE = "DELETE FROM " + FeConstants.INTERNAL_DB_NAME - + "." + "${tblName}" + " WHERE ${condition}"; + private static final String DELETE_TABLE_STATISTICS_TEMPLATE = "DELETE FROM " + FeConstants.INTERNAL_DB_NAME + + "." + "${tblName}" + " WHERE ${condition} AND `catalog_id` = '${catalogId}' AND `db_id` = '${dbId}'"; private static final String FETCH_RECENT_STATS_UPDATED_COL = "SELECT * FROM " @@ -95,21 +95,22 @@ public class StatisticsRepository { private static final String FETCH_STATS_PART_ID = "SELECT * FROM " + FeConstants.INTERNAL_DB_NAME + "." + StatisticConstants.STATISTIC_TBL_NAME - + " WHERE tbl_id = ${tblId}" + + " WHERE tbl_id = ${tblId} AND `catalog_id` = '${catalogId}' AND `db_id` = '${dbId}'" + " AND part_id IS NOT NULL"; private static final String QUERY_PARTITION_STATISTICS = "SELECT * FROM " + FeConstants.INTERNAL_DB_NAME + "." + StatisticConstants.STATISTIC_TBL_NAME + " WHERE " - + " ${inPredicate}" + + " ${inPredicate} AND `catalog_id` = '${catalogId}' AND `db_id` = '${dbId}'" + " AND part_id IS NOT NULL"; private static final String FETCH_TABLE_STATISTICS = "SELECT * FROM " + FeConstants.INTERNAL_DB_NAME + "." + StatisticConstants.STATISTIC_TBL_NAME - + " WHERE tbl_id = ${tblId}" + + " WHERE tbl_id = ${tblId} AND `catalog_id` = '${catalogId}' AND `db_id` = '${dbId}'" + " AND part_id IS NULL"; - public static ColumnStatistic queryColumnStatisticsByName(long tableId, long indexId, String colName) { - ResultRow resultRow = queryColumnStatisticById(tableId, indexId, colName); + public static ColumnStatistic queryColumnStatisticsByName( + long ctlId, long dbId, long tableId, long indexId, String colName) { + ResultRow resultRow = queryColumnStatisticById(ctlId, dbId, tableId, indexId, colName); if (resultRow == null) { return ColumnStatistic.UNKNOWN; } @@ -127,31 +128,35 @@ public static List queryColumnStatisticsByPartitions(TableName } partitionIds.add(partition.getId()); } - return queryPartitionStatistics(dbObjects.table.getId(), + return queryPartitionStatistics(dbObjects.catalog.getId(), dbObjects.db.getId(), dbObjects.table.getId(), colName, partitionIds).stream().map(ColumnStatistic::fromResultRow).collect( Collectors.toList()); } - public static List queryColumnStatisticsForTable(long tableId) - throws AnalysisException { + public static List queryColumnStatisticsForTable(long ctlId, long dbId, long tableId) { Map params = new HashMap<>(); params.put("tblId", String.valueOf(tableId)); + generateCtlDbIdParams(ctlId, dbId, params); List rows = StatisticsUtil.executeQuery(FETCH_TABLE_STATISTICS, params); return rows == null ? Collections.emptyList() : rows; } - public static ResultRow queryColumnStatisticById(long tblId, long indexId, String colName) { - return queryColumnStatisticById(tblId, indexId, colName, false); + private static ResultRow queryColumnStatisticById( + long ctlId, long dbId, long tblId, long indexId, String colName) { + return queryColumnStatisticById(ctlId, dbId, tblId, indexId, colName, false); } - public static ResultRow queryColumnHistogramById(long tblId, long indexId, String colName) { - return queryColumnStatisticById(tblId, indexId, colName, true); + private static ResultRow queryColumnHistogramById( + long ctlId, long dbId, long tblId, long indexId, String colName) { + return queryColumnStatisticById(ctlId, dbId, tblId, indexId, colName, true); } - private static ResultRow queryColumnStatisticById(long tblId, long indexId, String colName, boolean isHistogram) { + private static ResultRow queryColumnStatisticById(long ctlId, long dbId, long tblId, long indexId, String colName, + boolean isHistogram) { Map map = new HashMap<>(); String id = constructId(tblId, indexId, colName); map.put("id", StatisticsUtil.escapeSQL(id)); + generateCtlDbIdParams(ctlId, dbId, map); List rows = isHistogram ? StatisticsUtil.executeQuery(FETCH_COLUMN_HISTOGRAM_TEMPLATE, map) : StatisticsUtil.executeQuery(FETCH_COLUMN_STATISTIC_TEMPLATE, map); int size = rows.size(); @@ -161,19 +166,22 @@ private static ResultRow queryColumnStatisticById(long tblId, long indexId, Stri return size == 0 ? null : rows.get(0); } - public static List queryPartitionStatistics(long tblId, String colName, Set partIds) { + private static List queryPartitionStatistics( + long ctlId, long dbId, long tblId, String colName, Set partIds) { StringJoiner sj = new StringJoiner(","); for (Long partId : partIds) { sj.add("'" + constructId(tblId, -1, colName, partId) + "'"); } Map params = new HashMap<>(); params.put("idList", sj.toString()); + generateCtlDbIdParams(ctlId, dbId, params); List rows = StatisticsUtil.executeQuery(FETCH_PARTITIONS_STATISTIC_TEMPLATE, params); return rows == null ? Collections.emptyList() : rows; } - public static Histogram queryColumnHistogramByName(long tableId, long indexId, String colName) { - ResultRow resultRow = queryColumnHistogramById(tableId, indexId, colName); + private static Histogram queryColumnHistogramByName( + long ctlId, long dbId, long tableId, long indexId, String colName) { + ResultRow resultRow = queryColumnHistogramById(ctlId, dbId, tableId, indexId, colName); if (resultRow == null) { return Histogram.UNKNOWN; } @@ -188,19 +196,22 @@ private static String constructId(Object... params) { return stringJoiner.toString(); } - public static void dropStatistics(Set partIds) throws DdlException { - dropStatisticsByPartId(partIds, StatisticConstants.STATISTIC_TBL_NAME); + public static void dropStatisticsByPartIds(long ctlId, long dbId, long tblId, Set partIds) + throws DdlException { + dropStatisticsByPartId(ctlId, dbId, tblId, partIds, StatisticConstants.STATISTIC_TBL_NAME); } - public static void dropStatistics(long tblId, Set colNames) throws DdlException { + public static void dropStatisticsByColNames( + long ctlId, long dbId, long tblId, Set colNames) throws DdlException { if (colNames == null) { return; } - dropStatisticsByColName(tblId, colNames, StatisticConstants.STATISTIC_TBL_NAME); - dropStatisticsByColName(tblId, colNames, StatisticConstants.HISTOGRAM_TBL_NAME); + dropStatisticsByColName(ctlId, dbId, tblId, colNames, StatisticConstants.STATISTIC_TBL_NAME); + dropStatisticsByColName(ctlId, dbId, tblId, colNames, StatisticConstants.HISTOGRAM_TBL_NAME); } - public static void dropStatisticsByColName(long tblId, Set colNames, String statsTblName) + private static void dropStatisticsByColName( + long ctlId, long dbId, long tblId, Set colNames, String statsTblName) throws DdlException { Map params = new HashMap<>(); params.put("tblName", statsTblName); @@ -214,38 +225,43 @@ public static void dropStatisticsByColName(long tblId, Set colNames, Str inPredicate.append(","); columnCount++; if (columnCount == Config.max_allowed_in_element_num_of_delete) { - executeDropSql(inPredicate, tblId, params); + executeDropSql(inPredicate, ctlId, dbId, tblId, params); columnCount = 0; inPredicate.setLength(0); } } if (inPredicate.length() > 0) { - executeDropSql(inPredicate, tblId, params); + executeDropSql(inPredicate, ctlId, dbId, tblId, params); } } - public static void executeDropSql(StringBuilder inPredicate, long tblId, Map params) + private static void executeDropSql( + StringBuilder inPredicate, long ctlId, long dbId, long tblId, Map params) throws DdlException { if (inPredicate.length() > 0) { inPredicate.delete(inPredicate.length() - 1, inPredicate.length()); } String predicate = String.format("tbl_id = '%s' AND %s IN (%s)", tblId, "col_id", inPredicate); params.put("condition", predicate); + generateCtlDbIdParams(ctlId, dbId, params); try { - StatisticsUtil.execUpdate(new StringSubstitutor(params).replace(DROP_TABLE_STATISTICS_TEMPLATE)); + StatisticsUtil.execUpdate(new StringSubstitutor(params).replace(DELETE_TABLE_STATISTICS_TEMPLATE)); } catch (Exception e) { throw new DdlException(e.getMessage(), e); } } - public static void dropStatisticsByPartId(Set partIds, String statsTblName) throws DdlException { + private static void dropStatisticsByPartId( + long ctlId, long dbId, long tblId, Set partIds, String statsTblName) throws DdlException { Map params = new HashMap<>(); String right = StatisticsUtil.joinElementsToString(partIds, ","); String inPredicate = String.format(" part_id IN (%s)", right); params.put("tblName", statsTblName); params.put("condition", inPredicate); + generateCtlDbIdParams(ctlId, dbId, params); + params.put("tblId", String.valueOf(tblId)); try { - StatisticsUtil.execUpdate(new StringSubstitutor(params).replace(DROP_TABLE_STATISTICS_TEMPLATE)); + StatisticsUtil.execUpdate(new StringSubstitutor(params).replace(DELETE_TABLE_STATISTICS_TEMPLATE)); } catch (Exception e) { throw new DdlException(e.getMessage(), e); } @@ -313,7 +329,7 @@ public static void alterColumnStatistics(AlterColumnStatsStmt alterColumnStatsSt if (partitionIds.isEmpty()) { // update table granularity statistics params.put("partId", "NULL"); - StatisticsUtil.execUpdate(INSERT_INTO_COLUMN_STATISTICS, params); + StatisticsUtil.execUpdate(INSERT_INTO_COLUMN_STATISTICS_FOR_ALTER, params); ColStatsData data = new ColStatsData(constructId(objects.table.getId(), indexId, colName), objects.catalog.getId(), objects.db.getId(), objects.table.getId(), indexId, colName, null, columnStatistic); @@ -331,7 +347,7 @@ public static void alterColumnStatistics(AlterColumnStatsStmt alterColumnStatsSt for (Long partitionId : partitionIds) { HashMap partParams = Maps.newHashMap(params); partParams.put("partId", String.valueOf(partitionId)); - StatisticsUtil.execUpdate(INSERT_INTO_COLUMN_STATISTICS, partParams); + StatisticsUtil.execUpdate(INSERT_INTO_COLUMN_STATISTICS_FOR_ALTER, partParams); // TODO cache partition granular statistics // Env.getCurrentEnv().getStatisticsCache() // .updateColStatsCache(partitionId, -1, colName, builder.build()); @@ -350,10 +366,11 @@ public static List fetchStatsFullName(long limit, long offset) { return StatisticsUtil.execStatisticQuery(new StringSubstitutor(params).replace(FETCH_STATS_FULL_NAME)); } - public static Map> fetchColAndPartsForStats(long tblId) { + public static Map> fetchColAndPartsForStats(long ctlId, long dbId, long tblId) { Map params = Maps.newHashMap(); params.put("tblId", String.valueOf(tblId)); StringSubstitutor stringSubstitutor = new StringSubstitutor(params); + generateCtlDbIdParams(ctlId, dbId, params); String partSql = stringSubstitutor.replace(FETCH_STATS_PART_ID); List resultRows = StatisticsUtil.execStatisticQuery(partSql); @@ -376,11 +393,11 @@ public static Map> fetchColAndPartsForStats(long tblId) { return columnToPartitions; } - public static List loadColStats(long tableId, long idxId, String colName) { + public static List loadColStats(long ctlId, long dbId, long tableId, long idxId, String colName) { Map params = new HashMap<>(); String id = constructId(tableId, idxId, colName); params.put("id", StatisticsUtil.escapeSQL(id)); - + generateCtlDbIdParams(ctlId, dbId, params); return StatisticsUtil.execStatisticQuery(new StringSubstitutor(params) .replace(FETCH_COLUMN_STATISTIC_TEMPLATE)); } @@ -388,12 +405,23 @@ public static List loadColStats(long tableId, long idxId, String colN public static List loadPartStats(Collection keys) { String inPredicate = "CONCAT(tbl_id, '-', idx_id, '-', col_id) in (%s)"; StringJoiner sj = new StringJoiner(","); + long ctlId = -1; + long dbId = -1; + // ATTN: ctlId and dbId should be same in all keys for (StatisticsCacheKey statisticsCacheKey : keys) { sj.add("'" + statisticsCacheKey.toString() + "'"); + ctlId = statisticsCacheKey.catalogId; + dbId = statisticsCacheKey.dbId; } Map params = new HashMap<>(); params.put("inPredicate", String.format(inPredicate, sj.toString())); + generateCtlDbIdParams(ctlId, dbId, params); return StatisticsUtil.execStatisticQuery(new StringSubstitutor(params) .replace(QUERY_PARTITION_STATISTICS)); } + + private static void generateCtlDbIdParams(long ctdId, long dbId, Map params) { + params.put("catalogId", String.valueOf(ctdId)); + params.put("dbId", String.valueOf(dbId)); + } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/OtherJoinTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/OtherJoinTest.java index a9144574583b8c..3513db98de2bc9 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/OtherJoinTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/OtherJoinTest.java @@ -56,7 +56,7 @@ private void randomTest(int tableNum, int edgeNum) { plan = new LogicalProject(plan.getOutput(), plan); Set> res1 = hyperGraphBuilder.evaluate(plan); CascadesContext cascadesContext = MemoTestUtils.createCascadesContext(connectContext, plan); - hyperGraphBuilder.initStats(cascadesContext); + hyperGraphBuilder.initStats("tpch", cascadesContext); Plan optimizedPlan = PlanChecker.from(cascadesContext) .dpHypOptimize() .getBestPlanTree(); diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/jobs/joinorder/joinhint/DistributeHintTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/jobs/joinorder/joinhint/DistributeHintTest.java index 3779e56c496042..0b2ed8069ade83 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/jobs/joinorder/joinhint/DistributeHintTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/jobs/joinorder/joinhint/DistributeHintTest.java @@ -104,7 +104,7 @@ private void randomTest(int tableNum, int edgeNum, boolean withJoinHint, boolean Set> res1 = hyperGraphBuilder.evaluate(plan); if (!withLeading) { CascadesContext cascadesContext = MemoTestUtils.createCascadesContext(connectContext, plan); - hyperGraphBuilder.initStats(cascadesContext); + hyperGraphBuilder.initStats("tpch", cascadesContext); Plan optimizedPlan = PlanChecker.from(cascadesContext) .analyze() .optimize() @@ -128,7 +128,7 @@ private void randomTest(int tableNum, int edgeNum, boolean withJoinHint, boolean for (int i = 0; i < (tableNum * tableNum - 1); i++) { Plan leadingPlan = generateLeadingHintPlan(tableNum, plan); CascadesContext cascadesContext = MemoTestUtils.createCascadesContext(connectContext, leadingPlan); - hyperGraphBuilder.initStats(cascadesContext); + hyperGraphBuilder.initStats("tpch", cascadesContext); Plan optimizedPlan = PlanChecker.from(cascadesContext) .analyze() .optimize() diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/memo/RankTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/memo/RankTest.java index 57773bb0c4f6de..7d3cc0103b3742 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/memo/RankTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/memo/RankTest.java @@ -35,8 +35,10 @@ import java.util.Set; class RankTest extends TestWithFeService { + @Test - void test() { + void test() throws Exception { + createDatabase("test"); HyperGraphBuilder hyperGraphBuilder = new HyperGraphBuilder(Sets.newHashSet(JoinType.INNER_JOIN)); hyperGraphBuilder.init(0, 1, 2); Plan plan = hyperGraphBuilder @@ -45,7 +47,7 @@ void test() { .buildPlan(); plan = new LogicalProject(plan.getOutput(), plan); CascadesContext cascadesContext = MemoTestUtils.createCascadesContext(connectContext, plan); - hyperGraphBuilder.initStats(cascadesContext); + hyperGraphBuilder.initStats("test", cascadesContext); PhysicalPlan bestPlan = PlanChecker.from(cascadesContext) .optimize() .getBestPlanTree(); diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/sqltest/JoinOrderJobTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/sqltest/JoinOrderJobTest.java index 73c3a6145200ee..a2cb14e02080b7 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/sqltest/JoinOrderJobTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/sqltest/JoinOrderJobTest.java @@ -139,7 +139,7 @@ protected void test64TableJoin() { plan = new LogicalProject(plan.getOutput(), plan); CascadesContext cascadesContext = MemoTestUtils.createCascadesContext(connectContext, plan); Assertions.assertEquals(cascadesContext.getMemo().countMaxContinuousJoin(), 64); - hyperGraphBuilder.initStats(cascadesContext); + hyperGraphBuilder.initStats("test", cascadesContext); PlanChecker.from(cascadesContext) .optimize() .getBestPlanTree(); @@ -153,7 +153,7 @@ void test64CliqueJoin() { .randomBuildPlanWith(64, 64 * 63 / 2); plan = new LogicalProject(plan.getOutput(), plan); CascadesContext cascadesContext = MemoTestUtils.createCascadesContext(connectContext, plan); - hyperGraphBuilder.initStats(cascadesContext); + hyperGraphBuilder.initStats("test", cascadesContext); PlanChecker.from(cascadesContext) .rewrite() .dpHypOptimize() diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/util/HyperGraphBuilder.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/util/HyperGraphBuilder.java index 6a1da5a994ca4e..815cff0f2d9457 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/util/HyperGraphBuilder.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/util/HyperGraphBuilder.java @@ -18,6 +18,7 @@ package org.apache.doris.nereids.util; import org.apache.doris.catalog.Env; +import org.apache.doris.catalog.OlapTable; import org.apache.doris.common.Pair; import org.apache.doris.nereids.CascadesContext; import org.apache.doris.nereids.hint.DistributeHint; @@ -41,6 +42,7 @@ import org.apache.doris.statistics.StatisticsCacheKey; import com.google.common.base.Preconditions; +import com.google.common.base.Strings; import com.google.common.collect.ImmutableList; import com.google.common.collect.Lists; @@ -194,16 +196,21 @@ public HyperGraphBuilder init(int... rowCounts) { return this; } - public void initStats(CascadesContext context) { + public void initStats(String dbName, CascadesContext context) { for (Group group : context.getMemo().getGroups()) { GroupExpression groupExpression = group.getLogicalExpression(); if (groupExpression.getPlan() instanceof LogicalOlapScan) { LogicalOlapScan scan = (LogicalOlapScan) groupExpression.getPlan(); + OlapTable table = scan.getTable(); + if (Strings.isNullOrEmpty(table.getQualifiedDbName())) { + table.setQualifiedDbName(dbName); + } Statistics stats = injectRowcount((LogicalOlapScan) groupExpression.getPlan()); for (Expression expr : stats.columnStatistics().keySet()) { SlotReference slot = (SlotReference) expr; Env.getCurrentEnv().getStatisticsCache().putCache( - new StatisticsCacheKey(scan.getTable().getId(), -1, slot.getName()), + new StatisticsCacheKey(table.getDatabase().getCatalog().getId(), + table.getDatabase().getId(), table.getId(), -1, slot.getName()), stats.columnStatistics().get(expr)); } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/statistics/CacheTest.java b/fe/fe-core/src/test/java/org/apache/doris/statistics/CacheTest.java index f23b93624b4dae..729291d5323ff0 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/statistics/CacheTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/statistics/CacheTest.java @@ -200,9 +200,9 @@ public List execStatisticQuery(String sql) { }; StatisticsCache statisticsCache = new StatisticsCache(); - statisticsCache.refreshHistogramSync(0, -1, "col"); + statisticsCache.refreshHistogramSync(0, 0, 0, -1, "col"); Thread.sleep(10000); - Histogram histogram = statisticsCache.getHistogram(0, "col"); + Histogram histogram = statisticsCache.getHistogram(0, 0, 0, "col"); Assertions.assertNotNull(histogram); }