Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

[fix](statistics)Fix drop stats log editlog bug. Catch drop stats exception while truncate table. #40738

Merged
merged 1 commit into from
Sep 12, 2024
+235 −70
Merged

[fix](statistics)Fix drop stats log editlog bug. Catch drop stats exception while truncate table. #40738

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 2 additions & 10 deletions fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2866,11 +2866,7 @@ public void modifyTableLightSchemaChange(String rawSql, Database db, OlapTable o
}
Env.getCurrentEnv().getEditLog().logModifyTableAddOrDropInvertedIndices(info);
// Drop table column stats after light schema change finished.
try {
Env.getCurrentEnv().getAnalysisManager().dropStats(olapTable, null);
} catch (Exception e) {
LOG.info("Failed to drop stats after light schema change. Reason: {}", e.getMessage());
}
Env.getCurrentEnv().getAnalysisManager().dropStats(olapTable, null);

if (isDropIndex) {
// send drop rpc to be
Expand Down Expand Up @@ -2898,11 +2894,7 @@ public void modifyTableLightSchemaChange(String rawSql, Database db, OlapTable o
}
Env.getCurrentEnv().getEditLog().logModifyTableAddOrDropColumns(info);
// Drop table column stats after light schema change finished.
try {
Env.getCurrentEnv().getAnalysisManager().dropStats(olapTable, null);
} catch (Exception e) {
LOG.info("Failed to drop stats after light schema change. Reason: {}", e.getMessage());
}
Env.getCurrentEnv().getAnalysisManager().dropStats(olapTable, null);
}
LOG.info("finished modify table's add or drop or modify columns. table: {}, job: {}, is replay: {}",
olapTable.getName(), jobId, isReplay);
Expand Down
6 changes: 1 addition & 5 deletions fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeJobV2.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -641,11 +641,7 @@ protected void runRunningJob() throws AlterCancelException {
LOG.info("set table's state to NORMAL, table id: {}, job id: {}", tableId, jobId);
postProcessOriginIndex();
// Drop table column stats after schema change finished.
try {
Env.getCurrentEnv().getAnalysisManager().dropStats(tbl, null);
} catch (Exception e) {
LOG.info("Failed to drop stats after schema change finished. Reason: {}", e.getMessage());
}
Env.getCurrentEnv().getAnalysisManager().dropStats(tbl, null);
}

private void onFinished(OlapTable tbl) {
Expand Down
6 changes: 1 addition & 5 deletions fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5115,11 +5115,7 @@ private void renameColumn(Database db, OlapTable table, String colName,
indexIdToSchemaVersion);
editLog.logColumnRename(info);
LOG.info("rename coloumn[{}] to {}", colName, newColName);
try {
Env.getCurrentEnv().getAnalysisManager().dropStats(table, null);
} catch (Exception e) {
LOG.info("Failed to drop stats after rename column. Reason: {}", e.getMessage());
}
Env.getCurrentEnv().getAnalysisManager().dropStats(table, null);
}
}

Expand Down
2 changes: 1 addition & 1 deletion fe/fe-core/src/main/java/org/apache/doris/catalog/TabletStatMgr.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -182,7 +182,7 @@ protected void runAfterCatalogReady() {
index.setRowCount(indexRowCount);
LOG.debug("Table {} index {} all tablets reported[{}], row count {}",
olapTable.getName(), olapTable.getIndexNameById(index.getId()),
indexReported, tableRowCount);
indexReported, indexRowCount);
} // end for indices
} // end for partitions

Expand Down
6 changes: 0 additions & 6 deletions fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3314,17 +3314,11 @@ public TStatus invalidateStatsCache(TInvalidateFollowerStatsCacheRequest request
InvalidateStatsTarget target = GsonUtils.GSON.fromJson(request.key, InvalidateStatsTarget.class);
AnalysisManager analysisManager = Env.getCurrentEnv().getAnalysisManager();
TableStatsMeta tableStats = analysisManager.findTableStatsStatus(target.tableId);
if (tableStats == null) {
return new TStatus(TStatusCode.OK);
}
PartitionNames partitionNames = null;
if (target.partitions != null) {
partitionNames = new PartitionNames(false, new ArrayList<>(target.partitions));
}
if (target.isTruncate) {
if (partitionNames == null || partitionNames.isStar() || partitionNames.getPartitionNames() == null) {
tableStats.clearIndexesRowCount();
}
analysisManager.submitAsyncDropStatsTask(target.catalogId, target.dbId,
target.tableId, tableStats, partitionNames);
} else {
Expand Down
93 changes: 50 additions & 43 deletions fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisManager.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -59,6 +59,7 @@
import org.apache.doris.nereids.trees.expressions.Slot;
import org.apache.doris.nereids.trees.expressions.SlotReference;
import org.apache.doris.persist.AnalyzeDeletionLog;
import org.apache.doris.persist.TableStatsDeletionLog;
import org.apache.doris.persist.gson.GsonUtils;
import org.apache.doris.qe.ConnectContext;
import org.apache.doris.qe.ShowResultSet;
Expand Down Expand Up @@ -655,14 +656,21 @@ public void dropStats(DropStatsStmt dropStatsStmt) throws DdlException {
return;
}

TableStatsMeta tableStats = findTableStatsStatus(dropStatsStmt.getTblId());
if (tableStats == null) {
return;
}
Set<String> cols = dropStatsStmt.getColumnNames();
PartitionNames partitionNames = dropStatsStmt.getPartitionNames();
long catalogId = dropStatsStmt.getCatalogIdId();
long dbId = dropStatsStmt.getDbId();
long tblId = dropStatsStmt.getTblId();
TableStatsMeta tableStats = findTableStatsStatus(dropStatsStmt.getTblId());
if (tableStats == null) {
return;
TableIf table = StatisticsUtil.findTable(catalogId, dbId, tblId);
// Remove tableMetaStats if drop whole table stats.
if (cols == null && (!table.isPartitionedTable() || partitionNames == null
|| partitionNames.isStar() || partitionNames.getPartitionNames() == null)) {
removeTableStats(tblId);
Env.getCurrentEnv().getEditLog().logDeleteTableStats(new TableStatsDeletionLog(tblId));
}
invalidateLocalStats(catalogId, dbId, tblId, cols, tableStats, partitionNames);
// Drop stats ddl is master only operation.
Expand All @@ -674,26 +682,32 @@ public void dropStats(DropStatsStmt dropStatsStmt) throws DdlException {
StatisticsRepository.dropStatistics(catalogId, dbId, tblId, cols, partitions);
}

public void dropStats(TableIf table, PartitionNames partitionNames) throws DdlException {
TableStatsMeta tableStats = findTableStatsStatus(table.getId());
if (tableStats == null) {
return;
}
long catalogId = table.getDatabase().getCatalog().getId();
long dbId = table.getDatabase().getId();
long tableId = table.getId();
if (partitionNames == null || partitionNames.isStar() || partitionNames.getPartitionNames() == null) {
tableStats.clearIndexesRowCount();
}
submitAsyncDropStatsTask(catalogId, dbId, tableId, tableStats, partitionNames);
// Drop stats ddl is master only operation.
Set<String> partitions = null;
if (partitionNames != null && !partitionNames.isStar() && partitionNames.getPartitionNames() != null) {
partitions = new HashSet<>(partitionNames.getPartitionNames());
public void dropStats(TableIf table, PartitionNames partitionNames) {
try {
TableStatsMeta tableStats = findTableStatsStatus(table.getId());
if (tableStats == null) {
return;
}
long catalogId = table.getDatabase().getCatalog().getId();
long dbId = table.getDatabase().getId();
long tableId = table.getId();
if (!table.isPartitionedTable() || partitionNames == null
|| partitionNames.isStar() || partitionNames.getPartitionNames() == null) {
removeTableStats(tableId);
Env.getCurrentEnv().getEditLog().logDeleteTableStats(new TableStatsDeletionLog(tableId));
}
submitAsyncDropStatsTask(catalogId, dbId, tableId, tableStats, partitionNames);
// Drop stats ddl is master only operation.
Set<String> partitions = null;
if (partitionNames != null && !partitionNames.isStar() && partitionNames.getPartitionNames() != null) {
partitions = new HashSet<>(partitionNames.getPartitionNames());
}
// Drop stats ddl is master only operation.
invalidateRemoteStats(catalogId, dbId, tableId, null, partitions, true);
StatisticsRepository.dropStatistics(catalogId, dbId, table.getId(), null, partitions);
} catch (Throwable e) {
LOG.warn("Failed to drop stats for table {}", table.getName(), e);
}
// Drop stats ddl is master only operation.
invalidateRemoteStats(catalogId, dbId, tableId, null, partitions, true);
StatisticsRepository.dropStatistics(catalogId, dbId, table.getId(), null, partitions);
}

class DropStatsTask implements Runnable {
Expand Down Expand Up @@ -753,14 +767,9 @@ public void dropCachedStats(long catalogId, long dbId, long tableId) {

public void invalidateLocalStats(long catalogId, long dbId, long tableId, Set<String> columns,
TableStatsMeta tableStats, PartitionNames partitionNames) {
if (tableStats == null) {
return;
}
TableIf table = StatisticsUtil.findTable(catalogId, dbId, tableId);
StatisticsCache statsCache = Env.getCurrentEnv().getStatisticsCache();
boolean allColumn = false;
if (columns == null) {
allColumn = true;
columns = table.getSchemaAllIndexes(false)
.stream().map(Column::getName).collect(Collectors.toSet());
}
Expand Down Expand Up @@ -795,11 +804,16 @@ public void invalidateLocalStats(long catalogId, long dbId, long tableId, Set<St
indexName = olapTable.getIndexNameById(indexId);
}
}
statsCache.invalidateColumnStatsCache(catalogId, dbId, tableId, indexId, column);
if (allPartition) {
tableStats.removeColumn(indexName, column);
statsCache.invalidateColumnStatsCache(catalogId, dbId, tableId, indexId, column);
if (tableStats != null) {
tableStats.removeColumn(indexName, column);
}
}
ColStatsMeta columnStatsMeta = null;
if (tableStats != null) {
columnStatsMeta = tableStats.findColumnStatsMeta(indexName, column);
}
ColStatsMeta columnStatsMeta = tableStats.findColumnStatsMeta(indexName, column);
for (String part : partNames) {
statsCache.invalidatePartitionColumnStatsCache(catalogId, dbId, tableId, indexId, part, column);
if (columnStatsMeta != null && columnStatsMeta.partitionUpdateRows != null) {
Expand All @@ -811,14 +825,10 @@ public void invalidateLocalStats(long catalogId, long dbId, long tableId, Set<St
}
}
}
if (allColumn && allPartition) {
tableStats.removeAllColumn();
tableStats.clearIndexesRowCount();
removeTableStats(tableId);
if (tableStats != null) {
tableStats.userInjected = false;
tableStats.rowCount = table.getRowCount();
}
tableStats.updatedTime = 0;
tableStats.userInjected = false;
tableStats.rowCount = table.getRowCount();
}

public void invalidateRemoteStats(long catalogId, long dbId, long tableId,
Expand All @@ -829,18 +839,15 @@ public void invalidateRemoteStats(long catalogId, long dbId, long tableId,
request.key = GsonUtils.GSON.toJson(target);
StatisticsCache statisticsCache = Env.getCurrentEnv().getStatisticsCache();
SystemInfoService.HostInfo selfNode = Env.getCurrentEnv().getSelfNode();
boolean success = true;
for (Frontend frontend : Env.getCurrentEnv().getFrontends(null)) {
// Skip master
if (selfNode.getHost().equals(frontend.getHost())) {
continue;
}
success = success && statisticsCache.invalidateStats(frontend, request);
statisticsCache.invalidateStats(frontend, request);
}
if (!success) {
// If any rpc failed, use edit log to sync table stats to non-master FEs.
LOG.warn("Failed to invalidate all remote stats by rpc for table {}, use edit log.", tableId);
TableStatsMeta tableStats = findTableStatsStatus(tableId);
TableStatsMeta tableStats = findTableStatsStatus(tableId);
if (tableStats != null) {
logCreateTableStats(tableStats);
}
}
Expand Down
Loading
Loading