Skip to content

Commit

Permalink
[fix](statistics)Fix stats cleaner delete partition stats bug (#30648)
Browse files Browse the repository at this point in the history
When a partition in OlapTable is removed, we should use partition id to delete the related stats record in column_statistics. Before, it was using id, which may cause delete useful stats of other partition.
  • Loading branch information
Jibing-Li committed Jan 31, 2024
1 parent ffd6318 commit bf580f9
Showing 1 changed file with 8 additions and 4 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -170,12 +170,15 @@ private void deleteExpiredStats(ExpiredStats expiredStats, String tblName) {
doDelete("idx_id", expiredStats.expiredIdxId.stream()
.map(String::valueOf).collect(Collectors.toList()),
FeConstants.INTERNAL_DB_NAME + "." + tblName, false);
doDelete("part_id", expiredStats.expiredPartitionId.stream()
.map(String::valueOf).collect(Collectors.toList()),
FeConstants.INTERNAL_DB_NAME + "." + tblName, false);
doDelete("id", expiredStats.ids.stream()
.map(String::valueOf).collect(Collectors.toList()),
FeConstants.INTERNAL_DB_NAME + "." + tblName, false);
}

private void doDelete(String/*col name*/ colName, List<String> pred, String tblName, boolean taskOnly) {
private void doDelete(String colName, List<String> pred, String tblName, boolean taskOnly) {
String deleteTemplate = "DELETE FROM " + tblName + " WHERE ${left} IN (${right})";
if (CollectionUtils.isEmpty(pred)) {
return;
Expand Down Expand Up @@ -242,7 +245,7 @@ private long findExpiredStats(OlapTable statsTbl, ExpiredStats expiredStats, lon
continue;
}
if (!olapTable.getPartitionIds().contains(Long.parseLong(partId))) {
expiredStats.ids.add(id);
expiredStats.expiredPartitionId.add(Long.parseLong(partId));
}
} catch (Exception e) {
LOG.warn("Error occurred when retrieving expired stats", e);
Expand All @@ -257,16 +260,16 @@ private static class ExpiredStats {
Set<Long> expiredCatalog = new HashSet<>();
Set<Long> expiredDatabase = new HashSet<>();
Set<Long> expiredTable = new HashSet<>();

Set<Long> expiredIdxId = new HashSet<>();

Set<Long> expiredPartitionId = new HashSet<>();
Set<String> ids = new HashSet<>();

public boolean isFull() {
return expiredCatalog.size() >= Config.max_allowed_in_element_num_of_delete
|| expiredDatabase.size() >= Config.max_allowed_in_element_num_of_delete
|| expiredTable.size() >= Config.max_allowed_in_element_num_of_delete
|| expiredIdxId.size() >= Config.max_allowed_in_element_num_of_delete
|| expiredPartitionId.size() >= Config.max_allowed_in_element_num_of_delete
|| ids.size() >= Config.max_allowed_in_element_num_of_delete;
}

Expand All @@ -275,6 +278,7 @@ public boolean isEmpty() {
&& expiredDatabase.isEmpty()
&& expiredTable.isEmpty()
&& expiredIdxId.isEmpty()
&& expiredPartitionId.isEmpty()
&& ids.size() < Config.max_allowed_in_element_num_of_delete / 10;
}
}
Expand Down

0 comments on commit bf580f9

Please sign in to comment.