From 698e6249d2512190d463d85dd295b4734f91802b Mon Sep 17 00:00:00 2001 From: zclllhhjj Date: Thu, 29 Aug 2024 10:55:18 +0800 Subject: [PATCH] [Fix](load) fix commit txn timeout when loading to table with many tablet (#40031) fix commit txn timeout when loading to table with many tablet --- .../apache/doris/load/BrokerFileGroup.java | 16 +++++--- .../transaction/DatabaseTransactionMgr.java | 40 +++++++++++++------ 2 files changed, 38 insertions(+), 18 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/load/BrokerFileGroup.java b/fe/fe-core/src/main/java/org/apache/doris/load/BrokerFileGroup.java index ff27acadcac2f4..6676e0a2a1f6db 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/load/BrokerFileGroup.java +++ b/fe/fe-core/src/main/java/org/apache/doris/load/BrokerFileGroup.java @@ -149,12 +149,16 @@ public void parse(Database db, DataDescription dataDescription) throws DdlExcept } } - boolean isPartitionRestoring = olapTable.getPartitions().stream().anyMatch( - partition -> partition.getState() == PartitionState.RESTORE - ); - // restore table - if (!isPartitionRestoring && olapTable.getState() == OlapTableState.RESTORE) { - throw new DdlException("Table [" + olapTable.getName() + "] is under restore"); + // only do check when here's restore on this table now + if (olapTable.getState() == OlapTableState.RESTORE) { + boolean hasPartitionRestoring = olapTable.getPartitions().stream() + .anyMatch(partition -> partition.getState() == PartitionState.RESTORE); + // tbl RESTORE && all partition NOT RESTORE -> whole table restore + // tbl RESTORE && some partition RESTORE -> just partitions restore, NOT WHOLE TABLE + // so check wether the whole table restore here + if (!hasPartitionRestoring) { + throw new DdlException("Table [" + olapTable.getName() + "] is under restore"); + } } if (olapTable.getKeysType() != KeysType.AGG_KEYS && dataDescription.isNegative()) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/transaction/DatabaseTransactionMgr.java b/fe/fe-core/src/main/java/org/apache/doris/transaction/DatabaseTransactionMgr.java index 636fe582ed11a9..38514aff8d6e64 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/transaction/DatabaseTransactionMgr.java +++ b/fe/fe-core/src/main/java/org/apache/doris/transaction/DatabaseTransactionMgr.java @@ -504,7 +504,9 @@ private void checkCommitStatus(List tableList, TransactionState transacti List tabletIds = tabletCommitInfos.stream() .map(TabletCommitInfo::getTabletId).collect(Collectors.toList()); List tabletMetaList = tabletInvertedIndex.getTabletMetaList(tabletIds); + HashMap tableIdtoRestoring = new HashMap<>(); for (int i = 0; i < tabletMetaList.size(); i++) { + // get partition and table of this tablet TabletMeta tabletMeta = tabletMetaList.get(i); if (tabletMeta == TabletInvertedIndex.NOT_EXIST_TABLET_META) { continue; @@ -513,29 +515,43 @@ private void checkCommitStatus(List
tableList, TransactionState transacti long tableId = tabletMeta.getTableId(); OlapTable tbl = (OlapTable) idToTable.get(tableId); if (tbl == null) { - // this can happen when tableId == -1 (tablet being dropping) - // or table really not exist. + // this can happen when tableId == -1 (tablet being dropping) or table really not exist. continue; } + // check relative partition restore here long partitionId = tabletMeta.getPartitionId(); if (tbl.getPartition(partitionId) == null) { - // this can happen when partitionId == -1 (tablet being dropping) - // or partition really not exist. + // this can happen when partitionId == -1 (tablet being dropping) or partition really not exist. continue; - } else if (tbl.getPartition(partitionId).getState() == PartitionState.RESTORE) { + } + if (tbl.getPartition(partitionId).getState() == PartitionState.RESTORE) { // partition in restore process which can not load data throw new LoadException("Table [" + tbl.getName() + "], Partition [" + tbl.getPartition(partitionId).getName() + "] is in restore process. Can not load into it"); } - boolean isPartitionRestoring = tbl.getPartitions().stream().anyMatch( - partition -> partition.getState() == PartitionState.RESTORE - ); - // restore table - if (!isPartitionRestoring && tbl.getState() == OlapTableState.RESTORE) { - throw new LoadException("Table " + tbl.getName() + " is in restore process. " - + "Can not load into it"); + // only do check when here's restore on this table now + if (tbl.getState() == OlapTableState.RESTORE) { + boolean hasPartitionRestoring = false; + if (tableIdtoRestoring.containsKey(tableId)) { + hasPartitionRestoring = tableIdtoRestoring.get(tableId); + } else { + for (Partition partition : tbl.getPartitions()) { + if (partition.getState() == PartitionState.RESTORE) { + hasPartitionRestoring = true; + break; + } + } + tableIdtoRestoring.put(tableId, hasPartitionRestoring); + } + // tbl RESTORE && all partition NOT RESTORE -> whole table restore + // tbl RESTORE && some partition RESTORE -> just partitions restore, NOT WHOLE TABLE + // so check wether the whole table restore here + if (!hasPartitionRestoring) { + throw new LoadException( + "Table " + tbl.getName() + " is in restore process. " + "Can not load into it"); + } } if (!tableToPartition.containsKey(tableId)) {