diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalTable.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalTable.java index a9f2da13b4093b..8217f1c3a367df 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalTable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalTable.java @@ -53,6 +53,7 @@ import com.google.common.collect.Lists; import com.google.common.collect.Maps; import com.google.common.collect.Sets; +import org.apache.commons.collections.CollectionUtils; import org.apache.commons.collections.MapUtils; import org.apache.commons.lang3.StringUtils; import org.apache.hadoop.hive.metastore.api.ColumnStatisticsData; @@ -741,18 +742,17 @@ public Map getAndCopyPartitionItems() { return res; } - private HiveMetaStoreCache.HivePartitionValues getHivePartitionValues() { - HiveMetaStoreCache cache = Env.getCurrentEnv().getExtMetaCacheMgr() - .getMetaStoreCache((HMSExternalCatalog) getCatalog()); - return cache.getPartitionValues( - getDbName(), getName(), getPartitionColumnTypes()); - } - @Override public MTMVSnapshotIf getPartitionSnapshot(String partitionName, MTMVRefreshContext context) throws AnalysisException { - long partitionLastModifyTime = getPartitionLastModifyTime(partitionName); - return new MTMVTimestampSnapshot(partitionLastModifyTime); + HiveMetaStoreCache cache = Env.getCurrentEnv().getExtMetaCacheMgr() + .getMetaStoreCache((HMSExternalCatalog) getCatalog()); + HiveMetaStoreCache.HivePartitionValues hivePartitionValues = cache.getPartitionValues( + getDbName(), getName(), getPartitionColumnTypes()); + Long partitionId = getPartitionIdByNameOrAnalysisException(partitionName, hivePartitionValues); + HivePartition hivePartition = getHivePartitionByIdOrAnalysisException(partitionId, + hivePartitionValues, cache); + return new MTMVTimestampSnapshot(hivePartition.getLastModifiedTime()); } @Override @@ -760,45 +760,50 @@ public MTMVSnapshotIf getTableSnapshot(MTMVRefreshContext context) throws Analys if (getPartitionType() == PartitionType.UNPARTITIONED) { return new MTMVMaxTimestampSnapshot(getName(), getLastDdlTime()); } - String partitionName = ""; + Long maxPartitionId = 0L; long maxVersionTime = 0L; long visibleVersionTime; - for (Entry entry : getAndCopyPartitionItems().entrySet()) { - visibleVersionTime = getPartitionLastModifyTime(entry.getKey()); + HiveMetaStoreCache cache = Env.getCurrentEnv().getExtMetaCacheMgr() + .getMetaStoreCache((HMSExternalCatalog) getCatalog()); + HiveMetaStoreCache.HivePartitionValues hivePartitionValues = cache.getPartitionValues( + getDbName(), getName(), getPartitionColumnTypes()); + BiMap idToName = hivePartitionValues.getPartitionNameToIdMap().inverse(); + if (MapUtils.isEmpty(idToName)) { + throw new AnalysisException("partitions is empty for : " + getName()); + } + for (Long partitionId : idToName.keySet()) { + visibleVersionTime = getHivePartitionByIdOrAnalysisException(partitionId, hivePartitionValues, + cache).getLastModifiedTime(); if (visibleVersionTime > maxVersionTime) { maxVersionTime = visibleVersionTime; - partitionName = entry.getKey(); + maxPartitionId = partitionId; } } - return new MTMVMaxTimestampSnapshot(partitionName, maxVersionTime); + return new MTMVMaxTimestampSnapshot(idToName.get(maxPartitionId), maxVersionTime); } - private long getPartitionLastModifyTime(String partitionName) throws AnalysisException { - return getPartitionByName(partitionName).getLastModifiedTime(); - } - - private HivePartition getPartitionByName(String partitionName) throws AnalysisException { - PartitionItem item = getAndCopyPartitionItems().get(partitionName); - List> partitionValuesList = transferPartitionItemToPartitionValues(item); - List partitions = getPartitionsByPartitionValues(partitionValuesList); - if (partitions.size() != 1) { - throw new AnalysisException("partition not normal, size: " + partitions.size()); + private Long getPartitionIdByNameOrAnalysisException(String partitionName, + HiveMetaStoreCache.HivePartitionValues hivePartitionValues) + throws AnalysisException { + Long partitionId = hivePartitionValues.getPartitionNameToIdMap().get(partitionName); + if (partitionId == null) { + throw new AnalysisException("can not find partition: " + partitionName); } - return partitions.get(0); + return partitionId; } - private List getPartitionsByPartitionValues(List> partitionValuesList) { - HiveMetaStoreCache cache = Env.getCurrentEnv().getExtMetaCacheMgr() - .getMetaStoreCache((HMSExternalCatalog) getCatalog()); - return cache.getAllPartitionsWithCache(getDbName(), getName(), - partitionValuesList); - } - - private List> transferPartitionItemToPartitionValues(PartitionItem item) { - List> partitionValuesList = Lists.newArrayListWithCapacity(1); - partitionValuesList.add( - ((ListPartitionItem) item).getItems().get(0).getPartitionValuesAsStringListForHive()); - return partitionValuesList; + private HivePartition getHivePartitionByIdOrAnalysisException(Long partitionId, + HiveMetaStoreCache.HivePartitionValues hivePartitionValues, + HiveMetaStoreCache cache) throws AnalysisException { + List partitionValues = hivePartitionValues.getPartitionValuesMap().get(partitionId); + if (CollectionUtils.isEmpty(partitionValues)) { + throw new AnalysisException("can not find partitionValues: " + partitionId); + } + HivePartition partition = cache.getHivePartition(getDbName(), getName(), partitionValues); + if (partition == null) { + throw new AnalysisException("can not find partition: " + partitionId); + } + return partition; } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetaStoreCache.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetaStoreCache.java index b87c14afbc8a42..99338fb87ad23b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetaStoreCache.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetaStoreCache.java @@ -521,6 +521,10 @@ public List getFilesByPartitions(List partitions, return fileLists; } + public HivePartition getHivePartition(String dbName, String name, List partitionValues) { + return partitionCache.get(new PartitionCacheKey(dbName, name, partitionValues)); + } + public List getAllPartitionsWithCache(String dbName, String name, List> partitionValuesList) { return getAllPartitions(dbName, name, partitionValuesList, true);