diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetaStoreCache.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetaStoreCache.java index f639109356987f..061f948e5dd0ec 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetaStoreCache.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetaStoreCache.java @@ -348,21 +348,21 @@ private Map loadPartitions(Iterable inputFormat, - JobConf jobConf, - List partitionValues, - String bindBrokerName) throws UserException { + private FileCacheValue getFileCache(String location, String inputFormat, + JobConf jobConf, + List partitionValues, + String bindBrokerName) throws UserException { FileCacheValue result = new FileCacheValue(); RemoteFileSystem fs = Env.getCurrentEnv().getExtMetaCacheMgr().getFsCache().getRemoteFileSystem( new FileSystemCache.FileSystemCacheKey(LocationPath.getFSIdentity( - location, bindBrokerName), jobConf, bindBrokerName)); - result.setSplittable(HiveUtil.isSplittable(fs, inputFormat, location, jobConf)); + location, bindBrokerName), jobConf, bindBrokerName)); + result.setSplittable(HiveUtil.isSplittable(fs, inputFormat, location)); try { // For Tez engine, it may generate subdirectoies for "union" query. // So there may be files and directories in the table directory at the same time. eg: @@ -421,18 +421,18 @@ private FileCacheValue loadFiles(FileCacheKey key) { FileInputFormat.setInputPaths(jobConf, finalLocation.get()); try { FileCacheValue result; - InputFormat inputFormat = HiveUtil.getInputFormat(jobConf, key.inputFormat, false); // TODO: This is a temp config, will remove it after the HiveSplitter is stable. if (key.useSelfSplitter) { - result = getFileCache(finalLocation.get(), inputFormat, jobConf, - key.getPartitionValues(), key.bindBrokerName); + result = getFileCache(finalLocation.get(), key.inputFormat, jobConf, + key.getPartitionValues(), key.bindBrokerName); } else { + InputFormat inputFormat = HiveUtil.getInputFormat(jobConf, key.inputFormat, false); InputSplit[] splits; String remoteUser = jobConf.get(HdfsResource.HADOOP_USER_NAME); if (!Strings.isNullOrEmpty(remoteUser)) { UserGroupInformation ugi = UserGroupInformation.createRemoteUser(remoteUser); splits = ugi.doAs( - (PrivilegedExceptionAction) () -> inputFormat.getSplits(jobConf, 0)); + (PrivilegedExceptionAction) () -> inputFormat.getSplits(jobConf, 0)); } else { splits = inputFormat.getSplits(jobConf, 0 /* use hdfs block size as default */); } @@ -715,7 +715,7 @@ public void addPartitionsCache(String dbName, String tblName, List parti } public void dropPartitionsCache(String dbName, String tblName, List partitionNames, - boolean invalidPartitionCache) { + boolean invalidPartitionCache) { PartitionValueCacheKey key = new PartitionValueCacheKey(dbName, tblName, null); HivePartitionValues partitionValues = partitionValuesCache.getIfPresent(key); if (partitionValues == null) { @@ -839,17 +839,17 @@ public List getFilesByTransaction(List partitions RemoteFileSystem fs = Env.getCurrentEnv().getExtMetaCacheMgr().getFsCache().getRemoteFileSystem( new FileSystemCache.FileSystemCacheKey( LocationPath.getFSIdentity(location, bindBrokerName), - jobConf, bindBrokerName)); + jobConf, bindBrokerName)); RemoteFiles locatedFiles = fs.listLocatedFiles(location, true, false); if (delta.isDeleteDelta()) { List deleteDeltaFileNames = locatedFiles.files().stream().map(f -> f.getName()).filter( name -> name.startsWith(HIVE_TRANSACTIONAL_ORC_BUCKET_PREFIX)) - .collect(Collectors.toList()); + .collect(Collectors.toList()); deleteDeltas.add(new DeleteDeltaInfo(location, deleteDeltaFileNames)); continue; } locatedFiles.files().stream().filter( - f -> f.getName().startsWith(HIVE_TRANSACTIONAL_ORC_BUCKET_PREFIX)) + f -> f.getName().startsWith(HIVE_TRANSACTIONAL_ORC_BUCKET_PREFIX)) .forEach(fileCacheValue::addFile); } @@ -859,10 +859,10 @@ public List getFilesByTransaction(List partitions RemoteFileSystem fs = Env.getCurrentEnv().getExtMetaCacheMgr().getFsCache().getRemoteFileSystem( new FileSystemCache.FileSystemCacheKey( LocationPath.getFSIdentity(location, bindBrokerName), - jobConf, bindBrokerName)); + jobConf, bindBrokerName)); RemoteFiles locatedFiles = fs.listLocatedFiles(location, true, false); locatedFiles.files().stream().filter( - f -> f.getName().startsWith(HIVE_TRANSACTIONAL_ORC_BUCKET_PREFIX)) + f -> f.getName().startsWith(HIVE_TRANSACTIONAL_ORC_BUCKET_PREFIX)) .forEach(fileCacheValue::addFile); } fileCacheValue.setAcidInfo(new AcidInfo(partition.getPath(), deleteDeltas)); @@ -976,8 +976,8 @@ public FileCacheKey(String location, String inputFormat, List partitionV } public static FileCacheKey createDummyCacheKey(String dbName, String tblName, String location, - String inputFormat, boolean useSelfSplitter, - String bindBrokerName) { + String inputFormat, boolean useSelfSplitter, + String bindBrokerName) { FileCacheKey fileCacheKey = new FileCacheKey(location, inputFormat, null, bindBrokerName); fileCacheKey.dummyKey = dbName + "." + tblName; fileCacheKey.useSelfSplitter = useSelfSplitter; @@ -996,7 +996,7 @@ public boolean equals(Object obj) { return dummyKey.equals(((FileCacheKey) obj).dummyKey); } return location.equals(((FileCacheKey) obj).location) - && Objects.equals(partitionValues, ((FileCacheKey) obj).partitionValues); + && Objects.equals(partitionValues, ((FileCacheKey) obj).partitionValues); } @Override @@ -1115,10 +1115,10 @@ public static class HivePartitionValues { private Map> idToUniqueIdsMap; private Map idToPartitionItem; private Map> partitionValuesMap; - //multi pair + // multi pair private Map> uidToPartitionRange; private Map, UniqueId> rangeToId; - //single pair + // single pair private RangeMap singleColumnRangeMap; private Map> singleUidToColumnRangeMap; diff --git a/fe/fe-core/src/main/java/org/apache/doris/external/hive/util/HiveUtil.java b/fe/fe-core/src/main/java/org/apache/doris/external/hive/util/HiveUtil.java index f0f013fa1f0cb1..55243b9b148fe4 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/external/hive/util/HiveUtil.java +++ b/fe/fe-core/src/main/java/org/apache/doris/external/hive/util/HiveUtil.java @@ -185,11 +185,11 @@ private static Type convertHiveTypeToiveDoris(TypeInfo hiveTypeInfo) { } } - public static boolean isSplittable(RemoteFileSystem remoteFileSystem, InputFormat inputFormat, - String location, JobConf jobConf) throws UserException { + public static boolean isSplittable(RemoteFileSystem remoteFileSystem, String inputFormat, + String location) throws UserException { if (remoteFileSystem instanceof BrokerFileSystem) { return ((BrokerFileSystem) remoteFileSystem) - .isSplittable(location, inputFormat.getClass().getCanonicalName()); + .isSplittable(location, inputFormat); } return HMSExternalTable.SUPPORTED_HIVE_FILE_FORMATS.contains(inputFormat);