[fix](multicatlog) fix read hive/iceberg catalog on cosn & fix read d…

…ata via broker (#22087) * [fix](multicatlog) fix read hive/iceberg catalog on cosn & fix read data via broker * Update FileSystemFactory.java
apache · Aug 10, 2023 · f7d00d4 · f7d00d4
1 parent f2658dc
commit f7d00d4
Show file tree

Hide file tree

Showing 4 changed files with 18 additions and 9 deletions.
diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/FeConstants.java b/fe/fe-core/src/main/java/org/apache/doris/common/FeConstants.java
@@ -80,6 +80,7 @@ public class FeConstants {
     public static String FS_PREFIX_GCS = "gs";
     public static String FS_PREFIX_BOS = "bos";
     public static String FS_PREFIX_COS = "cos";
+    public static String FS_PREFIX_COSN = "cosn";
     public static String FS_PREFIX_OBS = "obs";
     public static String FS_PREFIX_OFS = "ofs";
     public static String FS_PREFIX_GFS = "gfs";

diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/util/S3Util.java b/fe/fe-core/src/main/java/org/apache/doris/common/util/S3Util.java
@@ -53,9 +53,10 @@ public class S3Util {
 
     public static boolean isObjStorage(String location) {
         return isObjStorageUseS3Client(location)
-                || location.startsWith(FeConstants.FS_PREFIX_COS)
-                || location.startsWith(FeConstants.FS_PREFIX_OSS)
-                || location.startsWith(FeConstants.FS_PREFIX_OBS);
+            // if treat cosn(tencent hadoop-cos) as a s3 file system, may bring incompatible issues
+            || (location.startsWith(FeConstants.FS_PREFIX_COS) && !location.startsWith(FeConstants.FS_PREFIX_COSN))
+            || location.startsWith(FeConstants.FS_PREFIX_OSS)
+            || location.startsWith(FeConstants.FS_PREFIX_OBS);
     }
 
     private static boolean isObjStorageUseS3Client(String location) {

diff --git a/fe/fe-core/src/main/java/org/apache/doris/fs/FileSystemFactory.java b/fe/fe-core/src/main/java/org/apache/doris/fs/FileSystemFactory.java
@@ -66,7 +66,8 @@ public static Pair<FileSystemType, String> getFSIdentity(String location) {
             fsType = FileSystemType.S3;
         } else if (location.startsWith(FeConstants.FS_PREFIX_HDFS) || location.startsWith(FeConstants.FS_PREFIX_GFS)) {
             fsType = FileSystemType.DFS;
-        } else if (location.startsWith(FeConstants.FS_PREFIX_OFS)) {
+        } else if (location.startsWith(FeConstants.FS_PREFIX_OFS) || location.startsWith(FeConstants.FS_PREFIX_COSN)) {
+            // ofs:// and cosn:// use the same underlying file system: Tencent Cloud HDFS, aka CHDFS)) {
             fsType = FileSystemType.OFS;
         } else if (location.startsWith(FeConstants.FS_PREFIX_JFS)) {
             fsType = FileSystemType.JFS;

diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/external/FileQueryScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/external/FileQueryScanNode.java
@@ -353,12 +353,16 @@ private void setLocationPropertiesIfNecessary(TFileType locationType, FileSplit
                 params.setHdfsParams(tHdfsParams);
             }
 
-            if (locationType == TFileType.FILE_BROKER && !params.isSetBrokerAddresses()) {
-                FsBroker broker = Env.getCurrentEnv().getBrokerMgr().getAnyAliveBroker();
-                if (broker == null) {
-                    throw new UserException("No alive broker.");
+            if (locationType == TFileType.FILE_BROKER) {
+                params.setProperties(locationProperties);
+
+                if (!params.isSetBrokerAddresses()) {
+                    FsBroker broker = Env.getCurrentEnv().getBrokerMgr().getAnyAliveBroker();
+                    if (broker == null) {
+                        throw new UserException("No alive broker.");
+                    }
+                    params.addToBrokerAddresses(new TNetworkAddress(broker.host, broker.port));
                 }
-                params.addToBrokerAddresses(new TNetworkAddress(broker.host, broker.port));
             }
         } else if (locationType == TFileType.FILE_S3 && !params.isSetProperties()) {
             params.setProperties(locationProperties);
@@ -450,6 +454,8 @@ protected static Optional<TFileType> getTFileType(String location) {
                 return Optional.of(TFileType.FILE_LOCAL);
             } else if (location.startsWith(FeConstants.FS_PREFIX_OFS)) {
                 return Optional.of(TFileType.FILE_BROKER);
+            } else if (location.startsWith(FeConstants.FS_PREFIX_COSN)) {
+                return Optional.of(TFileType.FILE_BROKER);
             } else if (location.startsWith(FeConstants.FS_PREFIX_GFS)) {
                 return Optional.of(TFileType.FILE_BROKER);
             } else if (location.startsWith(FeConstants.FS_PREFIX_JFS)) {