Skip to content

Commit

Permalink
[Fix](multi catalog)Fix hive partition contains special character bug (
Browse files Browse the repository at this point in the history
…#22541)

Hive partition path may contain special characters, need to encode it before creating a URI object based on the file path.
  • Loading branch information
Jibing-Li authored Aug 3, 2023
1 parent 3447a70 commit 479e62d
Showing 1 changed file with 13 additions and 3 deletions.
16 changes: 13 additions & 3 deletions fe/fe-core/src/main/java/org/apache/doris/common/util/S3Util.java
Original file line number Diff line number Diff line change
Expand Up @@ -39,8 +39,12 @@
import software.amazon.awssdk.services.s3.S3Client;
import software.amazon.awssdk.services.s3.S3Configuration;

import java.io.UnsupportedEncodingException;
import java.net.URI;
import java.net.URISyntaxException;
import java.net.URLDecoder;
import java.net.URLEncoder;
import java.nio.charset.StandardCharsets;
import java.time.Duration;
import java.util.Map;

Expand Down Expand Up @@ -85,15 +89,21 @@ private static String normalizedLocation(String location, Map<String, String> pr
return normalizedHdfsPath(location, props);
}
return location;
} catch (URISyntaxException e) {
} catch (URISyntaxException | UnsupportedEncodingException e) {
throw new RuntimeException(e.getMessage(), e);
}
}

private static String normalizedHdfsPath(String location, Map<String, String> props) throws URISyntaxException {
private static String normalizedHdfsPath(String location, Map<String, String> props)
throws URISyntaxException, UnsupportedEncodingException {
// Hive partition may contain special characters such as ' ', '<', '>' and so on.
// Need to encode these characters before creating URI.
// But doesn't encode '/' and ':' so that we can get the correct uri host.
location = URLEncoder.encode(location, StandardCharsets.UTF_8.name()).replace("%2F", "/").replace("%3A", ":");
URI normalizedUri = new URI(location);
// compatible with 'hdfs:///' or 'hdfs:/'
if (StringUtils.isEmpty(normalizedUri.getHost())) {
location = URLDecoder.decode(location, StandardCharsets.UTF_8.name());
String normalizedPrefix = HdfsResource.HDFS_PREFIX + "//";
String brokenPrefix = HdfsResource.HDFS_PREFIX + "/";
if (location.startsWith(brokenPrefix) && !location.startsWith(normalizedPrefix)) {
Expand All @@ -116,7 +126,7 @@ private static String normalizedHdfsPath(String location, Map<String, String> pr
}
}
}
return location;
return URLDecoder.decode(location, StandardCharsets.UTF_8.name());
}

/**
Expand Down

0 comments on commit 479e62d

Please sign in to comment.