Fixed the S3 configuration error in Resource (DataLinkDC#2670)

* Fixed the S3 configuration error in Resource * fix i81n
aiwenmo · Dec 17, 2023 · 6366ecd · 6366ecd
1 parent 57bddf5
commit 6366ecd
Show file tree

Hide file tree

Showing 8 changed files with 45 additions and 12 deletions.
diff --git a/dinky-admin/src/main/java/org/dinky/init/SystemInit.java b/dinky-admin/src/main/java/org/dinky/init/SystemInit.java
@@ -134,7 +134,8 @@ private void initResources() {
                         systemConfiguration.getResourcesHdfsUser(),
                         systemConfiguration.getResourcesHdfsDefaultFS(),
                         systemConfiguration.getResourcesOssAccessKey(),
-                        systemConfiguration.getResourcesOssRegion())
+                        systemConfiguration.getResourcesOssRegion(),
+                        systemConfiguration.getResourcesPathStyleAccess())
                 .forEach(x -> x.addParameterCheck(y -> {
                     if (Boolean.TRUE.equals(
                             systemConfiguration.getResourcesEnable().getValue())) {
@@ -156,6 +157,9 @@ private void initResources() {
                                 ossProperties.setRegion(systemConfiguration
                                         .getResourcesOssRegion()
                                         .getValue());
+                                ossProperties.setPathStyleAccess(systemConfiguration
+                                        .getResourcesPathStyleAccess()
+                                        .getValue());
                                 Singleton.get(OssResourceManager.class).setOssTemplate(new OssTemplate(ossProperties));
                                 break;
                             case HDFS:

diff --git a/dinky-common/pom.xml b/dinky-common/pom.xml
@@ -34,7 +34,6 @@
         <dependency>
             <groupId>com.amazonaws</groupId>
             <artifactId>aws-java-sdk-s3</artifactId>
-            <version>1.12.589</version>
         </dependency>
         <dependency>
             <groupId>com.github.xiaoymin</groupId>

diff --git a/dinky-common/src/main/java/org/dinky/data/enums/Status.java b/dinky-common/src/main/java/org/dinky/data/enums/Status.java
@@ -367,6 +367,8 @@ public enum Status {
     SYS_RESOURCE_SETTINGS_HDFS_ROOT_USER_NOTE(173, "sys.resource.settings.hdfs.root.user.note"),
     SYS_RESOURCE_SETTINGS_HDFS_FS_DEFAULTFS(174, "sys.resource.settings.hdfs.fs.defaultFS"),
     SYS_RESOURCE_SETTINGS_HDFS_FS_DEFAULTFS_NOTE(175, "sys.resource.settings.hdfs.fs.defaultFS.note"),
+    SYS_RESOURCE_SETTINGS_PATH_STYLE_ACCESS(176, "sys.resource.settings.oss.path.style.access"),
+    SYS_RESOURCE_SETTINGS_PATH_STYLE_ACCESS_NOTE(177, "sys.resource.settings.oss.path.style.access.note"),
 
     /**
      * gateway config

diff --git a/dinky-common/src/main/java/org/dinky/data/model/SystemConfiguration.java b/dinky-common/src/main/java/org/dinky/data/model/SystemConfiguration.java
@@ -251,6 +251,10 @@ public static Configuration.OptionBuilder key(Status status) {
             .stringType()
             .defaultValue("file:///")
             .note(Status.SYS_RESOURCE_SETTINGS_HDFS_FS_DEFAULTFS_NOTE);
+    private final Configuration<Boolean> resourcesPathStyleAccess = key(Status.SYS_RESOURCE_SETTINGS_PATH_STYLE_ACCESS)
+            .booleanType()
+            .defaultValue(true)
+            .note(Status.SYS_RESOURCE_SETTINGS_PATH_STYLE_ACCESS_NOTE);
 
     /**
      * Initialize after spring bean startup

diff --git a/dinky-common/src/main/resources/i18n/messages_en_US.properties b/dinky-common/src/main/resources/i18n/messages_en_US.properties
@@ -222,10 +222,10 @@ sys.metrics.settings.flink.gatherTimeout.note=Flink Metrics collection timeout p
 sys.resource.settings.base.enable=Whether to enable Resource
 sys.resource.settings.base.enable.note=Enable resource management function. If you switch storage mode, you need to turn off this switch. After the relevant configuration is completed, turn it on again.
 sys.resource.settings.base.upload.base.path=Root path of the upload directory
-sys.resource.settings.base.upload.base.path.note=Resources are stored on the HDFS/OSS path. Resource files will be stored in this base path. Configure it by yourself. Please ensure that the directory exists on the relevant storage system and has read capabilities. Write permission. recommend
+sys.resource.settings.base.upload.base.path.note=Resources are stored on the HDFS/OSS(S3) path. Resource files will be stored in this base path. Configure it by yourself. Please ensure that the directory exists on the relevant storage system and has read capabilities. Write permission. recommend
 sys.resource.settings.base.model=Storage model
-sys.resource.settings.base.model.note=Supports HDFS and OSS, it will take effect after switching the option, and resource files will be migrated at the same time.
-sys.resource.settings.oss.endpoint=URL of the object storage service
+sys.resource.settings.base.model.note=Supports HDFS and S3(Minio,OSS,COS ant etc...), it will take effect after switching the option, and resource files will be migrated at the same time.
+sys.resource.settings.oss.endpoint=URL of the object storage service(Endpoint)
 sys.resource.settings.oss.endpoint.note=For example: https://oss-cn-hangzhou.aliyuncs.com
 sys.resource.settings.oss.accessKey=Access key
 sys.resource.settings.oss.accessKey.note=Access key is like a user ID that uniquely identifies your account
@@ -235,6 +235,9 @@ sys.resource.settings.oss.bucketName=Bucket name
 sys.resource.settings.oss.bucketName.note=Default bucket name
 sys.resource.settings.oss.region=region
 sys.resource.settings.oss.region.note=region
+sys.resource.settings.oss.path.style.access=Path Style
+sys.resource.settings.oss.path.style.access.note=Whether to enable Path Style, Different providers (such as Alibaba Cloud OSS and Tencent Cloud COS) have different support conditions, please read the description of the provider
+
 sys.resource.settings.hdfs.root.user=HDFS operation user name
 sys.resource.settings.hdfs.root.user.note=HDFS operation user name
 sys.resource.settings.hdfs.fs.defaultFS=HDFS defaultFS

diff --git a/dinky-common/src/main/resources/i18n/messages_zh_CN.properties b/dinky-common/src/main/resources/i18n/messages_zh_CN.properties
@@ -222,10 +222,11 @@ sys.metrics.settings.flink.gatherTimeout.note=Flink Metrics 采集超时时长
 sys.resource.settings.base.enable=是否启用Resource
 sys.resource.settings.base.enable.note=启用资源管理功能，如果切换存储模式时，需关闭此开关，相关配置完成后，再开启
 sys.resource.settings.base.upload.base.path=上传目录的根路径
-sys.resource.settings.base.upload.base.path.note=资源存储在HDFS/OSS路径上，资源文件将存储到此基本路径，自行配置，请确保该目录存在于相关存储系统上并具有读写权限。
+sys.resource.settings.base.upload.base.path.note=资源存储在HDFS/OSS (S3)路径上，资源文件将存储到此基本路径，自行配置，请确保该目录存在于相关存储系统上并具有读写权限。
 sys.resource.settings.base.model=存储模式
-sys.resource.settings.base.model.note=支持HDFS、OSS，切换选项后即可生效，同时并迁移资源文件
-sys.resource.settings.oss.endpoint=对象存储服务的 URL
+sys.resource.settings.base.model.note=支持HDFS、S3(Minio、阿里云OSS、腾讯云COS等..)，切换选项后即可生效，同时并迁移资源文件
+
+sys.resource.settings.oss.endpoint=对象存储服务的 URL（Endpoint）
 sys.resource.settings.oss.endpoint.note=例如：https://oss-cn-hangzhou.aliyuncs.com
 sys.resource.settings.oss.accessKey=Access key
 sys.resource.settings.oss.accessKey.note=Access key就像用户ID，可以唯一标识你的账户
@@ -235,6 +236,9 @@ sys.resource.settings.oss.bucketName=存储桶名称
 sys.resource.settings.oss.bucketName.note=默认的存储桶名称
 sys.resource.settings.oss.region=区域
 sys.resource.settings.oss.region.note=区域,例如：oss-cn-hangzhou
+sys.resource.settings.oss.path.style.access=Path Style
+sys.resource.settings.oss.path.style.access.note=是否开启 path style, 不同的提供方（如阿里云oss，腾讯云cos）支持情况不同，请阅读提供方文档说明进行填写
+
 sys.resource.settings.hdfs.root.user=HDFS 操作用户名
 sys.resource.settings.hdfs.root.user.note=HDFS 操作用户名
 sys.resource.settings.hdfs.fs.defaultFS=HDFS defaultFS

diff --git a/dinky-web/src/pages/RegCenter/Resource/components/ResourcesUploadModal/index.tsx b/dinky-web/src/pages/RegCenter/Resource/components/ResourcesUploadModal/index.tsx
@@ -42,14 +42,18 @@ const ResourcesUploadModal: React.FC<ResourcesUploadModalProps> = (props) => {
     multiple: true,
     action: url + '?pid=' + pid,
     onChange: async (info) => {
-      const { status } = info.file;
+      const { status, response } = info.file;
       if (status !== 'uploading') {
         console.log(info.file, info.fileList);
       }
       if (status === 'done') {
-        await SuccessMessageAsync(
-          l('rc.resource.upload.success', '', { fileName: info.file.name })
-        );
+        if (response.success) {
+          await SuccessMessageAsync(
+            l('rc.resource.upload.success', '', { fileName: info.file.name })
+          );
+        } else {
+          await ErrorMessageAsync(response.msg);
+        }
       } else if (status === 'error') {
         await ErrorMessageAsync(l('rc.resource.upload.fail', '', { fileName: info.file.name }));
       }

diff --git a/pom.xml b/pom.xml
@@ -50,6 +50,7 @@
 
     <properties>
         <assertj.version>3.21.0</assertj.version>
+        <aws-java-sdk-s3.version>1.12.589</aws-java-sdk-s3.version>
         <build-helper-maven-plugin.version>1.7</build-helper-maven-plugin.version>
         <clickhouse.version>0.2.6</clickhouse.version>
         <commons-email>1.5</commons-email>
@@ -124,6 +125,11 @@
 
     <dependencyManagement>
         <dependencies>
+            <dependency>
+                <groupId>com.amazonaws</groupId>
+                <artifactId>aws-java-sdk-s3</artifactId>
+                <version>${aws-java-sdk-s3.version}</version>
+            </dependency>
             <dependency>
                 <groupId>org.commonmark</groupId>
                 <artifactId>commonmark</artifactId>
@@ -275,6 +281,7 @@
                 <artifactId>protobuf-java</artifactId>
                 <version>${protobuf-java.version}</version>
             </dependency>
+
             <!-- hutool java工具类库  -->
             <dependency>
                 <groupId>cn.hutool</groupId>
@@ -490,6 +497,12 @@
                 <groupId>org.dinky</groupId>
                 <artifactId>dinky-metadata-phoenix</artifactId>
                 <version>${project.version}</version>
+                <exclusions>
+                    <exclusion>
+                        <groupId>joda-time</groupId>
+                        <artifactId>joda-time</artifactId>
+                    </exclusion>
+                </exclusions>
             </dependency>
             <dependency>
                 <groupId>org.dinky</groupId>