[regression](multi-catalog) add aliyun dlf hive on oss and huawei obs…

… test case (apache#25650) add aliyun dlf hive on oss and huawei obs test case now obs cases have some problem, will not fix this at this PR, just add comment.
hello-stephen · Oct 24, 2023 · 40e430c · 40e430c
1 parent d4fca3c
commit 40e430c
Show file tree

Hide file tree

Showing 5 changed files with 141 additions and 0 deletions.
diff --git a/regression-test/conf/regression-conf.groovy b/regression-test/conf/regression-conf.groovy
@@ -169,9 +169,13 @@ extEsPort = 9200
 extEsUser = "*******"
 extEsPassword = "***********"
 
+enableObjStorageTest=false
 enableMaxComputeTest=false
 aliYunAk="***********"
+dlfUid="***********"
 aliYunSk="***********"
+hwYunAk="***********"
+hwYunSk="***********"
 
 s3Endpoint = "cos.ap-hongkong.myqcloud.com"
 s3BucketName = "doris-build-hk-1308700295"

diff --git a/regression-test/data/external_table_p2/hive/test_cloud_accessible_obs.out b/regression-test/data/external_table_p2/hive/test_cloud_accessible_obs.out
@@ -0,0 +1,7 @@
+-- This file is automatically generated. You should know what you did if you want to edit this
+-- !hms_q1 --
+123	22	234	123.324235	sawer	234.1234	a23f                                              	1234vb	false	2023-04-23 21:23:34.123	2023-04-23
+
+-- !hms_q2 --
+11625	62	234	2347.6236	awsali	546.2342	a23f                                              	1234vb	true	2023-04-22 21:21:34.123	2023-04-21	2023-04-24
+23621	23	234	345.12512356	aswwas	525.2352	a23f                                              	wsd	false	2023-04-23 21:23:34.123	2023-04-23	2023-04-22
diff --git a/regression-test/data/external_table_p2/hive/test_cloud_accessible_oss.out b/regression-test/data/external_table_p2/hive/test_cloud_accessible_oss.out
@@ -0,0 +1,10 @@
+-- This file is automatically generated. You should know what you did if you want to edit this
+-- !hms_q1 --
+123	22	234	123.324235	sawer	234.1234	a23f                                              	1234vb	false	2023-04-23 21:23:34.123	2023-04-23
+
+-- !hms_q2 --
+11625	62	234	2347.6236	awsali	546.2342	a23f                                              	1234vb	true	2023-04-22 21:21:34.123	2023-04-21	2023-04-24
+23621	23	234	345.12512356	aswwas	525.2352	a23f                                              	wsd	false	2023-04-23 21:23:34.123	2023-04-23	2023-04-22
+
+-- !dlf_q1 --
+26	AAAAAAAAKBAAAAAA	1997-08-16	2000-08-15	site_4	2450594	2446944	Unknown	William Carter	2	Also difficult women relax. Raw solutions exclude	Often well-known police can hand acres. Healthy, sensible problems make more silent recordings	Keith Williams	2	able                                              	892       	Cedar North	Drive          	Suite 280 	Midway	Williamson County	TN	31904     	United States	-5.00	0.07
diff --git a/regression-test/suites/external_table_p2/hive/test_cloud_accessible_obs.groovy b/regression-test/suites/external_table_p2/hive/test_cloud_accessible_obs.groovy
@@ -0,0 +1,47 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("test_cloud_accessible_obs", "p2,external,hive,external_remote,external_remote_hive") {
+    String enabled = context.config.otherConfigs.get("enableObjStorageTest")
+    if (enabled != null && enabled.equalsIgnoreCase("true")) {
+        String extHiveHmsHost = context.config.otherConfigs.get("extHiveHmsHost")
+        String extHiveHmsPort = context.config.otherConfigs.get("extHiveHmsPort")
+        String ak = context.config.otherConfigs.get("hwYunAk")
+        String sk = context.config.otherConfigs.get("hwYunSk")
+        String hms_catalog_name = "test_cloud_accessible_obs"
+        sql """drop catalog if exists ${hms_catalog_name};"""
+        sql """
+            CREATE CATALOG IF NOT EXISTS ${hms_catalog_name}
+            PROPERTIES ( 
+                'type' = 'hms',
+                'hive.metastore.uris' = 'thrift://${extHiveHmsHost}:${extHiveHmsPort}',
+                'obs.endpoint' = 'obs.cn-north-4.myhuaweicloud.com',
+                'obs.access_key' = '${ak}',
+                'obs.secret_key' = '${sk}'
+            );
+        """
+
+        logger.info("catalog " + hms_catalog_name + " created")
+        sql """switch ${hms_catalog_name};"""
+        logger.info("switched to catalog " + hms_catalog_name)
+        sql """ use cloud_accessible """
+        qt_hms_q1 """ select * from types_obs order by hms_int """
+        qt_hms_q2 """ select * from types_one_part_obs order by hms_int """
+
+        sql """drop catalog ${hms_catalog_name};"""
+    }
+}
diff --git a/regression-test/suites/external_table_p2/hive/test_cloud_accessible_oss.groovy b/regression-test/suites/external_table_p2/hive/test_cloud_accessible_oss.groovy
@@ -0,0 +1,73 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("test_cloud_accessible_oss", "p2,external,hive,external_remote,external_remote_hive") {
+    String enabled = context.config.otherConfigs.get("enableObjStorageTest")
+    if (enabled != null && enabled.equalsIgnoreCase("true")) {
+        String extHiveHmsHost = context.config.otherConfigs.get("extHiveHmsHost")
+        String extHiveHmsPort = context.config.otherConfigs.get("extHiveHmsPort")
+        String ak = context.config.otherConfigs.get("aliYunAk")
+        String sk = context.config.otherConfigs.get("aliYunSk")
+        String hms_catalog_name = "test_cloud_accessible_oss"
+        sql """drop catalog if exists ${hms_catalog_name};"""
+        sql """
+            CREATE CATALOG IF NOT EXISTS ${hms_catalog_name}
+            PROPERTIES ( 
+                'type' = 'hms',
+                'hive.metastore.uris' = 'thrift://${extHiveHmsHost}:${extHiveHmsPort}',
+                'oss.endpoint' = 'oss-cn-beijing.aliyuncs.com',
+                'oss.access_key' = '${ak}',
+                'oss.secret_key' = '${sk}'
+            );
+        """
+
+        logger.info("catalog " + hms_catalog_name + " created")
+        sql """switch ${hms_catalog_name};"""
+        logger.info("switched to catalog " + hms_catalog_name)
+        sql """ use cloud_accessible """
+        qt_hms_q1 """ select * from types_oss order by hms_int """
+        qt_hms_q2 """ select * from types_one_part_oss order by hms_int """
+
+        sql """drop catalog ${hms_catalog_name};"""
+
+        // dlf case
+        String dlf_catalog_name = "test_cloud_accessible_dlf"
+        String dlf_uid = context.config.otherConfigs.get("dlfUid")
+        sql """drop catalog if exists ${dlf_catalog_name};"""
+        sql """
+            CREATE CATALOG  IF NOT EXISTS ${dlf_catalog_name}
+            PROPERTIES (
+                "type"="hms",
+                "hive.metastore.type" = "dlf",
+                "dlf.endpoint" = "dlf.cn-beijing.aliyuncs.com",
+                "dlf.region" = "cn-beijing",
+                "dlf.proxy.mode" = "DLF_ONLY",
+                "dlf.uid" = "${dlf_uid}",
+                "dlf.access_key" = "${ak}",
+                "dlf.secret_key" = "${sk}",
+                "dlf.access.public" = "true"
+            );
+        """
+        logger.info("catalog " + dlf_catalog_name + " created")
+        sql """switch ${dlf_catalog_name};"""
+        logger.info("switched to catalog " + dlf_catalog_name)
+        sql """ use jz_datalake """
+        qt_dlf_q1 """ select * from web_site where web_site_id='AAAAAAAAKBAAAAAA' order by web_site_sk,web_site_id limit 1; """ // test char,date,varchar,double,decimal
+
+        sql """drop catalog ${dlf_catalog_name};"""
+    }
+}