diff --git a/regression-test/conf/regression-conf.groovy b/regression-test/conf/regression-conf.groovy index c501bfeba43f92..08d03632c373d4 100644 --- a/regression-test/conf/regression-conf.groovy +++ b/regression-test/conf/regression-conf.groovy @@ -105,11 +105,13 @@ brokerName = "broker_name" // broker load test config enableBrokerLoad=true -// for s3 releated cases, "aliyun" or "aliyun-internal" or "tencent" or "huawei" or "azure" +// for s3 releated cases, "aliyun" or "aliyun-internal" or "tencent" or "huawei" or "azure" or "gcp" +// if s3Source is set, s3Endpoint s3BucketName s3Region s3Provider will be filled with default value if not set s3Source="aliyun" // s3Endpoint = "" // s3BucketName = "" // s3Region = "" +// s3Provider = "" ak="***********" sk="***********" diff --git a/regression-test/framework/src/main/groovy/org/apache/doris/regression/Config.groovy b/regression-test/framework/src/main/groovy/org/apache/doris/regression/Config.groovy index e13b64f1101330..aa6e8a2ff720fa 100644 --- a/regression-test/framework/src/main/groovy/org/apache/doris/regression/Config.groovy +++ b/regression-test/framework/src/main/groovy/org/apache/doris/regression/Config.groovy @@ -468,7 +468,7 @@ class Config { log.info("withOutLoadData is ${config.withOutLoadData}".toString()) log.info("caseNamePrefix is ${config.caseNamePrefix}".toString()) log.info("dryRun is ${config.dryRun}".toString()) - def s3SourceList = ["aliyun", "aliyun-internal", "tencent", "huawei", "azure"] + def s3SourceList = ["aliyun", "aliyun-internal", "tencent", "huawei", "azure", "gcp"] if (s3SourceList.contains(config.s3Source)) { log.info("s3Source is ${config.s3Source}".toString()) log.info("s3Provider is ${config.otherConfigs.get("s3Provider")}".toString()) @@ -609,54 +609,74 @@ class Config { } if (config.otherConfigs.get("s3Provider") == null) { + def s3Provider = "OSS" if (config.s3Source == "aliyun" || config.s3Source == "aliyun-internal") { - config.otherConfigs.put("s3Provider", "OSS") + s3Provider = "OSS" } else if (config.s3Source == "tencent") { - config.otherConfigs.put("s3Provider", "COS") + s3Provider = "COS" } else if (config.s3Source == "huawei") { - config.otherConfigs.put("s3Provider", "OBS") + s3Provider = "OBS" } else if (config.s3Source == "azure") { - config.otherConfigs.put("s3Provider", "AZURE") + s3Provider = "AZURE" + } else if (config.s3Source == "gcp") { + s3Provider = "GCP" } + config.otherConfigs.put("s3Provider", "${s3Provider}") + log.info("Set s3Provider to '${s3Provider}' because not specify.".toString()) } if (config.otherConfigs.get("s3BucketName") == null) { + def s3BucketName = "doris-regression-hk" if (config.s3Source == "aliyun") { - config.otherConfigs.put("s3BucketName", "doris-regression-hk") + s3BucketName = "doris-regression-hk" } else if (config.s3Source == "aliyun-internal") { - config.otherConfigs.put("s3BucketName", "doris-regression") + s3BucketName = "doris-regression" } else if (config.s3Source == "tencent") { - config.otherConfigs.put("s3BucketName", "doris-build-1308700295") + s3BucketName = "doris-build-1308700295" } else if (config.s3Source == "huawei") { - config.otherConfigs.put("s3BucketName", "doris-build") + s3BucketName = "doris-build" } else if (config.s3Source == "azure") { - config.otherConfigs.put("s3BucketName", "qa-build") + s3BucketName = "qa-build" + } else if (config.s3Source == "gcp") { + s3BucketName = "doris-regression" } + config.otherConfigs.put("s3BucketName", "${s3BucketName}") + log.info("Set s3BucketName to '${s3BucketName}' because not specify.".toString()) } if (config.otherConfigs.get("s3Region") == null) { + def s3Region = "oss-cn-hongkong" if (config.s3Source == "aliyun") { - config.otherConfigs.put("s3Region", "oss-cn-hongkong") + s3Region = "oss-cn-hongkong" } else if (config.s3Source == "aliyun-internal") { - config.otherConfigs.put("s3Region", "oss-cn-beijing") + s3Region = "oss-cn-beijing" } else if (config.s3Source == "tencent") { - config.otherConfigs.put("s3Region", "ap-beijing") + s3Region = "ap-beijing" } else if (config.s3Source == "huawei") { - config.otherConfigs.put("s3Region", "cn-north-4") + s3Region = "cn-north-4" } else if (config.s3Source == "azure") { - config.otherConfigs.put("s3Region", "azure-region") + s3Region = "azure-region" + } else if (config.s3Source == "gcp") { + s3Region = "us-central1" } + config.otherConfigs.put("s3Region", "${s3Region}") + log.info("Set s3Region to '${s3Region}' because not specify.".toString()) } if (config.otherConfigs.get("s3Endpoint") == null) { + def s3Endpoint = "oss-cn-hongkong.aliyuncs.com" if (config.s3Source == "aliyun") { - config.otherConfigs.put("s3Endpoint", "oss-cn-hongkong.aliyuncs.com") + s3Endpoint = "oss-cn-hongkong.aliyuncs.com" } else if (config.s3Source == "aliyun-internal") { - config.otherConfigs.put("s3Endpoint", "oss-cn-beijing-internal.aliyuncs.com") + s3Endpoint = "oss-cn-beijing-internal.aliyuncs.com" } else if (config.s3Source == "tencent") { - config.otherConfigs.put("s3Endpoint", "cos.ap-beijing.myqcloud.com") + s3Endpoint = "cos.ap-beijing.myqcloud.com" } else if (config.s3Source == "huawei") { - config.otherConfigs.put("s3Endpoint", "obs.cn-north-4.myhuaweicloud.com") + s3Endpoint = "obs.cn-north-4.myhuaweicloud.com" } else if (config.s3Source == "azure") { - config.otherConfigs.put("s3Endpoint", "azure-endpoint") + s3Endpoint = "azure-endpoint" + } else if (config.s3Source == "gcp") { + s3Endpoint = "storage.googleapis.com" } + config.otherConfigs.put("s3Endpoint", "${s3Endpoint}") + log.info("Set s3Endpoint to '${s3Endpoint}' because not specify.".toString()) } if (config.caseNamePrefix == null) { diff --git a/regression-test/pipeline/cloud_p0/conf/regression-conf-custom.groovy b/regression-test/pipeline/cloud_p0/conf/regression-conf-custom.groovy index c90ec09d06daef..b9972df9f1646d 100644 --- a/regression-test/pipeline/cloud_p0/conf/regression-conf-custom.groovy +++ b/regression-test/pipeline/cloud_p0/conf/regression-conf-custom.groovy @@ -75,4 +75,4 @@ enableKafkaTest=true // trino-connector catalog test config enableTrinoConnectorTest = false -s3Provider = "COS" +s3Source = "aliyun" diff --git a/regression-test/pipeline/cloud_p1/conf/regression-conf-custom.groovy b/regression-test/pipeline/cloud_p1/conf/regression-conf-custom.groovy index 1b7c4f4c07c725..0583744a65015d 100644 --- a/regression-test/pipeline/cloud_p1/conf/regression-conf-custom.groovy +++ b/regression-test/pipeline/cloud_p1/conf/regression-conf-custom.groovy @@ -22,4 +22,4 @@ excludeDirectories = "000_the_start_sentinel_do_not_touch," + // keep this line max_failure_num = 50 -s3Provider = "COS" +s3Source = "aliyun" diff --git a/regression-test/pipeline/external/conf/regression-conf.groovy b/regression-test/pipeline/external/conf/regression-conf.groovy index 28956568e58934..93965b84219e97 100644 --- a/regression-test/pipeline/external/conf/regression-conf.groovy +++ b/regression-test/pipeline/external/conf/regression-conf.groovy @@ -138,10 +138,7 @@ es_8_port=39200 cacheDataPath = "/data/regression/" -s3Endpoint = "cos.ap-hongkong.myqcloud.com" -s3BucketName = "doris-build-hk-1308700295" -s3Region = "ap-hongkong" -s3Provider = "COS" +s3Source="aliyun" max_failure_num=50 diff --git a/regression-test/pipeline/p0/conf/regression-conf.groovy b/regression-test/pipeline/p0/conf/regression-conf.groovy index 1c2fb4d08b67dd..563a83d4ec0669 100644 --- a/regression-test/pipeline/p0/conf/regression-conf.groovy +++ b/regression-test/pipeline/p0/conf/regression-conf.groovy @@ -136,10 +136,7 @@ es_8_port=39200 cacheDataPath = "/data/regression/" -s3Endpoint = "cos.ap-hongkong.myqcloud.com" -s3BucketName = "doris-build-hk-1308700295" -s3Region = "ap-hongkong" -s3Provider = "COS" +s3Source="aliyun" //arrow flight sql test config extArrowFlightSqlHost = "127.0.0.1" diff --git a/regression-test/pipeline/p1/conf/regression-conf.groovy b/regression-test/pipeline/p1/conf/regression-conf.groovy index f85892e6834155..b7318f12e0f953 100644 --- a/regression-test/pipeline/p1/conf/regression-conf.groovy +++ b/regression-test/pipeline/p1/conf/regression-conf.groovy @@ -71,10 +71,7 @@ excludeDirectories = "000_the_start_sentinel_do_not_touch," + // keep this line cacheDataPath="/data/regression/" -s3Endpoint = "cos.ap-hongkong.myqcloud.com" -s3BucketName = "doris-build-hk-1308700295" -s3Region = "ap-hongkong" -s3Provider = "COS" +s3Source="aliyun" max_failure_num=0