Skip to content

Commit

Permalink
[regression](s3) add default conf for s3 releated cases
Browse files Browse the repository at this point in the history
  • Loading branch information
stephen committed Jul 16, 2024
1 parent b373135 commit ea40ff7
Show file tree
Hide file tree
Showing 2 changed files with 79 additions and 7 deletions.
14 changes: 8 additions & 6 deletions regression-test/conf/regression-conf.groovy
View file
Original file line number Diff line number Diff line change
Expand Up @@ -104,8 +104,14 @@ brokerName = "broker_name"

// broker load test config
enableBrokerLoad=true
ak=""
sk=""

// for s3 releated cases, "aliyun" or "aliyun-internal" or "tencent" or "huawei" or "azure"
s3Source="aliyun"
// s3Endpoint = ""
// s3BucketName = ""
// s3Region = ""
ak="***********"
sk="***********"

// jdbc connector test config
// To enable jdbc test, you need first start mysql/pg container.
Expand Down Expand Up @@ -194,10 +200,6 @@ aliYunSk="***********"
hwYunAk="***********"
hwYunSk="***********"

s3Endpoint = "cos.ap-hongkong.myqcloud.com"
s3BucketName = "doris-build-hk-1308700295"
s3Region = "ap-hongkong"

//arrow flight sql test config
extArrowFlightSqlHost = "127.0.0.1"
extArrowFlightSqlPort = 8080
Expand Down
72 changes: 71 additions & 1 deletion regression-test/framework/src/main/groovy/org/apache/doris/regression/Config.groovy
View file
Original file line number Diff line number Diff line change
Expand Up @@ -144,9 +144,12 @@ class Config {
public String kafkaBrokerList
public String cloudVersion

public String s3Source

Config() {}

Config(
String s3Source,
String caseNamePrefix,
String defaultDb,
String jdbcUrl,
Expand Down Expand Up @@ -199,6 +202,7 @@ class Config {
String clusterDir,
String kafkaBrokerList,
String cloudVersion) {
this.s3Source = s3Source
this.caseNamePrefix = caseNamePrefix
this.defaultDb = defaultDb
this.jdbcUrl = jdbcUrl
Expand Down Expand Up @@ -436,7 +440,6 @@ class Config {
}
log.info("recycleAddr : $config.recycleServiceHttpAddress, socketAddr : $config.recycleServiceHttpInetSocketAddress")


config.defaultDb = cmd.getOptionValue(defaultDbOpt, config.defaultDb)
config.jdbcUrl = cmd.getOptionValue(jdbcOpt, config.jdbcUrl)
config.jdbcUser = cmd.getOptionValue(userOpt, config.jdbcUser)
Expand Down Expand Up @@ -465,6 +468,16 @@ class Config {
log.info("withOutLoadData is ${config.withOutLoadData}".toString())
log.info("caseNamePrefix is ${config.caseNamePrefix}".toString())
log.info("dryRun is ${config.dryRun}".toString())
def s3SourceList = ["aliyun", "aliyun-internal", "tencent", "huawei", "azure"]
if (s3SourceList.contains(config.s3Source)) {
log.info("s3Source is ${config.s3Source}".toString())
log.info("s3Provider is ${config.otherConfigs.get("s3Provider")}".toString())
log.info("s3BucketName is ${config.otherConfigs.get("s3BucketName")}".toString())
log.info("s3Region is ${config.otherConfigs.get("s3Region")}".toString())
log.info("s3Endpoint is ${config.otherConfigs.get("s3Endpoint")}".toString())
} else {
throw new Exception("The s3Source '${config.s3Source}' is invalid, optional values ${s3SourceList}")
}

Properties props = cmd.getOptionProperties("conf")
config.otherConfigs.putAll(props)
Expand All @@ -477,6 +490,7 @@ class Config {

static Config fromConfigObject(ConfigObject obj) {
def config = new Config(
configToString(obj.s3Source),
configToString(obj.caseNamePrefix),
configToString(obj.defaultDb),
configToString(obj.jdbcUrl),
Expand Down Expand Up @@ -589,6 +603,62 @@ class Config {
}

static void fillDefaultConfig(Config config) {
if (config.s3Source == null) {
config.s3Source = "aliyun"
log.info("Set s3Source to 'aliyun' because not specify.".toString())
}

if (config.otherConfigs.get("s3Provider") == null) {
if (config.s3Source == "aliyun" || config.s3Source == "aliyun-internal") {
config.otherConfigs.put("s3Provider", "OSS")
} else if (config.s3Source == "tencent") {
config.otherConfigs.put("s3Provider", "COS")
} else if (config.s3Source == "huawei") {
config.otherConfigs.put("s3Provider", "OBS")
} else if (config.s3Source == "azure") {
config.otherConfigs.put("s3Provider", "AZURE")
}
}
if (config.otherConfigs.get("s3BucketName") == null) {
if (config.s3Source == "aliyun") {
config.otherConfigs.put("s3BucketName", "doris-regression-hk")
} else if (config.s3Source == "aliyun-internal") {
config.otherConfigs.put("s3BucketName", "doris-regression")
} else if (config.s3Source == "tencent") {
config.otherConfigs.put("s3BucketName", "doris-build-1308700295")
} else if (config.s3Source == "huawei") {
config.otherConfigs.put("s3BucketName", "doris-build")
} else if (config.s3Source == "azure") {
config.otherConfigs.put("s3BucketName", "doris-regression")
}
}
if (config.otherConfigs.get("s3Region") == null) {
if (config.s3Source == "aliyun") {
config.otherConfigs.put("s3Region", "oss-cn-hongkong")
} else if (config.s3Source == "aliyun-internal") {
config.otherConfigs.put("s3Region", "oss-cn-beijing")
} else if (config.s3Source == "tencent") {
config.otherConfigs.put("s3Region", "ap-beijing")
} else if (config.s3Source == "huawei") {
config.otherConfigs.put("s3Region", "cn-north-4")
} else if (config.s3Source == "azure") {
config.otherConfigs.put("s3Region", "azure-region")
}
}
if (config.otherConfigs.get("s3Endpoint") == null) {
if (config.s3Source == "aliyun") {
config.otherConfigs.put("s3Endpoint", "oss-cn-hongkong.aliyuncs.com")
} else if (config.s3Source == "aliyun-internal") {
config.otherConfigs.put("s3Endpoint", "oss-cn-beijing-internal.aliyuncs.com")
} else if (config.s3Source == "tencent") {
config.otherConfigs.put("s3Endpoint", "cos.ap-beijing.myqcloud.com")
} else if (config.s3Source == "huawei") {
config.otherConfigs.put("s3Endpoint", "obs.cn-north-4.myhuaweicloud.com")
} else if (config.s3Source == "azure") {
config.otherConfigs.put("s3Endpoint", "azure-endpoint")
}
}

if (config.caseNamePrefix == null) {
config.caseNamePrefix = ""
log.info("set caseNamePrefix to '' because not specify.".toString())
Expand Down

0 comments on commit ea40ff7

Please sign in to comment.